From 0066d28554106235d0f2a5709b5eaf85056a8c71 Mon Sep 17 00:00:00 2001 From: Ivan C Date: Sun, 21 May 2023 12:11:25 +0300 Subject: [PATCH 1/2] Added JSON Lines output --- README.md | 2 +- runner/subjs/options.go | 2 ++ runner/subjs/runner.go | 66 +++++++++++++++++++++++++++++++---------- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7b9fd35..06a60bd 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ $ subjs -h | `-t` | Timeout (in seconds) for http client (default 15) | `subjs -t 20` | | `-ua` | User-Agent to send in requests | `subjs -ua "Chrome..."` | | `-version` | Show version number | `subjs -version"` | - +| `-jsl` | JSON Lines output (contains source url as well) | `subjs -jsl`| ## Installation ### From Source: diff --git a/runner/subjs/options.go b/runner/subjs/options.go index 73a3fe8..7502527 100644 --- a/runner/subjs/options.go +++ b/runner/subjs/options.go @@ -11,6 +11,7 @@ type Options struct { Workers int Timeout int UserAgent string + JSONLines bool } func ParseOptions() *Options { @@ -19,6 +20,7 @@ func ParseOptions() *Options { flag.StringVar(&opts.UserAgent, "ua", "", "User-Agent to send in requests") flag.IntVar(&opts.Workers, "c", 10, "Number of concurrent workers") flag.IntVar(&opts.Timeout, "t", 15, "Timeout (in seconds) for http client") + flag.BoolVar(&opts.JSONLines, "jsl", false, "JSON Lines output") showVersion := flag.Bool("version", false, "Show version number") flag.Parse() if *showVersion { diff --git a/runner/subjs/runner.go b/runner/subjs/runner.go index 4a1a3a6..c7eddb7 100644 --- a/runner/subjs/runner.go +++ b/runner/subjs/runner.go @@ -15,7 +15,7 @@ import ( "github.com/PuerkitoBio/goquery" ) -const version = `1.0.2` +const version = `1.0.3` type SubJS struct { client *http.Client @@ -40,7 +40,7 @@ func (s *SubJS) Run() error { // otherwise read from file input, err = os.Open(s.opts.InputFile) if err != nil { - return fmt.Errorf("Could not open input file: %s", err) + return fmt.Errorf("Could not open input file: %s\"}", err) } defer input.Close() } @@ -104,9 +104,23 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) { //log.Fatalf("error parsing url: %v", err) return } - doc.Find("script").Each(func(index int, s *goquery.Selection) { - js, _ := s.Attr("src") - if js != "" { + doc.Find("script").Each(func(index int, z *goquery.Selection) { + js, _ := z.Attr("src") + if js != "" && s.opts.JSONLines { + if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { + js := fmt.Sprintf("{\"%s\": \"%s\"}", u, js) + results <- js + } else if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js) + results <- js + } else if strings.HasPrefix(js, "/") { + js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js) + results <- js + } else { + js := fmt.Sprintf("{\"%s\": \"%s://%s/%s\"}", u, u.Scheme, u.Host, js) + results <- js + } + } else if js != "" { if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { results <- js } else if strings.HasPrefix(js, "//") { @@ -120,21 +134,43 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) { results <- js } } - r := regexp.MustCompile(`[(\w./:)]*js`) - matches := r.FindAllString(s.Contents().Text(), -1) + r := regexp.MustCompile(`[(\w./:)]*\.js`) + matches := r.FindAllString(z.Contents().Text(), -1) for _, js := range matches { - if strings.HasPrefix(js, "//") { - js := fmt.Sprintf("%s:%s", u.Scheme, js) + if s.opts.JSONLines { + if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js) + results <- js + } else if strings.HasPrefix(js, "/") { + js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js) + results <- js + } + } else { + if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("%s:%s", u.Scheme, js) + results <- js + } else if strings.HasPrefix(js, "/") { + js := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, js) + results <- js + } + } + }}) + doc.Find("div").Each(func(index int, z *goquery.Selection) { + js, _ := z.Attr("data-script-src") + if js != "" && s.opts.JSONLines { + if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { + results <- js + } else if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js) results <- js } else if strings.HasPrefix(js, "/") { - js := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, js) + js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js) + results <- js + } else { + js := fmt.Sprintf("{\"%s\": \"%s://%s/%s\"}", u, u.Scheme, u.Host, js) results <- js } - } - }) - doc.Find("div").Each(func(index int, s *goquery.Selection) { - js, _ := s.Attr("data-script-src") - if js != "" { + } else if js != "" { if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { results <- js } else if strings.HasPrefix(js, "//") { From aa19abc2b241eb6498b266d25aa16d8f1362a6f6 Mon Sep 17 00:00:00 2001 From: Ivan C Date: Sun, 21 May 2023 22:45:26 +0300 Subject: [PATCH 2/2] Added Cookie Header --- README.md | 3 ++- runner/subjs/options.go | 2 ++ runner/subjs/runner.go | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 06a60bd..a4021d4 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,13 @@ $ subjs -h | `-ua` | User-Agent to send in requests | `subjs -ua "Chrome..."` | | `-version` | Show version number | `subjs -version"` | | `-jsl` | JSON Lines output (contains source url as well) | `subjs -jsl`| +| `-cookie` | Cookie to send in requests | `subjs -cookie "JSESSIONID...` | ## Installation ### From Source: ``` -$ GO111MODULE=on go get -u -v github.com/lc/subjs@latest +$ GO111MODULE=on go install -v github.com/lc/subjs@latest ``` ### From Binary diff --git a/runner/subjs/options.go b/runner/subjs/options.go index 7502527..b1335e9 100644 --- a/runner/subjs/options.go +++ b/runner/subjs/options.go @@ -12,12 +12,14 @@ type Options struct { Timeout int UserAgent string JSONLines bool + Cookie string } func ParseOptions() *Options { opts := &Options{} flag.StringVar(&opts.InputFile, "i", "", "Input file containing URLS") flag.StringVar(&opts.UserAgent, "ua", "", "User-Agent to send in requests") + flag.StringVar(&opts.Cookie, "cookie", "", "Cookie to send in requests") flag.IntVar(&opts.Workers, "c", 10, "Number of concurrent workers") flag.IntVar(&opts.Timeout, "t", 15, "Timeout (in seconds) for http client") flag.BoolVar(&opts.JSONLines, "jsl", false, "JSON Lines output") diff --git a/runner/subjs/runner.go b/runner/subjs/runner.go index c7eddb7..92eee7b 100644 --- a/runner/subjs/runner.go +++ b/runner/subjs/runner.go @@ -89,6 +89,9 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) { if s.opts.UserAgent != "" { req.Header.Add("User-Agent", s.opts.UserAgent) } + if s.opts.Cookie != "" { + req.Header.Add("Cookie", s.opts.Cookie) + } resp, err := s.client.Do(req) if err != nil { continue