diff --git a/cmd/goop.go b/cmd/goop.go index 445aeda..7e824df 100644 --- a/cmd/goop.go +++ b/cmd/goop.go @@ -11,6 +11,7 @@ import ( var force bool var keep bool var list bool +var useragent string var rootCmd = &cobra.Command{ Use: "goop", Short: "goop is a very fast tool to grab sources from exposed .git folders", @@ -21,12 +22,12 @@ var rootCmd = &cobra.Command{ dir = args[1] } if list { - if err := goop.CloneList(args[0], dir, force, keep); err != nil { + if err := goop.CloneList(args[0], dir, force, keep, useragent); err != nil { log.Error().Err(err).Msg("exiting") os.Exit(1) } } else { - if err := goop.Clone(args[0], dir, force, keep); err != nil { + if err := goop.Clone(args[0], dir, force, keep, useragent); err != nil { log.Error().Err(err).Msg("exiting") os.Exit(1) } @@ -38,6 +39,7 @@ func init() { rootCmd.PersistentFlags().BoolVarP(&force, "force", "f", false, "overrides DIR if it already exists") rootCmd.PersistentFlags().BoolVarP(&keep, "keep", "k", false, "keeps already downloaded files in DIR, useful if you keep being ratelimited by server") rootCmd.PersistentFlags().BoolVarP(&list, "list", "l", false, "allows you to supply the name of a file containing a list of domain names instead of just one domain") + rootCmd.PersistentFlags().StringVarP(&useragent, "useragent", "u", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36", "allows you to change the default User-Agent") } func Execute() { diff --git a/pkg/goop/clone.go b/pkg/goop/clone.go index c985af3..a9175ea 100644 --- a/pkg/goop/clone.go +++ b/pkg/goop/clone.go @@ -58,18 +58,31 @@ func proxyFromEnv() fasthttp.DialFunc { return nil } -var c = &fasthttp.Client{ - Name: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36", - MaxConnsPerHost: utils.MaxInt(maxConcurrency+250, fasthttp.DefaultMaxConnsPerHost), - TLSConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - NoDefaultUserAgentHeader: true, - MaxConnWaitTimeout: 10 * time.Second, - Dial: proxyFromEnv(), +var c *fasthttp.Client + +func CustomUE(useragent string) *fasthttp.Client { + if strings.Compare(useragent, "") != 0 { + c = &fasthttp.Client{ + Name: useragent, + MaxConnsPerHost: utils.MaxInt(maxConcurrency+250, fasthttp.DefaultMaxConnsPerHost), + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + NoDefaultUserAgentHeader: true, + MaxConnWaitTimeout: 10 * time.Second, + Dial: proxyFromEnv(), + } + } else { + c.Name = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36" + } + + return c } -func CloneList(listFile, baseDir string, force, keep bool) error { +func CloneList(listFile, baseDir string, force, keep bool, useragent string) error { + // Check for custom UserAgent + CustomUE(useragent) + lf, err := os.Open(listFile) if err != nil { return err @@ -92,14 +105,17 @@ func CloneList(listFile, baseDir string, force, keep bool) error { dir = utils.Url(dir, parsed.Host) } log.Info().Str("target", u).Str("dir", dir).Bool("force", force).Bool("keep", keep).Msg("starting download") - if err := Clone(u, dir, force, keep); err != nil { - log.Error().Str("target", u).Str("dir", dir).Bool("force", force).Bool("keep", keep).Msg("download failed") + if err := Clone(u, dir, force, keep, useragent); err != nil { + log.Error().Str("target", u).Str("dir", dir).Bool("force", force).Bool("keep", keep).Str("useragent", useragent).Msg("download failed") } } return nil } -func Clone(u, dir string, force, keep bool) error { +func Clone(u, dir string, force, keep bool, useragent string) error { + // Check for custom UserAgent + CustomUE(useragent) + baseUrl := strings.TrimSuffix(u, "/") baseUrl = strings.TrimSuffix(baseUrl, "/HEAD") baseUrl = strings.TrimSuffix(baseUrl, "/.git")