From 4a12286beb9dbb2ac2662954d1e6e28f8a217ef5 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 24 Mar 2023 14:20:31 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E8=A3=85random=20useragent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/option.go | 47 ++++++++-------- internal/pool.go | 6 ++- internal/runner.go | 130 +++++++++++++++++++++++---------------------- internal/utils.go | 23 ++++++++ pkg/config.go | 49 ++++++++--------- 5 files changed, 143 insertions(+), 112 deletions(-) diff --git a/internal/option.go b/internal/option.go index 180de99..0eb8013 100644 --- a/internal/option.go +++ b/internal/option.go @@ -126,29 +126,30 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } var err error r := &Runner{ - Progress: uiprogress.New(), - Threads: opt.Threads, - PoolSize: opt.PoolSize, - Mod: opt.Mod, - Timeout: opt.Timeout, - RateLimit: opt.RateLimit, - Deadline: opt.Deadline, - Headers: make(map[string]string), - Offset: opt.Offset, - Total: opt.Limit, - taskCh: make(chan *Task), - OutputCh: make(chan *pkg.Baseline, 100), - FuzzyCh: make(chan *pkg.Baseline, 100), - Fuzzy: opt.Fuzzy, - Force: opt.Force, - CheckOnly: opt.CheckOnly, - CheckPeriod: opt.CheckPeriod, - ErrPeriod: opt.ErrPeriod, - BreakThreshold: opt.BreakThreshold, - Crawl: opt.Crawl, - Active: opt.Active, - Bak: opt.Bak, - Common: opt.Common, + Progress: uiprogress.New(), + Threads: opt.Threads, + PoolSize: opt.PoolSize, + Mod: opt.Mod, + Timeout: opt.Timeout, + RateLimit: opt.RateLimit, + Deadline: opt.Deadline, + Headers: make(map[string]string), + Offset: opt.Offset, + Total: opt.Limit, + taskCh: make(chan *Task), + OutputCh: make(chan *pkg.Baseline, 100), + FuzzyCh: make(chan *pkg.Baseline, 100), + Fuzzy: opt.Fuzzy, + Force: opt.Force, + CheckOnly: opt.CheckOnly, + CheckPeriod: opt.CheckPeriod, + ErrPeriod: opt.ErrPeriod, + BreakThreshold: opt.BreakThreshold, + Crawl: opt.Crawl, + Active: opt.Active, + Bak: opt.Bak, + Common: opt.Common, + RandomUserAgent: opt.RandomUserAgent, } // log and bar diff --git a/internal/pool.go b/internal/pool.go index 1bcf6f3..4009435 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -16,6 +16,7 @@ import ( "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" "golang.org/x/time/rate" + "math/rand" "net/url" "path" "strings" @@ -61,7 +62,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { limiter: rate.NewLimiter(rate.Limit(config.RateLimit), 1), failedCount: 1, } - + rand.Seed(time.Now().UnixNano()) // 格式化dir, 保证至少有一个"/" if strings.HasSuffix(config.BaseURL, "/") { pool.dir = pool.url.Path @@ -257,7 +258,10 @@ func (pool *Pool) Invoke(v interface{}) { logs.Log.Error(err.Error()) return } + req.SetHeaders(pool.Headers) + req.SetHeader("User-Agent", RandomUA()) + start := time.Now() resp, reqerr := pool.client.Do(pool.ctx, req) if pool.ClientType == ihttp.FAST { diff --git a/internal/runner.go b/internal/runner.go index 9c17545..03f847b 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -36,74 +36,76 @@ type Runner struct { bar *uiprogress.Bar finished int - Tasks []*Task - URLList []string - Wordlist []string - Rules *rule.Program - AppendRules *rule.Program - Headers map[string]string - Fns []func(string) string - FilterExpr *vm.Program - MatchExpr *vm.Program - RecursiveExpr *vm.Program - RecuDepth int - Threads int - PoolSize int - ClientType int - Pools *ants.PoolWithFunc - PoolName map[string]bool - Timeout int - Mod string - Probes []string - OutputCh chan *pkg.Baseline - FuzzyCh chan *pkg.Baseline - Fuzzy bool - OutputFile *files.File - FuzzyFile *files.File - DumpFile *files.File - StatFile *files.File - Progress *uiprogress.Progress - Offset int - Limit int - RateLimit int - Total int - Deadline int - CheckPeriod int - ErrPeriod int - BreakThreshold int - Color bool - CheckOnly bool - Force bool - IgnoreWaf bool - Crawl bool - Active bool - Bak bool - Common bool + Tasks []*Task + URLList []string + Wordlist []string + Rules *rule.Program + AppendRules *rule.Program + Headers map[string]string + Fns []func(string) string + FilterExpr *vm.Program + MatchExpr *vm.Program + RecursiveExpr *vm.Program + RecuDepth int + Threads int + PoolSize int + ClientType int + Pools *ants.PoolWithFunc + PoolName map[string]bool + Timeout int + Mod string + Probes []string + OutputCh chan *pkg.Baseline + FuzzyCh chan *pkg.Baseline + Fuzzy bool + OutputFile *files.File + FuzzyFile *files.File + DumpFile *files.File + StatFile *files.File + Progress *uiprogress.Progress + Offset int + Limit int + RateLimit int + Total int + Deadline int + CheckPeriod int + ErrPeriod int + BreakThreshold int + Color bool + CheckOnly bool + Force bool + IgnoreWaf bool + Crawl bool + Active bool + Bak bool + Common bool + RandomUserAgent bool } func (r *Runner) PrepareConfig() *pkg.Config { config := &pkg.Config{ - Thread: r.Threads, - Timeout: r.Timeout, - RateLimit: r.RateLimit, - Headers: r.Headers, - Mod: pkg.ModMap[r.Mod], - OutputCh: r.OutputCh, - FuzzyCh: r.FuzzyCh, - Fuzzy: r.Fuzzy, - CheckPeriod: r.CheckPeriod, - ErrPeriod: int32(r.ErrPeriod), - BreakThreshold: int32(r.BreakThreshold), - MatchExpr: r.MatchExpr, - FilterExpr: r.FilterExpr, - RecuExpr: r.RecursiveExpr, - AppendRule: r.AppendRules, - IgnoreWaf: r.IgnoreWaf, - Crawl: r.Crawl, - Active: r.Active, - Bak: r.Bak, - Common: r.Common, - ClientType: r.ClientType, + Thread: r.Threads, + Timeout: r.Timeout, + RateLimit: r.RateLimit, + Headers: r.Headers, + Mod: pkg.ModMap[r.Mod], + OutputCh: r.OutputCh, + FuzzyCh: r.FuzzyCh, + Fuzzy: r.Fuzzy, + CheckPeriod: r.CheckPeriod, + ErrPeriod: int32(r.ErrPeriod), + BreakThreshold: int32(r.BreakThreshold), + MatchExpr: r.MatchExpr, + FilterExpr: r.FilterExpr, + RecuExpr: r.RecursiveExpr, + AppendRule: r.AppendRules, + IgnoreWaf: r.IgnoreWaf, + Crawl: r.Crawl, + Active: r.Active, + Bak: r.Bak, + Common: r.Common, + ClientType: r.ClientType, + RandomUserAgent: r.RandomUserAgent, } if config.ClientType == 0 { diff --git a/internal/utils.go b/internal/utils.go index b0bd2f6..a551644 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -6,12 +6,31 @@ import ( "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "io/ioutil" + "math/rand" "net/url" "path" "strconv" "strings" ) +var ( + randomUserAgent = []string{ + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254", + "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246", + "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", + } + uacount = len(randomUserAgent) +) + func parseExtension(s string) string { if i := strings.Index(s, "."); i != -1 { return s[i+1:] @@ -232,3 +251,7 @@ func FormatURL(base, u string) string { return relaPath(base, u) } } + +func RandomUA() string { + return randomUserAgent[rand.Intn(uacount)] +} diff --git a/pkg/config.go b/pkg/config.go index 17cfe9f..720dd3a 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -20,28 +20,29 @@ var ModMap = map[string]SprayMod{ } type Config struct { - BaseURL string - Thread int - Wordlist []string - Timeout int - RateLimit int - CheckPeriod int - ErrPeriod int32 - BreakThreshold int32 - Method string - Mod SprayMod - Headers map[string]string - ClientType int - MatchExpr *vm.Program - FilterExpr *vm.Program - RecuExpr *vm.Program - AppendRule *rule.Program - OutputCh chan *Baseline - FuzzyCh chan *Baseline - Fuzzy bool - IgnoreWaf bool - Crawl bool - Active bool - Bak bool - Common bool + BaseURL string + Thread int + Wordlist []string + Timeout int + RateLimit int + CheckPeriod int + ErrPeriod int32 + BreakThreshold int32 + Method string + Mod SprayMod + Headers map[string]string + ClientType int + MatchExpr *vm.Program + FilterExpr *vm.Program + RecuExpr *vm.Program + AppendRule *rule.Program + OutputCh chan *Baseline + FuzzyCh chan *Baseline + Fuzzy bool + IgnoreWaf bool + Crawl bool + Active bool + Bak bool + Common bool + RandomUserAgent bool }