From 693f7e4e715e24220e8a0a4cc9aecb7f2dbe7d3f Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 14 Oct 2024 01:54:57 +0800 Subject: [PATCH] add `--append-depth` limit append recu depth --- internal/option.go | 5 +- internal/pool/brutepool.go | 163 ++++++++++++++++++------------------- internal/pool/checkpool.go | 2 +- internal/pool/config.go | 69 ++++++++++++++++ internal/pool/pool.go | 2 +- internal/pool/type.go | 47 +++++++++++ internal/pool/types.go | 107 ------------------------ internal/runner.go | 27 +++--- 8 files changed, 216 insertions(+), 206 deletions(-) create mode 100644 internal/pool/config.go create mode 100644 internal/pool/type.go delete mode 100644 internal/pool/types.go diff --git a/internal/option.go b/internal/option.go index 87ec03f..e537389 100644 --- a/internal/option.go +++ b/internal/option.go @@ -115,6 +115,7 @@ type PluginOptions struct { CommonPlugin bool `long:"common" description:"Bool, enable common file found" config:"common"` CrawlPlugin bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"` CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth" config:"crawl-depth"` + AppendDepth int `long:"append-depth" default:"2" description:"Int, append depth" config:"append-depth"` } type ModeOptions struct { @@ -253,7 +254,6 @@ func (opt *Option) Prepare() error { logs.Log.Logf(pkg.LogVerbose, "Black Status: %v, WhiteStatus: %v, WAFStatus: %v", pkg.BlackStatus, pkg.WhiteStatus, pkg.WAFStatus) logs.Log.Logf(pkg.LogVerbose, "Fuzzy Status: %v, Unique Status: %v", pkg.FuzzyStatus, pkg.UniqueStatus) - pool.MaxCrawl = opt.CrawlDepth return nil } @@ -354,13 +354,12 @@ func (opt *Option) NewRunner() (*Runner, error) { var express string if opt.Recursive != "current.IsDir()" && opt.Depth != 0 { // 默认不打开递归, 除非指定了非默认的递归表达式 - pool.MaxRecursion = 1 + opt.Depth = 1 express = opt.Recursive } if opt.Depth != 0 { // 手动设置的depth优先级高于默认 - pool.MaxRecursion = opt.Depth express = opt.Recursive } diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index fbd9099..a5d270b 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -23,9 +23,6 @@ import ( ) var ( - MaxRedirect = 3 - MaxCrawl = 3 - MaxRecursion = 0 EnableAllFuzzy = false EnableAllUnique = false //AllowHostModSource = []parsers.SpraySource{parsers.WordSource, parsers.CheckSource, parsers.InitIndexSource, parsers.InitRandomSource} @@ -494,7 +491,7 @@ func (pool *BrutePool) Handler() { // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 if bl.IsValid { pool.Statistor.FoundNumber++ - if bl.RecuDepth < MaxRecursion { + if bl.RecuDepth < pool.MaxRecursionDepth { if pkg.CompareWithExpr(pool.RecuExpr, params) { bl.Recu = true } @@ -511,76 +508,6 @@ func (pool *BrutePool) Handler() { pool.analyzeDone = true } -func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { - if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource { - pool.wg.Done() - return - } - - go func() { - defer pool.wg.Done() - for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { - pool.addAddition(&Unit{ - path: pkg.Dir(bl.Url.Path) + u, - host: bl.Host, - source: parsers.AppendRuleSource, - }) - } - }() -} - -func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) { - if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource { - // 防止自身递归 - pool.wg.Done() - return - } - - go func() { - defer pool.wg.Done() - - for u := range NewBruteWords(pool.Config, pool.AppendWords).Output { - pool.addAddition(&Unit{ - path: pkg.SafePath(bl.Path, u), - host: bl.Host, - source: parsers.AppendSource, - }) - } - }() -} - -func (pool *BrutePool) doAppend(bl *pkg.Baseline) { - pool.wg.Add(2) - pool.doAppendWords(bl) - pool.doAppendRule(bl) -} - -func (pool *BrutePool) doActive() { - defer pool.wg.Done() - if pool.Mod == HostSpray { - return - } - for _, u := range pkg.ActivePath { - pool.addAddition(&Unit{ - path: pool.dir + u[1:], - source: parsers.FingerSource, - }) - } -} - -func (pool *BrutePool) doCommonFile() { - defer pool.wg.Done() - if pool.Mod == HostSpray { - return - } - for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output { - pool.addAddition(&Unit{ - path: pool.dir + u, - source: parsers.CommonFileSource, - }) - } -} - func (pool *BrutePool) checkRedirect(redirectURL string) bool { if pool.random.RedirectURL == "" { // 如果random的redirectURL为空, 此时该项 @@ -758,7 +685,7 @@ func (pool *BrutePool) doCheck() { } func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { - if !pool.Crawl || bl.ReqDepth >= MaxCrawl { + if !pool.Crawl || bl.ReqDepth >= pool.MaxCrawlDepth { return } @@ -788,7 +715,7 @@ func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { } func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) { - if bl.ReqDepth >= MaxCrawl { + if bl.ReqDepth >= pool.MaxCrawlDepth { pool.wg.Done() return } @@ -817,12 +744,12 @@ func (pool *BrutePool) doBak() { if pool.Mod == HostSpray { return } - for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output { - pool.addAddition(&Unit{ - path: pool.dir + w, - source: parsers.BakSource, - }) - } + //for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output { + // pool.addAddition(&Unit{ + // path: pool.dir + w, + // source: parsers.BakSource, + // }) + //} for w := range NewBruteDSL(pool.Config, "{?@bak_name}.{?@bak_ext}", nil).Output { pool.addAddition(&Unit{ @@ -831,3 +758,75 @@ func (pool *BrutePool) doBak() { }) } } + +func (pool *BrutePool) doAppend(bl *pkg.Baseline) { + pool.wg.Add(2) + pool.doAppendWords(bl) + pool.doAppendRule(bl) +} + +func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { + if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource || bl.ReqDepth >= pool.MaxAppendDepth { + pool.wg.Done() + return + } + + go func() { + defer pool.wg.Done() + for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { + pool.addAddition(&Unit{ + path: pkg.Dir(bl.Url.Path) + u, + host: bl.Host, + source: parsers.AppendRuleSource, + depth: bl.ReqDepth + 1, + }) + } + }() +} + +func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) { + if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource || bl.ReqDepth >= pool.MaxAppendDepth { + // 防止自身递归 + pool.wg.Done() + return + } + + go func() { + defer pool.wg.Done() + + for u := range NewBruteWords(pool.Config, pool.AppendWords).Output { + pool.addAddition(&Unit{ + path: pkg.SafePath(bl.Path, u), + host: bl.Host, + source: parsers.AppendSource, + depth: bl.RecuDepth + 1, + }) + } + }() +} + +func (pool *BrutePool) doActive() { + defer pool.wg.Done() + if pool.Mod == HostSpray { + return + } + for _, u := range pkg.ActivePath { + pool.addAddition(&Unit{ + path: pool.dir + u[1:], + source: parsers.FingerSource, + }) + } +} + +func (pool *BrutePool) doCommonFile() { + defer pool.wg.Done() + if pool.Mod == HostSpray { + return + } + for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output { + pool.addAddition(&Unit{ + path: pool.dir + u, + source: parsers.CommonFileSource, + }) + } +} diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 9d9d23f..62a9b36 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -181,7 +181,7 @@ func (pool *CheckPool) Handler() { } func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { - if depth >= MaxRedirect { + if depth >= pool.MaxRedirect { return } var reURL string diff --git a/internal/pool/config.go b/internal/pool/config.go new file mode 100644 index 0000000..9f49652 --- /dev/null +++ b/internal/pool/config.go @@ -0,0 +1,69 @@ +package pool + +import ( + "github.com/chainreactors/logs" + "github.com/chainreactors/spray/pkg" + "github.com/chainreactors/words" + "github.com/chainreactors/words/rule" + "github.com/expr-lang/expr/vm" + "sync" + "time" +) + +type Config struct { + BaseURL string + ProxyAddr string + Thread int + Wordlist []string + Timeout time.Duration + ProcessCh chan *pkg.Baseline + OutputCh chan *pkg.Baseline + FuzzyCh chan *pkg.Baseline + Outwg *sync.WaitGroup + RateLimit int + CheckPeriod int + ErrPeriod int32 + BreakThreshold int32 + Method string + Mod SprayMod + Headers map[string]string + ClientType int + MatchExpr *vm.Program + FilterExpr *vm.Program + RecuExpr *vm.Program + AppendRule *rule.Program + Fns []words.WordFunc + AppendWords []string + Fuzzy bool + IgnoreWaf bool + Crawl bool + Scope []string + Active bool + Bak bool + Common bool + RetryLimit int + RandomUserAgent bool + Random string + Index string + MaxRedirect int + MaxCrawlDepth int + MaxRecursionDepth int + MaxAppendDepth int +} + +func NewBruteWords(config *Config, list []string) *words.Worder { + word := words.NewWorderWithList(list) + word.Fns = config.Fns + word.Run() + return word +} + +func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder { + word, err := words.NewWorderWithDsl(dsl, params, nil) + if err != nil { + logs.Log.Error(err.Error()) + } + word.Fns = config.Fns + word.Run() + return word +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 85357ed..34f3a36 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -29,7 +29,7 @@ type BasePool struct { } func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) { - if depth >= MaxRedirect { + if depth >= pool.MaxRedirect { return } reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) diff --git a/internal/pool/type.go b/internal/pool/type.go new file mode 100644 index 0000000..ed4381f --- /dev/null +++ b/internal/pool/type.go @@ -0,0 +1,47 @@ +package pool + +import ( + "github.com/chainreactors/parsers" + "github.com/chainreactors/spray/pkg" +) + +func newUnit(path string, source parsers.SpraySource) *Unit { + return &Unit{path: path, source: source} +} + +type Unit struct { + number int + host string + path string + source parsers.SpraySource + retry int + frontUrl string + depth int +} + +func NewBaselines() *Baselines { + return &Baselines{ + baselines: map[int]*pkg.Baseline{}, + } +} + +type Baselines struct { + FailedBaselines []*pkg.Baseline + random *pkg.Baseline + index *pkg.Baseline + baselines map[int]*pkg.Baseline +} + +type SprayMod int + +const ( + PathSpray SprayMod = iota + 1 + HostSpray + ParamSpray + CustomSpray +) + +var ModMap = map[string]SprayMod{ + "path": PathSpray, + "host": HostSpray, +} diff --git a/internal/pool/types.go b/internal/pool/types.go deleted file mode 100644 index 87128da..0000000 --- a/internal/pool/types.go +++ /dev/null @@ -1,107 +0,0 @@ -package pool - -import ( - "github.com/chainreactors/logs" - "github.com/chainreactors/parsers" - "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/words" - "github.com/chainreactors/words/rule" - "github.com/expr-lang/expr/vm" - "sync" - "time" -) - -func newUnit(path string, source parsers.SpraySource) *Unit { - return &Unit{path: path, source: source} -} - -type Unit struct { - number int - host string - path string - source parsers.SpraySource - retry int - frontUrl string - depth int // redirect depth -} - -func NewBaselines() *Baselines { - return &Baselines{ - baselines: map[int]*pkg.Baseline{}, - } -} - -type Baselines struct { - FailedBaselines []*pkg.Baseline - random *pkg.Baseline - index *pkg.Baseline - baselines map[int]*pkg.Baseline -} - -type SprayMod int - -const ( - PathSpray SprayMod = iota + 1 - HostSpray - ParamSpray - CustomSpray -) - -var ModMap = map[string]SprayMod{ - "path": PathSpray, - "host": HostSpray, -} - -type Config struct { - BaseURL string - ProxyAddr string - Thread int - Wordlist []string - Timeout time.Duration - ProcessCh chan *pkg.Baseline - OutputCh chan *pkg.Baseline - FuzzyCh chan *pkg.Baseline - Outwg *sync.WaitGroup - RateLimit int - CheckPeriod int - ErrPeriod int32 - BreakThreshold int32 - Method string - Mod SprayMod - Headers map[string]string - ClientType int - MatchExpr *vm.Program - FilterExpr *vm.Program - RecuExpr *vm.Program - AppendRule *rule.Program - Fns []words.WordFunc - AppendWords []string - Fuzzy bool - IgnoreWaf bool - Crawl bool - Scope []string - Active bool - Bak bool - Common bool - RetryLimit int - RandomUserAgent bool - Random string - Index string -} - -func NewBruteWords(config *Config, list []string) *words.Worder { - word := words.NewWorderWithList(list) - word.Fns = config.Fns - word.Run() - return word -} - -func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder { - word, err := words.NewWorderWithDsl(dsl, params, nil) - if err != nil { - logs.Log.Error(err.Error()) - } - word.Fns = config.Fns - word.Run() - return word -} diff --git a/internal/runner.go b/internal/runner.go index 9a282ef..5b50e1e 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -51,7 +51,6 @@ type Runner struct { Count int // tasks total number Wordlist []string AppendWords []string - RecuDepth int ClientType int Probes []string Total int // wordlist total number @@ -81,17 +80,21 @@ func (r *Runner) PrepareConfig() *pool.Config { AppendWords: r.AppendWords, // 对有效目录追加字典 Fns: r.Fns, //IgnoreWaf: r.IgnoreWaf, - Crawl: r.CrawlPlugin, - Scope: r.Scope, - Active: r.Finger, - Bak: r.BakPlugin, - Common: r.CommonPlugin, - RetryLimit: r.RetryCount, - ClientType: r.ClientType, - RandomUserAgent: r.RandomUserAgent, - Random: r.Random, - Index: r.Index, - ProxyAddr: r.Proxy, + Crawl: r.CrawlPlugin, + Scope: r.Scope, + Active: r.Finger, + Bak: r.BakPlugin, + Common: r.CommonPlugin, + RetryLimit: r.RetryCount, + ClientType: r.ClientType, + RandomUserAgent: r.RandomUserAgent, + Random: r.Random, + Index: r.Index, + ProxyAddr: r.Proxy, + MaxRecursionDepth: r.Depth, + MaxRedirect: 3, + MaxAppendDepth: r.AppendDepth, + MaxCrawlDepth: r.CrawlDepth, } if config.ClientType == ihttp.Auto {