From 344e5604710ab8ff863570cb63ba72838621043b Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 14 Oct 2024 01:54:57 +0800 Subject: [PATCH 1/9] add `--append-depth` limit append recu depth --- internal/option.go | 5 +- internal/pool/brutepool.go | 163 ++++++++++++++++++------------------- internal/pool/checkpool.go | 2 +- internal/pool/config.go | 69 ++++++++++++++++ internal/pool/pool.go | 2 +- internal/pool/type.go | 47 +++++++++++ internal/pool/types.go | 107 ------------------------ internal/runner.go | 27 +++--- 8 files changed, 216 insertions(+), 206 deletions(-) create mode 100644 internal/pool/config.go create mode 100644 internal/pool/type.go delete mode 100644 internal/pool/types.go diff --git a/internal/option.go b/internal/option.go index 87ec03f..e537389 100644 --- a/internal/option.go +++ b/internal/option.go @@ -115,6 +115,7 @@ type PluginOptions struct { CommonPlugin bool `long:"common" description:"Bool, enable common file found" config:"common"` CrawlPlugin bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"` CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth" config:"crawl-depth"` + AppendDepth int `long:"append-depth" default:"2" description:"Int, append depth" config:"append-depth"` } type ModeOptions struct { @@ -253,7 +254,6 @@ func (opt *Option) Prepare() error { logs.Log.Logf(pkg.LogVerbose, "Black Status: %v, WhiteStatus: %v, WAFStatus: %v", pkg.BlackStatus, pkg.WhiteStatus, pkg.WAFStatus) logs.Log.Logf(pkg.LogVerbose, "Fuzzy Status: %v, Unique Status: %v", pkg.FuzzyStatus, pkg.UniqueStatus) - pool.MaxCrawl = opt.CrawlDepth return nil } @@ -354,13 +354,12 @@ func (opt *Option) NewRunner() (*Runner, error) { var express string if opt.Recursive != "current.IsDir()" && opt.Depth != 0 { // 默认不打开递归, 除非指定了非默认的递归表达式 - pool.MaxRecursion = 1 + opt.Depth = 1 express = opt.Recursive } if opt.Depth != 0 { // 手动设置的depth优先级高于默认 - pool.MaxRecursion = opt.Depth express = opt.Recursive } diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index fbd9099..a5d270b 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -23,9 +23,6 @@ import ( ) var ( - MaxRedirect = 3 - MaxCrawl = 3 - MaxRecursion = 0 EnableAllFuzzy = false EnableAllUnique = false //AllowHostModSource = []parsers.SpraySource{parsers.WordSource, parsers.CheckSource, parsers.InitIndexSource, parsers.InitRandomSource} @@ -494,7 +491,7 @@ func (pool *BrutePool) Handler() { // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 if bl.IsValid { pool.Statistor.FoundNumber++ - if bl.RecuDepth < MaxRecursion { + if bl.RecuDepth < pool.MaxRecursionDepth { if pkg.CompareWithExpr(pool.RecuExpr, params) { bl.Recu = true } @@ -511,76 +508,6 @@ func (pool *BrutePool) Handler() { pool.analyzeDone = true } -func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { - if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource { - pool.wg.Done() - return - } - - go func() { - defer pool.wg.Done() - for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { - pool.addAddition(&Unit{ - path: pkg.Dir(bl.Url.Path) + u, - host: bl.Host, - source: parsers.AppendRuleSource, - }) - } - }() -} - -func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) { - if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource { - // 防止自身递归 - pool.wg.Done() - return - } - - go func() { - defer pool.wg.Done() - - for u := range NewBruteWords(pool.Config, pool.AppendWords).Output { - pool.addAddition(&Unit{ - path: pkg.SafePath(bl.Path, u), - host: bl.Host, - source: parsers.AppendSource, - }) - } - }() -} - -func (pool *BrutePool) doAppend(bl *pkg.Baseline) { - pool.wg.Add(2) - pool.doAppendWords(bl) - pool.doAppendRule(bl) -} - -func (pool *BrutePool) doActive() { - defer pool.wg.Done() - if pool.Mod == HostSpray { - return - } - for _, u := range pkg.ActivePath { - pool.addAddition(&Unit{ - path: pool.dir + u[1:], - source: parsers.FingerSource, - }) - } -} - -func (pool *BrutePool) doCommonFile() { - defer pool.wg.Done() - if pool.Mod == HostSpray { - return - } - for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output { - pool.addAddition(&Unit{ - path: pool.dir + u, - source: parsers.CommonFileSource, - }) - } -} - func (pool *BrutePool) checkRedirect(redirectURL string) bool { if pool.random.RedirectURL == "" { // 如果random的redirectURL为空, 此时该项 @@ -758,7 +685,7 @@ func (pool *BrutePool) doCheck() { } func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { - if !pool.Crawl || bl.ReqDepth >= MaxCrawl { + if !pool.Crawl || bl.ReqDepth >= pool.MaxCrawlDepth { return } @@ -788,7 +715,7 @@ func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { } func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) { - if bl.ReqDepth >= MaxCrawl { + if bl.ReqDepth >= pool.MaxCrawlDepth { pool.wg.Done() return } @@ -817,12 +744,12 @@ func (pool *BrutePool) doBak() { if pool.Mod == HostSpray { return } - for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output { - pool.addAddition(&Unit{ - path: pool.dir + w, - source: parsers.BakSource, - }) - } + //for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output { + // pool.addAddition(&Unit{ + // path: pool.dir + w, + // source: parsers.BakSource, + // }) + //} for w := range NewBruteDSL(pool.Config, "{?@bak_name}.{?@bak_ext}", nil).Output { pool.addAddition(&Unit{ @@ -831,3 +758,75 @@ func (pool *BrutePool) doBak() { }) } } + +func (pool *BrutePool) doAppend(bl *pkg.Baseline) { + pool.wg.Add(2) + pool.doAppendWords(bl) + pool.doAppendRule(bl) +} + +func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { + if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource || bl.ReqDepth >= pool.MaxAppendDepth { + pool.wg.Done() + return + } + + go func() { + defer pool.wg.Done() + for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { + pool.addAddition(&Unit{ + path: pkg.Dir(bl.Url.Path) + u, + host: bl.Host, + source: parsers.AppendRuleSource, + depth: bl.ReqDepth + 1, + }) + } + }() +} + +func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) { + if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource || bl.ReqDepth >= pool.MaxAppendDepth { + // 防止自身递归 + pool.wg.Done() + return + } + + go func() { + defer pool.wg.Done() + + for u := range NewBruteWords(pool.Config, pool.AppendWords).Output { + pool.addAddition(&Unit{ + path: pkg.SafePath(bl.Path, u), + host: bl.Host, + source: parsers.AppendSource, + depth: bl.RecuDepth + 1, + }) + } + }() +} + +func (pool *BrutePool) doActive() { + defer pool.wg.Done() + if pool.Mod == HostSpray { + return + } + for _, u := range pkg.ActivePath { + pool.addAddition(&Unit{ + path: pool.dir + u[1:], + source: parsers.FingerSource, + }) + } +} + +func (pool *BrutePool) doCommonFile() { + defer pool.wg.Done() + if pool.Mod == HostSpray { + return + } + for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output { + pool.addAddition(&Unit{ + path: pool.dir + u, + source: parsers.CommonFileSource, + }) + } +} diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 9d9d23f..62a9b36 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -181,7 +181,7 @@ func (pool *CheckPool) Handler() { } func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { - if depth >= MaxRedirect { + if depth >= pool.MaxRedirect { return } var reURL string diff --git a/internal/pool/config.go b/internal/pool/config.go new file mode 100644 index 0000000..9f49652 --- /dev/null +++ b/internal/pool/config.go @@ -0,0 +1,69 @@ +package pool + +import ( + "github.com/chainreactors/logs" + "github.com/chainreactors/spray/pkg" + "github.com/chainreactors/words" + "github.com/chainreactors/words/rule" + "github.com/expr-lang/expr/vm" + "sync" + "time" +) + +type Config struct { + BaseURL string + ProxyAddr string + Thread int + Wordlist []string + Timeout time.Duration + ProcessCh chan *pkg.Baseline + OutputCh chan *pkg.Baseline + FuzzyCh chan *pkg.Baseline + Outwg *sync.WaitGroup + RateLimit int + CheckPeriod int + ErrPeriod int32 + BreakThreshold int32 + Method string + Mod SprayMod + Headers map[string]string + ClientType int + MatchExpr *vm.Program + FilterExpr *vm.Program + RecuExpr *vm.Program + AppendRule *rule.Program + Fns []words.WordFunc + AppendWords []string + Fuzzy bool + IgnoreWaf bool + Crawl bool + Scope []string + Active bool + Bak bool + Common bool + RetryLimit int + RandomUserAgent bool + Random string + Index string + MaxRedirect int + MaxCrawlDepth int + MaxRecursionDepth int + MaxAppendDepth int +} + +func NewBruteWords(config *Config, list []string) *words.Worder { + word := words.NewWorderWithList(list) + word.Fns = config.Fns + word.Run() + return word +} + +func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder { + word, err := words.NewWorderWithDsl(dsl, params, nil) + if err != nil { + logs.Log.Error(err.Error()) + } + word.Fns = config.Fns + word.Run() + return word +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 85357ed..34f3a36 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -29,7 +29,7 @@ type BasePool struct { } func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) { - if depth >= MaxRedirect { + if depth >= pool.MaxRedirect { return } reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) diff --git a/internal/pool/type.go b/internal/pool/type.go new file mode 100644 index 0000000..ed4381f --- /dev/null +++ b/internal/pool/type.go @@ -0,0 +1,47 @@ +package pool + +import ( + "github.com/chainreactors/parsers" + "github.com/chainreactors/spray/pkg" +) + +func newUnit(path string, source parsers.SpraySource) *Unit { + return &Unit{path: path, source: source} +} + +type Unit struct { + number int + host string + path string + source parsers.SpraySource + retry int + frontUrl string + depth int +} + +func NewBaselines() *Baselines { + return &Baselines{ + baselines: map[int]*pkg.Baseline{}, + } +} + +type Baselines struct { + FailedBaselines []*pkg.Baseline + random *pkg.Baseline + index *pkg.Baseline + baselines map[int]*pkg.Baseline +} + +type SprayMod int + +const ( + PathSpray SprayMod = iota + 1 + HostSpray + ParamSpray + CustomSpray +) + +var ModMap = map[string]SprayMod{ + "path": PathSpray, + "host": HostSpray, +} diff --git a/internal/pool/types.go b/internal/pool/types.go deleted file mode 100644 index 87128da..0000000 --- a/internal/pool/types.go +++ /dev/null @@ -1,107 +0,0 @@ -package pool - -import ( - "github.com/chainreactors/logs" - "github.com/chainreactors/parsers" - "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/words" - "github.com/chainreactors/words/rule" - "github.com/expr-lang/expr/vm" - "sync" - "time" -) - -func newUnit(path string, source parsers.SpraySource) *Unit { - return &Unit{path: path, source: source} -} - -type Unit struct { - number int - host string - path string - source parsers.SpraySource - retry int - frontUrl string - depth int // redirect depth -} - -func NewBaselines() *Baselines { - return &Baselines{ - baselines: map[int]*pkg.Baseline{}, - } -} - -type Baselines struct { - FailedBaselines []*pkg.Baseline - random *pkg.Baseline - index *pkg.Baseline - baselines map[int]*pkg.Baseline -} - -type SprayMod int - -const ( - PathSpray SprayMod = iota + 1 - HostSpray - ParamSpray - CustomSpray -) - -var ModMap = map[string]SprayMod{ - "path": PathSpray, - "host": HostSpray, -} - -type Config struct { - BaseURL string - ProxyAddr string - Thread int - Wordlist []string - Timeout time.Duration - ProcessCh chan *pkg.Baseline - OutputCh chan *pkg.Baseline - FuzzyCh chan *pkg.Baseline - Outwg *sync.WaitGroup - RateLimit int - CheckPeriod int - ErrPeriod int32 - BreakThreshold int32 - Method string - Mod SprayMod - Headers map[string]string - ClientType int - MatchExpr *vm.Program - FilterExpr *vm.Program - RecuExpr *vm.Program - AppendRule *rule.Program - Fns []words.WordFunc - AppendWords []string - Fuzzy bool - IgnoreWaf bool - Crawl bool - Scope []string - Active bool - Bak bool - Common bool - RetryLimit int - RandomUserAgent bool - Random string - Index string -} - -func NewBruteWords(config *Config, list []string) *words.Worder { - word := words.NewWorderWithList(list) - word.Fns = config.Fns - word.Run() - return word -} - -func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder { - word, err := words.NewWorderWithDsl(dsl, params, nil) - if err != nil { - logs.Log.Error(err.Error()) - } - word.Fns = config.Fns - word.Run() - return word -} diff --git a/internal/runner.go b/internal/runner.go index 9a282ef..5b50e1e 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -51,7 +51,6 @@ type Runner struct { Count int // tasks total number Wordlist []string AppendWords []string - RecuDepth int ClientType int Probes []string Total int // wordlist total number @@ -81,17 +80,21 @@ func (r *Runner) PrepareConfig() *pool.Config { AppendWords: r.AppendWords, // 对有效目录追加字典 Fns: r.Fns, //IgnoreWaf: r.IgnoreWaf, - Crawl: r.CrawlPlugin, - Scope: r.Scope, - Active: r.Finger, - Bak: r.BakPlugin, - Common: r.CommonPlugin, - RetryLimit: r.RetryCount, - ClientType: r.ClientType, - RandomUserAgent: r.RandomUserAgent, - Random: r.Random, - Index: r.Index, - ProxyAddr: r.Proxy, + Crawl: r.CrawlPlugin, + Scope: r.Scope, + Active: r.Finger, + Bak: r.BakPlugin, + Common: r.CommonPlugin, + RetryLimit: r.RetryCount, + ClientType: r.ClientType, + RandomUserAgent: r.RandomUserAgent, + Random: r.Random, + Index: r.Index, + ProxyAddr: r.Proxy, + MaxRecursionDepth: r.Depth, + MaxRedirect: 3, + MaxAppendDepth: r.AppendDepth, + MaxCrawlDepth: r.CrawlDepth, } if config.ClientType == ihttp.Auto { From e483bb44398f010d23724b6e0ebaf7f40c29979c Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 14 Oct 2024 02:20:16 +0800 Subject: [PATCH 2/9] baseline add from and parent prop --- go.mod | 2 +- go.sum | 2 ++ internal/pool/brutepool.go | 38 ++++++++++++++++++++++++++++++++++---- internal/pool/checkpool.go | 4 ++++ internal/pool/pool.go | 20 ++------------------ internal/pool/type.go | 10 ++++++++++ 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 638c2f5..fafa128 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f - github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65 + github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57 github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f github.com/expr-lang/expr v1.16.9 github.com/gookit/config/v2 v2.2.5 diff --git a/go.sum b/go.sum index 0647b56..f10bbfb 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84 h1:F6umsdHLx github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65 h1:subSvyczsErYMRnCD07s4Ub6zOSaw2xZ1/O9t3tHkuw= github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= +github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57 h1:KuijtekTNtSpQbKf2jqKp99gxnGQXffPeEF+EOHnXBE= +github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index a5d270b..3242329 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -322,9 +322,7 @@ func (pool *BrutePool) Invoke(v interface{}) { if !ihttp.CheckBodySize(int64(bl.BodyLength)) { bl.ExceedLength = true } - bl.Source = unit.source - bl.ReqDepth = unit.depth - bl.Number = unit.number + unit.Update(bl) bl.Spended = time.Since(start).Milliseconds() switch unit.source { case parsers.InitRandomSource: @@ -684,6 +682,26 @@ func (pool *BrutePool) doCheck() { } } +func (pool *BrutePool) doRedirect(bl *pkg.Baseline, depth int) { + if depth >= pool.MaxRedirect { + return + } + reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) + pool.wg.Add(1) + go func() { + defer pool.wg.Done() + pool.addAddition(&Unit{ + path: reURL, + parent: bl.Number, + host: bl.Host, + source: parsers.RedirectSource, + from: bl.Source, + frontUrl: bl.UrlString, + depth: depth + 1, + }) + }() +} + func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { if !pool.Crawl || bl.ReqDepth >= pool.MaxCrawlDepth { return @@ -705,8 +723,10 @@ func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { } pool.addAddition(&Unit{ path: u, + parent: bl.Number, host: bl.Host, source: parsers.CrawlSource, + from: bl.Source, depth: bl.ReqDepth + 1, }) } @@ -731,7 +751,13 @@ func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) { if _, ok := pool.scopeurls[u]; !ok { pool.urls.Store(u, nil) pool.wg.Add(1) - pool.scopePool.Invoke(&Unit{path: u, source: parsers.CrawlSource, depth: bl.ReqDepth + 1}) + pool.scopePool.Invoke(&Unit{ + path: u, + parent: bl.Number, + source: parsers.CrawlSource, + from: bl.Source, + depth: bl.ReqDepth + 1, + }) } pool.scopeLocker.Unlock() } @@ -776,8 +802,10 @@ func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { pool.addAddition(&Unit{ path: pkg.Dir(bl.Url.Path) + u, + parent: bl.Number, host: bl.Host, source: parsers.AppendRuleSource, + from: bl.Source, depth: bl.ReqDepth + 1, }) } @@ -797,8 +825,10 @@ func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) { for u := range NewBruteWords(pool.Config, pool.AppendWords).Output { pool.addAddition(&Unit{ path: pkg.SafePath(bl.Path, u), + parent: bl.Number, host: bl.Host, source: parsers.AppendSource, + from: bl.Source, depth: bl.RecuDepth + 1, }) } diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 62a9b36..1b84cdc 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -199,9 +199,11 @@ func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { go func() { pool.additionCh <- &Unit{ path: reURL, + parent: bl.Number, source: parsers.RedirectSource, frontUrl: bl.UrlString, depth: depth + 1, + from: bl.Source, } }() } @@ -221,8 +223,10 @@ func (pool *CheckPool) doUpgrade(bl *pkg.Baseline) { go func() { pool.additionCh <- &Unit{ path: reurl, + parent: bl.Number, source: parsers.UpgradeSource, depth: bl.ReqDepth + 1, + from: bl.Source, } }() } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 34f3a36..d183b79 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -28,24 +28,6 @@ type BasePool struct { isFallback atomic.Bool } -func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) { - if depth >= pool.MaxRedirect { - return - } - reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) - pool.wg.Add(1) - go func() { - defer pool.wg.Done() - pool.addAddition(&Unit{ - path: reURL, - host: bl.Host, - source: parsers.RedirectSource, - frontUrl: bl.UrlString, - depth: depth + 1, - }) - }() -} - func (pool *BasePool) doRetry(bl *pkg.Baseline) { if bl.Retry >= pool.RetryLimit { return @@ -55,8 +37,10 @@ func (pool *BasePool) doRetry(bl *pkg.Baseline) { defer pool.wg.Done() pool.addAddition(&Unit{ path: bl.Path, + parent: bl.Number, host: bl.Host, source: parsers.RetrySource, + from: bl.Source, retry: bl.Retry + 1, }) }() diff --git a/internal/pool/type.go b/internal/pool/type.go index ed4381f..9967169 100644 --- a/internal/pool/type.go +++ b/internal/pool/type.go @@ -11,14 +11,24 @@ func newUnit(path string, source parsers.SpraySource) *Unit { type Unit struct { number int + parent int host string path string + from parsers.SpraySource source parsers.SpraySource retry int frontUrl string depth int } +func (u *Unit) Update(bl *pkg.Baseline) { + bl.Number = u.number + bl.Parent = u.parent + bl.Host = u.host + bl.Path = u.path + bl.Source = u.source +} + func NewBaselines() *Baselines { return &Baselines{ baselines: map[int]*pkg.Baseline{}, From af82ae43b9372c3dec1d44a8925ceb5daae074db Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 14 Oct 2024 02:20:39 +0800 Subject: [PATCH 3/9] enhance probe output --- internal/format.go | 13 ++++++++----- internal/runner.go | 4 ++-- pkg/baseline.go | 9 +++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/format.go b/internal/format.go index 2908ab2..346a5f1 100644 --- a/internal/format.go +++ b/internal/format.go @@ -39,17 +39,20 @@ func Format(opts Option) { group[result.Url.Host] = append(group[result.Url.Host], &result) } - // 分组 - for _, results := range group { for _, result := range results { if !opts.Fuzzy && result.IsFuzzy { continue } - if !opts.NoColor { - logs.Log.Console(result.ColorString() + "\n") + if opts.OutputProbe == "" { + if !opts.NoColor { + logs.Log.Console(result.ColorString() + "\n") + } else { + logs.Log.Console(result.String() + "\n") + } } else { - logs.Log.Console(result.String() + "\n") + probes := strings.Split(opts.OutputProbe, ",") + logs.Log.Console(result.ProbeOutput(probes) + "\n") } } } diff --git a/internal/runner.go b/internal/runner.go index 5b50e1e..9879518 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -364,7 +364,7 @@ func (r *Runner) Output(bl *pkg.Baseline) { if r.Option.Json { out = bl.ToJson() } else if len(r.Probes) > 0 { - out = bl.Format(r.Probes) + out = bl.ProbeOutput(r.Probes) } else if r.Color { out = bl.ColorString() } else { @@ -385,7 +385,7 @@ func (r *Runner) Output(bl *pkg.Baseline) { } else if r.FileOutput == "full" { r.OutputFile.SafeWrite(bl.String() + "\n") } else { - r.OutputFile.SafeWrite(bl.Format(strings.Split(r.FileOutput, ",")) + "\n") + r.OutputFile.SafeWrite(bl.ProbeOutput(strings.Split(r.FileOutput, ",")) + "\n") } r.OutputFile.SafeSync() diff --git a/pkg/baseline.go b/pkg/baseline.go index 0099040..df5fe23 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -235,6 +235,15 @@ func (bl *Baseline) Compare(other *Baseline) int { return -1 } +func (bl *Baseline) ProbeOutput(format []string) string { + var s strings.Builder + for _, f := range format { + s.WriteString("\t") + s.WriteString(bl.Get(f)) + } + return strings.TrimSpace(s.String()) +} + var Distance uint8 = 5 // 数字越小越相似, 数字为0则为完全一致. func (bl *Baseline) FuzzyCompare(other *Baseline) bool { From 6bbc6141ac2d562fffb5526a670105f3f06fbe91 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 16 Oct 2024 14:47:29 +0800 Subject: [PATCH 4/9] enhance: add 404 default fuzzystatus, 429 waf status --- internal/option.go | 2 +- pkg/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/option.go b/internal/option.go index e537389..aff0f19 100644 --- a/internal/option.go +++ b/internal/option.go @@ -132,7 +132,7 @@ type ModeOptions struct { BreakThreshold int `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold" config:"error-threshold"` BlackStatus string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status" config:"black-status"` WhiteStatus string `long:"white-status" default:"200" description:"Strings (comma split), custom white status" config:"white-status"` - FuzzyStatus string `long:"fuzzy-status" default:"500,501,502,503,301,302" description:"Strings (comma split), custom fuzzy status" config:"fuzzy-status"` + FuzzyStatus string `long:"fuzzy-status" default:"500,501,502,503,301,302,404" description:"Strings (comma split), custom fuzzy status" config:"fuzzy-status"` UniqueStatus string `long:"unique-status" default:"403,200,404" description:"Strings (comma split), custom unique status" config:"unique-status"` Unique bool `long:"unique" description:"Bool, unique response" config:"unique"` RetryCount int `long:"retry" default:"0" description:"Int, retry count" config:"retry"` diff --git a/pkg/utils.go b/pkg/utils.go index 1de80bb..2390156 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -30,7 +30,7 @@ var ( WhiteStatus = []int{} // cmd input, 200 BlackStatus = []int{} // cmd input, 400,410 FuzzyStatus = []int{} // cmd input, 500,501,502,503 - WAFStatus = []int{493, 418, 1020, 406} + WAFStatus = []int{493, 418, 1020, 406, 429} UniqueStatus = []int{} // 相同unique的403表示命中了同一条acl, 相同unique的200表示default页面 // plugins From 5cb9aa119db6d1b1feed26bb99073443ec3c9fa0 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 30 Oct 2024 15:57:32 +0800 Subject: [PATCH 5/9] fix: not same domain filtered --- go.mod | 2 +- go.sum | 2 ++ internal/pool/brutepool.go | 32 +++++++++++++++++++++++++------- pkg/baseline.go | 25 +++++++++++++------------ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index fafa128..0a8a8c2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f - github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57 + github.com/chainreactors/parsers v0.0.0-20241016065831-bedaf68005f1 github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f github.com/expr-lang/expr v1.16.9 github.com/gookit/config/v2 v2.2.5 diff --git a/go.sum b/go.sum index f10bbfb..4a76f75 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65 h1:subSvyczs github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57 h1:KuijtekTNtSpQbKf2jqKp99gxnGQXffPeEF+EOHnXBE= github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= +github.com/chainreactors/parsers v0.0.0-20241016065831-bedaf68005f1 h1:Ka/KBrqAgwiL07TwYjtqF2DQ3x0fCxw1XHG+GFqMKEc= +github.com/chainreactors/parsers v0.0.0-20241016065831-bedaf68005f1/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 3242329..2317767 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -562,17 +562,26 @@ func (pool *BrutePool) PreCompare(resp *ihttp.Response) error { return nil } +func (pool *BrutePool) checkHostname(u string) bool { + if v, err := url.Parse(u); err == nil { + if v.Host == "" { + return true + } + if v.Host == pool.url.Host { + return true + } else { + return false + } + } + return true +} + func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { if !bl.IsValid { return false } var status = -1 - // 30x状态码的特殊处理 - if bl.RedirectURL != "" && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { - bl.Reason = pkg.ErrFuzzyRedirect.Error() - pool.putToFuzzy(bl) - return false - } + // 使用与baseline相同状态码, 需要在fuzzystatus中提前配置 base, ok := pool.baselines[bl.Status] // 挑选对应状态码的baseline进行compare if !ok { @@ -587,7 +596,15 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { } } - if ok { + // 30x状态码的特殊处理 + if bl.RedirectURL != "" { + if pool.checkHostname(bl.RedirectURL) && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { + bl.Reason = pkg.ErrFuzzyRedirect.Error() + return false + } + } + + if ok && !bl.IsBaseline { if status = base.Compare(bl); status == 1 { bl.Reason = pkg.ErrCompareFailed.Error() return false @@ -619,6 +636,7 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) { if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) { + bl.IsBaseline = true bl.Collect() pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取 pool.baselines[bl.Status] = bl diff --git a/pkg/baseline.go b/pkg/baseline.go index df5fe23..aacf1a0 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -113,18 +113,19 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba type Baseline struct { *parsers.SprayResult - Url *url.URL `json:"-"` - Dir bool `json:"-"` - Chunked bool `json:"-"` - Body BS `json:"-"` - Header BS `json:"-"` - Raw BS `json:"-"` - Response *http.Response `json:"-"` - Recu bool `json:"-"` - RecuDepth int `json:"-"` - URLs []string `json:"-"` - Collected bool `json:"-"` - Retry int `json:"-"` + Url *url.URL `json:"-"` + Dir bool `json:"-"` + Chunked bool `json:"-"` + Body BS `json:"-"` + Header BS `json:"-"` + Raw BS `json:"-"` + Response *http.Response `json:"-"` + Recu bool `json:"-"` + RecuDepth int `json:"-"` + URLs []string `json:"-"` + Collected bool `json:"-"` + Retry int `json:"-"` + IsBaseline bool `json:"-"` } func (bl *Baseline) IsDir() bool { From 0ca5c02de71c1df5957064cd5d8cc3497f7c6b79 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 30 Oct 2024 16:11:05 +0800 Subject: [PATCH 6/9] enhance: skip not same host redirect --- internal/pool/brutepool.go | 32 ++++++++++++++++++++------------ pkg/baseline.go | 1 + 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 2317767..e79ab6e 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -315,7 +315,7 @@ func (pool *BrutePool) Invoke(v interface{}) { // 手动处理重定向 if bl.IsValid && unit.source != parsers.CheckSource && bl.RedirectURL != "" { - //pool.wg.Add(1) + bl.SameDomain = pool.checkHost(bl.RedirectURL) pool.doRedirect(bl, unit.depth) } @@ -508,7 +508,7 @@ func (pool *BrutePool) Handler() { func (pool *BrutePool) checkRedirect(redirectURL string) bool { if pool.random.RedirectURL == "" { - // 如果random的redirectURL为空, 此时该项 + // 如果random的redirectURL为空, 忽略 return true } @@ -562,7 +562,9 @@ func (pool *BrutePool) PreCompare(resp *ihttp.Response) error { return nil } -func (pool *BrutePool) checkHostname(u string) bool { +// same host return true +// diff host return false +func (pool *BrutePool) checkHost(u string) bool { if v, err := url.Parse(u); err == nil { if v.Host == "" { return true @@ -582,8 +584,19 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { } var status = -1 + // 30x状态码的特殊处理 + if bl.RedirectURL != "" { + if bl.SameDomain && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { + bl.Reason = pkg.ErrFuzzyRedirect.Error() + return false + } + } + // 使用与baseline相同状态码, 需要在fuzzystatus中提前配置 base, ok := pool.baselines[bl.Status] // 挑选对应状态码的baseline进行compare + if bl.IsBaseline { + ok = false + } if !ok { if pool.random.Status == bl.Status { // 当other的状态码与base相同时, 会使用base @@ -596,15 +609,7 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { } } - // 30x状态码的特殊处理 - if bl.RedirectURL != "" { - if pool.checkHostname(bl.RedirectURL) && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { - bl.Reason = pkg.ErrFuzzyRedirect.Error() - return false - } - } - - if ok && !bl.IsBaseline { + if ok { if status = base.Compare(bl); status == 1 { bl.Reason = pkg.ErrCompareFailed.Error() return false @@ -704,6 +709,9 @@ func (pool *BrutePool) doRedirect(bl *pkg.Baseline, depth int) { if depth >= pool.MaxRedirect { return } + if !bl.SameDomain { + return // 不同域名的重定向不处理 + } reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) pool.wg.Add(1) go func() { diff --git a/pkg/baseline.go b/pkg/baseline.go index aacf1a0..d17e1b1 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -125,6 +125,7 @@ type Baseline struct { URLs []string `json:"-"` Collected bool `json:"-"` Retry int `json:"-"` + SameDomain bool `json:"-"` IsBaseline bool `json:"-"` } From 9e74a170967b2c39c91e0f791494cf1e2c25a057 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 1 Nov 2024 12:25:53 +0800 Subject: [PATCH 7/9] fix: path join not expect --- pkg/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/utils.go b/pkg/utils.go index 2390156..fdb098a 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -270,9 +270,9 @@ func CRC16Hash(data []byte) uint16 { func SafePath(dir, u string) string { hasSlash := strings.HasPrefix(u, "/") if hasSlash { - return path.Join(dir, u[1:]) + return dir + u[1:] } else { - return path.Join(dir, u) + return dir + u } } From 02162cffd6b6e8143676465446b97fede424372c Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 1 Nov 2024 12:27:31 +0800 Subject: [PATCH 8/9] revert: not same redirect banned --- internal/pool/brutepool.go | 11 ++++++----- pkg/baseline.go | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index e79ab6e..bd49aac 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -315,7 +315,7 @@ func (pool *BrutePool) Invoke(v interface{}) { // 手动处理重定向 if bl.IsValid && unit.source != parsers.CheckSource && bl.RedirectURL != "" { - bl.SameDomain = pool.checkHost(bl.RedirectURL) + bl.SameRedirectDomain = pool.checkHost(bl.RedirectURL) pool.doRedirect(bl, unit.depth) } @@ -586,7 +586,7 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { // 30x状态码的特殊处理 if bl.RedirectURL != "" { - if bl.SameDomain && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { + if bl.SameRedirectDomain && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { bl.Reason = pkg.ErrFuzzyRedirect.Error() return false } @@ -709,9 +709,10 @@ func (pool *BrutePool) doRedirect(bl *pkg.Baseline, depth int) { if depth >= pool.MaxRedirect { return } - if !bl.SameDomain { - return // 不同域名的重定向不处理 - } + + //if !bl.SameRedirectDomain { + // return // 不同域名的重定向不处理 + //} reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) pool.wg.Add(1) go func() { diff --git a/pkg/baseline.go b/pkg/baseline.go index d17e1b1..18df8c1 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -113,20 +113,20 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba type Baseline struct { *parsers.SprayResult - Url *url.URL `json:"-"` - Dir bool `json:"-"` - Chunked bool `json:"-"` - Body BS `json:"-"` - Header BS `json:"-"` - Raw BS `json:"-"` - Response *http.Response `json:"-"` - Recu bool `json:"-"` - RecuDepth int `json:"-"` - URLs []string `json:"-"` - Collected bool `json:"-"` - Retry int `json:"-"` - SameDomain bool `json:"-"` - IsBaseline bool `json:"-"` + Url *url.URL `json:"-"` + Dir bool `json:"-"` + Chunked bool `json:"-"` + Body BS `json:"-"` + Header BS `json:"-"` + Raw BS `json:"-"` + Response *http.Response `json:"-"` + Recu bool `json:"-"` + RecuDepth int `json:"-"` + URLs []string `json:"-"` + Collected bool `json:"-"` + Retry int `json:"-"` + SameRedirectDomain bool `json:"-"` + IsBaseline bool `json:"-"` } func (bl *Baseline) IsDir() bool { From de12d568cee48fb34c697d4ae08a634039f363c6 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 1 Nov 2024 12:30:35 +0800 Subject: [PATCH 9/9] enhance: add hard exit, https://github.com/chainreactors/spray/issues/78 --- cmd/cmd.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/cmd.go b/cmd/cmd.go index fcf3138..2029c77 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -149,6 +149,15 @@ func Spray() { } ctx, canceler := context.WithTimeout(context.Background(), time.Duration(runner.Deadline)*time.Second) + go func() { + select { + case <-ctx.Done(): + time.Sleep(10 * time.Second) + logs.Log.Errorf("deadline and timeout not work, hard exit!!!") + os.Exit(0) + } + }() + go func() { exitChan := make(chan os.Signal, 2) signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)