add --append-depth limit append recu depth

internal/option.go

@@ -115,6 +115,7 @@ type PluginOptions struct {
 	CommonPlugin  bool     `long:"common" description:"Bool, enable common file found" config:"common"`
 	CrawlPlugin   bool     `long:"crawl" description:"Bool, enable crawl" config:"crawl"`
 	CrawlDepth    int      `long:"crawl-depth" default:"3" description:"Int, crawl depth" config:"crawl-depth"`
+	AppendDepth   int      `long:"append-depth" default:"2" description:"Int, append depth" config:"append-depth"`
 }
 
 type ModeOptions struct {
@@ -253,7 +254,6 @@ func (opt *Option) Prepare() error {
 
 	logs.Log.Logf(pkg.LogVerbose, "Black Status: %v, WhiteStatus: %v, WAFStatus: %v", pkg.BlackStatus, pkg.WhiteStatus, pkg.WAFStatus)
 	logs.Log.Logf(pkg.LogVerbose, "Fuzzy Status: %v, Unique Status: %v", pkg.FuzzyStatus, pkg.UniqueStatus)
-	pool.MaxCrawl = opt.CrawlDepth
 
 	return nil
 }
@@ -354,13 +354,12 @@ func (opt *Option) NewRunner() (*Runner, error) {
 	var express string
 	if opt.Recursive != "current.IsDir()" && opt.Depth != 0 {
 		// 默认不打开递归, 除非指定了非默认的递归表达式
-		pool.MaxRecursion = 1
+		opt.Depth = 1
 		express = opt.Recursive
 	}
 
 	if opt.Depth != 0 {
 		// 手动设置的depth优先级高于默认
-		pool.MaxRecursion = opt.Depth
 		express = opt.Recursive
 	}
 

internal/pool/brutepool.go

@@ -23,9 +23,6 @@ import (
 )
 
 var (
-	MaxRedirect     = 3
-	MaxCrawl        = 3
-	MaxRecursion    = 0
 	EnableAllFuzzy  = false
 	EnableAllUnique = false
 	//AllowHostModSource = []parsers.SpraySource{parsers.WordSource, parsers.CheckSource, parsers.InitIndexSource, parsers.InitRandomSource}
@@ -494,7 +491,7 @@ func (pool *BrutePool) Handler() {
 		// 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度
 		if bl.IsValid {
 			pool.Statistor.FoundNumber++
-			if bl.RecuDepth < MaxRecursion {
+			if bl.RecuDepth < pool.MaxRecursionDepth {
 				if pkg.CompareWithExpr(pool.RecuExpr, params) {
 					bl.Recu = true
 				}
@@ -511,76 +508,6 @@ func (pool *BrutePool) Handler() {
 	pool.analyzeDone = true
 }
 
-func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) {
-	if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource {
-		pool.wg.Done()
-		return
-	}
-
-	go func() {
-		defer pool.wg.Done()
-		for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) {
-			pool.addAddition(&Unit{
-				path:   pkg.Dir(bl.Url.Path) + u,
-				host:   bl.Host,
-				source: parsers.AppendRuleSource,
-			})
-		}
-	}()
-}
-
-func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) {
-	if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource {
-		// 防止自身递归
-		pool.wg.Done()
-		return
-	}
-
-	go func() {
-		defer pool.wg.Done()
-
-		for u := range NewBruteWords(pool.Config, pool.AppendWords).Output {
-			pool.addAddition(&Unit{
-				path:   pkg.SafePath(bl.Path, u),
-				host:   bl.Host,
-				source: parsers.AppendSource,
-			})
-		}
-	}()
-}
-
-func (pool *BrutePool) doAppend(bl *pkg.Baseline) {
-	pool.wg.Add(2)
-	pool.doAppendWords(bl)
-	pool.doAppendRule(bl)
-}
-
-func (pool *BrutePool) doActive() {
-	defer pool.wg.Done()
-	if pool.Mod == HostSpray {
-		return
-	}
-	for _, u := range pkg.ActivePath {
-		pool.addAddition(&Unit{
-			path:   pool.dir + u[1:],
-			source: parsers.FingerSource,
-		})
-	}
-}
-
-func (pool *BrutePool) doCommonFile() {
-	defer pool.wg.Done()
-	if pool.Mod == HostSpray {
-		return
-	}
-	for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output {
-		pool.addAddition(&Unit{
-			path:   pool.dir + u,
-			source: parsers.CommonFileSource,
-		})
-	}
-}
-
 func (pool *BrutePool) checkRedirect(redirectURL string) bool {
 	if pool.random.RedirectURL == "" {
 		// 如果random的redirectURL为空, 此时该项
@@ -758,7 +685,7 @@ func (pool *BrutePool) doCheck() {
 }
 
 func (pool *BrutePool) doCrawl(bl *pkg.Baseline) {
-	if !pool.Crawl || bl.ReqDepth >= MaxCrawl {
+	if !pool.Crawl || bl.ReqDepth >= pool.MaxCrawlDepth {
 		return
 	}
 
@@ -788,7 +715,7 @@ func (pool *BrutePool) doCrawl(bl *pkg.Baseline) {
 }
 
 func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) {
-	if bl.ReqDepth >= MaxCrawl {
+	if bl.ReqDepth >= pool.MaxCrawlDepth {
 		pool.wg.Done()
 		return
 	}
@@ -817,12 +744,12 @@ func (pool *BrutePool) doBak() {
 	if pool.Mod == HostSpray {
 		return
 	}
-	for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output {
+	//for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output {
-		pool.addAddition(&Unit{
+	//	pool.addAddition(&Unit{
-			path:   pool.dir + w,
+	//		path:   pool.dir + w,
-			source: parsers.BakSource,
+	//		source: parsers.BakSource,
-		})
+	//	})
-	}
+	//}
 
 	for w := range NewBruteDSL(pool.Config, "{?@bak_name}.{?@bak_ext}", nil).Output {
 		pool.addAddition(&Unit{
@@ -831,3 +758,75 @@ func (pool *BrutePool) doBak() {
 		})
 	}
 }
+
+func (pool *BrutePool) doAppend(bl *pkg.Baseline) {
+	pool.wg.Add(2)
+	pool.doAppendWords(bl)
+	pool.doAppendRule(bl)
+}
+
+func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) {
+	if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource || bl.ReqDepth >= pool.MaxAppendDepth {
+		pool.wg.Done()
+		return
+	}
+
+	go func() {
+		defer pool.wg.Done()
+		for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) {
+			pool.addAddition(&Unit{
+				path:   pkg.Dir(bl.Url.Path) + u,
+				host:   bl.Host,
+				source: parsers.AppendRuleSource,
+				depth:  bl.ReqDepth + 1,
+			})
+		}
+	}()
+}
+
+func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) {
+	if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource || bl.ReqDepth >= pool.MaxAppendDepth {
+		// 防止自身递归
+		pool.wg.Done()
+		return
+	}
+
+	go func() {
+		defer pool.wg.Done()
+
+		for u := range NewBruteWords(pool.Config, pool.AppendWords).Output {
+			pool.addAddition(&Unit{
+				path:   pkg.SafePath(bl.Path, u),
+				host:   bl.Host,
+				source: parsers.AppendSource,
+				depth:  bl.RecuDepth + 1,
+			})
+		}
+	}()
+}
+
+func (pool *BrutePool) doActive() {
+	defer pool.wg.Done()
+	if pool.Mod == HostSpray {
+		return
+	}
+	for _, u := range pkg.ActivePath {
+		pool.addAddition(&Unit{
+			path:   pool.dir + u[1:],
+			source: parsers.FingerSource,
+		})
+	}
+}
+
+func (pool *BrutePool) doCommonFile() {
+	defer pool.wg.Done()
+	if pool.Mod == HostSpray {
+		return
+	}
+	for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output {
+		pool.addAddition(&Unit{
+			path:   pool.dir + u,
+			source: parsers.CommonFileSource,
+		})
+	}
+}

internal/pool/checkpool.go

@@ -181,7 +181,7 @@ func (pool *CheckPool) Handler() {
 }
 
 func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) {
-	if depth >= MaxRedirect {
+	if depth >= pool.MaxRedirect {
 		return
 	}
 	var reURL string

internal/pool/config.go

@@ -0,0 +1,69 @@
+package pool
+
+import (
+	"github.com/chainreactors/logs"
+	"github.com/chainreactors/spray/pkg"
+	"github.com/chainreactors/words"
+	"github.com/chainreactors/words/rule"
+	"github.com/expr-lang/expr/vm"
+	"sync"
+	"time"
+)
+
+type Config struct {
+	BaseURL           string
+	ProxyAddr         string
+	Thread            int
+	Wordlist          []string
+	Timeout           time.Duration
+	ProcessCh         chan *pkg.Baseline
+	OutputCh          chan *pkg.Baseline
+	FuzzyCh           chan *pkg.Baseline
+	Outwg             *sync.WaitGroup
+	RateLimit         int
+	CheckPeriod       int
+	ErrPeriod         int32
+	BreakThreshold    int32
+	Method            string
+	Mod               SprayMod
+	Headers           map[string]string
+	ClientType        int
+	MatchExpr         *vm.Program
+	FilterExpr        *vm.Program
+	RecuExpr          *vm.Program
+	AppendRule        *rule.Program
+	Fns               []words.WordFunc
+	AppendWords       []string
+	Fuzzy             bool
+	IgnoreWaf         bool
+	Crawl             bool
+	Scope             []string
+	Active            bool
+	Bak               bool
+	Common            bool
+	RetryLimit        int
+	RandomUserAgent   bool
+	Random            string
+	Index             string
+	MaxRedirect       int
+	MaxCrawlDepth     int
+	MaxRecursionDepth int
+	MaxAppendDepth    int
+}
+
+func NewBruteWords(config *Config, list []string) *words.Worder {
+	word := words.NewWorderWithList(list)
+	word.Fns = config.Fns
+	word.Run()
+	return word
+}
+
+func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder {
+	word, err := words.NewWorderWithDsl(dsl, params, nil)
+	if err != nil {
+		logs.Log.Error(err.Error())
+	}
+	word.Fns = config.Fns
+	word.Run()
+	return word
+}

internal/pool/pool.go

@@ -29,7 +29,7 @@ type BasePool struct {
 }
 
 func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) {
-	if depth >= MaxRedirect {
+	if depth >= pool.MaxRedirect {
 		return
 	}
 	reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL)

internal/pool/type.go

@@ -0,0 +1,47 @@
+package pool
+
+import (
+	"github.com/chainreactors/parsers"
+	"github.com/chainreactors/spray/pkg"
+)
+
+func newUnit(path string, source parsers.SpraySource) *Unit {
+	return &Unit{path: path, source: source}
+}
+
+type Unit struct {
+	number   int
+	host     string
+	path     string
+	source   parsers.SpraySource
+	retry    int
+	frontUrl string
+	depth    int
+}
+
+func NewBaselines() *Baselines {
+	return &Baselines{
+		baselines: map[int]*pkg.Baseline{},
+	}
+}
+
+type Baselines struct {
+	FailedBaselines []*pkg.Baseline
+	random          *pkg.Baseline
+	index           *pkg.Baseline
+	baselines       map[int]*pkg.Baseline
+}
+
+type SprayMod int
+
+const (
+	PathSpray SprayMod = iota + 1
+	HostSpray
+	ParamSpray
+	CustomSpray
+)
+
+var ModMap = map[string]SprayMod{
+	"path": PathSpray,
+	"host": HostSpray,
+}

internal/pool/types.go

@@ -1,107 +0,0 @@
-package pool
-
-import (
-	"github.com/chainreactors/logs"
-	"github.com/chainreactors/parsers"
-	"github.com/chainreactors/spray/pkg"
-	"github.com/chainreactors/words"
-	"github.com/chainreactors/words/rule"
-	"github.com/expr-lang/expr/vm"
-	"sync"
-	"time"
-)
-
-func newUnit(path string, source parsers.SpraySource) *Unit {
-	return &Unit{path: path, source: source}
-}
-
-type Unit struct {
-	number   int
-	host     string
-	path     string
-	source   parsers.SpraySource
-	retry    int
-	frontUrl string
-	depth    int // redirect depth
-}
-
-func NewBaselines() *Baselines {
-	return &Baselines{
-		baselines: map[int]*pkg.Baseline{},
-	}
-}
-
-type Baselines struct {
-	FailedBaselines []*pkg.Baseline
-	random          *pkg.Baseline
-	index           *pkg.Baseline
-	baselines       map[int]*pkg.Baseline
-}
-
-type SprayMod int
-
-const (
-	PathSpray SprayMod = iota + 1
-	HostSpray
-	ParamSpray
-	CustomSpray
-)
-
-var ModMap = map[string]SprayMod{
-	"path": PathSpray,
-	"host": HostSpray,
-}
-
-type Config struct {
-	BaseURL         string
-	ProxyAddr       string
-	Thread          int
-	Wordlist        []string
-	Timeout         time.Duration
-	ProcessCh       chan *pkg.Baseline
-	OutputCh        chan *pkg.Baseline
-	FuzzyCh         chan *pkg.Baseline
-	Outwg           *sync.WaitGroup
-	RateLimit       int
-	CheckPeriod     int
-	ErrPeriod       int32
-	BreakThreshold  int32
-	Method          string
-	Mod             SprayMod
-	Headers         map[string]string
-	ClientType      int
-	MatchExpr       *vm.Program
-	FilterExpr      *vm.Program
-	RecuExpr        *vm.Program
-	AppendRule      *rule.Program
-	Fns             []words.WordFunc
-	AppendWords     []string
-	Fuzzy           bool
-	IgnoreWaf       bool
-	Crawl           bool
-	Scope           []string
-	Active          bool
-	Bak             bool
-	Common          bool
-	RetryLimit      int
-	RandomUserAgent bool
-	Random          string
-	Index           string
-}
-
-func NewBruteWords(config *Config, list []string) *words.Worder {
-	word := words.NewWorderWithList(list)
-	word.Fns = config.Fns
-	word.Run()
-	return word
-}
-
-func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder {
-	word, err := words.NewWorderWithDsl(dsl, params, nil)
-	if err != nil {
-		logs.Log.Error(err.Error())
-	}
-	word.Fns = config.Fns
-	word.Run()
-	return word
-}

internal/runner.go

@@ -51,7 +51,6 @@ type Runner struct {
 	Count       int // tasks total number
 	Wordlist    []string
 	AppendWords []string
-	RecuDepth   int
 	ClientType  int
 	Probes      []string
 	Total       int // wordlist total number
@@ -81,17 +80,21 @@ func (r *Runner) PrepareConfig() *pool.Config {
 		AppendWords:    r.AppendWords, // 对有效目录追加字典
 		Fns:            r.Fns,
 		//IgnoreWaf:       r.IgnoreWaf,
-		Crawl:           r.CrawlPlugin,
+		Crawl:             r.CrawlPlugin,
-		Scope:           r.Scope,
+		Scope:             r.Scope,
-		Active:          r.Finger,
+		Active:            r.Finger,
-		Bak:             r.BakPlugin,
+		Bak:               r.BakPlugin,
-		Common:          r.CommonPlugin,
+		Common:            r.CommonPlugin,
-		RetryLimit:      r.RetryCount,
+		RetryLimit:        r.RetryCount,
-		ClientType:      r.ClientType,
+		ClientType:        r.ClientType,
-		RandomUserAgent: r.RandomUserAgent,
+		RandomUserAgent:   r.RandomUserAgent,
-		Random:          r.Random,
+		Random:            r.Random,
-		Index:           r.Index,
+		Index:             r.Index,
-		ProxyAddr:       r.Proxy,
+		ProxyAddr:         r.Proxy,
+		MaxRecursionDepth: r.Depth,
+		MaxRedirect:       3,
+		MaxAppendDepth:    r.AppendDepth,
+		MaxCrawlDepth:     r.CrawlDepth,
 	}
 
 	if config.ClientType == ihttp.Auto {