admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-09-15 11:40:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

实装--crawl-depth

Browse Source
This commit is contained in:
M09Ic 2023-01-10 01:09:00 +08:00
parent 1fca8c146f
commit f87acdf657
3 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/cmd.go
View File

@ -75,6 +75,7 @@ func Spray() {
pkg.Distance = uint8(option.SimhashDistance) pkg.Distance = uint8(option.SimhashDistance)
ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024 ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024
internal.MaxCrawl = option.CrawlDepth
if option.ReadAll { if option.ReadAll {
ihttp.DefaultMaxBodySize = 0 ihttp.DefaultMaxBodySize = 0
} }

4
internal/option.go
View File

@ -414,7 +414,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
} }
if opt.Recursive != "current.IsDir()" { if opt.Recursive != "current.IsDir()" {
maxRecursion = 1 MaxRecursion = 1
exp, err := expr.Compile(opt.Recursive) exp, err := expr.Compile(opt.Recursive)
if err != nil { if err != nil {
return nil, err return nil, err
@ -422,7 +422,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
r.RecursiveExpr = exp r.RecursiveExpr = exp
} }
if opt.Depth != 0 { if opt.Depth != 0 {
maxRecursion = opt.Depth MaxRecursion = opt.Depth
} }
// prepare header // prepare header

12
internal/pool.go
View File

@ -24,9 +24,9 @@ import (
var ( var (
max = 2147483647 max = 2147483647
maxRedirect = 3 MaxRedirect = 3
maxCrawl = 3 MaxCrawl = 3
maxRecursion = 0 MaxRecursion = 0
nilBaseline = &pkg.Baseline{} nilBaseline = &pkg.Baseline{}
) )
@ -107,7 +107,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
pool.wg.Add(2) pool.wg.Add(2)
pool.doCrawl(bl) pool.doCrawl(bl)
pool.doRule(bl) pool.doRule(bl)
if bl.RecuDepth < maxRecursion { if bl.RecuDepth < MaxRecursion {
if CompareWithExpr(pool.RecuExpr, params) { if CompareWithExpr(pool.RecuExpr, params) {
bl.Recu = true bl.Recu = true
} }
@ -473,7 +473,7 @@ func CompareWithExpr(exp *vm.Program, params map[string]interface{}) bool {
func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) { func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) {
defer pool.wg.Done() defer pool.wg.Done()
if depth >= maxRedirect { if depth >= MaxRedirect {
return return
} }
@ -546,7 +546,7 @@ func (pool *Pool) doCrawl(bl *pkg.Baseline) {
} else { } else {
// 通过map去重, 只有新的url才会进入到该逻辑 // 通过map去重, 只有新的url才会进入到该逻辑
pool.urls[u] = 1 pool.urls[u] = 1
if bl.ReqDepth < maxCrawl { if bl.ReqDepth < MaxCrawl {
pool.wg.Add(1) pool.wg.Add(1)
pool.addAddition(&Unit{ pool.addAddition(&Unit{
path: u[1:], path: u[1:],