diff --git a/internal/option.go b/internal/option.go index 0ab274d..aeb10e3 100644 --- a/internal/option.go +++ b/internal/option.go @@ -77,14 +77,15 @@ type RequestOptions struct { type ModeOptions struct { Advance bool `short:"a" long:"advance" description:"Bool, enable crawl and active"` - Force bool `long:"force" description:"Bool, skip error break"` - CheckOnly bool `long:"check-only" description:"Bool, check only"` - Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"` - Depth int `long:"depth" default:"0" description:"Int, recursive depth"` Active bool `long:"active" description:"Bool, enable active finger detect"` Crawl bool `long:"crawl" description:"Bool, enable crawl"` Bak bool `long:"bak" description:"Bool, enable bak found"` FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"` + Common bool `long:"common" description:"Bool, enable common file found"` + Force bool `long:"force" description:"Bool, skip error break"` + CheckOnly bool `long:"check-only" description:"Bool, check only"` + Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"` + Depth int `long:"depth" default:"0" description:"Int, recursive depth"` CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth"` CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request"` ErrPeriod int `long:"error-period" default:"10" description:"Int, check period when error"` @@ -136,6 +137,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Crawl: opt.Crawl, Active: opt.Active, Bak: opt.Bak, + Common: opt.Common, } if opt.Extracts != nil { @@ -181,6 +183,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.Crawl = true r.Active = true r.Bak = true + r.Common = true opt.AppendRule = append(opt.AppendRule, "filebak") } else if opt.FileBak { opt.AppendRule = append(opt.AppendRule, "filebak") diff --git a/internal/pool.go b/internal/pool.go index 92b255f..1e7162a 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -9,6 +9,7 @@ import ( "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/words" + "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" @@ -203,12 +204,19 @@ func (pool *Pool) Run(ctx context.Context, offset, limit int) { pool.wg.Add(1) go pool.doBak() } + + if pool.Common { + pool.wg.Add(1) + go pool.doCommonFile() + } + go func() { for { pool.wg.Wait() pool.closeCh <- struct{}{} } }() + Loop: for { select { @@ -284,7 +292,7 @@ func (pool *Pool) Invoke(v interface{}) { bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} pool.failedBaselines = append(pool.failedBaselines, bl) } else { - if unit.source <= 3 || unit.source == CrawlSource { + if unit.source <= 3 || unit.source == CrawlSource || unit.source == CommonFileSource { bl = pkg.NewBaseline(req.URI(), req.Host(), resp) } else { if pool.MatchExpr != nil { @@ -366,7 +374,7 @@ func (pool *Pool) Invoke(v interface{}) { case RedirectSource: bl.FrontURL = unit.frontUrl pool.tempCh <- bl - case CrawlSource, ActiveSource, RuleSource, BakSource: + default: pool.tempCh <- bl } } @@ -572,6 +580,30 @@ func (pool *Pool) doBak() { source: BakSource, }) } + + worder, err = words.NewWorderWithDsl("{@bak_name}.{@bak_ext}", nil, nil) + if err != nil { + return + } + worder.Run() + for w := range worder.C { + pool.wg.Add(1) + pool.addAddition(&Unit{ + path: safePath(pool.BaseURL, w), + source: BakSource, + }) + } +} + +func (pool *Pool) doCommonFile() { + defer pool.wg.Done() + for _, u := range mask.SpecialWords["common_file"] { + pool.wg.Add(1) + pool.addAddition(&Unit{ + path: safePath(pool.BaseURL, u), + source: CommonFileSource, + }) + } } func (pool *Pool) doCheck() { diff --git a/internal/runner.go b/internal/runner.go index f332581..e0c8203 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -75,6 +75,7 @@ type Runner struct { Crawl bool Active bool Bak bool + Common bool } func (r *Runner) PrepareConfig() *pkg.Config { @@ -97,6 +98,7 @@ func (r *Runner) PrepareConfig() *pkg.Config { Crawl: r.Crawl, Active: r.Active, Bak: r.Bak, + Common: r.Common, } if config.Mod == pkg.PathSpray { config.ClientType = ihttp.FAST diff --git a/internal/types.go b/internal/types.go index 4840e16..ddb7b00 100644 --- a/internal/types.go +++ b/internal/types.go @@ -55,6 +55,7 @@ const ( WafSource RuleSource BakSource + CommonFileSource ) func newUnit(path string, source int) *Unit { diff --git a/pkg/baseline.go b/pkg/baseline.go index 06d0fbc..8945fba 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -85,7 +85,9 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba bl.Host = host } - bl.Body = resp.Body() + body := resp.Body() + bl.Body = make([]byte, len(body)) + copy(bl.Body, body) bl.BodyLength = resp.ContentLength() bl.Header = resp.Header() bl.HeaderLength = len(bl.Header) diff --git a/pkg/config.go b/pkg/config.go index 8897e70..72d7615 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -42,4 +42,5 @@ type Config struct { Crawl bool Active bool Bak bool + Common bool }