diff --git a/go.mod b/go.mod index 8c1114e..96e2971 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/chainreactors/ipcs v0.0.13 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410 - github.com/chainreactors/words v0.3.2-0.20221214154622-381fc37abdf9 + github.com/chainreactors/words v0.3.2-0.20230105095023-67f7d4e9186a ) require ( diff --git a/go.sum b/go.sum index 85da0a7..ede6af4 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/chainreactors/words v0.3.2-0.20221214062855-48dff09b01ad h1:uL3TIQgvF github.com/chainreactors/words v0.3.2-0.20221214062855-48dff09b01ad/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/chainreactors/words v0.3.2-0.20221214154622-381fc37abdf9 h1:IUNopSuorfINmn4pOuSwZtxJbg8zsRIZ67a33SiYoQ0= github.com/chainreactors/words v0.3.2-0.20221214154622-381fc37abdf9/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/words v0.3.2-0.20230105095023-67f7d4e9186a h1:NoFfxJfPXiS2fzdmRIzWj4K+V7BRC2BAXlxQfckTeN0= +github.com/chainreactors/words v0.3.2-0.20230105095023-67f7d4e9186a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/option.go b/internal/option.go index 2d90dea..7e54e63 100644 --- a/internal/option.go +++ b/internal/option.go @@ -38,8 +38,9 @@ type InputOptions struct { Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"` Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt"` Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}"` - FilterRule string `long:"rule-filter" description:"String, filter rule, e.g.: --rule-filter '>8'"` Rules []string `short:"r" long:"rules" description:"Files, Multi, rule files, e.g.: -r rule1.txt -r rule2.txt"` + AppendRule string `long:"append-rule" description:"File, when found valid path , use append rule generator new word with current path"` + FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'"` } type FunctionOptions struct { @@ -274,13 +275,16 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } else if opt.FilterRule != "" { // if filter rule is not empty, set rules to ":", force to open filter mode r.Rules = rule.Compile(":", opt.FilterRule) + } else { + r.Rules = new(rule.Program) } - if len(r.Rules) > 0 { - r.Total = len(r.Wordlist) * len(r.Rules) + if len(r.Rules.Expressions) > 0 { + r.Total = len(r.Wordlist) * len(r.Rules.Expressions) } else { r.Total = len(r.Wordlist) } + pkg.DefaultStatistor = pkg.Statistor{ Word: opt.Word, WordCount: len(r.Wordlist), @@ -291,6 +295,13 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Total: r.Total, } + if opt.AppendRule != "" { + content, err := ioutil.ReadFile(opt.AppendRule) + if err != nil { + return nil, err + } + r.AppendRules = rule.Compile(string(content), "") + } // prepare task var tasks []*Task var taskfrom string diff --git a/internal/pool.go b/internal/pool.go index 07c3afe..60291b4 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -9,9 +9,11 @@ import ( "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/words" + "github.com/chainreactors/words/rule" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" "net/url" + "path" "strconv" "strings" "sync" @@ -36,7 +38,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { baselines: make(map[int]*pkg.Baseline), urls: make(map[string]int), tempCh: make(chan *pkg.Baseline, config.Thread), - checkCh: make(chan sourceType), + checkCh: make(chan int), additionCh: make(chan *Unit, 100), wg: sync.WaitGroup{}, initwg: sync.WaitGroup{}, @@ -83,6 +85,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { bl.RedirectURL = "/" + strings.TrimLeft(bl.RedirectURL, "/") bl.RedirectURL = pool.BaseURL + bl.RedirectURL } + pool.wg.Add(1) pool.doRedirect(bl, unit.depth) } pool.addFuzzyBaseline(bl) @@ -105,6 +108,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { pool.initwg.Done() case InitIndexSource: pool.index = bl + pool.wg.Add(1) pool.doCrawl(bl) pool.initwg.Done() case CheckSource: @@ -140,7 +144,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { case RedirectSource: bl.FrontURL = unit.frontUrl pool.tempCh <- bl - case CrawlSource, ActiveSource: + case CrawlSource, ActiveSource, RuleSource: pool.tempCh <- bl } @@ -196,7 +200,9 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 if bl.IsValid { - pool.doCrawl(bl) + pool.wg.Add(2) + go pool.doCrawl(bl) + go pool.doRule(bl) if bl.RecuDepth < maxRecursion { if CompareWithExpr(pool.RecuExpr, params) { bl.Recu = true @@ -221,7 +227,7 @@ type Pool struct { ctx context.Context cancel context.CancelFunc tempCh chan *pkg.Baseline // 待处理的baseline - checkCh chan sourceType // 独立的check管道, 防止与redirect/crawl冲突 + checkCh chan int // 独立的check管道, 防止与redirect/crawl冲突 additionCh chan *Unit reqCount int failedCount int @@ -302,6 +308,7 @@ func (pool *Pool) Run(ctx context.Context, offset, limit int) { } }() if pool.Active { + pool.wg.Add(1) go pool.doActive() } @@ -341,10 +348,12 @@ Loop: } } - for len(pool.additionCh) > 0 { - time.Sleep(time.Second) + for pool.analyzeDone { + time.Sleep(time.Duration(100) * time.Millisecond) } + pool.wg.Wait() + pool.Statistor.EndTime = time.Now().Unix() pool.Close() } @@ -435,6 +444,7 @@ func CompareWithExpr(exp *vm.Program, params map[string]interface{}) bool { } func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) { + defer pool.wg.Done() if depth >= maxRedirect { return } @@ -451,6 +461,10 @@ func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) { } func (pool *Pool) doCrawl(bl *pkg.Baseline) { + defer pool.wg.Done() + if !pool.Crawl { + return + } bl.CollectURL() for _, u := range bl.URLs { if strings.HasPrefix(u, "//") { @@ -467,7 +481,9 @@ func (pool *Pool) doCrawl(bl *pkg.Baseline) { pool.urls[u]++ } else { // 通过map去重, 只有新的url才会进入到该逻辑 + pool.locker.Lock() pool.urls[u] = 1 + pool.locker.Unlock() if bl.ReqDepth < maxCrawl { parsed, err := url.Parse(u) if err != nil { @@ -489,6 +505,7 @@ func (pool *Pool) doCrawl(bl *pkg.Baseline) { } func (pool *Pool) doActive() { + defer pool.wg.Done() for _, u := range pkg.ActivePath { pool.wg.Add(1) pool.additionCh <- &Unit{ @@ -514,10 +531,30 @@ func (pool *Pool) doCheck() { } } +func (pool *Pool) doRule(bl *pkg.Baseline) { + defer pool.wg.Done() + if pool.AppendRule == nil { + return + } + if bl.Source == int(RuleSource) || bl.Dir { + return + } + + for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { + pool.wg.Add(1) + pool.additionCh <- &Unit{ + path: path.Join(path.Dir(bl.Path), u), + source: RuleSource, + depth: 1, + } + } +} + func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) { if _, ok := pool.baselines[bl.Status]; !ok && IntsContains(FuzzyStatus, bl.Status) { bl.Collect() pool.locker.Lock() + pool.wg.Add(1) pool.doCrawl(bl) pool.baselines[bl.Status] = bl pool.locker.Unlock() diff --git a/internal/runner.go b/internal/runner.go index a46e538..8ccad09 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -38,7 +38,8 @@ type Runner struct { Tasks []*Task URLList []string Wordlist []string - Rules []rule.Expression + Rules *rule.Program + AppendRules *rule.Program Headers map[string]string Fns []func(string) string FilterExpr *vm.Program @@ -90,6 +91,7 @@ func (r *Runner) PrepareConfig() *pkg.Config { MatchExpr: r.MatchExpr, FilterExpr: r.FilterExpr, RecuExpr: r.RecursiveExpr, + AppendRule: r.AppendRules, IgnoreWaf: r.IgnoreWaf, Crawl: r.Crawl, Active: r.Active, @@ -180,7 +182,7 @@ func (r *Runner) Prepare(ctx context.Context) error { } else { pool.Statistor = pkg.NewStatistor(t.baseUrl) pool.worder = words.NewWorderWithFns(r.Wordlist, r.Fns) - pool.worder.Rules = r.Rules + pool.worder.Rules = r.Rules.Expressions } var limit int diff --git a/internal/types.go b/internal/types.go index 1311601..49524e6 100644 --- a/internal/types.go +++ b/internal/types.go @@ -44,10 +44,8 @@ func (e ErrorType) Error() string { } } -type sourceType int - const ( - CheckSource sourceType = iota + 1 + CheckSource = iota + 1 InitRandomSource InitIndexSource RedirectSource @@ -55,19 +53,20 @@ const ( ActiveSource WordSource WafSource + RuleSource ) -func newUnit(path string, source sourceType) *Unit { +func newUnit(path string, source int) *Unit { return &Unit{path: path, source: source} } -func newUnitWithNumber(path string, source sourceType, number int) *Unit { +func newUnitWithNumber(path string, source int, number int) *Unit { return &Unit{path: path, source: source} } type Unit struct { path string - source sourceType + source int frontUrl string depth int // redirect depth } diff --git a/internal/utils.go b/internal/utils.go index d985fe2..329aca9 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -103,5 +103,5 @@ func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, er rules.Write(content) rules.WriteString("\n") } - return rule.Compile(rules.String(), filter), nil + return rule.Compile(rules.String(), filter).Expressions, nil } diff --git a/pkg/baseline.go b/pkg/baseline.go index b7c1d7c..20399b2 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -47,6 +47,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { bl.Path = uu.Path bl.Url = uu } + bl.Dir = bl.IsDir() if resp.ClientType == ihttp.STANDARD { bl.Host = host } @@ -72,6 +73,7 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba bl.Path = uu.Path bl.Url = uu } + bl.Dir = bl.IsDir() if resp.ClientType == ihttp.STANDARD { bl.Host = host @@ -91,6 +93,7 @@ type Baseline struct { Url *url.URL `json:"-"` UrlString string `json:"url"` Path string `json:"path"` + Dir bool `json:"isdir"` Host string `json:"host"` Body []byte `json:"-"` BodyLength int `json:"body_length"` diff --git a/pkg/config.go b/pkg/config.go index 334025b..84f26a6 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -2,6 +2,7 @@ package pkg import ( "github.com/antonmedv/expr/vm" + "github.com/chainreactors/words/rule" ) type SprayMod int @@ -33,6 +34,7 @@ type Config struct { MatchExpr *vm.Program FilterExpr *vm.Program RecuExpr *vm.Program + AppendRule *rule.Program OutputCh chan *Baseline FuzzyCh chan *Baseline Fuzzy bool diff --git a/rule/filebak.txt b/rule/filebak.txt new file mode 100644 index 0000000..881f5b7 --- /dev/null +++ b/rule/filebak.txt @@ -0,0 +1,12 @@ +$~ +$b $a $k +$. $b $a $k +$. $b $a $k $2 +$. $o $l $d +$. $1 +$. $2 +$. $z $i $p +$. $t $a $r +$. $g $z +^. +^. $. $s $w $p \ No newline at end of file