From 350cf68e45b7b0cc72bea0737e8fafc80c643c42 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 6 Jan 2023 00:48:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0--bak,=20=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=AD=97=E5=85=B8=E7=88=86=E7=A0=B4=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/cmd.go | 4 --- go.mod | 2 +- go.sum | 6 +++++ internal/option.go | 4 ++- internal/pool.go | 65 ++++++++++++++++++++++++++++++++-------------- internal/runner.go | 11 +++++--- internal/types.go | 1 + internal/utils.go | 2 +- pkg/config.go | 1 + pkg/utils.go | 56 +++++++++++++++++++++++++++++++++++---- spray.go | 2 +- 11 files changed, 118 insertions(+), 36 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index f238b1e..ce95161 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -55,10 +55,6 @@ func Spray() { if err != nil { utils.Fatal(err.Error()) } - err = pkg.LoadRules() - if err != nil { - utils.Fatal(err.Error()) - } var runner *internal.Runner if option.ResumeFrom != "" { diff --git a/go.mod b/go.mod index 96e2971..b41ead6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/chainreactors/ipcs v0.0.13 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410 - github.com/chainreactors/words v0.3.2-0.20230105095023-67f7d4e9186a + github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a ) require ( diff --git a/go.sum b/go.sum index ede6af4..0c2bf43 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,12 @@ github.com/chainreactors/words v0.3.2-0.20221214154622-381fc37abdf9 h1:IUNopSuor github.com/chainreactors/words v0.3.2-0.20221214154622-381fc37abdf9/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/chainreactors/words v0.3.2-0.20230105095023-67f7d4e9186a h1:NoFfxJfPXiS2fzdmRIzWj4K+V7BRC2BAXlxQfckTeN0= github.com/chainreactors/words v0.3.2-0.20230105095023-67f7d4e9186a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/words v0.3.2-0.20230105160347-858217c41ce5 h1:uTiOYpgf4jz+/uwp+kAliLrOkVXjsC51pNmd4xH0uB4= +github.com/chainreactors/words v0.3.2-0.20230105160347-858217c41ce5/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/words v0.3.2-0.20230105161438-ec98bdc6906d h1:vWZwr3IaoEGEGE/IB0Im4gDqrOHpGK3szKOFDG4GFrc= +github.com/chainreactors/words v0.3.2-0.20230105161438-ec98bdc6906d/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U= +github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/option.go b/internal/option.go index 824a5a6..a5c967b 100644 --- a/internal/option.go +++ b/internal/option.go @@ -83,6 +83,7 @@ type ModeOptions struct { Depth int `long:"depth" default:"0" description:"Int, recursive depth"` Active bool `long:"active" description:"Bool, enable active finger detect"` Crawl bool `long:"crawl" description:"Bool, enable crawl"` + Bak bool `long:"bak" description:"Bool, enable bak found"` FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"` CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth"` CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request"` @@ -134,6 +135,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { BreakThreshold: opt.BreakThreshold, Crawl: opt.Crawl, Active: opt.Active, + Bak: opt.Bak, } if opt.Extracts != nil { @@ -254,7 +256,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { opt.Word += "{@ext}" } - r.Wordlist, err = mask.Run(opt.Word, dicts) + r.Wordlist, err = mask.Run(opt.Word, dicts, nil) if err != nil { return nil, err } diff --git a/internal/pool.go b/internal/pool.go index 60291b4..338e126 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -144,7 +144,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { case RedirectSource: bl.FrontURL = unit.frontUrl pool.tempCh <- bl - case CrawlSource, ActiveSource, RuleSource: + case CrawlSource, ActiveSource, RuleSource, BakSource: pool.tempCh <- bl } @@ -312,6 +312,11 @@ func (pool *Pool) Run(ctx context.Context, offset, limit int) { go pool.doActive() } + if pool.Bak { + pool.wg.Add(1) + go pool.doBak() + } + Loop: for { select { @@ -504,6 +509,25 @@ func (pool *Pool) doCrawl(bl *pkg.Baseline) { } } +func (pool *Pool) doRule(bl *pkg.Baseline) { + defer pool.wg.Done() + if pool.AppendRule == nil { + return + } + if bl.Source == int(RuleSource) || bl.Dir { + return + } + + for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { + pool.wg.Add(1) + pool.additionCh <- &Unit{ + path: path.Join(path.Dir(bl.Path), u), + source: RuleSource, + depth: 1, + } + } +} + func (pool *Pool) doActive() { defer pool.wg.Done() for _, u := range pkg.ActivePath { @@ -515,6 +539,26 @@ func (pool *Pool) doActive() { } } +func (pool *Pool) doBak() { + defer pool.wg.Done() + u, err := url.Parse(pool.BaseURL) + if err != nil { + return + } + worder, err := words.NewWorderWithDsl("{?0}.{@bak_ext}", [][]string{pkg.BakGenerator(u.Host)}, nil) + if err != nil { + return + } + worder.Run() + for w := range worder.C { + pool.wg.Add(1) + pool.additionCh <- &Unit{ + path: w, + source: BakSource, + } + } +} + func (pool *Pool) doCheck() { if pool.failedCount > pool.BreakThreshold { // 当报错次数超过上限是, 结束任务 @@ -531,25 +575,6 @@ func (pool *Pool) doCheck() { } } -func (pool *Pool) doRule(bl *pkg.Baseline) { - defer pool.wg.Done() - if pool.AppendRule == nil { - return - } - if bl.Source == int(RuleSource) || bl.Dir { - return - } - - for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { - pool.wg.Add(1) - pool.additionCh <- &Unit{ - path: path.Join(path.Dir(bl.Path), u), - source: RuleSource, - depth: 1, - } - } -} - func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) { if _, ok := pool.baselines[bl.Status]; !ok && IntsContains(FuzzyStatus, bl.Status) { bl.Collect() diff --git a/internal/runner.go b/internal/runner.go index 8ccad09..76f70d2 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -74,6 +74,7 @@ type Runner struct { IgnoreWaf bool Crawl bool Active bool + Bak bool } func (r *Runner) PrepareConfig() *pkg.Config { @@ -95,6 +96,7 @@ func (r *Runner) PrepareConfig() *pkg.Config { IgnoreWaf: r.IgnoreWaf, Crawl: r.Crawl, Active: r.Active, + Bak: r.Bak, } if config.Mod == pkg.PathSpray { config.ClientType = ihttp.FAST @@ -118,7 +120,8 @@ func (r *Runner) Prepare(ctx context.Context) error { r.poolwg.Done() return } - pool.worder = words.NewWorderWithFns(r.URLList, r.Fns) + pool.worder = words.NewWorder(r.URLList) + pool.worder.Fns = r.Fns pool.bar = pkg.NewBar("check", r.Total-r.Offset, r.Progress) pool.Run(ctx, r.Offset, r.Total) r.poolwg.Done() @@ -166,7 +169,8 @@ func (r *Runner) Prepare(ctx context.Context) error { r.Done() return } - pool.worder = words.NewWorderWithFns(wl, r.Fns) + pool.worder = words.NewWorder(wl) + pool.worder.Fns = r.Fns rules, err := loadRuleWithFiles(t.origin.RuleFiles, t.origin.RuleFilter) if err != nil { logs.Log.Error(err.Error()) @@ -181,7 +185,8 @@ func (r *Runner) Prepare(ctx context.Context) error { } } else { pool.Statistor = pkg.NewStatistor(t.baseUrl) - pool.worder = words.NewWorderWithFns(r.Wordlist, r.Fns) + pool.worder = words.NewWorder(r.Wordlist) + pool.worder.Fns = r.Fns pool.worder.Rules = r.Rules.Expressions } diff --git a/internal/types.go b/internal/types.go index 49524e6..b3961ca 100644 --- a/internal/types.go +++ b/internal/types.go @@ -54,6 +54,7 @@ const ( WordSource WafSource RuleSource + BakSource ) func newUnit(path string, source int) *Unit { diff --git a/internal/utils.go b/internal/utils.go index d383023..61f0ff4 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -101,7 +101,7 @@ func loadWordlist(word string, dictNames []string) ([]string, error) { if err != nil { return nil, err } - wl, err := mask.Run(word, dicts) + wl, err := mask.Run(word, dicts, nil) if err != nil { return nil, err } diff --git a/pkg/config.go b/pkg/config.go index 84f26a6..8897e70 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -41,4 +41,5 @@ type Config struct { IgnoreWaf bool Crawl bool Active bool + Bak bool } diff --git a/pkg/utils.go b/pkg/utils.go index 2dc9dec..a3ea522 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -3,7 +3,9 @@ package pkg import ( "encoding/json" "github.com/chainreactors/gogo/v2/pkg/fingers" + "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/ipcs" + "github.com/chainreactors/words/mask" "math/rand" "net/url" "os" @@ -33,6 +35,24 @@ var ( } ) +func StringsContains(s []string, e string) bool { + for _, v := range s { + if v == e { + return true + } + } + return false +} + +func IntsContains(s []int, e int) bool { + for _, v := range s { + if v == e { + return true + } + } + return false +} + func HasStdin() bool { stat, err := os.Stdin.Stat() if err != nil { @@ -98,6 +118,7 @@ func RandHost() string { func LoadTemplates() error { var err error + // load fingers Fingers, err = fingers.LoadFingers(LoadConfig("http")) if err != nil { return err @@ -126,18 +147,30 @@ func LoadTemplates() error { } } - return nil -} - -func LoadRules() error { + // load rule var data map[string]interface{} - err := json.Unmarshal(LoadConfig("rule"), &data) + err = json.Unmarshal(LoadConfig("rule"), &data) if err != nil { return err } for k, v := range data { Rules[k] = v.(string) } + + // load mask + var keywords map[string]interface{} + err = json.Unmarshal(LoadConfig("mask"), &keywords) + if err != nil { + return err + } + + for k, v := range keywords { + t := make([]string, len(v.([]interface{}))) + for i, vv := range v.([]interface{}) { + t[i] = utils.ToString(vv) + } + mask.SpecialWords[k] = t + } return nil } @@ -199,3 +232,16 @@ func URLJoin(base, uri string) string { return base + "/" + uri } } + +func BakGenerator(domain string) []string { + var possibilities []string + for first, _ := range domain { + for last, _ := range domain[first:] { + p := domain[first : first+last+1] + if !StringsContains(possibilities, p) { + possibilities = append(possibilities, p) + } + } + } + return possibilities +} diff --git a/spray.go b/spray.go index 8972553..a279e05 100644 --- a/spray.go +++ b/spray.go @@ -1,4 +1,4 @@ -//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule +//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask package main import "github.com/chainreactors/spray/cmd"