diff --git a/go.mod b/go.mod index 8731b14..41bb4b8 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,8 @@ require ( github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f - github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84 + github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65 github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f - github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 github.com/expr-lang/expr v1.16.9 github.com/gookit/config/v2 v2.2.5 github.com/jessevdk/go-flags v1.5.0 @@ -25,6 +24,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.1.0 // indirect + github.com/chainreactors/words v0.0.0-20240910083848-19a289e8984b // indirect github.com/facebookincubator/nvdtools v0.1.5 // indirect github.com/fatih/color v1.17.0 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect diff --git a/go.sum b/go.sum index d14c15a..0abf8c8 100644 --- a/go.sum +++ b/go.sum @@ -97,14 +97,22 @@ github.com/chainreactors/parsers v0.0.0-20240825154421-240ad7eba29d h1:/rtYIkD9F github.com/chainreactors/parsers v0.0.0-20240825154421-240ad7eba29d/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84 h1:F6umsdHLxKILxrH5wY2zVYwxMmb8XdCEc9apf7GWpOk= github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= +github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65 h1:subSvyczsErYMRnCD07s4Ub6zOSaw2xZ1/O9t3tHkuw= +github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f h1:2NKmadFYP9vCwC0YrazgttFACleOhxScTPzg0i76YAY= github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= +github.com/chainreactors/words v0.0.0-20240910083135-4b34aa283ee2 h1:7hVdyBKy8Obs4faaVhnQX1cUwZizeIA6wKHHX5OZTjA= +github.com/chainreactors/words v0.0.0-20240910083135-4b34aa283ee2/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= +github.com/chainreactors/words v0.0.0-20240910083848-19a289e8984b h1:OsZ1fyarW4NwK/Oi+Yf3nm/dTW0uX0UfxFjyky5Mb60= +github.com/chainreactors/words v0.0.0-20240910083848-19a289e8984b/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= +github.com/chainreactors/words v0.4.1-0.20240910075300-24fe4facd5a6 h1:QpUlRyzw5wn9avRbTHBKRhV06X79Yj/ghGKn2Gyb9v4= +github.com/chainreactors/words v0.4.1-0.20240910075300-24fe4facd5a6/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/internal/option.go b/internal/option.go index 6d1eb51..2660578 100644 --- a/internal/option.go +++ b/internal/option.go @@ -30,7 +30,6 @@ import ( var ( DefaultThreads = 20 - SkipChar = "%SKIP%" ) type Option struct { @@ -614,19 +613,10 @@ func (opt *Option) BuildWords(r *Runner) error { // 类似dirsearch中的 if opt.Extensions != "" { - r.AppendFunction(func(s string) []string { - exts := strings.Split(opt.Extensions, ",") - ss := make([]string, len(exts)) - for i, e := range exts { - if strings.Contains(s, "%EXT%") { - ss[i] = strings.Replace(s, "%EXT%", e, -1) - } - } - return ss - }) + r.AppendFunction(pkg.ParseEXTPlaceholderFunc(strings.Split(opt.Extensions, ","))) } else { r.AppendFunction(func(s string) []string { - if strings.Contains(s, "%EXT%") { + if strings.Contains(s, pkg.EXTChar) { return nil } return []string{s} @@ -669,13 +659,6 @@ func (opt *Option) BuildWords(r *Runner) error { }) } - // default skip function, skip %EXT% - r.AppendFunction(func(s string) []string { - if strings.Contains(s, "%EXT%") { - return nil - } - return []string{s} - }) if len(opt.Skips) > 0 { r.AppendFunction(func(s string) []string { for _, skip := range opt.Skips { diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index dad0c4a..bdf6582 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -9,7 +9,6 @@ import ( "github.com/chainreactors/spray/internal/ihttp" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/utils/iutils" - "github.com/chainreactors/words" "github.com/chainreactors/words/rule" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" @@ -200,7 +199,7 @@ func (pool *BrutePool) Run(offset, limit int) { Loop: for { select { - case w, ok := <-pool.Worder.C: + case w, ok := <-pool.Worder.Output: if !ok { done = true continue @@ -514,7 +513,7 @@ func (pool *BrutePool) Handler() { } func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { - if pool.AppendRule == nil || bl.Source == parsers.RuleSource { + if pool.AppendRule == nil || bl.Source == parsers.AppendRuleSource { pool.wg.Done() return } @@ -525,7 +524,7 @@ func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) { pool.addAddition(&Unit{ path: pkg.Dir(bl.Url.Path) + u, host: bl.Host, - source: parsers.RuleSource, + source: parsers.AppendRuleSource, }) } }() @@ -540,7 +539,8 @@ func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) { go func() { defer pool.wg.Done() - for _, u := range pool.AppendWords { + + for u := range NewBruteWords(pool.Config, pool.AppendWords).Output { pool.addAddition(&Unit{ path: pkg.SafePath(bl.Path, u), host: bl.Host, @@ -574,13 +574,7 @@ func (pool *BrutePool) doCommonFile() { if pool.Mod == HostSpray { return } - for _, u := range pkg.Dicts["common"] { - pool.addAddition(&Unit{ - path: pool.dir + u, - source: parsers.CommonFileSource, - }) - } - for _, u := range pkg.Dicts["log"] { + for u := range NewBruteWords(pool.Config, append(pkg.Dicts["common"], pkg.Dicts["log"]...)).Output { pool.addAddition(&Unit{ path: pool.dir + u, source: parsers.CommonFileSource, @@ -820,24 +814,14 @@ func (pool *BrutePool) doBak() { if pool.Mod == HostSpray { return } - worder, err := words.NewWorderWithDsl("{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil) - if err != nil { - return - } - worder.Run() - for w := range worder.C { + for w := range NewBruteDSL(pool.Config, "{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}).Output { pool.addAddition(&Unit{ path: pool.dir + w, source: parsers.BakSource, }) } - worder, err = words.NewWorderWithDsl("{?@bak_name}.{?@bak_ext}", nil, nil) - if err != nil { - return - } - worder.Run() - for w := range worder.C { + for w := range NewBruteDSL(pool.Config, "{?@bak_name}.{?@bak_ext}", nil).Output { pool.addAddition(&Unit{ path: pool.dir + w, source: parsers.BakSource, diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index a1cbb8b..81f1b43 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -67,7 +67,7 @@ func (pool *CheckPool) Run(ctx context.Context, offset, limit int) { Loop: for { select { - case u, ok := <-pool.Worder.C: + case u, ok := <-pool.Worder.Output: if !ok { done = true continue diff --git a/internal/pool/config.go b/internal/pool/config.go deleted file mode 100644 index 0857b49..0000000 --- a/internal/pool/config.go +++ /dev/null @@ -1,59 +0,0 @@ -package pool - -import ( - "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/words/rule" - "github.com/expr-lang/expr/vm" - "sync" - "time" -) - -type SprayMod int - -const ( - PathSpray SprayMod = iota + 1 - HostSpray - ParamSpray - CustomSpray -) - -var ModMap = map[string]SprayMod{ - "path": PathSpray, - "host": HostSpray, -} - -type Config struct { - BaseURL string - ProxyAddr string - Thread int - Wordlist []string - Timeout time.Duration - ProcessCh chan *pkg.Baseline - OutputCh chan *pkg.Baseline - FuzzyCh chan *pkg.Baseline - Outwg *sync.WaitGroup - RateLimit int - CheckPeriod int - ErrPeriod int32 - BreakThreshold int32 - Method string - Mod SprayMod - Headers map[string]string - ClientType int - MatchExpr *vm.Program - FilterExpr *vm.Program - RecuExpr *vm.Program - AppendRule *rule.Program - AppendWords []string - Fuzzy bool - IgnoreWaf bool - Crawl bool - Scope []string - Active bool - Bak bool - Common bool - RetryLimit int - RandomUserAgent bool - Random string - Index string -} diff --git a/internal/pool/types.go b/internal/pool/types.go index 791c838..87128da 100644 --- a/internal/pool/types.go +++ b/internal/pool/types.go @@ -1,8 +1,14 @@ package pool import ( + "github.com/chainreactors/logs" "github.com/chainreactors/parsers" "github.com/chainreactors/spray/pkg" + "github.com/chainreactors/words" + "github.com/chainreactors/words/rule" + "github.com/expr-lang/expr/vm" + "sync" + "time" ) func newUnit(path string, source parsers.SpraySource) *Unit { @@ -31,3 +37,71 @@ type Baselines struct { index *pkg.Baseline baselines map[int]*pkg.Baseline } + +type SprayMod int + +const ( + PathSpray SprayMod = iota + 1 + HostSpray + ParamSpray + CustomSpray +) + +var ModMap = map[string]SprayMod{ + "path": PathSpray, + "host": HostSpray, +} + +type Config struct { + BaseURL string + ProxyAddr string + Thread int + Wordlist []string + Timeout time.Duration + ProcessCh chan *pkg.Baseline + OutputCh chan *pkg.Baseline + FuzzyCh chan *pkg.Baseline + Outwg *sync.WaitGroup + RateLimit int + CheckPeriod int + ErrPeriod int32 + BreakThreshold int32 + Method string + Mod SprayMod + Headers map[string]string + ClientType int + MatchExpr *vm.Program + FilterExpr *vm.Program + RecuExpr *vm.Program + AppendRule *rule.Program + Fns []words.WordFunc + AppendWords []string + Fuzzy bool + IgnoreWaf bool + Crawl bool + Scope []string + Active bool + Bak bool + Common bool + RetryLimit int + RandomUserAgent bool + Random string + Index string +} + +func NewBruteWords(config *Config, list []string) *words.Worder { + word := words.NewWorderWithList(list) + word.Fns = config.Fns + word.Run() + return word +} + +func NewBruteDSL(config *Config, dsl string, params [][]string) *words.Worder { + word, err := words.NewWorderWithDsl(dsl, params, nil) + if err != nil { + logs.Log.Error(err.Error()) + } + word.Fns = config.Fns + word.Run() + return word +} diff --git a/internal/runner.go b/internal/runner.go index eeee03a..9a282ef 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -47,7 +47,7 @@ type Runner struct { DumpFile *files.File StatFile *files.File Progress *mpb.Progress - Fns []func(string) []string + Fns []words.WordFunc Count int // tasks total number Wordlist []string AppendWords []string @@ -79,6 +79,7 @@ func (r *Runner) PrepareConfig() *pool.Config { RecuExpr: r.RecursiveExpr, AppendRule: r.AppendRules, // 对有效目录追加规则, 根据rule生成 AppendWords: r.AppendWords, // 对有效目录追加字典 + Fns: r.Fns, //IgnoreWaf: r.IgnoreWaf, Crawl: r.CrawlPlugin, Scope: r.Scope, @@ -182,7 +183,7 @@ func (r *Runner) Prepare(ctx context.Context) error { brutePool.Statistor.Total = t.origin.sum } else { brutePool.Statistor = pkg.NewStatistor(t.baseUrl) - brutePool.Worder = words.NewWorder(r.Wordlist) + brutePool.Worder = words.NewWorderWithList(r.Wordlist) brutePool.Worder.Fns = r.Fns brutePool.Worder.Rules = r.Rules.Expressions } diff --git a/internal/types.go b/internal/types.go index 5e64727..9faf981 100644 --- a/internal/types.go +++ b/internal/types.go @@ -14,13 +14,13 @@ type Origin struct { sum int } -func (o *Origin) InitWorder(fns []func(string) []string) (*words.Worder, error) { +func (o *Origin) InitWorder(fns []words.WordFunc) (*words.Worder, error) { var worder *words.Worder wl, err := pkg.LoadWordlist(o.Word, o.Dictionaries) if err != nil { return nil, err } - worder = words.NewWorder(wl) + worder = words.NewWorderWithList(wl) worder.Fns = fns rules, err := pkg.LoadRuleWithFiles(o.RuleFiles, o.RuleFilter) if err != nil { diff --git a/pkg/parse.go b/pkg/parse.go new file mode 100644 index 0000000..88cfbda --- /dev/null +++ b/pkg/parse.go @@ -0,0 +1,20 @@ +package pkg + +import "strings" + +var ( + SkipChar = "%SKIP%" + EXTChar = "%EXT%" +) + +func ParseEXTPlaceholderFunc(exts []string) func(string) []string { + return func(s string) []string { + ss := make([]string, len(exts)) + for i, e := range exts { + if strings.Contains(s, EXTChar) { + ss[i] = strings.Replace(s, EXTChar, e, -1) + } + } + return ss + } +}