From 105c4263963288a0b728926de7c4f6a9ed6ef590 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 26 Aug 2024 01:20:03 +0800 Subject: [PATCH] refactor plugin --- cmd/cmd.go | 2 +- internal/option.go | 121 +++++++++++++++++++------------------ internal/pool/brutepool.go | 91 ++++++++++++++-------------- internal/runner.go | 10 ++- pkg/utils.go | 12 ++++ 5 files changed, 123 insertions(+), 113 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 44e115f..18a9b78 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -128,7 +128,7 @@ func Spray() { logs.Log.Errorf(err.Error()) return } - if option.ReadAll || runner.Crawl { + if option.ReadAll || runner.CrawlPlugin { ihttp.DefaultMaxBodySize = -1 } diff --git a/internal/option.go b/internal/option.go index ac9fe51..d7a0e51 100644 --- a/internal/option.go +++ b/internal/option.go @@ -111,12 +111,11 @@ type PluginOptions struct { Advance bool `short:"a" long:"advance" description:"Bool, enable all plugin" config:"all" ` Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" config:"extract"` ExtractConfig string `long:"extract-config" description:"String, extract config filename" config:"extract-config"` - Active bool `long:"active" description:"Bool, enable active finger path"` - Recon bool `long:"recon" description:"Bool, enable recon" config:"recon"` - Bak bool `long:"bak" description:"Bool, enable bak found" config:"bak"` - FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt" config:"file-bak"` - Common bool `long:"common" description:"Bool, enable common file found" config:"common"` - Crawl bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"` + ActivePlugin bool `long:"active" description:"Bool, enable active finger path"` + ReconPlugin bool `long:"recon" description:"Bool, enable recon" config:"recon"` + BakPlugin bool `long:"bak" description:"Bool, enable bak found" config:"bak"` + CommonPlugin bool `long:"common" description:"Bool, enable common file found" config:"common"` + CrawlPlugin bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"` CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth" config:"crawl-depth"` } @@ -310,52 +309,9 @@ func (opt *Option) NewRunner() (*Runner, error) { r.Threads = 1000 } - if opt.Recon { - pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"] - } - - if opt.Finger { - pkg.EnableAllFingerEngine = true - } - - // brute only - if opt.Advance { - r.Crawl = true - r.Finger = true - r.Bak = true - r.Common = true - r.Active = true - pkg.EnableAllFingerEngine = true - pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"] - r.bruteMod = true - opt.AppendRule = append(opt.AppendRule, "filebak") - } - - if opt.FileBak { - r.bruteMod = true - opt.AppendRule = append(opt.AppendRule, "filebak") - } - if opt.Common { - r.bruteMod = true - r.AppendWords = append(r.AppendWords, mask.SpecialWords["common_file"]...) - } - - if opt.Active { - r.bruteMod = true - r.AppendWords = append(r.AppendWords, pkg.ActivePath...) - } - - if opt.Crawl { - r.bruteMod = true - } - - opt.PrintPlugin() - if r.bruteMod { - logs.Log.Important("enabling brute mod, because of enabled brute plugin") - } - - if opt.NoScope { - r.Scope = []string{"*"} + err = opt.BuildPlugin(r) + if err != nil { + return nil, err } err = opt.BuildWords(r) @@ -494,25 +450,21 @@ func (opt *Option) NewRunner() (*Runner, error) { func (opt *Option) PrintPlugin() { var s strings.Builder - if opt.Crawl { + if opt.CrawlPlugin { s.WriteString("crawl enable; ") } if opt.Finger { s.WriteString("active fingerprint enable; ") } - if opt.Bak { + if opt.BakPlugin { s.WriteString("bak file enable; ") } - if opt.Common { + if opt.CommonPlugin { s.WriteString("common file enable; ") } - if opt.Recon { + if opt.ReconPlugin { s.WriteString("recon enable; ") } - if opt.FileBak { - s.WriteString("file bak enable; ") - } - if opt.RetryCount > 0 { s.WriteString("Retry Count: " + strconv.Itoa(opt.RetryCount)) } @@ -522,6 +474,55 @@ func (opt *Option) PrintPlugin() { } } +func (opt *Option) BuildPlugin(r *Runner) error { + // brute only + if opt.Advance { + opt.CrawlPlugin = true + opt.Finger = true + opt.BakPlugin = true + opt.CommonPlugin = true + opt.ActivePlugin = true + } + + if opt.ReconPlugin { + pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"] + } + + if opt.Finger { + pkg.EnableAllFingerEngine = true + } + + if opt.BakPlugin { + r.bruteMod = true + opt.AppendRule = append(opt.AppendRule, "filebak") + r.AppendWords = append(r.AppendWords, pkg.GetPresetWordList([]string{"bak_file"})...) + } + + if opt.CommonPlugin { + r.bruteMod = true + r.AppendWords = append(r.AppendWords, pkg.GetPresetWordList([]string{"common_file", "log_file"})...) + } + + if opt.ActivePlugin { + r.bruteMod = true + r.AppendWords = append(r.AppendWords, pkg.ActivePath...) + } + + if opt.CrawlPlugin { + r.bruteMod = true + } + + opt.PrintPlugin() + if r.bruteMod { + logs.Log.Important("enabling brute mod, because of enabled brute plugin") + } + + if opt.NoScope { + r.Scope = []string{"*"} + } + return nil +} + func (opt *Option) BuildWords(r *Runner) error { var dicts [][]string var err error diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 01018c1..0e19635 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -10,7 +10,6 @@ import ( "github.com/chainreactors/spray/pkg" "github.com/chainreactors/utils/iutils" "github.com/chainreactors/words" - "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" @@ -597,7 +596,7 @@ func (pool *BrutePool) doActive() { func (pool *BrutePool) doCommonFile() { defer pool.wg.Done() - for _, u := range mask.SpecialWords["common_file"] { + for _, u := range pkg.GetPresetWordList([]string{"common_file", "log_file"}) { pool.addAddition(&Unit{ path: pool.dir + u, source: parsers.CommonFileSource, @@ -685,6 +684,50 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { return true } +func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) { + if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) { + bl.Collect() + pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取 + pool.baselines[bl.Status] = bl + logs.Log.Logf(pkg.LogVerbose, "[baseline.%dinit] %s", bl.Status, bl.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) + } +} + +func (pool *BrutePool) recover() { + logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.wordOffset) + for i, bl := range pool.FailedBaselines { + if i > int(pool.BreakThreshold) { + break + } + logs.Log.Errorf("[failed.%d] %s", i, bl.String()) + } +} + +func (pool *BrutePool) Close() { + for pool.analyzeDone { + // 等待缓存的待处理任务完成 + time.Sleep(time.Duration(100) * time.Millisecond) + } + close(pool.additionCh) // 关闭addition管道 + close(pool.checkCh) // 关闭check管道 + pool.Statistor.EndTime = time.Now().Unix() + pool.Bar.Close() +} + +func (pool *BrutePool) safePath(u string) string { + // 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common + if pool.isDir { + return pkg.SafePath(pool.dir, u) + } else { + return pkg.SafePath(pool.url.Path+"/", u) + } +} + +func (pool *BrutePool) resetFailed() { + pool.failedCount = 1 + pool.FailedBaselines = nil +} + func (pool *BrutePool) doCheck() { if pool.failedCount > pool.BreakThreshold { // 当报错次数超过上限是, 结束任务 @@ -755,15 +798,6 @@ func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) { }() } -func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) { - if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) { - bl.Collect() - pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取 - pool.baselines[bl.Status] = bl - logs.Log.Logf(pkg.LogVerbose, "[baseline.%dinit] %s", bl.Status, bl.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) - } -} - func (pool *BrutePool) doBak() { defer pool.wg.Done() worder, err := words.NewWorderWithDsl("{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil) @@ -790,38 +824,3 @@ func (pool *BrutePool) doBak() { }) } } - -func (pool *BrutePool) recover() { - logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.wordOffset) - for i, bl := range pool.FailedBaselines { - if i > int(pool.BreakThreshold) { - break - } - logs.Log.Errorf("[failed.%d] %s", i, bl.String()) - } -} - -func (pool *BrutePool) Close() { - for pool.analyzeDone { - // 等待缓存的待处理任务完成 - time.Sleep(time.Duration(100) * time.Millisecond) - } - close(pool.additionCh) // 关闭addition管道 - close(pool.checkCh) // 关闭check管道 - pool.Statistor.EndTime = time.Now().Unix() - pool.Bar.Close() -} - -func (pool *BrutePool) safePath(u string) string { - // 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common - if pool.isDir { - return pkg.SafePath(pool.dir, u) - } else { - return pkg.SafePath(pool.url.Path+"/", u) - } -} - -func (pool *BrutePool) resetFailed() { - pool.failedCount = 1 - pool.FailedBaselines = nil -} diff --git a/internal/runner.go b/internal/runner.go index 3aa4337..aaf4aa3 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -84,11 +84,11 @@ func (r *Runner) PrepareConfig() *pool.Config { AppendRule: r.AppendRules, // 对有效目录追加规则, 根据rule生成 AppendWords: r.AppendWords, // 对有效目录追加字典 //IgnoreWaf: r.IgnoreWaf, - Crawl: r.Crawl, + Crawl: r.CrawlPlugin, Scope: r.Scope, Active: r.Finger, - Bak: r.Bak, - Common: r.Common, + Bak: r.BakPlugin, + Common: r.CommonPlugin, RetryLimit: r.RetryCount, ClientType: r.ClientType, RandomUserAgent: r.RandomUserAgent, @@ -418,9 +418,7 @@ func (r *Runner) OutputHandler() { if !ok { return } - if r.Fuzzy { - r.Output(bl) - } + r.Output(bl) r.outwg.Done() } } diff --git a/pkg/utils.go b/pkg/utils.go index 6591fab..71cc2cb 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -5,6 +5,7 @@ import ( "bytes" "github.com/chainreactors/logs" "github.com/chainreactors/utils/iutils" + "github.com/chainreactors/words/mask" "github.com/expr-lang/expr" "github.com/expr-lang/expr/vm" "math/rand" @@ -391,3 +392,14 @@ func ParseRawResponse(raw []byte) (*http.Response, error) { defer resp.Body.Close() return resp, nil } + +func GetPresetWordList(key []string) []string { + var wordlist []string + + for _, k := range key { + if v, ok := mask.SpecialWords[k]; ok { + wordlist = append(wordlist, v...) + } + } + return wordlist +}