From af687701a76f0be9a4587f038b72a2d7458d5c5d Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sun, 11 Dec 2022 00:24:28 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E5=AE=9E=E7=8E=B0=E4=BA=86?= =?UTF-8?q?=E9=80=92=E5=BD=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/option.go | 1 + internal/pool.go | 74 ++++++++++++++++++++++++++-------------------- internal/runner.go | 20 +++++++++++-- pkg/baseline.go | 9 ++++++ pkg/config.go | 6 ++-- 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/internal/option.go b/internal/option.go index 8a9673f..349d60b 100644 --- a/internal/option.go +++ b/internal/option.go @@ -458,4 +458,5 @@ type Task struct { baseUrl string offset int total int + depth int } diff --git a/internal/pool.go b/internal/pool.go index baa3460..1ba7799 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -24,6 +24,7 @@ var ( var max = 2147483647 var maxRedirect = 3 +var maxRecuDepth = 0 func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { pctx, cancel := context.WithCancel(ctx) @@ -89,6 +90,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { defer fasthttp.ReleaseRequest(req.FastRequest) } + // compare与各种错误处理 var bl *pkg.Baseline if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { pool.failedCount++ @@ -122,7 +124,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { bl.Spended = time.Since(start).Milliseconds() switch unit.source { case InitRandomSource: - pool.base = bl + pool.random = bl pool.addFuzzyBaseline(bl) pool.initwg.Done() case InitIndexSource: @@ -132,7 +134,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { if bl.ErrString != "" { logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, pool.BreakThreshold, bl.ErrString) pool.failedBaselines = append(pool.failedBaselines, bl) - } else if i := pool.base.Compare(bl); i < 1 { + } else if i := pool.random.Compare(bl); i < 1 { if i == 0 { logs.Log.Debug("[check.fuzzy] maybe trigger risk control, " + bl.String()) } else { @@ -167,6 +169,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { }) pool.pool = p + // 挂起一个异步的处理结果线程, 不干扰主线程的请求并发 go func() { for bl := range pool.tempCh { if _, ok := pool.Statistor.Counts[bl.Status]; ok { @@ -174,9 +177,22 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { } else { pool.Statistor.Counts[bl.Status] = 1 } + params := map[string]interface{}{ + "index": pool.index, + "random": pool.random, + "current": bl, + } + for _, status := range FuzzyStatus { + if bl, ok := pool.baselines[status]; ok { + params["bl"+strconv.Itoa(status)] = bl + } else { + params["bl"+strconv.Itoa(status)] = &pkg.Baseline{} + } + } + var status bool if pool.MatchExpr != nil { - if pool.CompareWithExpr(pool.MatchExpr, bl) { + if CompareWithExpr(pool.MatchExpr, params) { status = true } } else { @@ -187,7 +203,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { if status { pool.Statistor.FoundNumber++ - if pool.FilterExpr != nil && pool.CompareWithExpr(pool.FilterExpr, bl) { + if pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) { pool.Statistor.FilteredNumber++ bl.Reason = ErrCustomFilter.Error() bl.IsValid = false @@ -195,6 +211,13 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { } else { bl.IsValid = false } + + // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 + if bl.IsValid && pool.Mod == pkg.PathSpray && bl.RecuDepth < maxRecuDepth { + if CompareWithExpr(pool.RecuExpr, params) { + bl.Recu = true + } + } pool.OutputCh <- bl pool.wg.Done() } @@ -216,14 +239,14 @@ type Pool struct { reqCount int failedCount int failedBaselines []*pkg.Baseline - base *pkg.Baseline + random *pkg.Baseline index *pkg.Baseline baselines map[int]*pkg.Baseline - locker sync.Mutex analyzeDone bool genReq func(s string) (*ihttp.Request, error) check func() worder *words.Worder + locker sync.Mutex wg sync.WaitGroup initwg sync.WaitGroup // 初始化用, 之后改成锁 } @@ -243,25 +266,25 @@ func (p *Pool) Init() error { p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource)) p.initwg.Wait() // 检测基本访问能力 - if p.base.ErrString != "" { - return fmt.Errorf(p.base.String()) + if p.random.ErrString != "" { + return fmt.Errorf(p.random.String()) } - p.base.Collect() - logs.Log.Important("[baseline.random] " + p.base.String()) + p.random.Collect() + logs.Log.Important("[baseline.random] " + p.random.String()) - if p.base.RedirectURL != "" { + if p.random.RedirectURL != "" { // 自定协议升级 // 某些网站http会重定向到https, 如果发现随机目录出现这种情况, 则自定将baseurl升级为https - rurl, err := url.Parse(p.base.RedirectURL) - if err == nil && rurl.Hostname() == p.base.Url.Hostname() && p.base.Url.Scheme == "http" && rurl.Scheme == "https" { + rurl, err := url.Parse(p.random.RedirectURL) + if err == nil && rurl.Hostname() == p.random.Url.Hostname() && p.random.Url.Scheme == "http" && rurl.Scheme == "https" { logs.Log.Importantf("baseurl %s upgrade http to https", p.BaseURL) p.BaseURL = strings.Replace(p.BaseURL, "http", "https", 1) } } - if p.base.RedirectURL != "" { + if p.random.RedirectURL != "" { CheckRedirect = func(redirectURL string) bool { - if redirectURL == p.base.RedirectURL { + if redirectURL == p.random.RedirectURL { // 相同的RedirectURL将被认为是无效数据 return false } else { @@ -334,7 +357,7 @@ func (p *Pool) PreCompare(resp *ihttp.Response) error { // 如果为白名单状态码则直接返回 return nil } - if p.base != nil && p.base.Status != 200 && p.base.Status == status { + if p.random != nil && p.random.Status != 200 && p.random.Status == status { return ErrSameStatus } @@ -360,10 +383,10 @@ func (p *Pool) BaseCompare(bl *pkg.Baseline) bool { var status = -1 base, ok := p.baselines[bl.Status] // 挑选对应状态码的baseline进行compare if !ok { - if p.base.Status == bl.Status { + if p.random.Status == bl.Status { // 当other的状态码与base相同时, 会使用base ok = true - base = p.base + base = p.random } else if p.index.Status == bl.Status { // 当other的状态码与index相同时, 会使用index ok = true @@ -397,20 +420,7 @@ func (p *Pool) BaseCompare(bl *pkg.Baseline) bool { return true } -func (p *Pool) CompareWithExpr(exp *vm.Program, other *pkg.Baseline) bool { - params := map[string]interface{}{ - "index": p.index, - "base": p.base, - "current": other, - } - for _, status := range FuzzyStatus { - if bl, ok := p.baselines[status]; ok { - params["bl"+strconv.Itoa(status)] = bl - } else { - params["bl"+strconv.Itoa(status)] = &pkg.Baseline{} - } - } - +func CompareWithExpr(exp *vm.Program, params map[string]interface{}) bool { res, err := expr.Run(exp, params) if err != nil { logs.Log.Warn(err.Error()) diff --git a/internal/runner.go b/internal/runner.go index 7644930..87e16ec 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -37,9 +37,12 @@ type Runner struct { Fns []func(string) string FilterExpr *vm.Program MatchExpr *vm.Program + RecursiveExpr *vm.Program + RecuDepth int Threads int PoolSize int Pools *ants.PoolWithFunc + PoolName map[string]bool Timeout int Mod string Probes []string @@ -75,6 +78,7 @@ func (r *Runner) PrepareConfig() *pkg.Config { BreakThreshold: r.BreakThreshold, MatchExpr: r.MatchExpr, FilterExpr: r.FilterExpr, + RecuExpr: r.RecursiveExpr, } if config.Mod == pkg.PathSpray { config.ClientType = ihttp.FAST @@ -163,6 +167,16 @@ func (r *Runner) Prepare(ctx context.Context) error { return nil } +func (r *Runner) AddPool(task *Task) { + if _, ok := r.PoolName[task.baseUrl]; ok { + logs.Log.Importantf("already added pool, skip %s", task.baseUrl) + return + } + task.depth++ + r.poolwg.Add(1) + r.Pools.Invoke(task) +} + func (r *Runner) Run(ctx context.Context) { Loop: for { @@ -174,8 +188,7 @@ Loop: if !ok { break Loop } - r.poolwg.Add(1) - r.Pools.Invoke(t) + r.AddPool(t) } } @@ -267,6 +280,9 @@ func (r *Runner) Outputting() { if bl.IsValid { saveFunc(bl) + if bl.Recu { + r.AddPool(&Task{bl.UrlString, 0, r.Total, bl.RecuDepth + 1}) + } } else { logs.Log.Debug(bl.String()) } diff --git a/pkg/baseline.go b/pkg/baseline.go index 152c3ab..24e9e09 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -80,9 +80,18 @@ type Baseline struct { Reason string `json:"reason"` IsValid bool `json:"valid"` IsFuzzy bool `json:"fuzzy"` + RecuDepth int `json:"-"` + Recu bool `json:"-"` *parsers.Hashes } +func (bl *Baseline) IsDir() bool { + if strings.HasSuffix(bl.Path, "/") { + return true + } + return false +} + // Collect 深度收集信息 func (bl *Baseline) Collect() { if len(bl.Body) > 0 { diff --git a/pkg/config.go b/pkg/config.go index 030fe2e..8fe216d 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -36,7 +36,7 @@ type Config struct { Rules []rule.Expression MatchExpr *vm.Program FilterExpr *vm.Program - - OutputCh chan *Baseline - FuzzyCh chan *Baseline + RecuExpr *vm.Program + OutputCh chan *Baseline + FuzzyCh chan *Baseline }