From c3993fc8c2d257301ceda7ac72a65b4c5ce59145 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 4 Feb 2023 19:44:37 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=9A=E5=A4=84=E6=80=A7=E8=83=BD=E4=B8=8E?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96,=20=E8=83=BD=E6=9B=B4?= =?UTF-8?q?=E5=A5=BD=E7=9A=84=E6=94=AF=E6=8C=81=E4=B8=8D=E5=90=8C=E7=9A=84?= =?UTF-8?q?fuzz=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- go.sum | 2 + internal/pool.go | 184 ++++++++++++++++++++++++----------------------- pkg/baseline.go | 8 +++ 4 files changed, 107 insertions(+), 89 deletions(-) diff --git a/go.mod b/go.mod index 3277430..7b8e9bc 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/chainreactors/gogo/v2 v2.10.4 github.com/chainreactors/ipcs v0.0.13 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 - github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49 + github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599 github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361 ) diff --git a/go.sum b/go.sum index 9c88f8c..0e3ca44 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/chainreactors/parsers v0.3.1-0.20230201101112-af520af38b4d h1:gJaqiDa github.com/chainreactors/parsers v0.3.1-0.20230201101112-af520af38b4d/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49 h1:snsLbWcScZRQV66TZxHfECkb2htZzhW2EcF1c60AA9c= github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= +github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599 h1:9PwMZzN+RZDv2BUDvOG8e0N6W3XJQLVaP2AW6RD5mjM= +github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/chainreactors/words v0.4.1-0.20230203114605-f305deb098a2 h1:51GoU85MLp/s8IvXcKLeedSxypkvZBFJWIBUlGV+MiI= diff --git a/internal/pool.go b/internal/pool.go index 27ece16..85d8286 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -73,81 +73,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { pool.reqPool = p // 挂起一个异步的处理结果线程, 不干扰主线程的请求并发 - go func() { - for bl := range pool.tempCh { - if bl.IsValid { - pool.addFuzzyBaseline(bl) - } - if _, ok := pool.Statistor.Counts[bl.Status]; ok { - pool.Statistor.Counts[bl.Status]++ - } else { - pool.Statistor.Counts[bl.Status] = 1 - } - - if _, ok := pool.Statistor.Sources[bl.Source]; ok { - pool.Statistor.Sources[bl.Source]++ - } else { - pool.Statistor.Sources[bl.Source] = 1 - } - - var params map[string]interface{} - if pool.MatchExpr != nil || pool.FilterExpr != nil || pool.RecuExpr != nil { - params = map[string]interface{}{ - "index": pool.index, - "random": pool.random, - "current": bl, - } - for _, status := range FuzzyStatus { - if bl, ok := pool.baselines[status]; ok { - params["bl"+strconv.Itoa(status)] = bl - } else { - params["bl"+strconv.Itoa(status)] = nilBaseline - } - } - } - - var status bool - if pool.MatchExpr != nil { - if CompareWithExpr(pool.MatchExpr, params) { - status = true - } - } else { - if pool.BaseCompare(bl) { - status = true - } - } - - if status { - pool.Statistor.FoundNumber++ - if pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) { - pool.Statistor.FilteredNumber++ - bl.Reason = ErrCustomFilter.Error() - bl.IsValid = false - } - } else { - bl.IsValid = false - } - - // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 - if bl.IsValid { - pool.waiter.Add(2) - pool.doCrawl(bl) - pool.doRule(bl) - if bl.RecuDepth < MaxRecursion { - if CompareWithExpr(pool.RecuExpr, params) { - bl.Recu = true - } - } - } - if !pool.closed { - // 如果任务被取消, 所有还没处理的请求结果都会被丢弃 - pool.OutputCh <- bl - } - pool.waiter.Done() - } - - pool.analyzeDone = true - }() + go pool.Handler() return pool, nil } @@ -378,10 +304,9 @@ func (pool *Pool) Invoke(v interface{}) { pool.locker.Lock() pool.index = bl pool.locker.Unlock() - pool.waiter.Add(1) - pool.doCrawl(bl) if bl.Status == 200 || (bl.Status/100) == 3 { - pool.OutputCh <- bl + pool.waiter.Add(1) + pool.tempCh <- bl } pool.initwg.Done() case CheckSource: @@ -420,6 +345,85 @@ func (pool *Pool) Invoke(v interface{}) { } } +func (pool *Pool) Handler() { + for bl := range pool.tempCh { + if bl.IsValid { + pool.addFuzzyBaseline(bl) + } + if _, ok := pool.Statistor.Counts[bl.Status]; ok { + pool.Statistor.Counts[bl.Status]++ + } else { + pool.Statistor.Counts[bl.Status] = 1 + } + + if _, ok := pool.Statistor.Sources[bl.Source]; ok { + pool.Statistor.Sources[bl.Source]++ + } else { + pool.Statistor.Sources[bl.Source] = 1 + } + + var params map[string]interface{} + if pool.MatchExpr != nil || pool.FilterExpr != nil || pool.RecuExpr != nil { + params = map[string]interface{}{ + "index": pool.index, + "random": pool.random, + "current": bl, + } + for _, status := range FuzzyStatus { + if bl, ok := pool.baselines[status]; ok { + params["bl"+strconv.Itoa(status)] = bl + } else { + params["bl"+strconv.Itoa(status)] = nilBaseline + } + } + } + + var status bool + if pool.MatchExpr != nil { + if CompareWithExpr(pool.MatchExpr, params) { + status = true + } + } else { + if pool.BaseCompare(bl) { + status = true + } + } + + if status { + pool.Statistor.FoundNumber++ + if pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) { + pool.Statistor.FilteredNumber++ + bl.Reason = ErrCustomFilter.Error() + bl.IsValid = false + } + } else { + bl.IsValid = false + } + + if bl.IsValid || bl.IsFuzzy { + pool.waiter.Add(2) + pool.doCrawl(bl) + pool.doRule(bl) + } + // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 + if bl.IsValid { + if bl.RecuDepth < MaxRecursion { + if CompareWithExpr(pool.RecuExpr, params) { + bl.Recu = true + } + } + } + + if !pool.closed { + // 如果任务被取消, 所有还没处理的请求结果都会被丢弃 + pool.OutputCh <- bl + } + pool.waiter.Done() + } + + pool.analyzeDone = true +} + func (pool *Pool) checkRedirect(redirectURL string) bool { if pool.random.RedirectURL == "" { // 如果random的redirectURL为空, 此时该项 @@ -470,13 +474,12 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error { } func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool { - if !bl.IsValid { - return false - } var status = -1 base, ok := pool.baselines[bl.Status] // 挑选对应状态码的baseline进行compare if !ok { - if pool.random.Status == bl.Status { + if pool.index != nil { + + } else if pool.random.Status == bl.Status { // 当other的状态码与base相同时, 会使用base ok = true base = pool.random @@ -735,12 +738,17 @@ func (pool *Pool) Close() { func (pool *Pool) safePath(u string) string { // 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common hasSlash := strings.HasPrefix(u, "/") - if !pool.isDir && hasSlash { - // 如果path已经有"/", 则去掉 - return pool.dir + "/" + u - } else if pool.isDir && hasSlash { - return pool.dir + u[1:] + if hasSlash { + if pool.isDir { + return pool.dir + u[1:] + } else { + return pool.url.Path + u + } } else { - return pool.dir + u + if pool.isDir { + return pool.url.Path + u + } else { + return pool.url.Path + "/" + u + } } } diff --git a/pkg/baseline.go b/pkg/baseline.go index 92a0624..c2c500b 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -100,6 +100,7 @@ type Baseline struct { Recu bool `json:"-"` RecuDepth int `json:"-"` URLs []string `json:"-"` + Collected bool `json:"-"` } func (bl *Baseline) IsDir() bool { @@ -130,6 +131,13 @@ func (bl *Baseline) Collect() { } func (bl *Baseline) CollectURL() { + if bl.Collected { + // 防止重复收集 + return + } else { + bl.Collected = true + } + if len(bl.Body) == 0 { return }