From 3ff46e5e1f92a92fe12d5fe8f026c7835c59bc71 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 12 Jan 2023 19:21:35 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=86=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E4=B8=AD=E7=9A=84number=E5=AD=97=E6=AE=B5,=20=E7=8E=B0?= =?UTF-8?q?=E5=9C=A8=E8=83=BD=E6=AD=A3=E7=A1=AE=E5=88=A4=E6=96=AD=E4=BA=86?= =?UTF-8?q?,=20=E5=B9=B6=E4=B8=94addition=E4=B8=AD=E7=9A=84number=E4=BC=9A?= =?UTF-8?q?=E6=98=AF=E5=BD=93=E5=89=8D=E7=9A=84wordoffset,=20=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E7=94=A8=E6=9D=A5=E5=88=A4=E6=96=AD=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E5=8F=91=E7=94=9F=E7=9A=84=E4=BD=8D=E7=BD=AE.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/option.go | 6 +++--- internal/pool.go | 32 +++++++++++++++++--------------- internal/types.go | 5 +++++ pkg/baseline.go | 4 ++-- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/internal/option.go b/internal/option.go index e827a40..aae4de8 100644 --- a/internal/option.go +++ b/internal/option.go @@ -451,7 +451,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, err } } else if opt.AutoFile { - r.OutputFile, err = files.NewFile("result.json", true, false, true) + r.OutputFile, err = files.NewFile("result.json", false, false, true) if err != nil { return nil, err } @@ -463,7 +463,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, err } } else if opt.AutoFile { - r.FuzzyFile, err = files.NewFile("fuzzy.json", true, false, true) + r.FuzzyFile, err = files.NewFile("fuzzy.json", false, false, true) if err != nil { return nil, err } @@ -475,7 +475,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, err } } else if opt.Dump { - r.DumpFile, err = files.NewFile("dump.json", true, false, true) + r.DumpFile, err = files.NewFile("dump.json", false, false, true) if err != nil { return nil, err } diff --git a/internal/pool.go b/internal/pool.go index 4c018f0..3d157c5 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -54,7 +54,6 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { waiter: sync.WaitGroup{}, initwg: sync.WaitGroup{}, limiter: rate.NewLimiter(rate.Limit(config.RateLimit), 1), - reqCount: 1, failedCount: 1, } @@ -162,7 +161,7 @@ type Pool struct { tempCh chan *pkg.Baseline // 待处理的baseline checkCh chan int // 独立的check管道， 防止与redirect/crawl冲突 additionCh chan *Unit - reqCount int + wordOffset int failedCount int isFailed bool failedBaselines []*pkg.Baseline @@ -251,8 +250,8 @@ Loop: continue } pool.Statistor.End++ - if pool.reqCount < offset { - pool.reqCount++ + pool.wordOffset++ + if pool.wordOffset <= offset { continue } @@ -263,13 +262,13 @@ Loop: pool.waiter.Add(1) pool.urls[u] = struct{}{} - pool.reqPool.Invoke(newUnit(pool.safePath(u), WordSource)) // 原样的目录拼接, 输入了几个"/"就是几个, 适配java的目录解析 + pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(u), WordSource, pool.wordOffset)) // 原样的目录拼接, 输入了几个"/"就是几个, 适配java的目录解析 case source := <-pool.checkCh: pool.Statistor.CheckNumber++ if pool.Mod == pkg.HostSpray { - pool.reqPool.Invoke(newUnit(pkg.RandHost(), source)) + pool.reqPool.Invoke(newUnitWithNumber(pkg.RandHost(), source, pool.wordOffset)) } else if pool.Mod == pkg.PathSpray { - pool.reqPool.Invoke(newUnit(pool.safePath(pkg.RandPath()), source)) + pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(pkg.RandPath()), source, pool.wordOffset)) } case unit, ok := <-pool.additionCh: if !ok { @@ -280,6 +279,7 @@ Loop: pool.waiter.Done() } else { pool.urls[unit.path] = struct{}{} + unit.number = pool.wordOffset pool.reqPool.Invoke(unit) } case <-closeCh: @@ -347,6 +347,7 @@ func (pool *Pool) Invoke(v interface{}) { } bl.Source = unit.source bl.ReqDepth = unit.depth + bl.Number = unit.number bl.Spended = time.Since(start).Milliseconds() switch unit.source { case InitRandomSource: @@ -388,9 +389,7 @@ func (pool *Pool) Invoke(v interface{}) { case WordSource: // 异步进行性能消耗较大的深度对比 pool.tempCh <- bl - pool.reqCount++ - if pool.reqCount%pool.CheckPeriod == 0 { - pool.reqCount++ + if pool.wordOffset%pool.CheckPeriod == 0 { pool.doCheck() } else if pool.failedCount%pool.ErrPeriod == 0 { pool.failedCount++ @@ -581,7 +580,7 @@ func (pool *Pool) doRule(bl *pkg.Baseline) { pool.waiter.Done() return } - if bl.Source == int(RuleSource) { + if bl.Source == RuleSource { pool.waiter.Done() return } @@ -695,7 +694,7 @@ func (pool *Pool) resetFailed() { } func (pool *Pool) recover() { - logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.reqCount) + logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.wordOffset) for i, bl := range pool.failedBaselines { logs.Log.Errorf("[failed.%d] %s", i, bl.String()) } @@ -714,10 +713,13 @@ func (pool *Pool) Close() { func (pool *Pool) safePath(u string) string { // 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common - if !pool.isDir && !strings.HasPrefix(u, "/") { + hasSlash := strings.HasPrefix(u, "/") + if !pool.isDir && hasSlash { // 如果path已经有"/", 则去掉 - return pool.dir + u - } else { + return pool.dir + "/" + u + } else if pool.isDir && hasSlash { return pool.dir + u[1:] + } else { + return pool.dir + u } } diff --git a/internal/types.go b/internal/types.go index ddb7b00..821a178 100644 --- a/internal/types.go +++ b/internal/types.go @@ -62,7 +62,12 @@ func newUnit(path string, source int) *Unit { return &Unit{path: path, source: source} } +func newUnitWithNumber(path string, source int, number int) *Unit { + return &Unit{path: path, source: source, number: number} +} + type Unit struct { + number int path string source int frontUrl string diff --git a/pkg/baseline.go b/pkg/baseline.go index d676a3c..27dc262 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -91,6 +91,8 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba type Baseline struct { Number int `json:"number"` Url *url.URL `json:"-"` + IsValid bool `json:"valid"` + IsFuzzy bool `json:"fuzzy"` UrlString string `json:"url"` Path string `json:"path"` Dir bool `json:"-"` @@ -112,8 +114,6 @@ type Baseline struct { Extracteds Extracteds `json:"extracts"` ErrString string `json:"error"` Reason string `json:"reason"` - IsValid bool `json:"valid"` - IsFuzzy bool `json:"fuzzy"` Source int `json:"source"` ReqDepth int `json:"depth"` Distance uint8 `json:"distance"`