diff --git a/internal/pool.go b/internal/pool.go index 943a724..cd69817 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -133,14 +133,12 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { case CheckSource: if bl.ErrString != "" { logs.Log.Warnf("[check.error] %s maybe ip had banned, break (%d/%d), error: %s", pool.BaseURL, pool.failedCount, pool.BreakThreshold, bl.ErrString) - pool.failedBaselines = append(pool.failedBaselines, bl) } else if i := pool.random.Compare(bl); i < 1 { if i == 0 { logs.Log.Debug("[check.fuzzy] maybe trigger risk control, " + bl.String()) } else { logs.Log.Warn("[check.failed] maybe trigger risk control, " + bl.String()) } - pool.failedBaselines = append(pool.failedBaselines, bl) } else { pool.resetFailed() // 如果后续访问正常, 重置错误次数 @@ -459,8 +457,7 @@ func (p *Pool) resetFailed() { } func (p *Pool) recover() { - logs.Log.Errorf("failed request exceeds the threshold , task will exit. Breakpoint %d", p.reqCount) - logs.Log.Error("collecting failed check") + logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", p.BaseURL, p.reqCount) for i, bl := range p.failedBaselines { logs.Log.Errorf("[failed.%d] %s", i, bl.String()) } diff --git a/pkg/baseline.go b/pkg/baseline.go index b355ac1..2cf4501 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -1,6 +1,7 @@ package pkg import ( + "bytes" "encoding/json" "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/parsers" @@ -98,7 +99,7 @@ func (bl *Baseline) Collect() { if len(bl.Body) > 0 { bl.Title = utils.AsciiEncode(parsers.MatchTitle(string(bl.Body))) } - bl.Hashes = parsers.NewHashes(bl.Body) + bl.Hashes = parsers.NewHashes(bl.Raw) // todo extract bl.Extracteds = Extractors.Extract(string(bl.Raw)) bl.Frameworks = FingerDetect(string(bl.Raw)) @@ -116,7 +117,7 @@ func (bl *Baseline) Compare(other *Baseline) int { if i := bl.BodyLength - other.BodyLength; i < 16 || i > -16 { // 如果body length相等且md5相等, 则说明是同一个页面 - if bl.BodyMd5 == parsers.Md5Hash(other.Body) { + if bytes.Equal(bl.Body, other.Body) { // 如果length相等, md5也相等, 则判断为全同 return 1 } else {