实装了fuzzy-compare

优化输出与输出逻辑
极大的优化了compare性能

go.mod

@@ -3,24 +3,25 @@ module github.com/chainreactors/spray
 go 1.17
 
 require (
-	github.com/chainreactors/files v0.2.0
+	github.com/chainreactors/files v0.2.4
 	github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8
-	github.com/chainreactors/gogo/v2 v2.8.10
+	github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d
 	github.com/chainreactors/logs v0.6.1
-	github.com/chainreactors/parsers v0.0.2
+	github.com/chainreactors/parsers v0.2.7
 	github.com/chainreactors/words v0.1.1
 )
 
 require (
-	github.com/chainreactors/ipcs v0.0.9
+	github.com/chainreactors/ipcs v0.0.13
 	github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c
 	github.com/gosuri/uiprogress v0.0.1
 	github.com/jessevdk/go-flags v1.5.0
-	github.com/panjf2000/ants/v2 v2.5.0
+	github.com/panjf2000/ants/v2 v2.6.0
 	github.com/valyala/fasthttp v1.40.0
 )
 
 require (
+	github.com/M09ic/go-ntlmssp v1.2.9 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect
 	github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect
@@ -29,8 +30,8 @@ require (
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
+	golang.org/x/crypto v0.2.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	golang.org/x/net v0.2.0 // indirect
+	golang.org/x/sys v0.2.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )

go.sum

@@ -1,19 +1,29 @@
+github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew=
 github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM=
 github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
+github.com/chainreactors/files v0.2.3 h1:rDU53H9BPFyQWuIsjZdTm4m7e2/SSemobaU2QUVfLow=
+github.com/chainreactors/files v0.2.3/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
+github.com/chainreactors/files v0.2.4 h1:R0iCqjWLcwwLoSi87FpgUlpxZAd+W4ZLQF3lkoLWZi0=
+github.com/chainreactors/files v0.2.4/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
 github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 h1:kMFr1Hj+rkp1wBPIw2pcQvelO5GnA7r7wY3h6vJ1joA=
 github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8/go.mod h1:7NDvFERNiXsujaBPD6s4WXj52uKdfnF2zVHQtKXIEV4=
 github.com/chainreactors/gogo/v2 v2.8.10 h1:PlPZ1lTIcVl2IfE5GgIr4+hocwrTMyUasMDbrR+7TqI=
 github.com/chainreactors/gogo/v2 v2.8.10/go.mod h1:BQy0aEZXo8nSe/bMMsFeD2KU4vHa8CZ0FHrzuVkH/hU=
+github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d h1:LrbxNFvUrCooEIbQd4JicGFxRCs5KNov8xjJb8b7udw=
+github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d/go.mod h1:L8CXaQB23of7vIZaBuajfNwXBemSthOy8o4myPs4PO4=
 github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows=
 github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
+github.com/chainreactors/ipcs v0.0.13 h1:TZww7XRr4qZPWqy9DjBzcJgxtSUwT4TAbcho4156bRI=
+github.com/chainreactors/ipcs v0.0.13/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
 github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
 github.com/chainreactors/logs v0.6.1 h1:+wLVqxErUyWYayAx6x+m5LA6jQBiHcLdTAJtiOUMoZU=
 github.com/chainreactors/logs v0.6.1/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
-github.com/chainreactors/parsers v0.0.2 h1:lQ4dCWzUrVV8dUVKEAw8qlOU6O4Aju4sXtElAfIhbFI=
 github.com/chainreactors/parsers v0.0.2/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0=
+github.com/chainreactors/parsers v0.2.6/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
+github.com/chainreactors/parsers v0.2.7 h1:3iEuluL7gSDrElZWyf1KEiTgddgcoZC0IaIHb9KA3pk=
+github.com/chainreactors/parsers v0.2.7/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
 github.com/chainreactors/words v0.1.1 h1:Zw4HKFtYcIH5SfuCV0X6kj/A5sN99jrQD2ChUonLOV8=
 github.com/chainreactors/words v0.1.1/go.mod h1:jRcFgafTKqdkd1+StzPCTJG1ESrZHluXEO2eERdHBMQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,6 +48,8 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q=
 github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
+github.com/panjf2000/ants/v2 v2.6.0 h1:xOSpw42m+BMiJ2I33we7h6fYzG4DAlpE1xyI7VS2gxU=
+github.com/panjf2000/ants/v2 v2.6.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -53,10 +65,14 @@ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7Fw
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE=
+golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -68,6 +84,8 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -75,11 +93,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

internal/option.go

@@ -41,6 +41,8 @@ type OutputOptions struct {
 	Filters     map[string]string `long:"filter" description:"String, "`
 	Extracts    []string          `long:"extract" description:"String, "`
 	OutputFile  string            `short:"f" description:"String, output filename"`
+	FuzzyFile   string            `long:"fuzzy-file" description:"String, fuzzy output filename"`
+	Fuzzy       bool              `long:"fuzzy" description:"String, open fuzzy output"`
 	OutputProbe string            `long:"probe" description:"String, output format"`
 }
 
@@ -78,6 +80,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		Offset:   opt.Offset,
 		Limit:    opt.Limit,
 		URLList:  make(chan string),
+		OutputCh: make(chan *pkg.Baseline, 100),
+		FuzzyCh:  make(chan *pkg.Baseline, 100),
+		Fuzzy:    opt.Fuzzy,
 	}
 
 	err = pkg.LoadTemplates()
@@ -185,7 +190,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	if opt.RemoveExtensions != "" {
 		rexts := strings.Split(opt.ExcludeExtensions, ",")
 		r.Fns = append(r.Fns, func(s string) string {
-			if ext := parseExtension(s); SliceContains(rexts, ext) {
+			if ext := parseExtension(s); StringsContains(rexts, ext) {
 				return strings.TrimSuffix(s, "."+ext)
 			}
 			return s
@@ -195,7 +200,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	if opt.ExcludeExtensions != "" {
 		exexts := strings.Split(opt.ExcludeExtensions, ",")
 		r.Fns = append(r.Fns, func(s string) string {
-			if ext := parseExtension(s); SliceContains(exexts, ext) {
+			if ext := parseExtension(s); StringsContains(exexts, ext) {
 				return ""
 			}
 			return s
@@ -259,7 +264,16 @@ func parseExtension(s string) string {
 	return ""
 }
 
-func SliceContains(s []string, e string) bool {
+func StringsContains(s []string, e string) bool {
+	for _, v := range s {
+		if v == e {
+			return true
+		}
+	}
+	return false
+}
+
+func IntsContains(s []int, e int) bool {
 	for _, v := range s {
 		if v == e {
 			return true

internal/pool.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"fmt"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
@@ -21,7 +22,7 @@ var (
 
 var breakThreshold int = 20
 
-func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (*Pool, error) {
+func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
 	pctx, cancel := context.WithCancel(ctx)
 	pool := &Pool{
 		Config:      config,
@@ -29,8 +30,10 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 		cancel:      cancel,
 		client:      ihttp.NewClient(config.Thread, 2, config.ClientType),
 		worder:      words.NewWorder(config.Wordlist),
-		outputCh:    outputCh,
+		outputCh:    config.OutputCh,
-		tempCh:      make(chan *baseline, config.Thread),
+		fuzzyCh:     config.FuzzyCh,
+		baselines:   make(map[int]*pkg.Baseline),
+		tempCh:      make(chan *pkg.Baseline, config.Thread),
 		wg:          sync.WaitGroup{},
 		initwg:      sync.WaitGroup{},
 		checkPeriod: 100,
@@ -50,7 +53,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 
 			if pool.failedCount > breakThreshold {
 				// 当报错次数超过上限是, 结束任务
-				pool.Recover()
+				pool.recover()
 				pool.cancel()
 			}
 		}
@@ -65,7 +68,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 
 			if pool.failedCount > breakThreshold {
 				// 当报错次数超过上限是, 结束任务
-				pool.Recover()
+				pool.recover()
 				pool.cancel()
 			}
 		}
@@ -79,7 +82,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 			return
 		}
 
-		var bl *baseline
+		var bl *pkg.Baseline
 		resp, reqerr := pool.client.Do(pctx, req)
 		if pool.ClientType == ihttp.FAST {
 			defer fasthttp.ReleaseResponse(resp.FastResponse)
@@ -88,26 +91,27 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 
 		if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
 			pool.failedCount++
-			bl = &baseline{Url: pool.BaseURL + unit.path, Err: reqerr}
+			bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, Err: reqerr.Error(), Reason: ErrRequestFailed.Error()}
 			pool.failedBaselines = append(pool.failedBaselines, bl)
 		} else {
-			if err = pool.PreCompare(resp); err == nil || unit.source == CheckSource || unit.source == InitSource {
+			if err = pool.PreCompare(resp); unit.source == CheckSource || unit.source == InitSource || err == nil {
 				// 通过预对比跳过一些无用数据, 减少性能消耗
-				bl = NewBaseline(req.URI(), req.Host(), resp)
+				bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
+				pool.addFuzzyBaseline(bl)
 			} else {
-				bl = NewInvalidBaseline(req.URI(), req.Host(), resp)
+				bl = pkg.NewInvalidBaseline(req.URI(), req.Host(), resp, err.Error())
 			}
 		}
 
 		switch unit.source {
 		case InitSource:
 			pool.base = bl
+			pool.addFuzzyBaseline(bl)
 			pool.initwg.Done()
-			logs.Log.Important("[baseline] " + bl.String())
 			return
 		case CheckSource:
-			if bl.Err != nil {
+			if bl.Err != "" {
-				logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, breakThreshold, bl.Err.Error())
+				logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, breakThreshold, bl.Err)
 				pool.failedBaselines = append(pool.failedBaselines, bl)
 			} else if i := pool.base.Compare(bl); i < 1 {
 				if i == 0 {
@@ -118,7 +122,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 
 				pool.failedBaselines = append(pool.failedBaselines, bl)
 			} else {
-				pool.ResetFailed() // 如果后续访问正常, 重置错误次数
+				pool.resetFailed() // 如果后续访问正常, 重置错误次数
 				logs.Log.Debug("[check.pass] " + bl.String())
 			}
 
@@ -144,20 +148,21 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 
 type Pool struct {
 	*pkg.Config
-	client *ihttp.Client
+	client          *ihttp.Client
-	pool   *ants.PoolWithFunc
+	pool            *ants.PoolWithFunc
-	bar    *pkg.Bar
+	bar             *pkg.Bar
-	ctx    context.Context
+	ctx             context.Context
-	cancel context.CancelFunc
+	cancel          context.CancelFunc
-	//baseReq      *http.Request
+	outputCh        chan *pkg.Baseline // 输出的chan, 全局统一
-	base            *baseline
+	fuzzyCh         chan *pkg.Baseline
-	outputCh        chan *baseline // 输出的chan, 全局统一
+	tempCh          chan *pkg.Baseline // 待处理的baseline
-	tempCh          chan *baseline // 待处理的baseline
 	reqCount        int
 	failedCount     int
 	checkPeriod     int
 	errPeriod       int
-	failedBaselines []*baseline
+	failedBaselines []*pkg.Baseline
+	base            *pkg.Baseline
+	baselines       map[int]*pkg.Baseline
 	analyzeDone     bool
 	genReq          func(s string) (*ihttp.Request, error)
 	check           func()
@@ -173,12 +178,13 @@ func (p *Pool) Init() error {
 	// todo 分析baseline
 	// 检测基本访问能力
 
-	if p.base.Err != nil {
+	if p.base.Err != "" {
 		p.cancel()
-		return p.base.Err
+		return fmt.Errorf(p.base.String())
 	}
 
 	p.base.Collect()
+	logs.Log.Important("[baseline.init] " + p.base.String())
 	if p.base.RedirectURL != "" {
 		CheckRedirect = func(redirectURL string) bool {
 			if redirectURL == p.base.RedirectURL {
@@ -233,6 +239,10 @@ Loop:
 }
 
 func (p *Pool) PreCompare(resp *ihttp.Response) error {
+	if p.base != nil && p.base.Status != 200 && p.base.Status == resp.StatusCode() {
+		return ErrSameStatus
+	}
+
 	if !CheckStatusCode(resp.StatusCode()) {
 		return ErrBadStatus
 	}
@@ -251,22 +261,32 @@ func (p *Pool) PreCompare(resp *ihttp.Response) error {
 
 func (p *Pool) comparing() {
 	for bl := range p.tempCh {
+		if !bl.IsValid {
+			// precompare 确认无效数据直接送入管道
+			p.outputCh <- bl
+			continue
+		}
+
 		if p.base.Compare(bl) == 1 {
 			// 如果是同一个包则设置为无效包
 			bl.IsValid = false
 			p.outputCh <- bl
 			continue
-		}
+		} else if base, ok := p.baselines[bl.Status]; ok && base.Compare(bl) == 1 {
-		if !bl.IsValid {
+			bl.IsValid = false
-			// 已经时被precompare过滤的项目, 跳过collect, 直接认为是无效数据
+			bl.IsFuzzy = true
 			p.outputCh <- bl
+			p.fuzzyCh <- bl
 			continue
 		}
 
 		bl.Collect()
-		if p.EnableFuzzy && p.base.FuzzyEqual(bl) {
+		// todo fuzzy compare
+		if p.base.FuzzyCompare(bl) {
 			bl.IsValid = false
+			bl.IsFuzzy = true
 			p.outputCh <- bl
+			p.fuzzyCh <- bl
 			continue
 		}
 
@@ -276,12 +296,24 @@ func (p *Pool) comparing() {
 	p.analyzeDone = true
 }
 
-func (p *Pool) ResetFailed() {
+func (p *Pool) addFuzzyBaseline(bl *pkg.Baseline) {
+	if !IntsContains(FuzzyStatus, bl.Status) {
+		return
+	}
+
+	if _, ok := p.baselines[bl.Status]; !ok {
+		bl.Collect()
+		p.baselines[bl.Status] = bl
+		logs.Log.Importantf("[baseline.%dinit] %s", bl.Status, bl.String())
+	}
+}
+
+func (p *Pool) resetFailed() {
 	p.failedCount = 0
 	p.failedBaselines = nil
 }
 
-func (p *Pool) Recover() {
+func (p *Pool) recover() {
 	logs.Log.Errorf("failed request exceeds the threshold , task will exit. Breakpoint %d", p.reqCount)
 	logs.Log.Error("collecting failed check")
 	for i, bl := range p.failedBaselines {

internal/runner.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"github.com/chainreactors/files"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
@@ -15,22 +16,26 @@ var BlackStatus = []int{400, 404, 410}
 var FuzzyStatus = []int{403, 500, 501, 502, 503}
 
 type Runner struct {
-	URLList  chan string
+	URLList    chan string
-	Wordlist []string
+	Wordlist   []string
-	Headers  http.Header
+	Headers    http.Header
-	Fns      []func(string) string
+	Fns        []func(string) string
-	Threads  int
+	Threads    int
-	PoolSize int
+	PoolSize   int
-	Pools    *ants.PoolWithFunc
+	Pools      *ants.PoolWithFunc
-	poolwg   sync.WaitGroup
+	poolwg     sync.WaitGroup
-	Timeout  int
+	Timeout    int
-	Mod      string
+	Mod        string
-	Probes   []string
+	Probes     []string
-	OutputCh chan *baseline
+	OutputCh   chan *pkg.Baseline
-	Progress *uiprogress.Progress
+	FuzzyCh    chan *pkg.Baseline
-	Offset   int
+	Fuzzy      bool
-	Limit    int
+	OutputFile *files.File
-	Deadline int
+	FuzzyFile  *files.File
+	Progress   *uiprogress.Progress
+	Offset     int
+	Limit      int
+	Deadline   int
 }
 
 func (r *Runner) Prepare(ctx context.Context) error {
@@ -44,8 +49,6 @@ func (r *Runner) Prepare(ctx context.Context) error {
 		return true
 	}
 
-	r.OutputCh = make(chan *baseline, 100)
-
 	r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) {
 		u := i.(string)
 		config := &pkg.Config{
@@ -56,6 +59,8 @@ func (r *Runner) Prepare(ctx context.Context) error {
 			Headers:  r.Headers,
 			Mod:      pkg.ModMap[r.Mod],
 			Fns:      r.Fns,
+			OutputCh: r.OutputCh,
+			FuzzyCh:  r.FuzzyCh,
 		}
 
 		if config.Mod == pkg.PathSpray {
@@ -64,7 +69,7 @@ func (r *Runner) Prepare(ctx context.Context) error {
 			config.ClientType = ihttp.STANDARD
 		}
 
-		pool, err := NewPool(ctx, config, r.OutputCh)
+		pool, err := NewPool(ctx, config)
 		if err != nil {
 			logs.Log.Error(err.Error())
 			return
@@ -83,7 +88,7 @@ func (r *Runner) Prepare(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	go r.Outputting()
+	r.Outputting()
 	return nil
 }
 
@@ -107,31 +112,51 @@ Loop:
 	for {
 		if len(r.OutputCh) == 0 {
 			close(r.OutputCh)
-			return
+			break
+		}
+	}
+
+	for {
+		if len(r.FuzzyCh) == 0 {
+			close(r.FuzzyCh)
+			break
 		}
 	}
 }
 
 func (r *Runner) Outputting() {
-	var outFunc func(baseline2 *baseline)
+	go func() {
-	if len(r.Probes) > 0 {
+		var outFunc func(*pkg.Baseline)
-		outFunc = func(bl *baseline) {
+		if len(r.Probes) > 0 {
-			logs.Log.Console("[+] " + bl.Format(r.Probes) + "\n")
+			outFunc = func(bl *pkg.Baseline) {
-		}
+				logs.Log.Console("[+] " + bl.Format(r.Probes) + "\n")
-	} else {
+			}
-		outFunc = func(bl *baseline) {
+		} else {
-			logs.Log.Console("[+] " + bl.String() + "\n")
+			outFunc = func(bl *pkg.Baseline) {
-		}
+				logs.Log.Console("[+] " + bl.String() + "\n")
-	}
-
-	for {
-		select {
-		case bl := <-r.OutputCh:
-			if bl.IsValid {
-				outFunc(bl)
-			} else {
-				logs.Log.Debug(bl.String())
 			}
 		}
-	}
+
+		for {
+			select {
+			case bl := <-r.OutputCh:
+				if bl.IsValid {
+					outFunc(bl)
+				} else {
+					logs.Log.Debug(bl.String())
+				}
+			}
+		}
+	}()
+
+	go func() {
+		for {
+			select {
+			case bl := <-r.FuzzyCh:
+				if r.Fuzzy {
+					logs.Log.Console("[baseline.fuzzy] " + bl.String() + "\n")
+				}
+			}
+		}
+	}()
 }

internal/types.go

@@ -4,6 +4,8 @@ type ErrorType uint
 
 const (
 	ErrBadStatus ErrorType = iota
+	ErrSameStatus
+	ErrRequestFailed
 	ErrWaf
 	ErrRedirect
 )
@@ -12,6 +14,10 @@ func (e ErrorType) Error() string {
 	switch e {
 	case ErrBadStatus:
 		return "bad status"
+	case ErrSameStatus:
+		return "same status"
+	case ErrRequestFailed:
+		return "request failed"
 	case ErrWaf:
 		return "maybe ban of waf"
 	case ErrRedirect:

pkg/baseline.go

@@ -1,17 +1,16 @@
-package internal
+package pkg
 
 import (
 	"encoding/json"
 	"github.com/chainreactors/parsers"
-	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
 	"net/url"
 	"strconv"
 	"strings"
 )
 
-func NewBaseline(u, host string, resp *ihttp.Response) *baseline {
+func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
-	bl := &baseline{
+	bl := &Baseline{
 		Url:     u,
 		Status:  resp.StatusCode(),
 		IsValid: true,
@@ -33,11 +32,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *baseline {
 	return bl
 }
 
-func NewInvalidBaseline(u, host string, resp *ihttp.Response) *baseline {
+func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline {
-	bl := &baseline{
+	bl := &Baseline{
 		Url:     u,
 		Status:  resp.StatusCode(),
 		IsValid: false,
+		Reason:  reason,
 	}
 
 	uu, err := url.Parse(u)
@@ -56,40 +56,42 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response) *baseline {
 	return bl
 }
 
-type baseline struct {
+type Baseline struct {
-	Url          string         `json:"url"`
+	Url          string     `json:"url"`
-	Path         string         `json:"path"`
+	Path         string     `json:"path"`
-	Host         string         `json:"host"`
+	Host         string     `json:"host"`
-	Body         []byte         `json:"-"`
+	Body         []byte     `json:"-"`
-	BodyLength   int            `json:"body_length"`
+	BodyLength   int        `json:"body_length"`
-	Header       []byte         `json:"-"`
+	Header       []byte     `json:"-"`
-	Raw          []byte         `json:"-"`
+	Raw          []byte     `json:"-"`
-	HeaderLength int            `json:"header_length"`
+	HeaderLength int        `json:"header_length"`
-	RedirectURL  string         `json:"redirect_url"`
+	RedirectURL  string     `json:"redirect_url"`
-	Status       int            `json:"status"`
+	Status       int        `json:"status"`
-	IsDynamicUrl bool           `json:"is_dynamic_url"` // 判断是否存在动态的url
+	IsDynamicUrl bool       `json:"is_dynamic_url"` // 判断是否存在动态的url
-	Spended      int            `json:"spended"`        // 耗时, 毫秒
+	Spended      int        `json:"spended"`        // 耗时, 毫秒
-	Title        string         `json:"title"`
+	Title        string     `json:"title"`
-	Frameworks   pkg.Frameworks `json:"frameworks"`
+	Frameworks   Frameworks `json:"frameworks"`
-	Extracteds   pkg.Extracteds `json:"extracts"`
+	Extracteds   Extracteds `json:"extracts"`
-	Err          error          `json:"-"`
+	Err          string     `json:"error"`
-	IsValid      bool           `json:"-"`
+	Reason       string     `json:"reason"`
+	IsValid      bool       `json:"valid"`
+	IsFuzzy      bool       `json:"fuzzy"`
 	*parsers.Hashes
 }
 
 // Collect 深度收集信息
-func (bl *baseline) Collect() {
+func (bl *Baseline) Collect() {
 	if len(bl.Body) > 0 {
 		bl.Title = parsers.MatchTitle(string(bl.Body))
 	}
 	bl.Hashes = parsers.NewHashes(bl.Raw)
 	// todo extract
-	bl.Extracteds = pkg.Extractors.Extract(string(bl.Raw))
+	bl.Extracteds = Extractors.Extract(string(bl.Raw))
-	bl.Frameworks = pkg.FingerDetect(string(bl.Raw))
+	bl.Frameworks = FingerDetect(string(bl.Raw))
 }
 
 // Compare if this equal other return true
-func (bl *baseline) Compare(other *baseline) int {
+func (bl *Baseline) Compare(other *Baseline) int {
 	if other.RedirectURL != "" && bl.RedirectURL == other.RedirectURL {
 		// 如果重定向url不为空, 且与base不相同, 则说明不是同一个页面
 		return 1
@@ -114,12 +116,15 @@ func (bl *baseline) Compare(other *baseline) int {
 	return -1
 }
 
-func (bl *baseline) FuzzyEqual(other *baseline) bool {
+func (bl *Baseline) FuzzyCompare(other *Baseline) bool {
 	// todo 模糊匹配
+	if parsers.SimhashCompare(other.BodySimhash, bl.BodySimhash) < 3 {
+		return true
+	}
 	return false
 }
 
-func (bl *baseline) Get(key string) string {
+func (bl *Baseline) Get(key string) string {
 	switch key {
 	case "url":
 		return bl.Url
@@ -160,7 +165,7 @@ func (bl *baseline) Get(key string) string {
 	}
 }
 
-func (bl *baseline) Additional(key string) string {
+func (bl *Baseline) Additional(key string) string {
 	if v := bl.Get(key); v != "" {
 		return " [" + v + "] "
 	} else {
@@ -168,16 +173,20 @@ func (bl *baseline) Additional(key string) string {
 	}
 }
 
-func (bl *baseline) Format(probes []string) string {
+func (bl *Baseline) Format(probes []string) string {
 	var line strings.Builder
 	line.WriteString(bl.Url)
 	if bl.Host != "" {
 		line.WriteString(" (" + bl.Host + ")")
 	}
 
-	if bl.Err != nil {
+	if bl.Reason != "" {
-		line.WriteString("err: ")
+		line.WriteString(" ,")
-		line.WriteString(bl.Err.Error())
+		line.WriteString(bl.Reason)
+	}
+	if bl.Err != "" {
+		line.WriteString(" ,err: ")
+		line.WriteString(bl.Err)
 		return line.String()
 	}
 
@@ -189,7 +198,7 @@ func (bl *baseline) Format(probes []string) string {
 	return line.String()
 }
 
-func (bl *baseline) String() string {
+func (bl *Baseline) String() string {
 	var line strings.Builder
 	//line.WriteString("[+] ")
 	line.WriteString(bl.Url)
@@ -197,9 +206,9 @@ func (bl *baseline) String() string {
 		line.WriteString(" (" + bl.Host + ")")
 	}
 
-	if bl.Err != nil {
+	if bl.Err != "" {
 		line.WriteString("err: ")
-		line.WriteString(bl.Err.Error())
+		line.WriteString(bl.Err)
 		return line.String()
 	}
 
@@ -218,7 +227,7 @@ func (bl *baseline) String() string {
 	return line.String()
 }
 
-func (bl *baseline) Jsonify() string {
+func (bl *Baseline) Jsonify() string {
 	bs, err := json.Marshal(bl)
 	if err != nil {
 		return ""

pkg/config.go

@@ -30,4 +30,6 @@ type Config struct {
 	EnableFuzzy bool
 	ClientType  int
 	Fns         []func(string) string
+	OutputCh    chan *Baseline
+	FuzzyCh     chan *Baseline
 }