From 7a5ec12541c335f8d29ba31212c67ddb698ef645 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 10 Nov 2022 21:03:07 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E8=A3=85=E4=BA=86fuzzy-compare=20?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=BE=93=E5=87=BA=E4=B8=8E=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E9=80=BB=E8=BE=91=20=E6=9E=81=E5=A4=A7=E7=9A=84=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E4=BA=86compare=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 17 +++--- go.sum | 24 ++++++-- internal/option.go | 20 ++++++- internal/pool.go | 96 ++++++++++++++++++++----------- internal/runner.go | 105 +++++++++++++++++++++------------- internal/types.go | 6 ++ {internal => pkg}/baseline.go | 87 +++++++++++++++------------- pkg/config.go | 2 + 8 files changed, 231 insertions(+), 126 deletions(-) rename {internal => pkg}/baseline.go (65%) diff --git a/go.mod b/go.mod index 42b4ca1..37d85dc 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,25 @@ module github.com/chainreactors/spray go 1.17 require ( - github.com/chainreactors/files v0.2.0 + github.com/chainreactors/files v0.2.4 github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 - github.com/chainreactors/gogo/v2 v2.8.10 + github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d github.com/chainreactors/logs v0.6.1 - github.com/chainreactors/parsers v0.0.2 + github.com/chainreactors/parsers v0.2.7 github.com/chainreactors/words v0.1.1 ) require ( - github.com/chainreactors/ipcs v0.0.9 + github.com/chainreactors/ipcs v0.0.13 github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c github.com/gosuri/uiprogress v0.0.1 github.com/jessevdk/go-flags v1.5.0 - github.com/panjf2000/ants/v2 v2.5.0 + github.com/panjf2000/ants/v2 v2.6.0 github.com/valyala/fasthttp v1.40.0 ) require ( + github.com/M09ic/go-ntlmssp v1.2.9 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect @@ -29,8 +30,8 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/twmb/murmur3 v1.1.6 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + golang.org/x/crypto v0.2.0 // indirect + golang.org/x/net v0.2.0 // indirect + golang.org/x/sys v0.2.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 339fe8f..18040a4 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,29 @@ +github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew= github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM= github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= +github.com/chainreactors/files v0.2.3 h1:rDU53H9BPFyQWuIsjZdTm4m7e2/SSemobaU2QUVfLow= +github.com/chainreactors/files v0.2.3/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= +github.com/chainreactors/files v0.2.4 h1:R0iCqjWLcwwLoSi87FpgUlpxZAd+W4ZLQF3lkoLWZi0= +github.com/chainreactors/files v0.2.4/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 h1:kMFr1Hj+rkp1wBPIw2pcQvelO5GnA7r7wY3h6vJ1joA= github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8/go.mod h1:7NDvFERNiXsujaBPD6s4WXj52uKdfnF2zVHQtKXIEV4= github.com/chainreactors/gogo/v2 v2.8.10 h1:PlPZ1lTIcVl2IfE5GgIr4+hocwrTMyUasMDbrR+7TqI= github.com/chainreactors/gogo/v2 v2.8.10/go.mod h1:BQy0aEZXo8nSe/bMMsFeD2KU4vHa8CZ0FHrzuVkH/hU= +github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d h1:LrbxNFvUrCooEIbQd4JicGFxRCs5KNov8xjJb8b7udw= +github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d/go.mod h1:L8CXaQB23of7vIZaBuajfNwXBemSthOy8o4myPs4PO4= github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows= github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII= +github.com/chainreactors/ipcs v0.0.13 h1:TZww7XRr4qZPWqy9DjBzcJgxtSUwT4TAbcho4156bRI= +github.com/chainreactors/ipcs v0.0.13/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII= github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= github.com/chainreactors/logs v0.6.1 h1:+wLVqxErUyWYayAx6x+m5LA6jQBiHcLdTAJtiOUMoZU= github.com/chainreactors/logs v0.6.1/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= -github.com/chainreactors/parsers v0.0.2 h1:lQ4dCWzUrVV8dUVKEAw8qlOU6O4Aju4sXtElAfIhbFI= github.com/chainreactors/parsers v0.0.2/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0= +github.com/chainreactors/parsers v0.2.6/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= +github.com/chainreactors/parsers v0.2.7 h1:3iEuluL7gSDrElZWyf1KEiTgddgcoZC0IaIHb9KA3pk= +github.com/chainreactors/parsers v0.2.7/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= github.com/chainreactors/words v0.1.1 h1:Zw4HKFtYcIH5SfuCV0X6kj/A5sN99jrQD2ChUonLOV8= github.com/chainreactors/words v0.1.1/go.mod h1:jRcFgafTKqdkd1+StzPCTJG1ESrZHluXEO2eERdHBMQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -38,6 +48,8 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q= github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= +github.com/panjf2000/ants/v2 v2.6.0 h1:xOSpw42m+BMiJ2I33we7h6fYzG4DAlpE1xyI7VS2gxU= +github.com/panjf2000/ants/v2 v2.6.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -53,10 +65,14 @@ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7Fw golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE= +golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -68,6 +84,8 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -75,11 +93,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/option.go b/internal/option.go index f515b22..af7e919 100644 --- a/internal/option.go +++ b/internal/option.go @@ -41,6 +41,8 @@ type OutputOptions struct { Filters map[string]string `long:"filter" description:"String, "` Extracts []string `long:"extract" description:"String, "` OutputFile string `short:"f" description:"String, output filename"` + FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename"` + Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output"` OutputProbe string `long:"probe" description:"String, output format"` } @@ -78,6 +80,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Offset: opt.Offset, Limit: opt.Limit, URLList: make(chan string), + OutputCh: make(chan *pkg.Baseline, 100), + FuzzyCh: make(chan *pkg.Baseline, 100), + Fuzzy: opt.Fuzzy, } err = pkg.LoadTemplates() @@ -185,7 +190,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if opt.RemoveExtensions != "" { rexts := strings.Split(opt.ExcludeExtensions, ",") r.Fns = append(r.Fns, func(s string) string { - if ext := parseExtension(s); SliceContains(rexts, ext) { + if ext := parseExtension(s); StringsContains(rexts, ext) { return strings.TrimSuffix(s, "."+ext) } return s @@ -195,7 +200,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if opt.ExcludeExtensions != "" { exexts := strings.Split(opt.ExcludeExtensions, ",") r.Fns = append(r.Fns, func(s string) string { - if ext := parseExtension(s); SliceContains(exexts, ext) { + if ext := parseExtension(s); StringsContains(exexts, ext) { return "" } return s @@ -259,7 +264,16 @@ func parseExtension(s string) string { return "" } -func SliceContains(s []string, e string) bool { +func StringsContains(s []string, e string) bool { + for _, v := range s { + if v == e { + return true + } + } + return false +} + +func IntsContains(s []int, e int) bool { for _, v := range s { if v == e { return true diff --git a/internal/pool.go b/internal/pool.go index e293d1b..b482d35 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -2,6 +2,7 @@ package internal import ( "context" + "fmt" "github.com/chainreactors/logs" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" @@ -21,7 +22,7 @@ var ( var breakThreshold int = 20 -func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (*Pool, error) { +func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { pctx, cancel := context.WithCancel(ctx) pool := &Pool{ Config: config, @@ -29,8 +30,10 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( cancel: cancel, client: ihttp.NewClient(config.Thread, 2, config.ClientType), worder: words.NewWorder(config.Wordlist), - outputCh: outputCh, - tempCh: make(chan *baseline, config.Thread), + outputCh: config.OutputCh, + fuzzyCh: config.FuzzyCh, + baselines: make(map[int]*pkg.Baseline), + tempCh: make(chan *pkg.Baseline, config.Thread), wg: sync.WaitGroup{}, initwg: sync.WaitGroup{}, checkPeriod: 100, @@ -50,7 +53,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( if pool.failedCount > breakThreshold { // 当报错次数超过上限是, 结束任务 - pool.Recover() + pool.recover() pool.cancel() } } @@ -65,7 +68,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( if pool.failedCount > breakThreshold { // 当报错次数超过上限是, 结束任务 - pool.Recover() + pool.recover() pool.cancel() } } @@ -79,7 +82,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( return } - var bl *baseline + var bl *pkg.Baseline resp, reqerr := pool.client.Do(pctx, req) if pool.ClientType == ihttp.FAST { defer fasthttp.ReleaseResponse(resp.FastResponse) @@ -88,26 +91,27 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { pool.failedCount++ - bl = &baseline{Url: pool.BaseURL + unit.path, Err: reqerr} + bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, Err: reqerr.Error(), Reason: ErrRequestFailed.Error()} pool.failedBaselines = append(pool.failedBaselines, bl) } else { - if err = pool.PreCompare(resp); err == nil || unit.source == CheckSource || unit.source == InitSource { + if err = pool.PreCompare(resp); unit.source == CheckSource || unit.source == InitSource || err == nil { // 通过预对比跳过一些无用数据, 减少性能消耗 - bl = NewBaseline(req.URI(), req.Host(), resp) + bl = pkg.NewBaseline(req.URI(), req.Host(), resp) + pool.addFuzzyBaseline(bl) } else { - bl = NewInvalidBaseline(req.URI(), req.Host(), resp) + bl = pkg.NewInvalidBaseline(req.URI(), req.Host(), resp, err.Error()) } } switch unit.source { case InitSource: pool.base = bl + pool.addFuzzyBaseline(bl) pool.initwg.Done() - logs.Log.Important("[baseline] " + bl.String()) return case CheckSource: - if bl.Err != nil { - logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, breakThreshold, bl.Err.Error()) + if bl.Err != "" { + logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, breakThreshold, bl.Err) pool.failedBaselines = append(pool.failedBaselines, bl) } else if i := pool.base.Compare(bl); i < 1 { if i == 0 { @@ -118,7 +122,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( pool.failedBaselines = append(pool.failedBaselines, bl) } else { - pool.ResetFailed() // 如果后续访问正常, 重置错误次数 + pool.resetFailed() // 如果后续访问正常, 重置错误次数 logs.Log.Debug("[check.pass] " + bl.String()) } @@ -144,20 +148,21 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( type Pool struct { *pkg.Config - client *ihttp.Client - pool *ants.PoolWithFunc - bar *pkg.Bar - ctx context.Context - cancel context.CancelFunc - //baseReq *http.Request - base *baseline - outputCh chan *baseline // 输出的chan, 全局统一 - tempCh chan *baseline // 待处理的baseline + client *ihttp.Client + pool *ants.PoolWithFunc + bar *pkg.Bar + ctx context.Context + cancel context.CancelFunc + outputCh chan *pkg.Baseline // 输出的chan, 全局统一 + fuzzyCh chan *pkg.Baseline + tempCh chan *pkg.Baseline // 待处理的baseline reqCount int failedCount int checkPeriod int errPeriod int - failedBaselines []*baseline + failedBaselines []*pkg.Baseline + base *pkg.Baseline + baselines map[int]*pkg.Baseline analyzeDone bool genReq func(s string) (*ihttp.Request, error) check func() @@ -173,12 +178,13 @@ func (p *Pool) Init() error { // todo 分析baseline // 检测基本访问能力 - if p.base.Err != nil { + if p.base.Err != "" { p.cancel() - return p.base.Err + return fmt.Errorf(p.base.String()) } p.base.Collect() + logs.Log.Important("[baseline.init] " + p.base.String()) if p.base.RedirectURL != "" { CheckRedirect = func(redirectURL string) bool { if redirectURL == p.base.RedirectURL { @@ -233,6 +239,10 @@ Loop: } func (p *Pool) PreCompare(resp *ihttp.Response) error { + if p.base != nil && p.base.Status != 200 && p.base.Status == resp.StatusCode() { + return ErrSameStatus + } + if !CheckStatusCode(resp.StatusCode()) { return ErrBadStatus } @@ -251,22 +261,32 @@ func (p *Pool) PreCompare(resp *ihttp.Response) error { func (p *Pool) comparing() { for bl := range p.tempCh { + if !bl.IsValid { + // precompare 确认无效数据直接送入管道 + p.outputCh <- bl + continue + } + if p.base.Compare(bl) == 1 { // 如果是同一个包则设置为无效包 bl.IsValid = false p.outputCh <- bl continue - } - if !bl.IsValid { - // 已经时被precompare过滤的项目, 跳过collect, 直接认为是无效数据 + } else if base, ok := p.baselines[bl.Status]; ok && base.Compare(bl) == 1 { + bl.IsValid = false + bl.IsFuzzy = true p.outputCh <- bl + p.fuzzyCh <- bl continue } bl.Collect() - if p.EnableFuzzy && p.base.FuzzyEqual(bl) { + // todo fuzzy compare + if p.base.FuzzyCompare(bl) { bl.IsValid = false + bl.IsFuzzy = true p.outputCh <- bl + p.fuzzyCh <- bl continue } @@ -276,12 +296,24 @@ func (p *Pool) comparing() { p.analyzeDone = true } -func (p *Pool) ResetFailed() { +func (p *Pool) addFuzzyBaseline(bl *pkg.Baseline) { + if !IntsContains(FuzzyStatus, bl.Status) { + return + } + + if _, ok := p.baselines[bl.Status]; !ok { + bl.Collect() + p.baselines[bl.Status] = bl + logs.Log.Importantf("[baseline.%dinit] %s", bl.Status, bl.String()) + } +} + +func (p *Pool) resetFailed() { p.failedCount = 0 p.failedBaselines = nil } -func (p *Pool) Recover() { +func (p *Pool) recover() { logs.Log.Errorf("failed request exceeds the threshold , task will exit. Breakpoint %d", p.reqCount) logs.Log.Error("collecting failed check") for i, bl := range p.failedBaselines { diff --git a/internal/runner.go b/internal/runner.go index 73e1bfd..b17dc88 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -2,6 +2,7 @@ package internal import ( "context" + "github.com/chainreactors/files" "github.com/chainreactors/logs" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" @@ -15,22 +16,26 @@ var BlackStatus = []int{400, 404, 410} var FuzzyStatus = []int{403, 500, 501, 502, 503} type Runner struct { - URLList chan string - Wordlist []string - Headers http.Header - Fns []func(string) string - Threads int - PoolSize int - Pools *ants.PoolWithFunc - poolwg sync.WaitGroup - Timeout int - Mod string - Probes []string - OutputCh chan *baseline - Progress *uiprogress.Progress - Offset int - Limit int - Deadline int + URLList chan string + Wordlist []string + Headers http.Header + Fns []func(string) string + Threads int + PoolSize int + Pools *ants.PoolWithFunc + poolwg sync.WaitGroup + Timeout int + Mod string + Probes []string + OutputCh chan *pkg.Baseline + FuzzyCh chan *pkg.Baseline + Fuzzy bool + OutputFile *files.File + FuzzyFile *files.File + Progress *uiprogress.Progress + Offset int + Limit int + Deadline int } func (r *Runner) Prepare(ctx context.Context) error { @@ -44,8 +49,6 @@ func (r *Runner) Prepare(ctx context.Context) error { return true } - r.OutputCh = make(chan *baseline, 100) - r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) { u := i.(string) config := &pkg.Config{ @@ -56,6 +59,8 @@ func (r *Runner) Prepare(ctx context.Context) error { Headers: r.Headers, Mod: pkg.ModMap[r.Mod], Fns: r.Fns, + OutputCh: r.OutputCh, + FuzzyCh: r.FuzzyCh, } if config.Mod == pkg.PathSpray { @@ -64,7 +69,7 @@ func (r *Runner) Prepare(ctx context.Context) error { config.ClientType = ihttp.STANDARD } - pool, err := NewPool(ctx, config, r.OutputCh) + pool, err := NewPool(ctx, config) if err != nil { logs.Log.Error(err.Error()) return @@ -83,7 +88,7 @@ func (r *Runner) Prepare(ctx context.Context) error { if err != nil { return err } - go r.Outputting() + r.Outputting() return nil } @@ -107,31 +112,51 @@ Loop: for { if len(r.OutputCh) == 0 { close(r.OutputCh) - return + break + } + } + + for { + if len(r.FuzzyCh) == 0 { + close(r.FuzzyCh) + break } } } func (r *Runner) Outputting() { - var outFunc func(baseline2 *baseline) - if len(r.Probes) > 0 { - outFunc = func(bl *baseline) { - logs.Log.Console("[+] " + bl.Format(r.Probes) + "\n") - } - } else { - outFunc = func(bl *baseline) { - logs.Log.Console("[+] " + bl.String() + "\n") - } - } - - for { - select { - case bl := <-r.OutputCh: - if bl.IsValid { - outFunc(bl) - } else { - logs.Log.Debug(bl.String()) + go func() { + var outFunc func(*pkg.Baseline) + if len(r.Probes) > 0 { + outFunc = func(bl *pkg.Baseline) { + logs.Log.Console("[+] " + bl.Format(r.Probes) + "\n") + } + } else { + outFunc = func(bl *pkg.Baseline) { + logs.Log.Console("[+] " + bl.String() + "\n") } } - } + + for { + select { + case bl := <-r.OutputCh: + if bl.IsValid { + outFunc(bl) + } else { + logs.Log.Debug(bl.String()) + } + } + } + }() + + go func() { + for { + select { + case bl := <-r.FuzzyCh: + if r.Fuzzy { + logs.Log.Console("[baseline.fuzzy] " + bl.String() + "\n") + } + } + } + }() } diff --git a/internal/types.go b/internal/types.go index b931664..0001589 100644 --- a/internal/types.go +++ b/internal/types.go @@ -4,6 +4,8 @@ type ErrorType uint const ( ErrBadStatus ErrorType = iota + ErrSameStatus + ErrRequestFailed ErrWaf ErrRedirect ) @@ -12,6 +14,10 @@ func (e ErrorType) Error() string { switch e { case ErrBadStatus: return "bad status" + case ErrSameStatus: + return "same status" + case ErrRequestFailed: + return "request failed" case ErrWaf: return "maybe ban of waf" case ErrRedirect: diff --git a/internal/baseline.go b/pkg/baseline.go similarity index 65% rename from internal/baseline.go rename to pkg/baseline.go index f2a9086..7c674c1 100644 --- a/internal/baseline.go +++ b/pkg/baseline.go @@ -1,17 +1,16 @@ -package internal +package pkg import ( "encoding/json" "github.com/chainreactors/parsers" - "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" "net/url" "strconv" "strings" ) -func NewBaseline(u, host string, resp *ihttp.Response) *baseline { - bl := &baseline{ +func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { + bl := &Baseline{ Url: u, Status: resp.StatusCode(), IsValid: true, @@ -33,11 +32,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *baseline { return bl } -func NewInvalidBaseline(u, host string, resp *ihttp.Response) *baseline { - bl := &baseline{ +func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline { + bl := &Baseline{ Url: u, Status: resp.StatusCode(), IsValid: false, + Reason: reason, } uu, err := url.Parse(u) @@ -56,40 +56,42 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response) *baseline { return bl } -type baseline struct { - Url string `json:"url"` - Path string `json:"path"` - Host string `json:"host"` - Body []byte `json:"-"` - BodyLength int `json:"body_length"` - Header []byte `json:"-"` - Raw []byte `json:"-"` - HeaderLength int `json:"header_length"` - RedirectURL string `json:"redirect_url"` - Status int `json:"status"` - IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url - Spended int `json:"spended"` // 耗时, 毫秒 - Title string `json:"title"` - Frameworks pkg.Frameworks `json:"frameworks"` - Extracteds pkg.Extracteds `json:"extracts"` - Err error `json:"-"` - IsValid bool `json:"-"` +type Baseline struct { + Url string `json:"url"` + Path string `json:"path"` + Host string `json:"host"` + Body []byte `json:"-"` + BodyLength int `json:"body_length"` + Header []byte `json:"-"` + Raw []byte `json:"-"` + HeaderLength int `json:"header_length"` + RedirectURL string `json:"redirect_url"` + Status int `json:"status"` + IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url + Spended int `json:"spended"` // 耗时, 毫秒 + Title string `json:"title"` + Frameworks Frameworks `json:"frameworks"` + Extracteds Extracteds `json:"extracts"` + Err string `json:"error"` + Reason string `json:"reason"` + IsValid bool `json:"valid"` + IsFuzzy bool `json:"fuzzy"` *parsers.Hashes } // Collect 深度收集信息 -func (bl *baseline) Collect() { +func (bl *Baseline) Collect() { if len(bl.Body) > 0 { bl.Title = parsers.MatchTitle(string(bl.Body)) } bl.Hashes = parsers.NewHashes(bl.Raw) // todo extract - bl.Extracteds = pkg.Extractors.Extract(string(bl.Raw)) - bl.Frameworks = pkg.FingerDetect(string(bl.Raw)) + bl.Extracteds = Extractors.Extract(string(bl.Raw)) + bl.Frameworks = FingerDetect(string(bl.Raw)) } // Compare if this equal other return true -func (bl *baseline) Compare(other *baseline) int { +func (bl *Baseline) Compare(other *Baseline) int { if other.RedirectURL != "" && bl.RedirectURL == other.RedirectURL { // 如果重定向url不为空, 且与base不相同, 则说明不是同一个页面 return 1 @@ -114,12 +116,15 @@ func (bl *baseline) Compare(other *baseline) int { return -1 } -func (bl *baseline) FuzzyEqual(other *baseline) bool { +func (bl *Baseline) FuzzyCompare(other *Baseline) bool { // todo 模糊匹配 + if parsers.SimhashCompare(other.BodySimhash, bl.BodySimhash) < 3 { + return true + } return false } -func (bl *baseline) Get(key string) string { +func (bl *Baseline) Get(key string) string { switch key { case "url": return bl.Url @@ -160,7 +165,7 @@ func (bl *baseline) Get(key string) string { } } -func (bl *baseline) Additional(key string) string { +func (bl *Baseline) Additional(key string) string { if v := bl.Get(key); v != "" { return " [" + v + "] " } else { @@ -168,16 +173,20 @@ func (bl *baseline) Additional(key string) string { } } -func (bl *baseline) Format(probes []string) string { +func (bl *Baseline) Format(probes []string) string { var line strings.Builder line.WriteString(bl.Url) if bl.Host != "" { line.WriteString(" (" + bl.Host + ")") } - if bl.Err != nil { - line.WriteString("err: ") - line.WriteString(bl.Err.Error()) + if bl.Reason != "" { + line.WriteString(" ,") + line.WriteString(bl.Reason) + } + if bl.Err != "" { + line.WriteString(" ,err: ") + line.WriteString(bl.Err) return line.String() } @@ -189,7 +198,7 @@ func (bl *baseline) Format(probes []string) string { return line.String() } -func (bl *baseline) String() string { +func (bl *Baseline) String() string { var line strings.Builder //line.WriteString("[+] ") line.WriteString(bl.Url) @@ -197,9 +206,9 @@ func (bl *baseline) String() string { line.WriteString(" (" + bl.Host + ")") } - if bl.Err != nil { + if bl.Err != "" { line.WriteString("err: ") - line.WriteString(bl.Err.Error()) + line.WriteString(bl.Err) return line.String() } @@ -218,7 +227,7 @@ func (bl *baseline) String() string { return line.String() } -func (bl *baseline) Jsonify() string { +func (bl *Baseline) Jsonify() string { bs, err := json.Marshal(bl) if err != nil { return "" diff --git a/pkg/config.go b/pkg/config.go index 025a023..818ec69 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -30,4 +30,6 @@ type Config struct { EnableFuzzy bool ClientType int Fns []func(string) string + OutputCh chan *Baseline + FuzzyCh chan *Baseline }