mirror of
https://github.com/chainreactors/spray.git
synced 2025-05-06 18:51:22 +00:00
实装了fuzzy-compare
优化输出与输出逻辑 极大的优化了compare性能
This commit is contained in:
M09Ic
parent
89899ebef5
commit
6f1ca49408
17
go.mod
17
go.mod
@ -3,24 +3,25 @@ module github.com/chainreactors/spray
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/chainreactors/files v0.2.0
|
||||
github.com/chainreactors/files v0.2.4
|
||||
github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8
|
||||
github.com/chainreactors/gogo/v2 v2.8.10
|
||||
github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d
|
||||
github.com/chainreactors/logs v0.6.1
|
||||
github.com/chainreactors/parsers v0.0.2
|
||||
github.com/chainreactors/parsers v0.2.7
|
||||
github.com/chainreactors/words v0.1.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/chainreactors/ipcs v0.0.9
|
||||
github.com/chainreactors/ipcs v0.0.13
|
||||
github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c
|
||||
github.com/gosuri/uiprogress v0.0.1
|
||||
github.com/jessevdk/go-flags v1.5.0
|
||||
github.com/panjf2000/ants/v2 v2.5.0
|
||||
github.com/panjf2000/ants/v2 v2.6.0
|
||||
github.com/valyala/fasthttp v1.40.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/M09ic/go-ntlmssp v1.2.9 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect
|
||||
github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect
|
||||
@ -29,8 +30,8 @@ require (
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/twmb/murmur3 v1.1.6 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
golang.org/x/crypto v0.2.0 // indirect
|
||||
golang.org/x/net v0.2.0 // indirect
|
||||
golang.org/x/sys v0.2.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
sigs.k8s.io/yaml v1.3.0 // indirect
|
||||
)
|
||||
|
||||
24
go.sum
24
go.sum
@ -1,19 +1,29 @@
|
||||
github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew=
|
||||
github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM=
|
||||
github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
|
||||
github.com/chainreactors/files v0.2.3 h1:rDU53H9BPFyQWuIsjZdTm4m7e2/SSemobaU2QUVfLow=
|
||||
github.com/chainreactors/files v0.2.3/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
|
||||
github.com/chainreactors/files v0.2.4 h1:R0iCqjWLcwwLoSi87FpgUlpxZAd+W4ZLQF3lkoLWZi0=
|
||||
github.com/chainreactors/files v0.2.4/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
|
||||
github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 h1:kMFr1Hj+rkp1wBPIw2pcQvelO5GnA7r7wY3h6vJ1joA=
|
||||
github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8/go.mod h1:7NDvFERNiXsujaBPD6s4WXj52uKdfnF2zVHQtKXIEV4=
|
||||
github.com/chainreactors/gogo/v2 v2.8.10 h1:PlPZ1lTIcVl2IfE5GgIr4+hocwrTMyUasMDbrR+7TqI=
|
||||
github.com/chainreactors/gogo/v2 v2.8.10/go.mod h1:BQy0aEZXo8nSe/bMMsFeD2KU4vHa8CZ0FHrzuVkH/hU=
|
||||
github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d h1:LrbxNFvUrCooEIbQd4JicGFxRCs5KNov8xjJb8b7udw=
|
||||
github.com/chainreactors/gogo/v2 v2.9.5-0.20221110124606-bb8c89742d4d/go.mod h1:L8CXaQB23of7vIZaBuajfNwXBemSthOy8o4myPs4PO4=
|
||||
github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows=
|
||||
github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
|
||||
github.com/chainreactors/ipcs v0.0.13 h1:TZww7XRr4qZPWqy9DjBzcJgxtSUwT4TAbcho4156bRI=
|
||||
github.com/chainreactors/ipcs v0.0.13/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
|
||||
github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
|
||||
github.com/chainreactors/logs v0.6.1 h1:+wLVqxErUyWYayAx6x+m5LA6jQBiHcLdTAJtiOUMoZU=
|
||||
github.com/chainreactors/logs v0.6.1/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
|
||||
github.com/chainreactors/parsers v0.0.2 h1:lQ4dCWzUrVV8dUVKEAw8qlOU6O4Aju4sXtElAfIhbFI=
|
||||
github.com/chainreactors/parsers v0.0.2/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0=
|
||||
github.com/chainreactors/parsers v0.2.6/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
|
||||
github.com/chainreactors/parsers v0.2.7 h1:3iEuluL7gSDrElZWyf1KEiTgddgcoZC0IaIHb9KA3pk=
|
||||
github.com/chainreactors/parsers v0.2.7/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
|
||||
github.com/chainreactors/words v0.1.1 h1:Zw4HKFtYcIH5SfuCV0X6kj/A5sN99jrQD2ChUonLOV8=
|
||||
github.com/chainreactors/words v0.1.1/go.mod h1:jRcFgafTKqdkd1+StzPCTJG1ESrZHluXEO2eERdHBMQ=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -38,6 +48,8 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q=
|
||||
github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
|
||||
github.com/panjf2000/ants/v2 v2.6.0 h1:xOSpw42m+BMiJ2I33we7h6fYzG4DAlpE1xyI7VS2gxU=
|
||||
github.com/panjf2000/ants/v2 v2.6.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@ -53,10 +65,14 @@ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7Fw
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE=
|
||||
golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -68,6 +84,8 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -75,11 +93,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
|
||||
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
|
||||
|
||||
@ -41,6 +41,8 @@ type OutputOptions struct {
|
||||
Filters map[string]string `long:"filter" description:"String, "`
|
||||
Extracts []string `long:"extract" description:"String, "`
|
||||
OutputFile string `short:"f" description:"String, output filename"`
|
||||
FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename"`
|
||||
Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output"`
|
||||
OutputProbe string `long:"probe" description:"String, output format"`
|
||||
}
|
||||
|
||||
@ -78,6 +80,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
|
||||
Offset: opt.Offset,
|
||||
Limit: opt.Limit,
|
||||
URLList: make(chan string),
|
||||
OutputCh: make(chan *pkg.Baseline, 100),
|
||||
FuzzyCh: make(chan *pkg.Baseline, 100),
|
||||
Fuzzy: opt.Fuzzy,
|
||||
}
|
||||
|
||||
err = pkg.LoadTemplates()
|
||||
@ -185,7 +190,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
|
||||
if opt.RemoveExtensions != "" {
|
||||
rexts := strings.Split(opt.ExcludeExtensions, ",")
|
||||
r.Fns = append(r.Fns, func(s string) string {
|
||||
if ext := parseExtension(s); SliceContains(rexts, ext) {
|
||||
if ext := parseExtension(s); StringsContains(rexts, ext) {
|
||||
return strings.TrimSuffix(s, "."+ext)
|
||||
}
|
||||
return s
|
||||
@ -195,7 +200,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
|
||||
if opt.ExcludeExtensions != "" {
|
||||
exexts := strings.Split(opt.ExcludeExtensions, ",")
|
||||
r.Fns = append(r.Fns, func(s string) string {
|
||||
if ext := parseExtension(s); SliceContains(exexts, ext) {
|
||||
if ext := parseExtension(s); StringsContains(exexts, ext) {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
@ -259,7 +264,16 @@ func parseExtension(s string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func SliceContains(s []string, e string) bool {
|
||||
func StringsContains(s []string, e string) bool {
|
||||
for _, v := range s {
|
||||
if v == e {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IntsContains(s []int, e int) bool {
|
||||
for _, v := range s {
|
||||
if v == e {
|
||||
return true
|
||||
|
||||
@ -2,6 +2,7 @@ package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chainreactors/logs"
|
||||
"github.com/chainreactors/spray/pkg"
|
||||
"github.com/chainreactors/spray/pkg/ihttp"
|
||||
@ -21,7 +22,7 @@ var (
|
||||
|
||||
var breakThreshold int = 20
|
||||
|
||||
func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (*Pool, error) {
|
||||
func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
|
||||
pctx, cancel := context.WithCancel(ctx)
|
||||
pool := &Pool{
|
||||
Config: config,
|
||||
@ -29,8 +30,10 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
|
||||
cancel: cancel,
|
||||
client: ihttp.NewClient(config.Thread, 2, config.ClientType),
|
||||
worder: words.NewWorder(config.Wordlist),
|
||||
outputCh: outputCh,
|
||||
tempCh: make(chan *baseline, config.Thread),
|
||||
outputCh: config.OutputCh,
|
||||
fuzzyCh: config.FuzzyCh,
|
||||
baselines: make(map[int]*pkg.Baseline),
|
||||
tempCh: make(chan *pkg.Baseline, config.Thread),
|
||||
wg: sync.WaitGroup{},
|
||||
initwg: sync.WaitGroup{},
|
||||
checkPeriod: 100,
|
||||
@ -50,7 +53,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
|
||||
|
||||
if pool.failedCount > breakThreshold {
|
||||
// 当报错次数超过上限是, 结束任务
|
||||
pool.Recover()
|
||||
pool.recover()
|
||||
pool.cancel()
|
||||
}
|
||||
}
|
||||
@ -65,7 +68,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
|
||||
|
||||
if pool.failedCount > breakThreshold {
|
||||
// 当报错次数超过上限是, 结束任务
|
||||
pool.Recover()
|
||||
pool.recover()
|
||||
pool.cancel()
|
||||
}
|
||||
}
|
||||
@ -79,7 +82,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
|
||||
return
|
||||
}
|
||||
|
||||
var bl *baseline
|
||||
var bl *pkg.Baseline
|
||||
resp, reqerr := pool.client.Do(pctx, req)
|
||||
if pool.ClientType == ihttp.FAST {
|
||||
defer fasthttp.ReleaseResponse(resp.FastResponse)
|
||||
@ -88,26 +91,27 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
|
||||
|
||||
if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
|
||||
pool.failedCount++
|
||||
bl = &baseline{Url: pool.BaseURL + unit.path, Err: reqerr}
|
||||
bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, Err: reqerr.Error(), Reason: ErrRequestFailed.Error()}
|
||||
pool.failedBaselines = append(pool.failedBaselines, bl)
|
||||
} else {
|
||||
if err = pool.PreCompare(resp); err == nil || unit.source == CheckSource || unit.source == InitSource {
|
||||
if err = pool.PreCompare(resp); unit.source == CheckSource || unit.source == InitSource || err == nil {
|
||||
// 通过预对比跳过一些无用数据, 减少性能消耗
|
||||
bl = NewBaseline(req.URI(), req.Host(), resp)
|
||||
bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
|
||||
pool.addFuzzyBaseline(bl)
|
||||
} else {
|
||||
bl = NewInvalidBaseline(req.URI(), req.Host(), resp)
|
||||
bl = pkg.NewInvalidBaseline(req.URI(), req.Host(), resp, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
switch unit.source {
|
||||
case InitSource:
|
||||
pool.base = bl
|
||||
pool.addFuzzyBaseline(bl)
|
||||
pool.initwg.Done()
|
||||
logs.Log.Important("[baseline] " + bl.String())
|
||||
return
|
||||
case CheckSource:
|
||||
if bl.Err != nil {
|
||||
logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, breakThreshold, bl.Err.Error())
|
||||
if bl.Err != "" {
|
||||
logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, breakThreshold, bl.Err)
|
||||
pool.failedBaselines = append(pool.failedBaselines, bl)
|
||||
} else if i := pool.base.Compare(bl); i < 1 {
|
||||
if i == 0 {
|
||||
@ -118,7 +122,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
|
||||
|
||||
pool.failedBaselines = append(pool.failedBaselines, bl)
|
||||
} else {
|
||||
pool.ResetFailed() // 如果后续访问正常, 重置错误次数
|
||||
pool.resetFailed() // 如果后续访问正常, 重置错误次数
|
||||
logs.Log.Debug("[check.pass] " + bl.String())
|
||||
}
|
||||
|
||||
@ -149,15 +153,16 @@ type Pool struct {
|
||||
bar *pkg.Bar
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
//baseReq *http.Request
|
||||
base *baseline
|
||||
outputCh chan *baseline // 输出的chan, 全局统一
|
||||
tempCh chan *baseline // 待处理的baseline
|
||||
outputCh chan *pkg.Baseline // 输出的chan, 全局统一
|
||||
fuzzyCh chan *pkg.Baseline
|
||||
tempCh chan *pkg.Baseline // 待处理的baseline
|
||||
reqCount int
|
||||
failedCount int
|
||||
checkPeriod int
|
||||
errPeriod int
|
||||
failedBaselines []*baseline
|
||||
failedBaselines []*pkg.Baseline
|
||||
base *pkg.Baseline
|
||||
baselines map[int]*pkg.Baseline
|
||||
analyzeDone bool
|
||||
genReq func(s string) (*ihttp.Request, error)
|
||||
check func()
|
||||
@ -173,12 +178,13 @@ func (p *Pool) Init() error {
|
||||
// todo 分析baseline
|
||||
// 检测基本访问能力
|
||||
|
||||
if p.base.Err != nil {
|
||||
if p.base.Err != "" {
|
||||
p.cancel()
|
||||
return p.base.Err
|
||||
return fmt.Errorf(p.base.String())
|
||||
}
|
||||
|
||||
p.base.Collect()
|
||||
logs.Log.Important("[baseline.init] " + p.base.String())
|
||||
if p.base.RedirectURL != "" {
|
||||
CheckRedirect = func(redirectURL string) bool {
|
||||
if redirectURL == p.base.RedirectURL {
|
||||
@ -233,6 +239,10 @@ Loop:
|
||||
}
|
||||
|
||||
func (p *Pool) PreCompare(resp *ihttp.Response) error {
|
||||
if p.base != nil && p.base.Status != 200 && p.base.Status == resp.StatusCode() {
|
||||
return ErrSameStatus
|
||||
}
|
||||
|
||||
if !CheckStatusCode(resp.StatusCode()) {
|
||||
return ErrBadStatus
|
||||
}
|
||||
@ -251,22 +261,32 @@ func (p *Pool) PreCompare(resp *ihttp.Response) error {
|
||||
|
||||
func (p *Pool) comparing() {
|
||||
for bl := range p.tempCh {
|
||||
if !bl.IsValid {
|
||||
// precompare 确认无效数据直接送入管道
|
||||
p.outputCh <- bl
|
||||
continue
|
||||
}
|
||||
|
||||
if p.base.Compare(bl) == 1 {
|
||||
// 如果是同一个包则设置为无效包
|
||||
bl.IsValid = false
|
||||
p.outputCh <- bl
|
||||
continue
|
||||
}
|
||||
if !bl.IsValid {
|
||||
// 已经时被precompare过滤的项目, 跳过collect, 直接认为是无效数据
|
||||
} else if base, ok := p.baselines[bl.Status]; ok && base.Compare(bl) == 1 {
|
||||
bl.IsValid = false
|
||||
bl.IsFuzzy = true
|
||||
p.outputCh <- bl
|
||||
p.fuzzyCh <- bl
|
||||
continue
|
||||
}
|
||||
|
||||
bl.Collect()
|
||||
if p.EnableFuzzy && p.base.FuzzyEqual(bl) {
|
||||
// todo fuzzy compare
|
||||
if p.base.FuzzyCompare(bl) {
|
||||
bl.IsValid = false
|
||||
bl.IsFuzzy = true
|
||||
p.outputCh <- bl
|
||||
p.fuzzyCh <- bl
|
||||
continue
|
||||
}
|
||||
|
||||
@ -276,12 +296,24 @@ func (p *Pool) comparing() {
|
||||
p.analyzeDone = true
|
||||
}
|
||||
|
||||
func (p *Pool) ResetFailed() {
|
||||
func (p *Pool) addFuzzyBaseline(bl *pkg.Baseline) {
|
||||
if !IntsContains(FuzzyStatus, bl.Status) {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := p.baselines[bl.Status]; !ok {
|
||||
bl.Collect()
|
||||
p.baselines[bl.Status] = bl
|
||||
logs.Log.Importantf("[baseline.%dinit] %s", bl.Status, bl.String())
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pool) resetFailed() {
|
||||
p.failedCount = 0
|
||||
p.failedBaselines = nil
|
||||
}
|
||||
|
||||
func (p *Pool) Recover() {
|
||||
func (p *Pool) recover() {
|
||||
logs.Log.Errorf("failed request exceeds the threshold , task will exit. Breakpoint %d", p.reqCount)
|
||||
logs.Log.Error("collecting failed check")
|
||||
for i, bl := range p.failedBaselines {
|
||||
|
||||
@ -2,6 +2,7 @@ package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/chainreactors/files"
|
||||
"github.com/chainreactors/logs"
|
||||
"github.com/chainreactors/spray/pkg"
|
||||
"github.com/chainreactors/spray/pkg/ihttp"
|
||||
@ -26,7 +27,11 @@ type Runner struct {
|
||||
Timeout int
|
||||
Mod string
|
||||
Probes []string
|
||||
OutputCh chan *baseline
|
||||
OutputCh chan *pkg.Baseline
|
||||
FuzzyCh chan *pkg.Baseline
|
||||
Fuzzy bool
|
||||
OutputFile *files.File
|
||||
FuzzyFile *files.File
|
||||
Progress *uiprogress.Progress
|
||||
Offset int
|
||||
Limit int
|
||||
@ -44,8 +49,6 @@ func (r *Runner) Prepare(ctx context.Context) error {
|
||||
return true
|
||||
}
|
||||
|
||||
r.OutputCh = make(chan *baseline, 100)
|
||||
|
||||
r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) {
|
||||
u := i.(string)
|
||||
config := &pkg.Config{
|
||||
@ -56,6 +59,8 @@ func (r *Runner) Prepare(ctx context.Context) error {
|
||||
Headers: r.Headers,
|
||||
Mod: pkg.ModMap[r.Mod],
|
||||
Fns: r.Fns,
|
||||
OutputCh: r.OutputCh,
|
||||
FuzzyCh: r.FuzzyCh,
|
||||
}
|
||||
|
||||
if config.Mod == pkg.PathSpray {
|
||||
@ -64,7 +69,7 @@ func (r *Runner) Prepare(ctx context.Context) error {
|
||||
config.ClientType = ihttp.STANDARD
|
||||
}
|
||||
|
||||
pool, err := NewPool(ctx, config, r.OutputCh)
|
||||
pool, err := NewPool(ctx, config)
|
||||
if err != nil {
|
||||
logs.Log.Error(err.Error())
|
||||
return
|
||||
@ -83,7 +88,7 @@ func (r *Runner) Prepare(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go r.Outputting()
|
||||
r.Outputting()
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -107,19 +112,27 @@ Loop:
|
||||
for {
|
||||
if len(r.OutputCh) == 0 {
|
||||
close(r.OutputCh)
|
||||
return
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
if len(r.FuzzyCh) == 0 {
|
||||
close(r.FuzzyCh)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Outputting() {
|
||||
var outFunc func(baseline2 *baseline)
|
||||
go func() {
|
||||
var outFunc func(*pkg.Baseline)
|
||||
if len(r.Probes) > 0 {
|
||||
outFunc = func(bl *baseline) {
|
||||
outFunc = func(bl *pkg.Baseline) {
|
||||
logs.Log.Console("[+] " + bl.Format(r.Probes) + "\n")
|
||||
}
|
||||
} else {
|
||||
outFunc = func(bl *baseline) {
|
||||
outFunc = func(bl *pkg.Baseline) {
|
||||
logs.Log.Console("[+] " + bl.String() + "\n")
|
||||
}
|
||||
}
|
||||
@ -134,4 +147,16 @@ func (r *Runner) Outputting() {
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case bl := <-r.FuzzyCh:
|
||||
if r.Fuzzy {
|
||||
logs.Log.Console("[baseline.fuzzy] " + bl.String() + "\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@ -4,6 +4,8 @@ type ErrorType uint
|
||||
|
||||
const (
|
||||
ErrBadStatus ErrorType = iota
|
||||
ErrSameStatus
|
||||
ErrRequestFailed
|
||||
ErrWaf
|
||||
ErrRedirect
|
||||
)
|
||||
@ -12,6 +14,10 @@ func (e ErrorType) Error() string {
|
||||
switch e {
|
||||
case ErrBadStatus:
|
||||
return "bad status"
|
||||
case ErrSameStatus:
|
||||
return "same status"
|
||||
case ErrRequestFailed:
|
||||
return "request failed"
|
||||
case ErrWaf:
|
||||
return "maybe ban of waf"
|
||||
case ErrRedirect:
|
||||
|
||||
@ -1,17 +1,16 @@
|
||||
package internal
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/chainreactors/parsers"
|
||||
"github.com/chainreactors/spray/pkg"
|
||||
"github.com/chainreactors/spray/pkg/ihttp"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func NewBaseline(u, host string, resp *ihttp.Response) *baseline {
|
||||
bl := &baseline{
|
||||
func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
|
||||
bl := &Baseline{
|
||||
Url: u,
|
||||
Status: resp.StatusCode(),
|
||||
IsValid: true,
|
||||
@ -33,11 +32,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *baseline {
|
||||
return bl
|
||||
}
|
||||
|
||||
func NewInvalidBaseline(u, host string, resp *ihttp.Response) *baseline {
|
||||
bl := &baseline{
|
||||
func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline {
|
||||
bl := &Baseline{
|
||||
Url: u,
|
||||
Status: resp.StatusCode(),
|
||||
IsValid: false,
|
||||
Reason: reason,
|
||||
}
|
||||
|
||||
uu, err := url.Parse(u)
|
||||
@ -56,7 +56,7 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response) *baseline {
|
||||
return bl
|
||||
}
|
||||
|
||||
type baseline struct {
|
||||
type Baseline struct {
|
||||
Url string `json:"url"`
|
||||
Path string `json:"path"`
|
||||
Host string `json:"host"`
|
||||
@ -70,26 +70,28 @@ type baseline struct {
|
||||
IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url
|
||||
Spended int `json:"spended"` // 耗时, 毫秒
|
||||
Title string `json:"title"`
|
||||
Frameworks pkg.Frameworks `json:"frameworks"`
|
||||
Extracteds pkg.Extracteds `json:"extracts"`
|
||||
Err error `json:"-"`
|
||||
IsValid bool `json:"-"`
|
||||
Frameworks Frameworks `json:"frameworks"`
|
||||
Extracteds Extracteds `json:"extracts"`
|
||||
Err string `json:"error"`
|
||||
Reason string `json:"reason"`
|
||||
IsValid bool `json:"valid"`
|
||||
IsFuzzy bool `json:"fuzzy"`
|
||||
*parsers.Hashes
|
||||
}
|
||||
|
||||
// Collect 深度收集信息
|
||||
func (bl *baseline) Collect() {
|
||||
func (bl *Baseline) Collect() {
|
||||
if len(bl.Body) > 0 {
|
||||
bl.Title = parsers.MatchTitle(string(bl.Body))
|
||||
}
|
||||
bl.Hashes = parsers.NewHashes(bl.Raw)
|
||||
// todo extract
|
||||
bl.Extracteds = pkg.Extractors.Extract(string(bl.Raw))
|
||||
bl.Frameworks = pkg.FingerDetect(string(bl.Raw))
|
||||
bl.Extracteds = Extractors.Extract(string(bl.Raw))
|
||||
bl.Frameworks = FingerDetect(string(bl.Raw))
|
||||
}
|
||||
|
||||
// Compare if this equal other return true
|
||||
func (bl *baseline) Compare(other *baseline) int {
|
||||
func (bl *Baseline) Compare(other *Baseline) int {
|
||||
if other.RedirectURL != "" && bl.RedirectURL == other.RedirectURL {
|
||||
// 如果重定向url不为空, 且与base不相同, 则说明不是同一个页面
|
||||
return 1
|
||||
@ -114,12 +116,15 @@ func (bl *baseline) Compare(other *baseline) int {
|
||||
return -1
|
||||
}
|
||||
|
||||
func (bl *baseline) FuzzyEqual(other *baseline) bool {
|
||||
func (bl *Baseline) FuzzyCompare(other *Baseline) bool {
|
||||
// todo 模糊匹配
|
||||
if parsers.SimhashCompare(other.BodySimhash, bl.BodySimhash) < 3 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (bl *baseline) Get(key string) string {
|
||||
func (bl *Baseline) Get(key string) string {
|
||||
switch key {
|
||||
case "url":
|
||||
return bl.Url
|
||||
@ -160,7 +165,7 @@ func (bl *baseline) Get(key string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func (bl *baseline) Additional(key string) string {
|
||||
func (bl *Baseline) Additional(key string) string {
|
||||
if v := bl.Get(key); v != "" {
|
||||
return " [" + v + "] "
|
||||
} else {
|
||||
@ -168,16 +173,20 @@ func (bl *baseline) Additional(key string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func (bl *baseline) Format(probes []string) string {
|
||||
func (bl *Baseline) Format(probes []string) string {
|
||||
var line strings.Builder
|
||||
line.WriteString(bl.Url)
|
||||
if bl.Host != "" {
|
||||
line.WriteString(" (" + bl.Host + ")")
|
||||
}
|
||||
|
||||
if bl.Err != nil {
|
||||
line.WriteString("err: ")
|
||||
line.WriteString(bl.Err.Error())
|
||||
if bl.Reason != "" {
|
||||
line.WriteString(" ,")
|
||||
line.WriteString(bl.Reason)
|
||||
}
|
||||
if bl.Err != "" {
|
||||
line.WriteString(" ,err: ")
|
||||
line.WriteString(bl.Err)
|
||||
return line.String()
|
||||
}
|
||||
|
||||
@ -189,7 +198,7 @@ func (bl *baseline) Format(probes []string) string {
|
||||
return line.String()
|
||||
}
|
||||
|
||||
func (bl *baseline) String() string {
|
||||
func (bl *Baseline) String() string {
|
||||
var line strings.Builder
|
||||
//line.WriteString("[+] ")
|
||||
line.WriteString(bl.Url)
|
||||
@ -197,9 +206,9 @@ func (bl *baseline) String() string {
|
||||
line.WriteString(" (" + bl.Host + ")")
|
||||
}
|
||||
|
||||
if bl.Err != nil {
|
||||
if bl.Err != "" {
|
||||
line.WriteString("err: ")
|
||||
line.WriteString(bl.Err.Error())
|
||||
line.WriteString(bl.Err)
|
||||
return line.String()
|
||||
}
|
||||
|
||||
@ -218,7 +227,7 @@ func (bl *baseline) String() string {
|
||||
return line.String()
|
||||
}
|
||||
|
||||
func (bl *baseline) Jsonify() string {
|
||||
func (bl *Baseline) Jsonify() string {
|
||||
bs, err := json.Marshal(bl)
|
||||
if err != nil {
|
||||
return ""
|
||||
@ -30,4 +30,6 @@ type Config struct {
|
||||
EnableFuzzy bool
|
||||
ClientType int
|
||||
Fns []func(string) string
|
||||
OutputCh chan *Baseline
|
||||
FuzzyCh chan *Baseline
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user