From c70d26fd84e2e80eed1ab3a3ca1a31e2e4798d89 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 6 Mar 2024 17:55:18 +0800 Subject: [PATCH 01/24] fix read body bug when MaxBodyLength == 0 --- cmd/cmd.go | 9 +++++++-- internal/ihttp/client.go | 14 ++++++++++++-- internal/ihttp/response.go | 12 ++++++------ internal/option.go | 2 +- internal/pool/brutepool.go | 2 +- pkg/baseline.go | 6 +++--- 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 087afcf..88d1875 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -89,7 +89,12 @@ func Spray() { // 初始化全局变量 pkg.Distance = uint8(option.SimhashDistance) - ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024 + if option.MaxBodyLength == -1 { + ihttp.DefaultMaxBodySize = -1 + } else { + ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024 + } + pool.MaxCrawl = option.CrawlDepth var runner *internal.Runner @@ -103,7 +108,7 @@ func Spray() { return } if option.ReadAll || runner.Crawl { - ihttp.DefaultMaxBodySize = 0 + ihttp.DefaultMaxBodySize = -1 } ctx, canceler := context.WithTimeout(context.Background(), time.Duration(runner.Deadline)*time.Second) diff --git a/internal/ihttp/client.go b/internal/ihttp/client.go index 974f95a..714fc77 100644 --- a/internal/ihttp/client.go +++ b/internal/ihttp/client.go @@ -16,9 +16,19 @@ import ( ) var ( - DefaultMaxBodySize = 1024 * 100 // 100k + DefaultMaxBodySize int64 = 1024 * 100 // 100k ) +func CheckBodySize(size int64) bool { + if DefaultMaxBodySize == -1 { + return true + } + if DefaultMaxBodySize == 0 { + return false + } + return size < DefaultMaxBodySize +} + const ( Auto = iota FAST @@ -41,7 +51,7 @@ func NewClient(config *ClientConfig) *Client { //ReadTimeout: time.Duration(timeout) * time.Second, //WriteTimeout: time.Duration(timeout) * time.Second, ReadBufferSize: 16384, // 16k - MaxResponseBodySize: DefaultMaxBodySize, + MaxResponseBodySize: int(DefaultMaxBodySize), NoDefaultUserAgentHeader: true, DisablePathNormalizing: true, DisableHeaderNamesNormalizing: true, diff --git a/internal/ihttp/response.go b/internal/ihttp/response.go index 6ca4432..1de39af 100644 --- a/internal/ihttp/response.go +++ b/internal/ihttp/response.go @@ -29,7 +29,7 @@ func (r *Response) Body() []byte { if r.FastResponse != nil { return r.FastResponse.Body() } else if r.StandardResponse != nil { - if DefaultMaxBodySize == 0 { + if DefaultMaxBodySize == -1 { body, err := io.ReadAll(r.StandardResponse.Body) if err != nil { return nil @@ -37,10 +37,10 @@ func (r *Response) Body() []byte { return body } else { var body []byte - if r.StandardResponse.ContentLength > 0 && r.StandardResponse.ContentLength < int64(DefaultMaxBodySize) { + if r.StandardResponse.ContentLength > 0 && CheckBodySize(r.StandardResponse.ContentLength) { body = make([]byte, r.StandardResponse.ContentLength) } else { - body = make([]byte, DefaultMaxBodySize) + return nil } n, err := io.ReadFull(r.StandardResponse.Body, body) @@ -62,11 +62,11 @@ func (r *Response) Body() []byte { } } -func (r *Response) ContentLength() int { +func (r *Response) ContentLength() int64 { if r.FastResponse != nil { - return r.FastResponse.Header.ContentLength() + return int64(r.FastResponse.Header.ContentLength()) } else if r.StandardResponse != nil { - return int(r.StandardResponse.ContentLength) + return r.StandardResponse.ContentLength } else { return 0 } diff --git a/internal/option.go b/internal/option.go index 2b58495..06fe958 100644 --- a/internal/option.go +++ b/internal/option.go @@ -91,7 +91,7 @@ type RequestOptions struct { RandomUserAgent bool `long:"random-agent" description:"Bool, use random with default user-agent" config:"random-useragent"` Cookie []string `long:"cookie" description:"Strings, custom cookie" config:"cookies"` ReadAll bool `long:"read-all" description:"Bool, read all response body" config:"read-all"` - MaxBodyLength int `long:"max-length" default:"100" description:"Int, max response body length (kb), default 100k, e.g. -max-length 1000" config:"max-body-length"` + MaxBodyLength int64 `long:"max-length" default:"100" description:"Int, max response body length (kb), -1 read-all, 0 not read body, default 100k, e.g. --max-length 1000" config:"max-length"` } type PluginOptions struct { diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index bde820d..2634d88 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -356,7 +356,7 @@ func (pool *BrutePool) Invoke(v interface{}) { pool.doRedirect(bl, unit.depth) } - if ihttp.DefaultMaxBodySize != 0 && bl.BodyLength > ihttp.DefaultMaxBodySize { + if !ihttp.CheckBodySize(int64(bl.BodyLength)) { bl.ExceedLength = true } bl.Source = unit.source diff --git a/pkg/baseline.go b/pkg/baseline.go index 6a8eb79..71b2114 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -35,7 +35,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { copy(bl.Header, header) bl.HeaderLength = len(bl.Header) - if i := resp.ContentLength(); i != 0 && i <= ihttp.DefaultMaxBodySize { + if i := resp.ContentLength(); ihttp.CheckBodySize(i) { body := resp.Body() bl.Body = make([]byte, len(body)) copy(bl.Body, body) @@ -44,7 +44,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { bl.Chunked = true bl.BodyLength = len(bl.Body) } else { - bl.BodyLength = i + bl.BodyLength = int(i) } } @@ -86,7 +86,7 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba // 无效数据也要读取body, 否则keep-alive不生效 resp.Body() - bl.BodyLength = resp.ContentLength() + bl.BodyLength = int(resp.ContentLength()) bl.RedirectURL = string(resp.GetHeader("Location")) bl.Dir = bl.IsDir() From ab830a5e987d16c597636ac1a710e9fc83e27875 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 6 Mar 2024 18:14:07 +0800 Subject: [PATCH 02/24] support --no-dict flag --- internal/option.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/option.go b/internal/option.go index 06fe958..f2a200f 100644 --- a/internal/option.go +++ b/internal/option.go @@ -48,14 +48,15 @@ type InputOptions struct { PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` CIDRs string `long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` //Raw string `long:"raw" description:"File, input raw request filename"` + NoDict bool `long:"no-dict" description:"Bool, no dictionary"` Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt" config:"dictionaries"` - Offset int `long:"offset" description:"Int, wordlist offset"` - Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"` Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"` Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt" config:"rules"` AppendRule []string `long:"append-rule" description:"Files, when found valid path , use append rule generator new word with current path" config:"append-rules"` FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'" config:"filter-rule"` AppendFile []string `long:"append-file" description:"Files, when found valid path , use append file new word with current path" config:"append-files"` + Offset int `long:"offset" description:"Int, wordlist offset"` + Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"` } type FunctionOptions struct { @@ -307,7 +308,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { // prepare word dicts := make([][]string, len(opt.Dictionaries)) - if len(opt.Dictionaries) == 0 { + if len(opt.Dictionaries) == 0 && !opt.NoDict { dicts = append(dicts, pkg.LoadDefaultDict()) logs.Log.Warn("not set any dictionary, use default dictionary: https://github.com/maurosoria/dirsearch/blob/master/db/dicc.txt") } else { From 16f149dadffb29c90e2e99ced42206dc33b44fa0 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 6 Mar 2024 18:26:59 +0800 Subject: [PATCH 03/24] enhance crawl print --- pkg/baseline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/baseline.go b/pkg/baseline.go index 71b2114..8b1bcad 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -188,7 +188,7 @@ func (bl *Baseline) CollectURL() { } bl.URLs = iutils.StringsUnique(bl.URLs) - if bl.URLs != nil { + if len(bl.URLs) != 0 { bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{ Name: "crawl", ExtractResult: bl.URLs, From 9e82bb1ab397e3a373e83442c9ddbd1732ef2625 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 00:24:30 +0800 Subject: [PATCH 04/24] use mpb replace uiprogress --- go.mod | 7 +++-- go.sum | 15 +++++++--- internal/option.go | 8 +++--- internal/pool/brutepool.go | 2 +- internal/runner.go | 35 ++++++++++++++--------- pkg/bar.go | 57 +++++++++++++++++++++----------------- 6 files changed, 74 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 46cf2ab..ce986df 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,10 @@ require ( github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 github.com/goccy/go-yaml v1.11.2 github.com/gookit/config/v2 v2.2.5 - github.com/gosuri/uiprogress v0.0.1 github.com/jessevdk/go-flags v1.5.0 github.com/panjf2000/ants/v2 v2.7.0 github.com/valyala/fasthttp v1.43.0 + github.com/vbauerster/mpb/v8 v8.7.2 golang.org/x/net v0.21.0 golang.org/x/time v0.3.0 sigs.k8s.io/yaml v1.4.0 @@ -25,6 +25,8 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect @@ -33,11 +35,12 @@ require ( github.com/go-playground/validator/v10 v10.14.1 // indirect github.com/gookit/color v1.5.4 // indirect github.com/gookit/goutil v0.6.15 // indirect - github.com/gosuri/uilive v0.0.4 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index aa8c2c2..9453d17 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= @@ -53,10 +57,6 @@ github.com/gookit/goutil v0.6.15 h1:mMQ0ElojNZoyPD0eVROk5QXJPh2uKR4g06slgPDF5Jo= github.com/gookit/goutil v0.6.15/go.mod h1:qdKdYEHQdEtyH+4fNdQNZfJHhI0jUZzHxQVAV3DaMDY= github.com/gookit/ini/v2 v2.2.3 h1:nSbN+x9OfQPcMObTFP+XuHt8ev6ndv/fWWqxFhPMu2E= github.com/gookit/ini/v2 v2.2.3/go.mod h1:Vu6p7P7xcfmb8KYu3L0ek8bqu/Im63N81q208SCCZY4= -github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= -github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= -github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= -github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -69,12 +69,17 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/panjf2000/ants/v2 v2.7.0 h1:Y3Bgpfo9HDkBoHNVFbMfY5mAvi5TAA17y3HbzQ74p5Y= github.com/panjf2000/ants/v2 v2.7.0/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -92,6 +97,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g= github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vbauerster/mpb/v8 v8.7.2 h1:SMJtxhNho1MV3OuFgS1DAzhANN1Ejc5Ct+0iSaIkB14= +github.com/vbauerster/mpb/v8 v8.7.2/go.mod h1:ZFnrjzspgDHoxYLGvxIruiNk73GNTPG4YHgVNpR10VY= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/internal/option.go b/internal/option.go index f2a200f..c1daa4c 100644 --- a/internal/option.go +++ b/internal/option.go @@ -15,7 +15,7 @@ import ( "github.com/chainreactors/utils/iutils" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" - "github.com/gosuri/uiprogress" + "github.com/vbauerster/mpb/v8" "io/ioutil" "net/url" "os" @@ -23,6 +23,7 @@ import ( "strconv" "strings" "sync" + "time" ) var ( @@ -152,7 +153,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, err } r := &Runner{ - Progress: uiprogress.New(), + Progress: mpb.New(mpb.WithRefreshRate(100 * time.Millisecond)), Threads: opt.Threads, PoolSize: opt.PoolSize, Mod: opt.Mod, @@ -195,8 +196,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.Color = false } if !(opt.Quiet || opt.NoBar) { - r.Progress.Start() - logs.Log.SetOutput(r.Progress.Bypass()) + logs.Log.SetOutput(r.Progress) } // configuration diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 2634d88..3b5ccee 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -237,7 +237,7 @@ Loop: pool.Statistor.End++ if w == "" { pool.Statistor.Skipped++ - continue + pool.Bar.Done() } pool.wordOffset++ diff --git a/internal/runner.go b/internal/runner.go index 56c4eca..23b122d 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -2,7 +2,6 @@ package internal import ( "context" - "fmt" "github.com/antonmedv/expr/vm" "github.com/chainreactors/files" "github.com/chainreactors/logs" @@ -11,8 +10,9 @@ import ( "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words" "github.com/chainreactors/words/rule" - "github.com/gosuri/uiprogress" "github.com/panjf2000/ants/v2" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" "sync" "time" ) @@ -33,7 +33,7 @@ type Runner struct { outwg *sync.WaitGroup outputCh chan *pkg.Baseline fuzzyCh chan *pkg.Baseline - bar *uiprogress.Bar + bar *mpb.Bar finished int Tasks chan *Task @@ -61,7 +61,7 @@ type Runner struct { FuzzyFile *files.File DumpFile *files.File StatFile *files.File - Progress *uiprogress.Progress + Progress *mpb.Progress Offset int Limit int RateLimit int @@ -157,7 +157,7 @@ func (r *Runner) Prepare(ctx context.Context) error { }() pool.Worder = words.NewWorderWithChan(ch) pool.Worder.Fns = r.Fns - pool.Bar = pkg.NewBar("check", r.Count-r.Offset, r.Progress) + pool.Bar = pkg.NewBar("check", r.Count-r.Offset, pool.Statistor, r.Progress) pool.Run(ctx, r.Offset, r.Count) r.poolwg.Done() }) @@ -171,12 +171,21 @@ func (r *Runner) Prepare(ctx context.Context) error { }() if r.Count > 0 { - r.bar = r.Progress.AddBar(r.Count) - r.bar.PrependCompleted() - r.bar.PrependFunc(func(b *uiprogress.Bar) string { - return fmt.Sprintf("total progressive: %d/%d ", r.finished, r.Count) - }) - r.bar.AppendElapsed() + prompt := "total progressive:" + r.bar = r.Progress.AddBar(int64(r.Count), + mpb.BarFillerClearOnComplete(), // 可选：当进度条完成时清除 + mpb.PrependDecorators( + // 显示自定义的信息，比如下载速度和进度 + decor.Name(prompt, decor.WC{W: len(prompt) + 1, C: decor.DindentRight}), // 这里调整了装饰器的参数 + decor.OnComplete( // 当进度完成时显示的文本 + decor.Counters(0, "% d/% d"), " done!", + ), + ), + mpb.AppendDecorators( + // 显示经过的时间 + decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 4}), + ), + ) } r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) { @@ -219,7 +228,7 @@ func (r *Runner) Prepare(ctx context.Context) error { } else { limit = pool.Statistor.Total } - pool.Bar = pkg.NewBar(config.BaseURL, limit-pool.Statistor.Offset, r.Progress) + pool.Bar = pkg.NewBar(config.BaseURL, limit-pool.Statistor.Offset, pool.Statistor, r.Progress) logs.Log.Importantf("[pool] task: %s, total %d words, %d threads, proxy: %s", pool.BaseURL, limit-pool.Statistor.Offset, pool.Thread, pool.ProxyAddr) err = pool.Init() if err != nil { @@ -331,7 +340,7 @@ Loop: } func (r *Runner) Done() { - r.bar.Incr() + r.bar.Increment() r.finished++ r.poolwg.Done() } diff --git a/pkg/bar.go b/pkg/bar.go index ac335e2..98612e3 100644 --- a/pkg/bar.go +++ b/pkg/bar.go @@ -1,45 +1,50 @@ package pkg import ( - "fmt" "github.com/chainreactors/go-metrics" - "github.com/gosuri/uiprogress" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" ) -func NewBar(u string, total int, progress *uiprogress.Progress) *Bar { - bar := &Bar{ - Bar: progress.AddBar(total), +func NewBar(u string, total int, stat *Statistor, p *mpb.Progress) *Bar { + m := metrics.NewMeter() + metrics.Register(u, m) + + // 在mpb v8中，Name装饰器的使用方式略有不同 + bar := p.AddBar(int64(total), + mpb.BarFillerClearOnComplete(), + mpb.BarRemoveOnComplete(), + mpb.PrependDecorators( + // 显示自定义的信息，比如下载速度和进度 + decor.Name(u, decor.WC{W: len(u) + 1, C: decor.DindentRight}), // 这里调整了装饰器的参数 + decor.Counters(0, "% d/% d"), + ), + mpb.AppendDecorators( + // 显示经过的时间 + decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 4}), + ), + ) + + return &Bar{ url: u, - m: metrics.NewMeter(), + bar: bar, + m: m, } - - metrics.Register(bar.url, bar.m) - bar.PrependCompleted() - bar.PrependFunc(func(b *uiprogress.Bar) string { - return fmt.Sprintf("%f/s %d/%d", bar.m.Rate1(), bar.m.Count(), bar.Bar.Total) - }) - bar.PrependFunc(func(b *uiprogress.Bar) string { - return u - }) - bar.AppendElapsed() - - return bar } type Bar struct { - url string - total int - close bool - *uiprogress.Bar - m metrics.Meter + url string + bar *mpb.Bar + m metrics.Meter } func (bar *Bar) Done() { bar.m.Mark(1) - bar.Incr() + bar.bar.Increment() } func (bar *Bar) Close() { - metrics.Unregister(bar.url) - bar.close = true + //metrics.Unregister(bar.url) + // 标记进度条为完成状态 + //bar.bar.Abort(false) } From b1e42e763da70c4797ac8ab28fa1b98d0bbb7477 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 02:55:51 +0800 Subject: [PATCH 05/24] optimize config. 1. add flag --init to init config.yaml 2. default load config.yaml if this file exist --- cmd/cmd.go | 21 +++- config.yaml | 238 ++++++++++++++++++++++++++++----------------- internal/config.go | 88 ++++++++++++++--- internal/option.go | 37 +++---- 4 files changed, 257 insertions(+), 127 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 88d1875..0f833a6 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "github.com/chainreactors/files" "github.com/chainreactors/logs" "github.com/chainreactors/spray/internal" "github.com/chainreactors/spray/internal/ihttp" @@ -63,13 +64,25 @@ func Spray() { logs.Important: logs.GreenBold, pkg.LogVerbose: logs.Green, }) - - if option.Config != "" { - err := internal.LoadConfig(option.Config, &option) + if option.InitConfig { + configStr := internal.InitDefaultConfig(&option, 0) + err := os.WriteFile("config.yaml", []byte(configStr), 0o744) if err != nil { - logs.Log.Error(err.Error()) + logs.Log.Warn("cannot create config: config.yaml, " + err.Error()) return } + return + } + if option.Config != "" { + if !files.IsExist(option.Config) { + logs.Log.Warnf("config file %s not found", option.Config) + } else { + err := internal.LoadConfig(option.Config, &option) + if err != nil { + logs.Log.Error(err.Error()) + return + } + } } if option.Version { diff --git a/config.yaml b/config.yaml index 3280a11..d6e89ce 100644 --- a/config.yaml +++ b/config.yaml @@ -1,94 +1,154 @@ input: - append-files: [] # Files, when found valid path, use append file new word with current path - append-rules: [] # Files, when found valid path, use append rule generator new word with current path - dictionaries: [] # Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt - filter-rule: "" # String, filter rule, e.g.: --rule-filter '>8 <4' - rules: [] # Files, rule files, e.g.: -r rule1.txt -r rule2.txt - word: "" # String, word generate dsl, e.g.: -w test{?ld#4} - + # Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt + dictionaries: [] + # Bool, no dictionary + no-dict: true + # String, word generate dsl, e.g.: -w test{?ld#4} + word: "" + # Files, rule files, e.g.: -r rule1.txt -r rule2.txt + rules: [] + # Files, when found valid path , use append rule generator new word with current path + append-rules: [] + # String, filter rule, e.g.: --rule-filter '>8 <4' + filter-rule: "" + # Files, when found valid path , use append file new word with current path + append-files: [] functions: - extension: "" # String, add extensions (separated by commas), e.g.: -e jsp,jspx - exclude-extension: "" # String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx - force-extension: false # Bool, force add extensions - remove-extension: "" # String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx - prefix: [] # Strings, add prefix, e.g.: --prefix aaa --prefix bbb - suffix: [] # Strings, add suffix, e.g.: --suffix aaa --suffix bbb - upper: false # Bool, upper wordlist, e.g.: --uppercase - lower: false # Bool, lower wordlist, e.g.: --lowercase - replace: null # Strings, replace string, e.g.: --replace aaa:bbb --replace ccc:ddd - skip: [ ] # String, skip word when generate. rule, e.g.: --skip aaa - -misc: - mod: path # String, path/host spray - client: auto # String, Client type - thread: 20 # Int, number of threads per pool - pool: 5 # Int, Pool size - timeout: 5 # Int, timeout with request (seconds) - deadline: 999999 # Int, deadline (seconds) - proxy: "" # String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080 - quiet: false # Bool, Quiet - debug: false # Bool, output debug info - verbose: [] # Bool, log verbose level, default 0, level1: -v, level2 -vv - no-bar: false # Bool, No progress bar - no-color: false # Bool, no color - -mode: - # status - black-status: "400,410" # Strings (comma split), custom black status - fuzzy-status: "500,501,502,503" # Strings (comma split), custom fuzzy status - unique-status: "403,200,404" # Strings (comma split), custom unique status - white-status: "200" # Strings (comma split), custom white status - - # check - check-only: false # Bool, check only - check-period: 200 # Int, check period when request - error-period: 10 # Int, check period when error - error-threshold: 20 # Int, break when the error exceeds the threshold - - # recursive - recursive: current.IsDir() # String, custom recursive rule, e.g.: --recursive current.IsDir() - depth: 0 # Int, recursive depth - - # crawl - scope: [] # String, custom scope, e.g.: --scope *.example.com - no-scope: false # Bool, no scope - - # other - index: / # String, custom index path - random: "" # String, custom random path - unique: false # Bool, unique response - distance: 5 # Int, simhash distance for unique response - force: false # Bool, skip error break - rate-limit: 0 # Int, request rate limit (rate/s), e.g.: --rate-limit 100 - retry: 0 # Int, retry count - + # String, add extensions (separated by commas), e.g.: -e jsp,jspx + extension: "" + # Bool, force add extensions + force-extension: false + # String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx + exclude-extension: "" + # String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx + remove-extension: "" + # Bool, upper wordlist, e.g.: --uppercase + upper: false + # Bool, lower wordlist, e.g.: --lowercase + lower: false + # Strings, add prefix, e.g.: --prefix aaa --prefix bbb + prefix: [] + # Strings, add suffix, e.g.: --suffix aaa --suffix bbb + suffix: [] + # Strings, replace string, e.g.: --replace aaa:bbb --replace ccc:ddd + replace: {} + # String, skip word when generate. rule, e.g.: --skip aaa + skip: [] output: - output-file: "" # String, output filename - auto-file: false # Bool, auto generator output and fuzzy filename - dump: false # Bool, dump all request - dump-file: "" # String, dump all request, and write to filename - fuzzy: false # Bool, open fuzzy output - fuzzy-file: "" # String, fuzzy output filename - filter: "" # String, custom filter function, e.g.: --filter 'current.Body contains "hello"' - match: "" # String, custom match function, e.g.: --match 'current.Status != 200'' - format: "" # String, output format, e.g.: --format 1.json - output_probe: "" # String, output probes - + # String, custom match function, e.g.: --match 'current.Status != 200'' + match: "" + # String, custom filter function, e.g.: --filter 'current.Body contains "hello"' + filter: "" + # String, open fuzzy output + fuzzy: false + # String, output filename + output-file: "" + # String, fuzzy output filename + fuzzy-file: "" + # String, dump all request, and write to filename + dump-file: "" + # Bool, dump all request + dump: false + # Bool, auto generator output and fuzzy filename + auto-file: false + # String, output format, e.g.: --format 1.json + format: "" + # String, output format + output_probe: "" plugins: - all: false # Bool, enable all plugin - bak: false # Bool, enable bak found - common: false # Bool, enable common file found - crawl: false # Bool, enable crawl - crawl-depth: 3 # Int, crawl depth - extract: [] # Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?) - file-bak: false # Bool, enable valid result bak found, equal --append-rule rule/filebak.txt - finger: false # Bool, enable active finger detect - recon: false # Bool, enable recon - + # Bool, enable all plugin + all: false + # Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?) + extract: [] + # String, extract config filename + extract-config: "" + # Bool, enable recon + recon: false + # Bool, enable active finger detect + finger: false + # Bool, enable bak found + bak: false + # Bool, enable valid result bak found, equal --append-rule rule/filebak.txt + file-bak: false + # Bool, enable common file found + common: false + # Bool, enable crawl + crawl: false + # Int, crawl depth + crawl-depth: 3 request: - cookies: [] # Strings, custom cookie - headers: [] # Strings, custom headers, e.g.: --headers 'Auth: example_auth' - max-body-length: 100 # Int, max response body length (kb), default 100k, e.g. -max-length 1000 - useragent: "" # String, custom user-agent, e.g.: --user-agent Custom - random-useragent: false # Bool, use random with default user-agent - read-all: false # Bool, read all response body + # Strings, custom headers, e.g.: --headers 'Auth: example_auth' + headers: [] + # String, custom user-agent, e.g.: --user-agent Custom + useragent: "" + # Bool, use random with default user-agent + random-useragent: false + # Strings, custom cookie + cookies: [] + # Bool, read all response body + read-all: false + # Int, max response body length (kb), -1 read-all, 0 not read body, default 100k, e.g. --max-length 1000 + max-length: 100 +mode: + # Int, request rate limit (rate/s), e.g.: --rate-limit 100 + rate-limit: 0 + # Bool, skip error break + force: false + # Bool, check only + check-only: false + # Bool, no scope + no-scope: false + # String, custom scope, e.g.: --scope *.example.com + scope: [] + # String,custom recursive rule, e.g.: --recursive current.IsDir() + recursive: current.IsDir() + # Int, recursive depth + depth: 0 + # String, custom index path + index: / + # String, custom random path + random: "" + # Int, check period when request + check-period: 200 + # Int, check period when error + error-period: 10 + # Int, break when the error exceeds the threshold + error-threshold: 20 + # Strings (comma split),custom black status + black-status: 400,410 + # Strings (comma split), custom white status + white-status: 200 + # Strings (comma split), custom fuzzy status + fuzzy-status: 500,501,502,503 + # Strings (comma split), custom unique status + unique-status: 403,200,404 + # Bool, unique response + unique: false + # Int, retry count + retry: 0 + distance: 5 +misc: + # String, path/host spray + mod: path + # String, Client type + client: auto + # Int, deadline (seconds) + deadline: 999999 + # Int, timeout with request (seconds) + timeout: 5 + # Int, Pool size + pool: 5 + # Int, number of threads per pool + thread: 20 + # Bool, output debug info + debug: false + # Bool, log verbose level ,default 0, level1: -v level2 -vv + verbose: [] + # Bool, Quiet + quiet: false + # Bool, no color + no-color: false + # Bool, No progress bar + no-bar: false + # String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080 + proxy: "" diff --git a/internal/config.go b/internal/config.go index c79df68..295f880 100644 --- a/internal/config.go +++ b/internal/config.go @@ -2,10 +2,10 @@ package internal import ( "fmt" - "github.com/goccy/go-yaml" "github.com/gookit/config/v2" "reflect" "strconv" + "strings" ) //var ( @@ -101,14 +101,14 @@ func setFieldValue(field reflect.Value) interface{} { } } -// extractConfigAndDefaults 提取带有 `config` 和 `default` 标签的字段 -func extractConfigAndDefaults(v reflect.Value, result map[string]interface{}) { +func extractConfigAndDefaults(v reflect.Value, result map[string]interface{}, comments map[string]string) { t := v.Type() for i := 0; i < v.NumField(); i++ { field := v.Field(i) fieldType := t.Field(i) configTag := fieldType.Tag.Get("config") defaultTag := fieldType.Tag.Get("default") + descriptionTag := fieldType.Tag.Get("description") // 读取description标签 if configTag != "" { var value interface{} @@ -117,30 +117,86 @@ func extractConfigAndDefaults(v reflect.Value, result map[string]interface{}) { } else { value = setFieldValue(field) } + fullPath := configTag // 在递归情况下，您可能需要构建完整的路径 if field.Kind() == reflect.Struct { nestedResult := make(map[string]interface{}) - extractConfigAndDefaults(field, nestedResult) + nestedComments := make(map[string]string) + extractConfigAndDefaults(field, nestedResult, nestedComments) result[configTag] = nestedResult + for k, v := range nestedComments { + comments[fullPath+"."+k] = v // 保留嵌套注释的路径 + } } else { result[configTag] = value + if descriptionTag != "" { + comments[fullPath] = descriptionTag + } } } } } -func initDefaultConfig(cfg interface{}) (string, error) { +func InitDefaultConfig(cfg interface{}, indentLevel int) string { + var yamlStr strings.Builder v := reflect.ValueOf(cfg) - if v.Kind() != reflect.Struct { - return "", fmt.Errorf("expected a struct, got %s", v.Kind()) + if v.Kind() == reflect.Ptr { + v = v.Elem() // 解引用指针 + } + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + fieldType := t.Field(i) + configTag := fieldType.Tag.Get("config") + if configTag == "" { + continue // 忽略没有config标签的字段 + } + + defaultTag := fieldType.Tag.Get("default") + descriptionTag := fieldType.Tag.Get("description") + + // 添加注释 + if descriptionTag != "" { + yamlStr.WriteString(fmt.Sprintf("%s# %s\n", strings.Repeat(" ", indentLevel*2), descriptionTag)) + } + + // 准备值 + valueStr := prepareValue(fieldType.Type.Kind(), defaultTag) + + // 根据字段类型进行处理 + switch field.Kind() { + case reflect.Struct: + // 对于嵌套结构体，递归生成YAML + yamlStr.WriteString(fmt.Sprintf("%s%s:\n%s", strings.Repeat(" ", indentLevel*2), configTag, InitDefaultConfig(field.Interface(), indentLevel+1))) + default: + // 直接生成键值对 + yamlStr.WriteString(fmt.Sprintf("%s%s: %s\n", strings.Repeat(" ", indentLevel*2), configTag, valueStr)) + } } - result := make(map[string]interface{}) - extractConfigAndDefaults(v, result) - - yamlData, err := yaml.Marshal(result) - if err != nil { - return "", err - } - - return string(yamlData), nil + return yamlStr.String() +} + +// prepareValue 根据字段类型和default标签的值，准备最终的值字符串 +func prepareValue(kind reflect.Kind, defaultVal string) string { + if defaultVal != "" { + return defaultVal + } + // 根据类型返回默认空值 + switch kind { + case reflect.Bool: + return "false" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return "0" + case reflect.Float32, reflect.Float64: + return "0.0" + case reflect.Slice, reflect.Array: + return "[]" + case reflect.String: + return `""` + case reflect.Struct, reflect.Map: + return "{}" + default: + return `""` + } } diff --git a/internal/option.go b/internal/option.go index c1daa4c..5ce52c0 100644 --- a/internal/option.go +++ b/internal/option.go @@ -32,8 +32,8 @@ var ( ) type Option struct { - InputOptions `group:"Input Options" config:"input" default:""` - FunctionOptions `group:"Function Options" config:"functions" default:""` + InputOptions `group:"Input Options" config:"input" ` + FunctionOptions `group:"Function Options" config:"functions" ` OutputOptions `group:"Output Options" config:"output"` PluginOptions `group:"Plugin Options" config:"plugins"` RequestOptions `group:"Request Options" config:"request"` @@ -43,14 +43,14 @@ type Option struct { type InputOptions struct { ResumeFrom string `long:"resume" description:"File, resume filename" ` - Config string `short:"c" long:"config" description:"File, config filename"` + Config string `short:"c" long:"config" default:"config.yaml" description:"File, config filename"` URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` URLFile string `short:"l" long:"list" description:"File, input filename"` PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` CIDRs string `long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` //Raw string `long:"raw" description:"File, input raw request filename"` - NoDict bool `long:"no-dict" description:"Bool, no dictionary"` Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt" config:"dictionaries"` + NoDict bool `long:"no-dict" description:"Bool, no dictionary" config:"no-dict"` Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"` Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt" config:"rules"` AppendRule []string `long:"append-rule" description:"Files, when found valid path , use append rule generator new word with current path" config:"append-rules"` @@ -132,19 +132,20 @@ type ModeOptions struct { } type MiscOptions struct { - Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray" config:"mod"` - Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type" config:"client"` - Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)" config:"deadline"` // todo 总的超时时间,适配云函数的deadline - Timeout int `long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"` - PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size" config:"pool"` - Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool" config:"thread"` - Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"` - Version bool `long:"version" description:"Bool, show version"` - Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"` - Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet" config:"quiet"` - NoColor bool `long:"no-color" description:"Bool, no color" config:"no-color"` - NoBar bool `long:"no-bar" description:"Bool, No progress bar" config:"no-bar"` - Proxy string `long:"proxy" default:"" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"` + Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray" config:"mod"` + Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type" config:"client"` + Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)" config:"deadline"` // todo 总的超时时间,适配云函数的deadline + Timeout int `long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"` + PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size" config:"pool"` + Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool" config:"thread"` + Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"` + Version bool `long:"version" description:"Bool, show version"` + Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"` + Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet" config:"quiet"` + NoColor bool `long:"no-color" description:"Bool, no color" config:"no-color"` + NoBar bool `long:"no-bar" description:"Bool, No progress bar" config:"no-bar"` + Proxy string `long:"proxy" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"` + InitConfig bool `long:"init" description:"Bool, init config file"` } func (opt *Option) PrepareRunner() (*Runner, error) { @@ -308,7 +309,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { // prepare word dicts := make([][]string, len(opt.Dictionaries)) - if len(opt.Dictionaries) == 0 && !opt.NoDict { + if len(opt.Dictionaries) == 0 && opt.Word == "" && !opt.NoDict { dicts = append(dicts, pkg.LoadDefaultDict()) logs.Log.Warn("not set any dictionary, use default dictionary: https://github.com/maurosoria/dirsearch/blob/master/db/dicc.txt") } else { From 4e78e55b6ed2fdc794eaed2d8c704a1e5995ccd5 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 04:15:28 +0800 Subject: [PATCH 06/24] fix config load priority, now config < cmd enhance bar print fix config.yaml default --- cmd/cmd.go | 22 ++++++++++++++-------- config.yaml | 14 +++++++------- internal/option.go | 20 ++++++++++---------- internal/runner.go | 42 ++++++++++++++++++++++++++---------------- pkg/bar.go | 35 +++++++++++++++++++++++------------ 5 files changed, 80 insertions(+), 53 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 0f833a6..17aea49 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -18,9 +18,19 @@ import ( ) var ver = "v0.9.5" +var DefaultConfig = "config.yaml" func Spray() { var option internal.Option + + if files.IsExist(DefaultConfig) { + err := internal.LoadConfig(DefaultConfig, &option) + if err != nil { + logs.Log.Error(err.Error()) + return + } + } + parser := flags.NewParser(&option, flags.Default) parser.Usage = ` @@ -74,14 +84,10 @@ func Spray() { return } if option.Config != "" { - if !files.IsExist(option.Config) { - logs.Log.Warnf("config file %s not found", option.Config) - } else { - err := internal.LoadConfig(option.Config, &option) - if err != nil { - logs.Log.Error(err.Error()) - return - } + err := internal.LoadConfig(option.Config, &option) + if err != nil { + logs.Log.Error(err.Error()) + return } } diff --git a/config.yaml b/config.yaml index d6e89ce..72d45df 100644 --- a/config.yaml +++ b/config.yaml @@ -2,7 +2,7 @@ input: # Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt dictionaries: [] # Bool, no dictionary - no-dict: true + no-dict: false # String, word generate dsl, e.g.: -w test{?ld#4} word: "" # Files, rule files, e.g.: -r rule1.txt -r rule2.txt @@ -55,6 +55,12 @@ output: format: "" # String, output format output_probe: "" + # Bool, Quiet + quiet: false + # Bool, no color + no-color: false + # Bool, No progress bar + no-bar: false plugins: # Bool, enable all plugin all: false @@ -144,11 +150,5 @@ misc: debug: false # Bool, log verbose level ,default 0, level1: -v level2 -vv verbose: [] - # Bool, Quiet - quiet: false - # Bool, no color - no-color: false - # Bool, No progress bar - no-bar: false # String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080 proxy: "" diff --git a/internal/option.go b/internal/option.go index 5ce52c0..b739a38 100644 --- a/internal/option.go +++ b/internal/option.go @@ -43,7 +43,7 @@ type Option struct { type InputOptions struct { ResumeFrom string `long:"resume" description:"File, resume filename" ` - Config string `short:"c" long:"config" default:"config.yaml" description:"File, config filename"` + Config string `short:"c" long:"config" description:"File, config filename"` URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` URLFile string `short:"l" long:"list" description:"File, input filename"` PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` @@ -85,6 +85,9 @@ type OutputOptions struct { AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" config:"auto-file"` Format string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json" config:"format"` OutputProbe string `short:"o" long:"probe" description:"String, output format" config:"output_probe"` + Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet" config:"quiet"` + NoColor bool `long:"no-color" description:"Bool, no color" config:"no-color"` + NoBar bool `long:"no-bar" description:"Bool, No progress bar" config:"no-bar"` } type RequestOptions struct { @@ -141,9 +144,6 @@ type MiscOptions struct { Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"` Version bool `long:"version" description:"Bool, show version"` Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"` - Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet" config:"quiet"` - NoColor bool `long:"no-color" description:"Bool, no color" config:"no-color"` - NoBar bool `long:"no-bar" description:"Bool, No progress bar" config:"no-bar"` Proxy string `long:"proxy" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"` InitConfig bool `long:"init" description:"Bool, init config file"` } @@ -154,7 +154,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, err } r := &Runner{ - Progress: mpb.New(mpb.WithRefreshRate(100 * time.Millisecond)), Threads: opt.Threads, PoolSize: opt.PoolSize, Mod: opt.Mod, @@ -197,6 +196,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.Color = false } if !(opt.Quiet || opt.NoBar) { + r.Progress = mpb.New(mpb.WithRefreshRate(100 * time.Millisecond)) logs.Log.SetOutput(r.Progress) } @@ -334,11 +334,11 @@ func (opt *Option) PrepareRunner() (*Runner, error) { opt.Word += "}" } - if opt.Suffixes != nil { + if len(opt.Suffixes) != 0 { mask.SpecialWords["suffix"] = opt.Suffixes opt.Word += "{@suffix}" } - if opt.Prefixes != nil { + if len(opt.Prefixes) != 0 { mask.SpecialWords["prefix"] = opt.Prefixes opt.Word = "{@prefix}" + opt.Word } @@ -362,7 +362,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { logs.Log.Logf(pkg.LogVerbose, "Parsed %d words by %s", len(r.Wordlist), opt.Word) } - if opt.Rules != nil { + if len(opt.Rules) != 0 { rules, err := loadRuleAndCombine(opt.Rules) if err != nil { return nil, err @@ -391,7 +391,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Total: r.Total, } - if opt.AppendRule != nil { + if len(opt.AppendRule) != 0 { content, err := loadRuleAndCombine(opt.AppendRule) if err != nil { return nil, err @@ -399,7 +399,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.AppendRules = rule.Compile(string(content), "") } - if opt.AppendFile != nil { + if len(opt.AppendFile) != 0 { var bs bytes.Buffer for _, f := range opt.AppendFile { content, err := ioutil.ReadFile(f) diff --git a/internal/runner.go b/internal/runner.go index 23b122d..8700f65 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -171,21 +171,7 @@ func (r *Runner) Prepare(ctx context.Context) error { }() if r.Count > 0 { - prompt := "total progressive:" - r.bar = r.Progress.AddBar(int64(r.Count), - mpb.BarFillerClearOnComplete(), // 可选：当进度条完成时清除 - mpb.PrependDecorators( - // 显示自定义的信息，比如下载速度和进度 - decor.Name(prompt, decor.WC{W: len(prompt) + 1, C: decor.DindentRight}), // 这里调整了装饰器的参数 - decor.OnComplete( // 当进度完成时显示的文本 - decor.Counters(0, "% d/% d"), " done!", - ), - ), - mpb.AppendDecorators( - // 显示经过的时间 - decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 4}), - ), - ) + r.addBar(r.Count) } r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) { @@ -339,8 +325,32 @@ Loop: time.Sleep(100 * time.Millisecond) // 延迟100ms, 等所有数据处理完毕 } +func (r *Runner) addBar(total int) { + if r.Progress == nil { + return + } + + prompt := "total progressive:" + r.bar = r.Progress.AddBar(int64(total), + mpb.BarFillerClearOnComplete(), // 可选：当进度条完成时清除 + mpb.PrependDecorators( + // 显示自定义的信息，比如下载速度和进度 + decor.Name(prompt, decor.WC{W: len(prompt) + 1, C: decor.DindentRight}), // 这里调整了装饰器的参数 + decor.OnComplete( // 当进度完成时显示的文本 + decor.Counters(0, "% d/% d"), " done!", + ), + ), + mpb.AppendDecorators( + // 显示经过的时间 + decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 4}), + ), + ) +} + func (r *Runner) Done() { - r.bar.Increment() + if r.bar != nil { + r.bar.Increment() + } r.finished++ r.poolwg.Done() } diff --git a/pkg/bar.go b/pkg/bar.go index 98612e3..fe20e11 100644 --- a/pkg/bar.go +++ b/pkg/bar.go @@ -1,26 +1,31 @@ package pkg import ( - "github.com/chainreactors/go-metrics" + "fmt" "github.com/vbauerster/mpb/v8" "github.com/vbauerster/mpb/v8/decor" + "time" ) func NewBar(u string, total int, stat *Statistor, p *mpb.Progress) *Bar { - m := metrics.NewMeter() - metrics.Register(u, m) - - // 在mpb v8中，Name装饰器的使用方式略有不同 + if p == nil { + return &Bar{ + url: u, + } + } bar := p.AddBar(int64(total), mpb.BarFillerClearOnComplete(), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( - // 显示自定义的信息，比如下载速度和进度 decor.Name(u, decor.WC{W: len(u) + 1, C: decor.DindentRight}), // 这里调整了装饰器的参数 - decor.Counters(0, "% d/% d"), + decor.NewAverageSpeed(0, "% .0f/s ", time.Now()), + decor.Counters(0, "%d/%d"), + decor.Any(func(s decor.Statistics) string { + return fmt.Sprintf(" found: %d", stat.FoundNumber) + }), ), mpb.AppendDecorators( - // 显示经过的时间 + decor.Percentage(), decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 4}), ), ) @@ -28,23 +33,29 @@ func NewBar(u string, total int, stat *Statistor, p *mpb.Progress) *Bar { return &Bar{ url: u, bar: bar, - m: m, + //m: m, } } type Bar struct { url string bar *mpb.Bar - m metrics.Meter + //m metrics.Meter } func (bar *Bar) Done() { - bar.m.Mark(1) + //bar.m.Mark(1) + if bar.bar == nil { + return + } bar.bar.Increment() } func (bar *Bar) Close() { //metrics.Unregister(bar.url) // 标记进度条为完成状态 - //bar.bar.Abort(false) + if bar.bar == nil { + return + } + bar.bar.Abort(false) } From ec3ee45b89744fdc15a31a83772270f3eeee0f19 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 04:24:00 +0800 Subject: [PATCH 07/24] add config load log --- cmd/cmd.go | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 17aea49..058a74b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -17,13 +17,22 @@ import ( "time" ) -var ver = "v0.9.5" +var ver = "v0.9.6" var DefaultConfig = "config.yaml" +func init() { + logs.Log.SetColorMap(map[logs.Level]func(string) string{ + logs.Info: logs.PurpleBold, + logs.Important: logs.GreenBold, + pkg.LogVerbose: logs.Green, + }) +} + func Spray() { var option internal.Option if files.IsExist(DefaultConfig) { + logs.Log.Warnf("config.yaml exist, loading") err := internal.LoadConfig(DefaultConfig, &option) if err != nil { logs.Log.Error(err.Error()) @@ -68,19 +77,17 @@ func Spray() { } else if len(option.Verbose) > 0 { logs.Log.SetLevel(pkg.LogVerbose) } - - logs.Log.SetColorMap(map[logs.Level]func(string) string{ - logs.Info: logs.PurpleBold, - logs.Important: logs.GreenBold, - pkg.LogVerbose: logs.Green, - }) if option.InitConfig { configStr := internal.InitDefaultConfig(&option, 0) - err := os.WriteFile("config.yaml", []byte(configStr), 0o744) + err := os.WriteFile(DefaultConfig, []byte(configStr), 0o744) if err != nil { logs.Log.Warn("cannot create config: config.yaml, " + err.Error()) return } + if files.IsExist(DefaultConfig) { + logs.Log.Warn("override default config: ./config.yaml") + } + logs.Log.Info("init default config: ./config.yaml") return } if option.Config != "" { @@ -89,6 +96,11 @@ func Spray() { logs.Log.Error(err.Error()) return } + if files.IsExist(DefaultConfig) { + logs.Log.Warnf("custom config %s, override default config", option.Config) + } else { + logs.Log.Important("load config: " + option.Config) + } } if option.Version { From 3da923b2a880980a7d3f0dd5374c39f7a4f3a8af Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 15:02:20 +0800 Subject: [PATCH 08/24] update README.md --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 79c51eb..d1f54c7 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,9 @@ * 超强的性能, 在本地测试极限性能的场景下, 能超过ffuf与feroxbruster的性能50%以上. 实际情况受到网络的影响, 感受没有这么明确. 但在多目标下可以感受到明显的区别. * 基于掩码的字典生成 * 基于规则的字典生成 -* 动态智能过滤 -* 全量[gogo](https://github.com/chainreactors/gogo)的指纹识别, 全量的[fingerprinthub](https://github.com/0x727/FingerprintHub)指纹 -* 自定义信息提取, 如ip,js, title, hash以及自定义的正则表达式 -* 自定义过滤策略 +* 动态智能过滤, 自定义过滤策略 +* 全量[gogo](https://github.com/chainreactors/gogo)的指纹识别, 全量的[fingerprinthub](https://github.com/0x727/FingerprintHub),[wappalyzer](https://github.com/projectdiscovery/wappalyzergo)指纹 +* 自定义信息提取, 内置敏感信息提取规则 * 自定义输出格式与内容 * *nix的命令行设计, 轻松与其他工具联动 * 多角度的自动被ban,被waf判断 @@ -84,14 +83,16 @@ go build . 1. [x] 模糊对比 2. [x] 断点续传 3. [x] 简易爬虫 -4. [ ] 支持http2 +4. [x] 支持http2 5. [ ] auto-tune, 自动调整并发数量 6. [x] 可自定义的递归配置 7. [x] 参考[feroxbuster](https://github.com/epi052/feroxbuster)的`--collect-backups`, 自动爆破有效目录的备份 -8. [ ] 支持socks/http代理, 不建议使用, 优先级较低. 代理的keep-alive会带来严重的性能下降 +8. [x] 支持socks/http代理, 不建议使用, 优先级较低. 代理的keep-alive会带来严重的性能下降 9. [ ] 云函数化, chainreactors工具链的通用分布式解决方案. ## Thanks * [fuzzuli](https://github.com/musana/fuzzuli) 提供了一个备份文件字典生成思路 -* [fingerprinthub](https://github.com/0x727/FingerprintHub) 作为指纹库的补充 \ No newline at end of file +* [fingerprinthub](https://github.com/0x727/FingerprintHub) 作为指纹库的补充 +* [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) 作为指纹库补充 +* [dirsearch](https://github.com/maurosoria/dirsearch) 提供了默认字典 \ No newline at end of file From 006e1af2dbacc47e59de735583a474bfcd6f46e7 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 15:26:30 +0800 Subject: [PATCH 09/24] fix --check-only panic --- internal/pool/brutepool.go | 13 ------------- internal/pool/checkpool.go | 14 +++++++++----- internal/pool/{unit.go => types.go} | 18 +++++++++++++++++- 3 files changed, 26 insertions(+), 19 deletions(-) rename internal/pool/{unit.go => types.go} (54%) diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 3b5ccee..16d3e11 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -764,16 +764,3 @@ func (pool *BrutePool) resetFailed() { pool.failedCount = 1 pool.FailedBaselines = nil } - -func NewBaselines() *Baselines { - return &Baselines{ - baselines: map[int]*pkg.Baseline{}, - } -} - -type Baselines struct { - FailedBaselines []*pkg.Baseline - random *pkg.Baseline - index *pkg.Baseline - baselines map[int]*pkg.Baseline -} diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 56aeb00..89678f0 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -2,6 +2,7 @@ package pool import ( "context" + "errors" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" "github.com/chainreactors/spray/internal/ihttp" @@ -19,9 +20,10 @@ func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) { pctx, cancel := context.WithCancel(ctx) pool := &CheckPool{ &This{ - Config: config, - ctx: pctx, - Cancel: cancel, + Config: config, + Statistor: pkg.NewStatistor(""), + ctx: pctx, + Cancel: cancel, client: ihttp.NewClient(&ihttp.ClientConfig{ Thread: config.Thread, Type: config.ClientType, @@ -112,7 +114,7 @@ func (pool *CheckPool) Invoke(v interface{}) { defer fasthttp.ReleaseRequest(req.FastRequest) } - if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { + if reqerr != nil && !errors.Is(reqerr, fasthttp.ErrBodyTooLarge) { pool.failedCount++ bl = &pkg.Baseline{ SprayResult: &parsers.SprayResult{ @@ -156,9 +158,11 @@ func (pool *CheckPool) Invoke(v interface{}) { } } + if bl.Source == parsers.CheckSource { + pool.Bar.Done() + } pool.reqCount++ pool.wg.Done() - pool.Bar.Done() } func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { diff --git a/internal/pool/unit.go b/internal/pool/types.go similarity index 54% rename from internal/pool/unit.go rename to internal/pool/types.go index 06ff692..46cbea4 100644 --- a/internal/pool/unit.go +++ b/internal/pool/types.go @@ -1,6 +1,9 @@ package pool -import "github.com/chainreactors/parsers" +import ( + "github.com/chainreactors/parsers" + "github.com/chainreactors/spray/pkg" +) func newUnit(path string, source parsers.SpraySource) *Unit { return &Unit{path: path, source: source} @@ -18,3 +21,16 @@ type Unit struct { frontUrl string depth int // redirect depth } + +func NewBaselines() *Baselines { + return &Baselines{ + baselines: map[int]*pkg.Baseline{}, + } +} + +type Baselines struct { + FailedBaselines []*pkg.Baseline + random *pkg.Baseline + index *pkg.Baseline + baselines map[int]*pkg.Baseline +} From 13530eee5df51af6c6948dd0e99002d4c46ba1c5 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 7 Mar 2024 15:40:25 +0800 Subject: [PATCH 10/24] update README.md --- README.md | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d1f54c7..15dfa5e 100644 --- a/README.md +++ b/README.md @@ -26,27 +26,57 @@ [**Document**](https://chainreactors.github.io/wiki/spray/start) -基本使用, 从字典中读取目录进行爆破 +### 基本使用 + +**从字典中读取目录进行爆破** `spray -u http://example.com -d wordlist1.txt -d wordlist2.txt` -通过掩码生成字典进行爆破 +**通过掩码生成字典进行爆破** `spray -u http://example.com -w "/aaa/bbb{?l#4}/ccc"` -通过规则生成字典爆破. 规则文件格式参考hashcat的字典生成规则 +**通过规则生成字典爆破** + +规则文件格式参考hashcat的字典生成规则 `spray -u http://example.com -r rule.txt -d 1.txt` -批量爆破 +**批量爆破多个目标** `spray -l url.txt -r rule.txt -d 1.txt` -断点续传 +**断点续传** `spray --resume stat.json` -被动url收集 +### 高级用法 + +**check-only 模式** + +类似ehole/httpx这类对单页面信息收集的模式. 会有针对性的性能优化. 默认使用[templates](https://github.com/chainreactors/templates/tree/master/fingers)指纹库. 可以使用`--finger`打开第三方指纹库的匹配 + +`spray -l url.txt --check-only` + +**启用拓展指纹识别** + +会进行主动探测常见的指纹目录, 并额外启用fingerprinthub与wappalyzer拓展指纹库 + +`spray -u http://example.com --finger ` + +**启用爬虫** + +`spray -u http://example.com --crawl` + +**扫描备份文件与常见通用文件** + +`spray -u http://example.com --bak --common` + +**启用所有插件** + +`spray -u http://example.com -a` + +**被动url收集** 参见: https://github.com/chainreactors/urlfounder From b2d85a7698f00610913e7fdf4b7d5c314d5443f5 Mon Sep 17 00:00:00 2001 From: M09ic Date: Fri, 26 Apr 2024 18:57:02 +0800 Subject: [PATCH 11/24] fix abs path parse --- cmd/cmd.go | 2 +- go.mod | 2 +- internal/option.go | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 058a74b..7c12828 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -32,7 +32,7 @@ func Spray() { var option internal.Option if files.IsExist(DefaultConfig) { - logs.Log.Warnf("config.yaml exist, loading") + logs.Log.Debug("config.yaml exist, loading") err := internal.LoadConfig(DefaultConfig, &option) if err != nil { logs.Log.Error(err.Error()) diff --git a/go.mod b/go.mod index ce986df..b0f0753 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/chainreactors/spray -go 1.21.3 +go 1.21 require github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 diff --git a/internal/option.go b/internal/option.go index b739a38..a5e30aa 100644 --- a/internal/option.go +++ b/internal/option.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "net/url" "os" + "path/filepath" "regexp" "strconv" "strings" @@ -417,7 +418,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } ports := utils.ParsePort(opt.PortRange) - // prepare task tasks := make(chan *Task, opt.PoolSize) var taskfrom string @@ -486,14 +486,13 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } else if opt.URLFile != "" { file, err = os.Open(opt.URLFile) if err != nil { - logs.Log.Error(err.Error()) + return nil, err } - taskfrom = opt.URLFile + taskfrom = filepath.Base(opt.URLFile) } else if files.HasStdin() { file = os.Stdin taskfrom = "stdin" } - if file != nil { content, err := ioutil.ReadAll(file) if err != nil { From 75680c21f4e988b0cac67f4f3b46aeed3ca83bdd Mon Sep 17 00:00:00 2001 From: M09ic Date: Thu, 16 May 2024 18:35:39 +0800 Subject: [PATCH 12/24] update dependency --- go.mod | 47 +++++++++++--------- go.sum | 118 +++++++++++++++++++++++++++---------------------- pkg/fingers.go | 2 +- pkg/load.go | 2 +- pkg/nuclei.go | 1 - 5 files changed, 92 insertions(+), 78 deletions(-) delete mode 100644 pkg/nuclei.go diff --git a/go.mod b/go.mod index b0f0753..aff852f 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,24 @@ module github.com/chainreactors/spray -go 1.21 +go 1.22 -require github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 +toolchain go1.22.2 require ( - github.com/antonmedv/expr v1.12.5 + github.com/antonmedv/expr v1.15.5 github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 - github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f + github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f - github.com/chainreactors/parsers v0.0.0-20240304115854-f71473f7b510 + github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 - github.com/goccy/go-yaml v1.11.2 github.com/gookit/config/v2 v2.2.5 github.com/jessevdk/go-flags v1.5.0 - github.com/panjf2000/ants/v2 v2.7.0 - github.com/valyala/fasthttp v1.43.0 - github.com/vbauerster/mpb/v8 v8.7.2 - golang.org/x/net v0.21.0 - golang.org/x/time v0.3.0 + github.com/panjf2000/ants/v2 v2.9.1 + github.com/valyala/fasthttp v1.53.0 + github.com/vbauerster/mpb/v8 v8.7.3 + golang.org/x/net v0.25.0 + golang.org/x/time v0.5.0 sigs.k8s.io/yaml v1.4.0 ) @@ -28,25 +27,31 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect - github.com/go-playground/validator/v10 v10.14.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-yaml v1.11.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/gookit/color v1.5.4 // indirect github.com/gookit/goutil v0.6.15 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 9453d17..a5d64ca 100644 --- a/go.sum +++ b/go.sum @@ -4,35 +4,34 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E= -github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= +github.com/antonmedv/expr v1.15.5 h1:y0Iz3cEwmpRz5/r3w4qQR0MfIqJGdGM1zbhD/v0G5Vg= +github.com/antonmedv/expr v1.15.5/go.mod h1:0E/6TxnOlRNp81GMzX9QfDPAmHo2Phg00y4JUv1ihsE= github.com/chainreactors/files v0.0.0-20230731174853-acee21c8c45a/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/files v0.0.0-20231102192550-a652458cee26/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 h1:8Plpi6haQbU8NzH+JtU6bkGDWF/OeC+GFj8DIDuY5yk= github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= -github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f h1:dkg/RUxRptaUdP8ZoQ9maN0vDyNSSuf8k4mTOHddD7c= github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= -github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 h1:kMFr1Hj+rkp1wBPIw2pcQvelO5GnA7r7wY3h6vJ1joA= -github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8/go.mod h1:7NDvFERNiXsujaBPD6s4WXj52uKdfnF2zVHQtKXIEV4= +github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2 h1:DHnjmpeZYJWEsUqr6s0wqu1MYcWRFvHGYNS04dTzhdg= +github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= -github.com/chainreactors/parsers v0.0.0-20240304115854-f71473f7b510 h1:4o04P6MSAWkW1my4ogThJQgc0LeStaZo+3EvV24P+cU= -github.com/chainreactors/parsers v0.0.0-20240304115854-f71473f7b510/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg= +github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1FGWD7gVZuF0cufxd7i9HJ2gltUewxXCfvs4= +github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg= github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 h1:oFpBQBegsjNhKrzcjJUrqOgIJoa3Bc9OUmdRmYFHcXg= github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 h1:QdH1w8MnoAEnXp+CGqwroCRhAs+gu5OnIyW+qnK8Ibg= github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 h1:4U+x+EB1P66zwYgTjxWXSOT8vF+651Ksr1lojiCZnT8= github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5/go.mod h1:poR/Cp00iqtqu9ltFwl6C00sKC0HY13u/Gh05ZBmP54= github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c h1:mucYYQn+sMGNSxidhleonzAdwL203RxhjJGnxQU4NWU= @@ -43,12 +42,13 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= -github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= -github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I= +github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gookit/config/v2 v2.2.5 h1:RECbYYbtherywmzn3LNeu9NA5ZqhD7MSKEMsJ7l+MpU= @@ -59,34 +59,44 @@ github.com/gookit/ini/v2 v2.2.3 h1:nSbN+x9OfQPcMObTFP+XuHt8ev6ndv/fWWqxFhPMu2E= github.com/gookit/ini/v2 v2.2.3/go.mod h1:Vu6p7P7xcfmb8KYu3L0ek8bqu/Im63N81q208SCCZY4= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/panjf2000/ants/v2 v2.7.0 h1:Y3Bgpfo9HDkBoHNVFbMfY5mAvi5TAA17y3HbzQ74p5Y= -github.com/panjf2000/ants/v2 v2.7.0/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= +github.com/panjf2000/ants/v2 v2.9.1 h1:Q5vh5xohbsZXGcD6hhszzGqB7jSSc2/CRr3QKIga8Kw= +github.com/panjf2000/ants/v2 v2.9.1/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -94,76 +104,76 @@ github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g= -github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vbauerster/mpb/v8 v8.7.2 h1:SMJtxhNho1MV3OuFgS1DAzhANN1Ejc5Ct+0iSaIkB14= -github.com/vbauerster/mpb/v8 v8.7.2/go.mod h1:ZFnrjzspgDHoxYLGvxIruiNk73GNTPG4YHgVNpR10VY= +github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4zc= +github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= +github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0= +github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= -golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/fingers.go b/pkg/fingers.go index d6a669b..a8509ac 100644 --- a/pkg/fingers.go +++ b/pkg/fingers.go @@ -8,7 +8,7 @@ import ( // gogo fingers engine func FingersDetect(content []byte) common.Frameworks { - frames, _ := FingerEngine.FingersEngine.Match(bytes.ToLower(content), "") + frames, _ := FingerEngine.FingersEngine.HTTPMatch(bytes.ToLower(content), "") return frames } diff --git a/pkg/load.go b/pkg/load.go index 6b14bda..aa90d6c 100644 --- a/pkg/load.go +++ b/pkg/load.go @@ -28,7 +28,7 @@ func LoadTemplates() error { if err != nil { return err } - for _, f := range FingerEngine.FingersEngine.Fingers { + for _, f := range FingerEngine.FingersEngine.HTTPFingers { for _, rule := range f.Rules { if rule.SendDataStr != "" { ActivePath = append(ActivePath, rule.SendDataStr) diff --git a/pkg/nuclei.go b/pkg/nuclei.go deleted file mode 100644 index c1caffe..0000000 --- a/pkg/nuclei.go +++ /dev/null @@ -1 +0,0 @@ -package pkg From f755fc3816b443c702e08a5b9d72d1d5a425993d Mon Sep 17 00:00:00 2001 From: M09ic Date: Fri, 17 May 2024 17:47:11 +0800 Subject: [PATCH 13/24] support -x/--method custom http method support --raw parser input from raw http --- internal/ihttp/request.go | 5 +++-- internal/option.go | 38 ++++++++++++++++++++++++++++++-------- internal/pool/brutepool.go | 15 +++++++++------ internal/pool/checkpool.go | 10 +++++----- internal/pool/pool.go | 26 +++++++++++++------------- internal/runner.go | 2 ++ 6 files changed, 62 insertions(+), 34 deletions(-) diff --git a/internal/ihttp/request.go b/internal/ihttp/request.go index 6fe89d2..34b72ab 100644 --- a/internal/ihttp/request.go +++ b/internal/ihttp/request.go @@ -5,13 +5,14 @@ import ( "net/http" ) -func BuildPathRequest(clientType int, base, path string) (*Request, error) { +func BuildPathRequest(clientType int, base, path, method string) (*Request, error) { if clientType == FAST { req := fasthttp.AcquireRequest() + req.Header.SetMethod(method) req.SetRequestURI(base + path) return &Request{FastRequest: req, ClientType: FAST}, nil } else { - req, err := http.NewRequest("GET", base+path, nil) + req, err := http.NewRequest(method, base+path, nil) return &Request{StandardRequest: req, ClientType: STANDARD}, err } } diff --git a/internal/option.go b/internal/option.go index a5e30aa..fc35556 100644 --- a/internal/option.go +++ b/internal/option.go @@ -1,6 +1,7 @@ package internal import ( + "bufio" "bytes" "errors" "fmt" @@ -17,6 +18,7 @@ import ( "github.com/chainreactors/words/rule" "github.com/vbauerster/mpb/v8" "io/ioutil" + "net/http" "net/url" "os" "path/filepath" @@ -43,13 +45,13 @@ type Option struct { } type InputOptions struct { - ResumeFrom string `long:"resume" description:"File, resume filename" ` - Config string `short:"c" long:"config" description:"File, config filename"` - URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` - URLFile string `short:"l" long:"list" description:"File, input filename"` - PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` - CIDRs string `long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` - //Raw string `long:"raw" description:"File, input raw request filename"` + ResumeFrom string `long:"resume" description:"File, resume filename" ` + Config string `short:"c" long:"config" description:"File, config filename"` + URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` + URLFile string `short:"l" long:"list" description:"File, input filename"` + PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` + CIDRs string `long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` + RawFile string `long:"raw" description:"File, input raw request filename"` Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt" config:"dictionaries"` NoDict bool `long:"no-dict" description:"Bool, no dictionary" config:"no-dict"` Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"` @@ -92,6 +94,7 @@ type OutputOptions struct { } type RequestOptions struct { + Method string `short:"x" long:"method" default:"GET" description:"String, request method, e.g.: --method POST" config:"method"` Headers []string `long:"header" description:"Strings, custom headers, e.g.: --headers 'Auth: example_auth'" config:"headers"` UserAgent string `long:"user-agent" description:"String, custom user-agent, e.g.: --user-agent Custom" config:"useragent"` RandomUserAgent bool `long:"random-agent" description:"Bool, use random with default user-agent" config:"random-useragent"` @@ -162,6 +165,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { RateLimit: opt.RateLimit, Deadline: opt.Deadline, Headers: make(map[string]string), + Method: opt.Method, Offset: opt.Offset, Total: opt.Limit, taskCh: make(chan *Task), @@ -459,6 +463,24 @@ func (opt *Option) PrepareRunner() (*Runner, error) { taskfrom = "cmd" r.Count = len(opt.URL) + } else if opt.RawFile != "" { + raw, err := os.Open(opt.RawFile) + if err != nil { + return nil, err + } + + req, err := http.ReadRequest(bufio.NewReader(raw)) + if err != nil { + return nil, err + } + go func() { + opt.GenerateTasks(tasks, fmt.Sprintf("http://%s%s", req.Host, req.URL.String()), ports) + close(tasks) + }() + r.Method = req.Method + for k, _ := range req.Header { + r.Headers[k] = req.Header.Get(k) + } } else if opt.CIDRs != "" { if len(ports) == 0 { ports = []string{"80", "443"} @@ -739,7 +761,7 @@ func (opt *Option) Validate() error { return errors.New("--resume and --depth cannot be used at the same time") } - if opt.ResumeFrom == "" && opt.URL == nil && opt.URLFile == "" && opt.CIDRs == "" { + if opt.ResumeFrom == "" && opt.URL == nil && opt.URLFile == "" && opt.CIDRs == "" && opt.RawFile == "" { return fmt.Errorf("without any target, please use -u/-l/-c/--resume to set targets") } return nil diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 16d3e11..e3777d6 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -2,6 +2,7 @@ package pool import ( "context" + "errors" "fmt" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" @@ -37,7 +38,7 @@ func NewBrutePool(ctx context.Context, config *Config) (*BrutePool, error) { pctx, cancel := context.WithCancel(ctx) pool := &BrutePool{ Baselines: NewBaselines(), - This: &This{ + BasePool: &BasePool{ Config: config, ctx: pctx, Cancel: cancel, @@ -83,7 +84,7 @@ func NewBrutePool(ctx context.Context, config *Config) (*BrutePool, error) { type BrutePool struct { *Baselines - *This + *BasePool base string // url的根目录, 在爬虫或者redirect时, 会需要用到根目录进行拼接 isDir bool url *url.URL @@ -125,7 +126,7 @@ func (pool *BrutePool) genReq(mod SprayMod, s string) (*ihttp.Request, error) { if mod == HostSpray { return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s) } else if mod == PathSpray { - return ihttp.BuildPathRequest(pool.ClientType, pool.base, s) + return ihttp.BuildPathRequest(pool.ClientType, pool.base, s, pool.Method) } return nil, fmt.Errorf("unknown mod") } @@ -311,7 +312,9 @@ func (pool *BrutePool) Invoke(v interface{}) { } req.SetHeaders(pool.Headers) - req.SetHeader("User-Agent", pkg.RandomUA()) + if pool.RandomUserAgent { + req.SetHeader("User-Agent", pkg.RandomUA()) + } start := time.Now() resp, reqerr := pool.client.Do(pool.ctx, req) @@ -322,7 +325,7 @@ func (pool *BrutePool) Invoke(v interface{}) { // compare与各种错误处理 var bl *pkg.Baseline - if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { + if reqerr != nil && !errors.Is(reqerr, fasthttp.ErrBodyTooLarge) { atomic.AddInt32(&pool.failedCount, 1) atomic.AddInt32(&pool.Statistor.FailedNumber, 1) bl = &pkg.Baseline{ @@ -422,7 +425,7 @@ func (pool *BrutePool) Invoke(v interface{}) { func (pool *BrutePool) NoScopeInvoke(v interface{}) { defer pool.wg.Done() unit := v.(*Unit) - req, err := ihttp.BuildPathRequest(pool.ClientType, unit.path, "") + req, err := ihttp.BuildPathRequest(pool.ClientType, unit.path, "", pool.Method) if err != nil { logs.Log.Error(err.Error()) return diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 89678f0..5cb0998 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -19,7 +19,7 @@ import ( func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) { pctx, cancel := context.WithCancel(ctx) pool := &CheckPool{ - &This{ + &BasePool{ Config: config, Statistor: pkg.NewStatistor(""), ctx: pctx, @@ -38,12 +38,12 @@ func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) { pool.Headers = map[string]string{"Connection": "close"} p, _ := ants.NewPoolWithFunc(config.Thread, pool.Invoke) - pool.This.Pool = p + pool.BasePool.Pool = p return pool, nil } type CheckPool struct { - *This + *BasePool } func (pool *CheckPool) Run(ctx context.Context, offset, limit int) { @@ -81,12 +81,12 @@ Loop: } pool.wg.Add(1) - _ = pool.This.Pool.Invoke(newUnit(u, parsers.CheckSource)) + _ = pool.BasePool.Pool.Invoke(newUnit(u, parsers.CheckSource)) case u, ok := <-pool.additionCh: if !ok { continue } - _ = pool.This.Pool.Invoke(u) + _ = pool.BasePool.Pool.Invoke(u) case <-pool.closeCh: break Loop case <-ctx.Done(): diff --git a/internal/pool/pool.go b/internal/pool/pool.go index f4bb8ab..6cfefe8 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -14,7 +14,7 @@ import ( "sync" ) -type This struct { +type BasePool struct { *Config Statistor *pkg.Statistor Pool *ants.PoolWithFunc @@ -31,7 +31,7 @@ type This struct { wg sync.WaitGroup } -func (pool *This) doRedirect(bl *pkg.Baseline, depth int) { +func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) { if depth >= MaxRedirect { return } @@ -48,7 +48,7 @@ func (pool *This) doRedirect(bl *pkg.Baseline, depth int) { }() } -func (pool *This) doRule(bl *pkg.Baseline) { +func (pool *BasePool) doRule(bl *pkg.Baseline) { if pool.AppendRule == nil { pool.wg.Done() return @@ -69,7 +69,7 @@ func (pool *This) doRule(bl *pkg.Baseline) { }() } -func (pool *This) doAppendWords(bl *pkg.Baseline) { +func (pool *BasePool) doAppendWords(bl *pkg.Baseline) { if pool.AppendWords == nil { pool.wg.Done() return @@ -90,7 +90,7 @@ func (pool *This) doAppendWords(bl *pkg.Baseline) { }() } -func (pool *This) doRetry(bl *pkg.Baseline) { +func (pool *BasePool) doRetry(bl *pkg.Baseline) { if bl.Retry >= pool.Retry { return } @@ -105,7 +105,7 @@ func (pool *This) doRetry(bl *pkg.Baseline) { }() } -func (pool *This) doActive() { +func (pool *BasePool) doActive() { defer pool.wg.Done() for _, u := range pkg.ActivePath { pool.addAddition(&Unit{ @@ -115,7 +115,7 @@ func (pool *This) doActive() { } } -func (pool *This) doCommonFile() { +func (pool *BasePool) doCommonFile() { defer pool.wg.Done() for _, u := range mask.SpecialWords["common_file"] { pool.addAddition(&Unit{ @@ -125,7 +125,7 @@ func (pool *This) doCommonFile() { } } -func (pool *This) addAddition(u *Unit) { +func (pool *BasePool) addAddition(u *Unit) { // 强行屏蔽报错, 防止goroutine泄露 pool.wg.Add(1) defer func() { @@ -135,25 +135,25 @@ func (pool *This) addAddition(u *Unit) { pool.additionCh <- u } -func (pool *This) Close() { +func (pool *BasePool) Close() { pool.Bar.Close() } -func (pool *This) genReq(s string) (*ihttp.Request, error) { +func (pool *BasePool) genReq(s string) (*ihttp.Request, error) { if pool.Mod == HostSpray { return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s) } else if pool.Mod == PathSpray { - return ihttp.BuildPathRequest(pool.ClientType, pool.BaseURL, s) + return ihttp.BuildPathRequest(pool.ClientType, pool.BaseURL, s, pool.Method) } return nil, fmt.Errorf("unknown mod") } -func (pool *This) putToOutput(bl *pkg.Baseline) { +func (pool *BasePool) putToOutput(bl *pkg.Baseline) { pool.OutLocker.Add(1) pool.OutputCh <- bl } -func (pool *This) putToFuzzy(bl *pkg.Baseline) { +func (pool *BasePool) putToFuzzy(bl *pkg.Baseline) { pool.OutLocker.Add(1) bl.IsFuzzy = true pool.FuzzyCh <- bl diff --git a/internal/runner.go b/internal/runner.go index 8700f65..e28a99c 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -43,6 +43,7 @@ type Runner struct { AppendRules *rule.Program AppendWords []string Headers map[string]string + Method string Fns []func(string) []string FilterExpr *vm.Program MatchExpr *vm.Program @@ -92,6 +93,7 @@ func (r *Runner) PrepareConfig() *pool.Config { Timeout: r.Timeout, RateLimit: r.RateLimit, Headers: r.Headers, + Method: r.Method, Mod: pool.ModMap[r.Mod], OutputCh: r.outputCh, FuzzyCh: r.fuzzyCh, From ea090aa6bd362abb2bc28b7a7420d26330c64579 Mon Sep 17 00:00:00 2001 From: M09ic Date: Fri, 17 May 2024 17:47:42 +0800 Subject: [PATCH 14/24] fix proxy config bug when use http proxy --- internal/ihttp/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/ihttp/client.go b/internal/ihttp/client.go index 714fc77..0c00336 100644 --- a/internal/ihttp/client.go +++ b/internal/ihttp/client.go @@ -143,7 +143,6 @@ func customDialFunc(proxyAddr string, timeout time.Duration) fasthttp.DialFunc { return nil } if strings.ToLower(u.Scheme) == "socks5" { - return func(addr string) (net.Conn, error) { dialer, err := proxy.SOCKS5("tcp", u.Host, nil, proxy.Direct) if err != nil { @@ -165,6 +164,6 @@ func customDialFunc(proxyAddr string, timeout time.Duration) fasthttp.DialFunc { return conn, nil } } else { - return fasthttpproxy.FasthttpHTTPDialerTimeout(proxyAddr, timeout) + return fasthttpproxy.FasthttpHTTPDialerTimeout(u.Host, timeout) } } From 9bb4ae7341357714a37dbdd6f7ab32920e73967c Mon Sep 17 00:00:00 2001 From: M09ic Date: Fri, 17 May 2024 17:54:52 +0800 Subject: [PATCH 15/24] update words --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index aff852f..108440b 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 - github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 + github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 github.com/gookit/config/v2 v2.2.5 github.com/jessevdk/go-flags v1.5.0 github.com/panjf2000/ants/v2 v2.9.1 diff --git a/go.sum b/go.sum index a5d64ca..b2d143e 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 h1:oFpBQBegsjN github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 h1:QdH1w8MnoAEnXp+CGqwroCRhAs+gu5OnIyW+qnK8Ibg= github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= +github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw= +github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From cbb8e2f29505f4d59b530ce70c1d5888e4247165 Mon Sep 17 00:00:00 2001 From: M09ic Date: Tue, 21 May 2024 18:05:27 +0800 Subject: [PATCH 16/24] misc update, bar and finger panic --- go.mod | 2 +- go.sum | 2 ++ pkg/bar.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 108440b..89338f8 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.2 require ( github.com/antonmedv/expr v1.15.5 github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 - github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2 + github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 diff --git a/go.sum b/go.sum index b2d143e..65e049a 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8/go.mod h1:/Xa9 github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2 h1:DHnjmpeZYJWEsUqr6s0wqu1MYcWRFvHGYNS04dTzhdg= github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= +github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18 h1:gapflZSatRTQZ2c72wFBSGNEN57pwzUqm+vhlthIkOw= +github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= diff --git a/pkg/bar.go b/pkg/bar.go index fe20e11..49e3f4d 100644 --- a/pkg/bar.go +++ b/pkg/bar.go @@ -57,5 +57,5 @@ func (bar *Bar) Close() { if bar.bar == nil { return } - bar.bar.Abort(false) + bar.bar.Abort(true) } From 4c8c00416bdc3ae4bb00850667ca642bc94e079b Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 30 May 2024 18:55:02 +0800 Subject: [PATCH 17/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E4=B8=AA?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E8=BE=93=E5=85=A5=E5=80=BC=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/option.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/option.go b/internal/option.go index fc35556..b374d3a 100644 --- a/internal/option.go +++ b/internal/option.go @@ -443,15 +443,16 @@ func (opt *Option) PrepareRunner() (*Runner, error) { // 根据不同的输入类型生成任务 if len(opt.URL) == 1 { - u, err := url.Parse(opt.URL[0]) - if err != nil { - u, _ = url.Parse("http://" + opt.URL[0]) - } + //u, err := fixUrl(opt.URL[0]) + //if err != nil { + // return nil, err + //} go func() { - opt.GenerateTasks(tasks, u.String(), ports) + opt.GenerateTasks(tasks, opt.URL[0], ports) close(tasks) }() - taskfrom = u.Host + parsed, _ := url.Parse(opt.URL[0]) + taskfrom = parsed.Host r.Count = 1 } else if len(opt.URL) > 1 { go func() { @@ -771,7 +772,7 @@ func (opt *Option) Validate() error { func (opt *Option) GenerateTasks(ch chan *Task, u string, ports []string) { parsed, err := url.Parse(u) if err != nil { - logs.Log.Warn(err.Error()) + logs.Log.Warnf("parse %s, %s ", u, err.Error()) return } @@ -784,7 +785,7 @@ func (opt *Option) GenerateTasks(ch chan *Task, u string, ports []string) { } if len(ports) == 0 { - ch <- &Task{baseUrl: u} + ch <- &Task{baseUrl: parsed.String()} return } From cd34c7b2dd8f646e073700c3a8c0d21ff629897e Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 30 May 2024 23:33:57 +0800 Subject: [PATCH 18/24] fix checkpool bar thread safe bug --- internal/ihttp/client.go | 11 ++--- internal/option.go | 10 ++--- internal/pool/brutepool.go | 13 +++--- internal/pool/checkpool.go | 90 +++++++++++++++++++++----------------- internal/pool/config.go | 1 + internal/pool/pool.go | 3 +- spray.go | 1 + 7 files changed, 68 insertions(+), 61 deletions(-) diff --git a/internal/ihttp/client.go b/internal/ihttp/client.go index 0c00336..a2fa8f4 100644 --- a/internal/ihttp/client.go +++ b/internal/ihttp/client.go @@ -48,8 +48,8 @@ func NewClient(config *ClientConfig) *Client { MaxConnsPerHost: config.Thread * 3 / 2, MaxIdleConnDuration: config.Timeout, //MaxConnWaitTimeout: time.Duration(timeout) * time.Second, - //ReadTimeout: time.Duration(timeout) * time.Second, - //WriteTimeout: time.Duration(timeout) * time.Second, + //ReadTimeout: config.Timeout * time.Second, + //WriteTimeout: config.Timeout * time.Second, ReadBufferSize: 16384, // 16k MaxResponseBodySize: int(DefaultMaxBodySize), NoDefaultUserAgentHeader: true, @@ -68,9 +68,10 @@ func NewClient(config *ClientConfig) *Client { Renegotiation: tls.RenegotiateOnceAsClient, InsecureSkipVerify: true, }, - MaxConnsPerHost: config.Thread * 3 / 2, - IdleConnTimeout: config.Timeout, - ReadBufferSize: 16384, // 16k + TLSHandshakeTimeout: config.Timeout, + MaxConnsPerHost: config.Thread * 3 / 2, + IdleConnTimeout: config.Timeout, + ReadBufferSize: 16384, // 16k }, Timeout: config.Timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { diff --git a/internal/option.go b/internal/option.go index b374d3a..46eb2f8 100644 --- a/internal/option.go +++ b/internal/option.go @@ -142,7 +142,7 @@ type MiscOptions struct { Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray" config:"mod"` Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type" config:"client"` Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)" config:"deadline"` // todo 总的超时时间,适配云函数的deadline - Timeout int `long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"` + Timeout int `short:"T" long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"` PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size" config:"pool"` Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool" config:"thread"` Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"` @@ -169,9 +169,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Offset: opt.Offset, Total: opt.Limit, taskCh: make(chan *Task), - outputCh: make(chan *pkg.Baseline, 100), + outputCh: make(chan *pkg.Baseline, 256), outwg: &sync.WaitGroup{}, - fuzzyCh: make(chan *pkg.Baseline, 100), + fuzzyCh: make(chan *pkg.Baseline, 256), Fuzzy: opt.Fuzzy, Force: opt.Force, CheckOnly: opt.CheckOnly, @@ -443,10 +443,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) { // 根据不同的输入类型生成任务 if len(opt.URL) == 1 { - //u, err := fixUrl(opt.URL[0]) - //if err != nil { - // return nil, err - //} go func() { opt.GenerateTasks(tasks, opt.URL[0], ports) close(tasks) diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index e3777d6..a9e4562 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -50,6 +50,7 @@ func NewBrutePool(ctx context.Context, config *Config) (*BrutePool, error) { }), additionCh: make(chan *Unit, config.Thread), closeCh: make(chan struct{}), + processCh: make(chan *pkg.Baseline, config.Thread), wg: sync.WaitGroup{}, }, base: u.Scheme + "://" + u.Host, @@ -58,7 +59,6 @@ func NewBrutePool(ctx context.Context, config *Config) (*BrutePool, error) { scopeurls: make(map[string]struct{}), uniques: make(map[uint16]struct{}), - handlerCh: make(chan *pkg.Baseline, config.Thread), checkCh: make(chan struct{}, config.Thread), initwg: sync.WaitGroup{}, limiter: rate.NewLimiter(rate.Limit(config.RateLimit), 1), @@ -91,8 +91,7 @@ type BrutePool struct { reqPool *ants.PoolWithFunc scopePool *ants.PoolWithFunc - handlerCh chan *pkg.Baseline // 待处理的baseline - checkCh chan struct{} // 独立的check管道， 防止与redirect/crawl冲突 + checkCh chan struct{} // 独立的check管道， 防止与redirect/crawl冲突 closed bool wordOffset int failedCount int32 @@ -406,7 +405,7 @@ func (pool *BrutePool) Invoke(v interface{}) { case parsers.WordSource: // 异步进行性能消耗较大的深度对比 - pool.handlerCh <- bl + pool.processCh <- bl if int(pool.Statistor.ReqTotal)%pool.CheckPeriod == 0 { pool.doCheck() } else if pool.failedCount%pool.ErrPeriod == 0 { @@ -416,9 +415,9 @@ func (pool *BrutePool) Invoke(v interface{}) { pool.Bar.Done() case parsers.RedirectSource: bl.FrontURL = unit.frontUrl - pool.handlerCh <- bl + pool.processCh <- bl default: - pool.handlerCh <- bl + pool.processCh <- bl } } @@ -454,7 +453,7 @@ func (pool *BrutePool) NoScopeInvoke(v interface{}) { } func (pool *BrutePool) Handler() { - for bl := range pool.handlerCh { + for bl := range pool.processCh { if bl.IsValid { pool.addFuzzyBaseline(bl) } diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 5cb0998..45aa58a 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -2,13 +2,11 @@ package pool import ( "context" - "errors" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" "github.com/chainreactors/spray/internal/ihttp" "github.com/chainreactors/spray/pkg" "github.com/panjf2000/ants/v2" - "github.com/valyala/fasthttp" "net/url" "strings" "sync" @@ -18,6 +16,7 @@ import ( // 类似httpx的无状态, 无scope, 无并发池的检测模式 func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) { pctx, cancel := context.WithCancel(ctx) + config.ClientType = ihttp.STANDARD pool := &CheckPool{ &BasePool{ Config: config, @@ -31,14 +30,16 @@ func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) { ProxyAddr: config.ProxyAddr, }), wg: sync.WaitGroup{}, - additionCh: make(chan *Unit, 100), + additionCh: make(chan *Unit, 1024), closeCh: make(chan struct{}), + processCh: make(chan *pkg.Baseline, config.Thread), }, } pool.Headers = map[string]string{"Connection": "close"} p, _ := ants.NewPoolWithFunc(config.Thread, pool.Invoke) pool.BasePool.Pool = p + go pool.Handler() return pool, nil } @@ -100,21 +101,32 @@ Loop: } func (pool *CheckPool) Invoke(v interface{}) { + defer func() { + pool.reqCount++ + pool.wg.Done() + }() + unit := v.(*Unit) req, err := pool.genReq(unit.path) if err != nil { - logs.Log.Error(err.Error()) + logs.Log.Debug(err.Error()) + bl := &pkg.Baseline{ + SprayResult: &parsers.SprayResult{ + UrlString: unit.path, + IsValid: false, + ErrString: err.Error(), + Reason: pkg.ErrUrlError.Error(), + ReqDepth: unit.depth, + }, + } + pool.processCh <- bl + return } req.SetHeaders(pool.Headers) start := time.Now() var bl *pkg.Baseline resp, reqerr := pool.client.Do(pool.ctx, req) - if pool.ClientType == ihttp.FAST { - defer fasthttp.ReleaseResponse(resp.FastResponse) - defer fasthttp.ReleaseRequest(req.FastRequest) - } - - if reqerr != nil && !errors.Is(reqerr, fasthttp.ErrBodyTooLarge) { + if reqerr != nil { pool.failedCount++ bl = &pkg.Baseline{ SprayResult: &parsers.SprayResult{ @@ -125,13 +137,8 @@ func (pool *CheckPool) Invoke(v interface{}) { ReqDepth: unit.depth, }, } - - if strings.Contains(reqerr.Error(), "timed out") || strings.Contains(reqerr.Error(), "actively refused") { - - } else { - pool.doUpgrade(bl) - } - + logs.Log.Debugf("%s, %s", unit.path, reqerr.Error()) + pool.doUpgrade(bl) } else { bl = pkg.NewBaseline(req.URI(), req.Host(), resp) bl.Collect() @@ -139,32 +146,33 @@ func (pool *CheckPool) Invoke(v interface{}) { bl.ReqDepth = unit.depth bl.Source = unit.source bl.Spended = time.Since(start).Milliseconds() - - // 手动处理重定向 - if bl.IsValid { - if bl.RedirectURL != "" { - pool.doRedirect(bl, unit.depth) - pool.putToFuzzy(bl) - } else if bl.Status == 400 { - pool.doUpgrade(bl) - pool.putToFuzzy(bl) - } else { - params := map[string]interface{}{ - "current": bl, - } - if pool.MatchExpr == nil || pkg.CompareWithExpr(pool.MatchExpr, params) { - pool.putToOutput(bl) - } - } - } - - if bl.Source == parsers.CheckSource { - pool.Bar.Done() - } - pool.reqCount++ - pool.wg.Done() + pool.processCh <- bl } +func (pool *CheckPool) Handler() { + for bl := range pool.processCh { + if bl.IsValid { + if bl.RedirectURL != "" { + pool.doRedirect(bl, bl.ReqDepth) + pool.putToFuzzy(bl) + } else if bl.Status == 400 { + pool.doUpgrade(bl) + pool.putToFuzzy(bl) + } else { + params := map[string]interface{}{ + "current": bl, + } + if pool.MatchExpr != nil && pkg.CompareWithExpr(pool.MatchExpr, params) { + bl.IsValid = true + } + } + } + if bl.Source == parsers.CheckSource { + pool.Bar.Done() + } + pool.putToOutput(bl) + } +} func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { if depth >= MaxRedirect { return diff --git a/internal/pool/config.go b/internal/pool/config.go index bbf2460..ee79bb7 100644 --- a/internal/pool/config.go +++ b/internal/pool/config.go @@ -27,6 +27,7 @@ type Config struct { Thread int Wordlist []string Timeout int + ProcessCh chan *pkg.Baseline OutputCh chan *pkg.Baseline FuzzyCh chan *pkg.Baseline OutLocker *sync.WaitGroup diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 6cfefe8..ebbf385 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -20,9 +20,10 @@ type BasePool struct { Pool *ants.PoolWithFunc Bar *pkg.Bar Worder *words.Worder + Cancel context.CancelFunc client *ihttp.Client ctx context.Context - Cancel context.CancelFunc + processCh chan *pkg.Baseline // 待处理的baseline dir string reqCount int failedCount int diff --git a/spray.go b/spray.go index 71b68c7..c57a737 100644 --- a/spray.go +++ b/spray.go @@ -5,6 +5,7 @@ import ( "github.com/chainreactors/spray/cmd" "github.com/gookit/config/v2" "github.com/gookit/config/v2/yaml" + //_ "net/http/pprof" ) func init() { From a965412c1abadf0c1c169b305e8e4e779be365fe Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 3 Jun 2024 15:30:09 +0800 Subject: [PATCH 19/24] fix wappalyzer panic --- go.mod | 4 ++-- go.sum | 4 ++++ internal/pool/checkpool.go | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 89338f8..c395645 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ toolchain go1.22.2 require ( github.com/antonmedv/expr v1.15.5 github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 - github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18 + github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc - github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 + github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 github.com/gookit/config/v2 v2.2.5 github.com/jessevdk/go-flags v1.5.0 diff --git a/go.sum b/go.sum index 65e049a..f3b2a6a 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2 h1:DHnjmpeZY github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18 h1:gapflZSatRTQZ2c72wFBSGNEN57pwzUqm+vhlthIkOw= github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= +github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541 h1:aDY5A+G53En6t3Pr4tbl+vxJle2p1VJsqHVlyhnWU8s= +github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541/go.mod h1:s3lvNYcSW7NfM1inpgyn/wY3UEqQIvp6gE6BthFaOVo= github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= @@ -24,6 +26,8 @@ github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1F github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg= github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 h1:oFpBQBegsjNhKrzcjJUrqOgIJoa3Bc9OUmdRmYFHcXg= github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= +github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 h1:+awuysRKLmdLQbVK+HPSOGvO3dFGdNSbM2jyLh+VYOA= +github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 h1:QdH1w8MnoAEnXp+CGqwroCRhAs+gu5OnIyW+qnK8Ibg= github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw= diff --git a/internal/pool/checkpool.go b/internal/pool/checkpool.go index 45aa58a..b13402a 100644 --- a/internal/pool/checkpool.go +++ b/internal/pool/checkpool.go @@ -173,6 +173,7 @@ func (pool *CheckPool) Handler() { pool.putToOutput(bl) } } + func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { if depth >= MaxRedirect { return From b84535a06b5f43702e8ca1a5d165c879d5f251a7 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Tue, 4 Jun 2024 16:24:36 +0800 Subject: [PATCH 20/24] fix NewBaseline panic --- pkg/baseline.go | 7 ++++++- pkg/errors.go | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/baseline.go b/pkg/baseline.go index 8b1bcad..6f3f12f 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -50,7 +50,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { bl.Raw = append(bl.Header, bl.Body...) bl.Response, err = ParseRawResponse(bl.Raw) - + if err != nil { + bl.IsValid = false + bl.Reason = ErrResponseError.Error() + bl.ErrString = err.Error() + return bl + } if r := bl.Response.Header.Get("Location"); r != "" { bl.RedirectURL = r } else { diff --git a/pkg/errors.go b/pkg/errors.go index 1dd3575..e441543 100644 --- a/pkg/errors.go +++ b/pkg/errors.go @@ -16,6 +16,7 @@ const ( ErrFuzzyRedirect ErrFuzzyNotUnique ErrUrlError + ErrResponseError ) var ErrMap = map[ErrorType]string{ @@ -32,6 +33,7 @@ var ErrMap = map[ErrorType]string{ ErrFuzzyRedirect: "fuzzy redirect", ErrFuzzyNotUnique: "not unique", ErrUrlError: "url parse error", + ErrResponseError: "response parse error", } func (e ErrorType) Error() string { From cbf60343bb7bf0b63480c9c4482259261b8d3f4b Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 6 Jun 2024 18:11:45 +0800 Subject: [PATCH 21/24] fix expr not work bug --- go.mod | 2 +- go.sum | 11 ++--------- internal/option.go | 8 ++++---- internal/pool/config.go | 2 +- internal/runner.go | 2 +- internal/utils.go | 29 ++++++++++++++--------------- pkg/utils.go | 4 ++-- 7 files changed, 25 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index c395645..b6407da 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.22 toolchain go1.22.2 require ( - github.com/antonmedv/expr v1.15.5 github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 + github.com/expr-lang/expr v1.16.9 github.com/gookit/config/v2 v2.2.5 github.com/jessevdk/go-flags v1.5.0 github.com/panjf2000/ants/v2 v2.9.1 diff --git a/go.sum b/go.sum index f3b2a6a..9143dc4 100644 --- a/go.sum +++ b/go.sum @@ -6,17 +6,11 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/antonmedv/expr v1.15.5 h1:y0Iz3cEwmpRz5/r3w4qQR0MfIqJGdGM1zbhD/v0G5Vg= -github.com/antonmedv/expr v1.15.5/go.mod h1:0E/6TxnOlRNp81GMzX9QfDPAmHo2Phg00y4JUv1ihsE= github.com/chainreactors/files v0.0.0-20230731174853-acee21c8c45a/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/files v0.0.0-20231102192550-a652458cee26/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 h1:8Plpi6haQbU8NzH+JtU6bkGDWF/OeC+GFj8DIDuY5yk= github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= -github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2 h1:DHnjmpeZYJWEsUqr6s0wqu1MYcWRFvHGYNS04dTzhdg= -github.com/chainreactors/fingers v0.0.0-20240425063355-2ed9bd9149d2/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= -github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18 h1:gapflZSatRTQZ2c72wFBSGNEN57pwzUqm+vhlthIkOw= -github.com/chainreactors/fingers v0.0.0-20240521093617-f8ffb3ecdc18/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE= github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541 h1:aDY5A+G53En6t3Pr4tbl+vxJle2p1VJsqHVlyhnWU8s= github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541/go.mod h1:s3lvNYcSW7NfM1inpgyn/wY3UEqQIvp6gE6BthFaOVo= github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= @@ -24,18 +18,17 @@ github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMv github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1FGWD7gVZuF0cufxd7i9HJ2gltUewxXCfvs4= github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg= -github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3 h1:oFpBQBegsjNhKrzcjJUrqOgIJoa3Bc9OUmdRmYFHcXg= github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 h1:+awuysRKLmdLQbVK+HPSOGvO3dFGdNSbM2jyLh+VYOA= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= -github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37 h1:QdH1w8MnoAEnXp+CGqwroCRhAs+gu5OnIyW+qnK8Ibg= -github.com/chainreactors/words v0.4.1-0.20240220104223-153f52e53f37/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= +github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= diff --git a/internal/option.go b/internal/option.go index 46eb2f8..a262073 100644 --- a/internal/option.go +++ b/internal/option.go @@ -5,7 +5,6 @@ import ( "bytes" "errors" "fmt" - "github.com/antonmedv/expr" "github.com/chainreactors/files" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" @@ -16,6 +15,7 @@ import ( "github.com/chainreactors/utils/iutils" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" + "github.com/expr-lang/expr" "github.com/vbauerster/mpb/v8" "io/ioutil" "net/http" @@ -632,7 +632,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { logs.Log.Logf(pkg.LogVerbose, "Loaded %d dictionaries and %d decorators", len(opt.Dictionaries), len(r.Fns)) if opt.Match != "" { - exp, err := expr.Compile(opt.Match, expr.Patch(&bytesPatcher{})) + exp, err := expr.Compile(opt.Match) if err != nil { return nil, err } @@ -640,7 +640,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } if opt.Filter != "" { - exp, err := expr.Compile(opt.Filter, expr.Patch(&bytesPatcher{})) + exp, err := expr.Compile(opt.Filter) if err != nil { return nil, err } @@ -662,7 +662,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } if express != "" { - exp, err := expr.Compile(express, expr.Patch(&bytesPatcher{})) + exp, err := expr.Compile(express) if err != nil { return nil, err } diff --git a/internal/pool/config.go b/internal/pool/config.go index ee79bb7..93c13fa 100644 --- a/internal/pool/config.go +++ b/internal/pool/config.go @@ -1,9 +1,9 @@ package pool import ( - "github.com/antonmedv/expr/vm" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words/rule" + "github.com/expr-lang/expr/vm" "sync" ) diff --git a/internal/runner.go b/internal/runner.go index e28a99c..1d0e59d 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -2,7 +2,6 @@ package internal import ( "context" - "github.com/antonmedv/expr/vm" "github.com/chainreactors/files" "github.com/chainreactors/logs" "github.com/chainreactors/spray/internal/ihttp" @@ -10,6 +9,7 @@ import ( "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words" "github.com/chainreactors/words/rule" + "github.com/expr-lang/expr/vm" "github.com/panjf2000/ants/v2" "github.com/vbauerster/mpb/v8" "github.com/vbauerster/mpb/v8/decor" diff --git a/internal/utils.go b/internal/utils.go index d745f02..f36946c 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -2,7 +2,6 @@ package internal import ( "bytes" - "github.com/antonmedv/expr/ast" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" @@ -145,20 +144,20 @@ func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, er return rule.Compile(rules.String(), filter).Expressions, nil } -type bytesPatcher struct{} - -func (p *bytesPatcher) Visit(node *ast.Node) { - switch (*node).(type) { - case *ast.MemberNode: - ast.Patch(node, &ast.CallNode{ - Callee: &ast.MemberNode{ - Node: *node, - Name: "String", - Property: &ast.StringNode{Value: "String"}, - }, - }) - } -} +//type bytesPatcher struct{} +// +//func (p *bytesPatcher) Visit(node *ast.Node) { +// switch (*node).(type) { +// case *ast.MemberNode: +// ast.Patch(node, &ast.CallNode{ +// Callee: &ast.MemberNode{ +// Node: *node, +// Name: "String", +// Property: &ast.StringNode{Value: "String"}, +// }, +// }) +// } +//} func wrapWordsFunc(f func(string) string) func(string) []string { return func(s string) []string { diff --git a/pkg/utils.go b/pkg/utils.go index fbf8ee6..6591fab 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -3,10 +3,10 @@ package pkg import ( "bufio" "bytes" - "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" "github.com/chainreactors/logs" "github.com/chainreactors/utils/iutils" + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" "math/rand" "net/http" "net/url" From 229ef1f4245ffc06a93687d0e8ece8983aefbcac Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 6 Jun 2024 18:21:26 +0800 Subject: [PATCH 22/24] fix socks auth not work bug --- internal/ihttp/client.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/ihttp/client.go b/internal/ihttp/client.go index a2fa8f4..5127c0d 100644 --- a/internal/ihttp/client.go +++ b/internal/ihttp/client.go @@ -145,7 +145,16 @@ func customDialFunc(proxyAddr string, timeout time.Duration) fasthttp.DialFunc { } if strings.ToLower(u.Scheme) == "socks5" { return func(addr string) (net.Conn, error) { - dialer, err := proxy.SOCKS5("tcp", u.Host, nil, proxy.Direct) + var auth *proxy.Auth + username := u.User.Username() + password, ok := u.User.Password() + if ok { + auth = &proxy.Auth{ + User: username, + Password: password, + } + } + dialer, err := proxy.SOCKS5("tcp", u.Host, auth, proxy.Direct) if err != nil { return nil, err } From 0d24501a8aaf39effa743e6b0959cfc94733193f Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sun, 23 Jun 2024 16:47:11 +0800 Subject: [PATCH 23/24] adapt goreleaser2.0 --- .github/workflows/gorelease.yml | 2 +- .goreleaser.yml | 4 +++- templates | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gorelease.yml b/.github/workflows/gorelease.yml index 2f4b945..4eed61f 100644 --- a/.github/workflows/gorelease.yml +++ b/.github/workflows/gorelease.yml @@ -33,7 +33,7 @@ jobs: with: distribution: goreleaser version: latest - args: release --rm-dist --skip-validate + args: release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GOPATH: "/home/runner/go" diff --git a/.goreleaser.yml b/.goreleaser.yml index d86356a..36ed09b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -3,7 +3,6 @@ project_name: spray before: hooks: - go mod tidy - - go get sigs.k8s.io/yaml - go generate builds: @@ -37,6 +36,9 @@ upx: - enabled: true goos: [linux, windows] + goarch: + - amd64 + - "386" archives: - diff --git a/templates b/templates index 89dd024..d4db8ff 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 89dd0246cdbb1ce4604e2538a661e826e8b41142 +Subproject commit d4db8ffbae6f0d52b18fea2bcabe7b2c29c2153d From 784776edb372771bea85177f091aab8acca64fb8 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sun, 23 Jun 2024 16:59:44 +0800 Subject: [PATCH 24/24] fix prefix,suffix, ext, bak not work bug --- internal/option.go | 6 +++--- internal/pool/brutepool.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/option.go b/internal/option.go index a262073..c82e6a7 100644 --- a/internal/option.go +++ b/internal/option.go @@ -341,11 +341,11 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if len(opt.Suffixes) != 0 { mask.SpecialWords["suffix"] = opt.Suffixes - opt.Word += "{@suffix}" + opt.Word += "{?@suffix}" } if len(opt.Prefixes) != 0 { mask.SpecialWords["prefix"] = opt.Prefixes - opt.Word = "{@prefix}" + opt.Word + opt.Word = "{?@prefix}" + opt.Word } if opt.ForceExtension && opt.Extensions != "" { @@ -356,7 +356,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } } mask.SpecialWords["ext"] = exts - opt.Word += "{@ext}" + opt.Word += "{?@ext}" } r.Wordlist, err = mask.Run(opt.Word, dicts, nil) diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index a9e4562..d7d6d84 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -710,7 +710,7 @@ func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) { func (pool *BrutePool) doBak() { defer pool.wg.Done() - worder, err := words.NewWorderWithDsl("{?0}.{@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil) + worder, err := words.NewWorderWithDsl("{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil) if err != nil { return } @@ -722,7 +722,7 @@ func (pool *BrutePool) doBak() { }) } - worder, err = words.NewWorderWithDsl("{@bak_name}.{@bak_ext}", nil, nil) + worder, err = words.NewWorderWithDsl("{?@bak_name}.{?@bak_ext}", nil, nil) if err != nil { return }