diff --git a/.github/workflows/gorelease.yml b/.github/workflows/gorelease.yml index 3f3602a..2f4b945 100644 --- a/.github/workflows/gorelease.yml +++ b/.github/workflows/gorelease.yml @@ -17,11 +17,16 @@ jobs: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} submodules: recursive + + - name: Install upx + run: sudo apt install upx -y + continue-on-error: true + - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.21 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v4 diff --git a/.goreleaser.yml b/.goreleaser.yml index 346f366..d86356a 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -33,12 +33,15 @@ builds: - all=-trimpath={{.Env.GOPATH}} no_unique_dist_dir: true +upx: + - + enabled: true + goos: [linux, windows] + archives: - name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" format: binary - replacements: - amd64_v1: amd64 checksum: name_template: "{{ .ProjectName }}_checksums.txt" diff --git a/cmd/cmd.go b/cmd/cmd.go index dc07a80..f7a82d9 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -5,10 +5,11 @@ import ( "fmt" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" - "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/spray/internal" + "github.com/chainreactors/spray/internal/ihttp" + "github.com/chainreactors/spray/internal/pool" "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/spray/pkg/ihttp" + "github.com/chainreactors/utils/iutils" "github.com/jessevdk/go-flags" "os" "os/signal" @@ -17,7 +18,7 @@ import ( "time" ) -var ver = "" +var ver = "v0.9.3" func Spray() { var option internal.Option @@ -51,6 +52,28 @@ func Spray() { return } + // logs + logs.AddLevel(pkg.LogVerbose, "verbose", "[=] %s {{suffix}}") + if option.Debug { + logs.Log.SetLevel(logs.Debug) + } else if len(option.Verbose) > 0 { + logs.Log.SetLevel(pkg.LogVerbose) + } + + logs.Log.SetColorMap(map[logs.Level]func(string) string{ + logs.Info: logs.PurpleBold, + logs.Important: logs.GreenBold, + pkg.LogVerbose: logs.Green, + }) + + if option.Config != "" { + err := internal.LoadConfig(option.Config, &option) + if err != nil { + logs.Log.Error(err.Error()) + return + } + } + if option.Version { fmt.Println(ver) return @@ -58,7 +81,7 @@ func Spray() { if option.Format != "" { internal.Format(option.Format, !option.NoColor) - os.Exit(0) + return } err = pkg.LoadTemplates() @@ -80,30 +103,25 @@ func Spray() { } } } - // 一些全局变量初始化 - if option.Debug { - logs.Log.Level = logs.Debug - } - logs.DefaultColorMap[logs.Info] = logs.PurpleBold - logs.DefaultColorMap[logs.Important] = logs.GreenBold + // 初始化全局变量 pkg.Distance = uint8(option.SimhashDistance) ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024 - internal.MaxCrawl = option.CrawlDepth - if option.ReadAll { - ihttp.DefaultMaxBodySize = 0 - } + pool.MaxCrawl = option.CrawlDepth + var runner *internal.Runner if option.ResumeFrom != "" { runner, err = option.PrepareRunner() } else { runner, err = option.PrepareRunner() } - if err != nil { logs.Log.Errorf(err.Error()) return } + if option.ReadAll || runner.Crawl { + ihttp.DefaultMaxBodySize = 0 + } ctx, canceler := context.WithTimeout(context.Background(), time.Duration(runner.Deadline)*time.Second) diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..3280a11 --- /dev/null +++ b/config.yaml @@ -0,0 +1,94 @@ +input: + append-files: [] # Files, when found valid path, use append file new word with current path + append-rules: [] # Files, when found valid path, use append rule generator new word with current path + dictionaries: [] # Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt + filter-rule: "" # String, filter rule, e.g.: --rule-filter '>8 <4' + rules: [] # Files, rule files, e.g.: -r rule1.txt -r rule2.txt + word: "" # String, word generate dsl, e.g.: -w test{?ld#4} + +functions: + extension: "" # String, add extensions (separated by commas), e.g.: -e jsp,jspx + exclude-extension: "" # String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx + force-extension: false # Bool, force add extensions + remove-extension: "" # String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx + prefix: [] # Strings, add prefix, e.g.: --prefix aaa --prefix bbb + suffix: [] # Strings, add suffix, e.g.: --suffix aaa --suffix bbb + upper: false # Bool, upper wordlist, e.g.: --uppercase + lower: false # Bool, lower wordlist, e.g.: --lowercase + replace: null # Strings, replace string, e.g.: --replace aaa:bbb --replace ccc:ddd + skip: [ ] # String, skip word when generate. rule, e.g.: --skip aaa + +misc: + mod: path # String, path/host spray + client: auto # String, Client type + thread: 20 # Int, number of threads per pool + pool: 5 # Int, Pool size + timeout: 5 # Int, timeout with request (seconds) + deadline: 999999 # Int, deadline (seconds) + proxy: "" # String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080 + quiet: false # Bool, Quiet + debug: false # Bool, output debug info + verbose: [] # Bool, log verbose level, default 0, level1: -v, level2 -vv + no-bar: false # Bool, No progress bar + no-color: false # Bool, no color + +mode: + # status + black-status: "400,410" # Strings (comma split), custom black status + fuzzy-status: "500,501,502,503" # Strings (comma split), custom fuzzy status + unique-status: "403,200,404" # Strings (comma split), custom unique status + white-status: "200" # Strings (comma split), custom white status + + # check + check-only: false # Bool, check only + check-period: 200 # Int, check period when request + error-period: 10 # Int, check period when error + error-threshold: 20 # Int, break when the error exceeds the threshold + + # recursive + recursive: current.IsDir() # String, custom recursive rule, e.g.: --recursive current.IsDir() + depth: 0 # Int, recursive depth + + # crawl + scope: [] # String, custom scope, e.g.: --scope *.example.com + no-scope: false # Bool, no scope + + # other + index: / # String, custom index path + random: "" # String, custom random path + unique: false # Bool, unique response + distance: 5 # Int, simhash distance for unique response + force: false # Bool, skip error break + rate-limit: 0 # Int, request rate limit (rate/s), e.g.: --rate-limit 100 + retry: 0 # Int, retry count + +output: + output-file: "" # String, output filename + auto-file: false # Bool, auto generator output and fuzzy filename + dump: false # Bool, dump all request + dump-file: "" # String, dump all request, and write to filename + fuzzy: false # Bool, open fuzzy output + fuzzy-file: "" # String, fuzzy output filename + filter: "" # String, custom filter function, e.g.: --filter 'current.Body contains "hello"' + match: "" # String, custom match function, e.g.: --match 'current.Status != 200'' + format: "" # String, output format, e.g.: --format 1.json + output_probe: "" # String, output probes + +plugins: + all: false # Bool, enable all plugin + bak: false # Bool, enable bak found + common: false # Bool, enable common file found + crawl: false # Bool, enable crawl + crawl-depth: 3 # Int, crawl depth + extract: [] # Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?) + file-bak: false # Bool, enable valid result bak found, equal --append-rule rule/filebak.txt + finger: false # Bool, enable active finger detect + recon: false # Bool, enable recon + +request: + cookies: [] # Strings, custom cookie + headers: [] # Strings, custom headers, e.g.: --headers 'Auth: example_auth' + max-body-length: 100 # Int, max response body length (kb), default 100k, e.g. -max-length 1000 + useragent: "" # String, custom user-agent, e.g.: --user-agent Custom + random-useragent: false # Bool, use random with default user-agent + read-all: false # Bool, read all response body diff --git a/go.mod b/go.mod index 08a3965..2df1ade 100644 --- a/go.mod +++ b/go.mod @@ -1,38 +1,47 @@ module github.com/chainreactors/spray -go 1.17 +go 1.19 -require ( - github.com/chainreactors/files v0.2.5-0.20230310102018-3d10f74c7d6b - github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 - github.com/chainreactors/gogo/v2 v2.11.1-0.20230327070928-b5ff67ac46c7 - github.com/chainreactors/logs v0.7.1-0.20230316032643-ed7d85ca234f - github.com/chainreactors/parsers v0.3.1-0.20230403160559-9ed502452575 - github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2 -) +require github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 require ( github.com/antonmedv/expr v1.12.5 - github.com/chainreactors/ipcs v0.0.13 - github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56 + github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 + github.com/chainreactors/gogo/v2 v2.11.12-0.20231228061950-116583962e30 + github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f + github.com/chainreactors/parsers v0.0.0-20240208143911-65866d5bbc6d + github.com/chainreactors/utils v0.0.0-20231031063336-9477f1b23886 + github.com/chainreactors/words v0.4.1-0.20240208114042-a1c5053345b0 + github.com/gookit/config/v2 v2.2.5 github.com/gosuri/uiprogress v0.0.1 github.com/jessevdk/go-flags v1.5.0 github.com/panjf2000/ants/v2 v2.7.0 github.com/valyala/fasthttp v1.43.0 + golang.org/x/net v0.6.0 golang.org/x/time v0.3.0 - sigs.k8s.io/yaml v1.3.0 ) require ( + dario.cat/mergo v1.0.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/fatih/color v1.14.1 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect + github.com/goccy/go-yaml v1.11.2 // indirect + github.com/gookit/color v1.5.4 // indirect + github.com/gookit/goutil v0.6.15 // indirect github.com/gosuri/uilive v0.0.4 // indirect github.com/klauspost/compress v1.15.10 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect - github.com/twmb/murmur3 v1.1.6 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/sys v0.2.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect ) diff --git a/go.sum b/go.sum index 3ff1998..3c3401d 100644 --- a/go.sum +++ b/go.sum @@ -1,38 +1,39 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/M09ic/go-ntlmssp v0.0.0-20230312133735-dcccd454dfe0/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E= github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= -github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= -github.com/chainreactors/files v0.2.3/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= -github.com/chainreactors/files v0.2.5-0.20230310102018-3d10f74c7d6b h1:FRKGDHJrXrYfHnoehgE98vBoKvMpa/8/+d4wG0Zgpg4= -github.com/chainreactors/files v0.2.5-0.20230310102018-3d10f74c7d6b/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= +github.com/chainreactors/files v0.0.0-20230731174853-acee21c8c45a/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= +github.com/chainreactors/files v0.0.0-20231102192550-a652458cee26/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= +github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 h1:8Plpi6haQbU8NzH+JtU6bkGDWF/OeC+GFj8DIDuY5yk= +github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 h1:kMFr1Hj+rkp1wBPIw2pcQvelO5GnA7r7wY3h6vJ1joA= github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8/go.mod h1:7NDvFERNiXsujaBPD6s4WXj52uKdfnF2zVHQtKXIEV4= -github.com/chainreactors/gogo/v2 v2.11.1-0.20230327070928-b5ff67ac46c7 h1:3G8ExdfyXiP83WOzYPIEComWu2ZqKmmqAQxdq92F+Gs= -github.com/chainreactors/gogo/v2 v2.11.1-0.20230327070928-b5ff67ac46c7/go.mod h1:hhPu1b7UjMobE+4gAjevJ9ixQbvVK2Z3lKqoy9MPK/g= -github.com/chainreactors/ipcs v0.0.13 h1:TZww7XRr4qZPWqy9DjBzcJgxtSUwT4TAbcho4156bRI= -github.com/chainreactors/ipcs v0.0.13/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII= -github.com/chainreactors/logs v0.6.1/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= -github.com/chainreactors/logs v0.7.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= -github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= -github.com/chainreactors/logs v0.7.1-0.20230316032643-ed7d85ca234f h1:exuFhz7uiKPB/JTS9AcMuUwgs8nfJNz5eG9P6ObVwlM= -github.com/chainreactors/logs v0.7.1-0.20230316032643-ed7d85ca234f/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= -github.com/chainreactors/neutron v0.0.0-20230227122754-80dc76323a1c/go.mod h1:GjZPKmcyVoQvngG+GBHxXbpXBcjIcvHGO9xj/VXRf3w= -github.com/chainreactors/parsers v0.3.0/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= -github.com/chainreactors/parsers v0.3.1-0.20230313041950-25d5f9059c79/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= -github.com/chainreactors/parsers v0.3.1-0.20230403160559-9ed502452575 h1:uHE9O8x70FXwge5p68U/lGC9Xs8Leg8hWJR9PHKGzsk= -github.com/chainreactors/parsers v0.3.1-0.20230403160559-9ed502452575/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= -github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56 h1:1uhvEh7Of4fQJXRMsfGEZGy5NcETsM2yataQ0oYSw0k= -github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56/go.mod h1:NKSu1V6EC4wa8QHtPfiJHlH9VjGfUQOx5HADK0xry3Y= -github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2 h1:/v8gTORQIRJl2lgNt82OOeP/04QZyNTGKcmjfstVN5E= -github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/gogo/v2 v2.11.12-0.20231228061950-116583962e30 h1:Zh96ERETgkygSLUZ2NZ7Zi7lDcNf8jqImz+0aXCDsHY= +github.com/chainreactors/gogo/v2 v2.11.12-0.20231228061950-116583962e30/go.mod h1:XAGU3kpCiA3ZZzp/JS2kCigk9jIM3SC6NcOBdQ2DYa4= +github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= +github.com/chainreactors/logs v0.0.0-20231220102821-19f082ce37c1/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= +github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k= +github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= +github.com/chainreactors/neutron v0.0.0-20231221064706-fd6aaac9c50b/go.mod h1:Q6xCl+KaPtCDIziAHegFxdHOvg6DgpA6hcUWRnQKDPk= +github.com/chainreactors/parsers v0.0.0-20231218072716-fb441aff745f/go.mod h1:ZHEkgxKf9DXoley2LUjdJkiSw08MC3vcJTxfqwYt2LU= +github.com/chainreactors/parsers v0.0.0-20231220104848-3a0b5a5bd8dc/go.mod h1:V2w16sBSSiBlmsDR4A0Q9PIk9+TP/6coTXv6olvTI6M= +github.com/chainreactors/parsers v0.0.0-20240208143911-65866d5bbc6d h1:NFZLic9KNL1KdyvZFatRufXV9FJ3AXmKgTFQQ6Sz+Vk= +github.com/chainreactors/parsers v0.0.0-20240208143911-65866d5bbc6d/go.mod h1:IS0hrYnccfJKU0NA12zdZk4mM7k/Qt4qnzMnFGBFLZI= +github.com/chainreactors/utils v0.0.0-20231031063336-9477f1b23886 h1:lS2T/uE9tg1MNDPrb44wawbNlD24zBlWoG0H+ZdwDAk= +github.com/chainreactors/utils v0.0.0-20231031063336-9477f1b23886/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= +github.com/chainreactors/words v0.4.1-0.20240208114042-a1c5053345b0 h1:7aAfDhZDLs6uiWNzYa68L4uzBX7ZIj7IT8v+AlmmpHw= +github.com/chainreactors/words v0.4.1-0.20240208114042-a1c5053345b0/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 h1:4U+x+EB1P66zwYgTjxWXSOT8vF+651Ksr1lojiCZnT8= github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5/go.mod h1:poR/Cp00iqtqu9ltFwl6C00sKC0HY13u/Gh05ZBmP54= @@ -40,8 +41,21 @@ github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c h1:mucYYQn+sMGNSx github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c/go.mod h1:gO3u2bjRAgUaLdQd2XK+3oooxrheOAx1BzS7WmPzw1s= github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 h1:11wFcswN+37U+ByjxdKzsRY5KzNqqq5Uk5ztxnLOc7w= github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7/go.mod h1:wSsK4VOECOSfSYTzkBFw+iGY7wj59e7X96ABtNj9aCQ= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= +github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/gookit/config/v2 v2.2.5 h1:RECbYYbtherywmzn3LNeu9NA5ZqhD7MSKEMsJ7l+MpU= +github.com/gookit/config/v2 v2.2.5/go.mod h1:NeX+yiNYn6Ei10eJvCQFXuHEPIE/IPS8bqaFIsszzaM= +github.com/gookit/goutil v0.6.15 h1:mMQ0ElojNZoyPD0eVROk5QXJPh2uKR4g06slgPDF5Jo= +github.com/gookit/goutil v0.6.15/go.mod h1:qdKdYEHQdEtyH+4fNdQNZfJHhI0jUZzHxQVAV3DaMDY= +github.com/gookit/ini/v2 v2.2.3 h1:nSbN+x9OfQPcMObTFP+XuHt8ev6ndv/fWWqxFhPMu2E= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= @@ -56,9 +70,15 @@ github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrD github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= github.com/panjf2000/ants/v2 v2.7.0 h1:Y3Bgpfo9HDkBoHNVFbMfY5mAvi5TAA17y3HbzQ74p5Y= @@ -73,10 +93,10 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= -github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -85,15 +105,31 @@ github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -101,25 +137,41 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/config.go b/internal/config.go new file mode 100644 index 0000000..c79df68 --- /dev/null +++ b/internal/config.go @@ -0,0 +1,146 @@ +package internal + +import ( + "fmt" + "github.com/goccy/go-yaml" + "github.com/gookit/config/v2" + "reflect" + "strconv" +) + +//var ( +// defaultConfigPath = ".config/spray/" +// defaultConfigFile = "config.yaml" +//) +// +//func LoadDefault(v interface{}) { +// dir, err := os.UserHomeDir() +// if err != nil { +// logs.Log.Error(err.Error()) +// return +// } +// if !files.IsExist(filepath.Join(dir, defaultConfigPath, defaultConfigFile)) { +// err := os.MkdirAll(filepath.Join(dir, defaultConfigPath), 0o700) +// if err != nil { +// logs.Log.Error(err.Error()) +// return +// } +// f, err := os.Create(filepath.Join(dir, defaultConfigPath, defaultConfigFile)) +// if err != nil { +// logs.Log.Error(err.Error()) +// return +// } +// err = LoadConfig(filepath.Join(dir, defaultConfigPath, defaultConfigFile), v) +// if err != nil { +// logs.Log.Error(err.Error()) +// return +// } +// var buf bytes.Buffer +// _, err = config.DumpTo(&buf, config.Yaml) +// if err != nil { +// logs.Log.Error(err.Error()) +// return +// } +// fmt.Println(buf.String()) +// f.Sync() +// } +//} + +func LoadConfig(filename string, v interface{}) error { + err := config.LoadFiles(filename) + if err != nil { + return err + } + err = config.Decode(v) + if err != nil { + return err + } + return nil +} + +func convertToFieldType(fieldType reflect.StructField, defaultVal string) interface{} { + switch fieldType.Type.Kind() { + case reflect.Bool: + val, err := strconv.ParseBool(defaultVal) + if err == nil { + return val + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val, err := strconv.ParseInt(defaultVal, 10, 64) + if err == nil { + return val + } + case reflect.Float32, reflect.Float64: + val, err := strconv.ParseFloat(defaultVal, 64) + if err == nil { + return val + } + case reflect.String: + return defaultVal + // 可以根据需要扩展其他类型 + } + return nil // 如果转换失败或类型不受支持，返回nil +} + +func setFieldValue(field reflect.Value) interface{} { + switch field.Kind() { + case reflect.Bool: + return false + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return 0 + case reflect.Float32, reflect.Float64: + return 0.0 + case reflect.Slice, reflect.Array: + return []interface{}{} // 返回一个空切片 + case reflect.String: + return "" + case reflect.Struct: + return make(map[string]interface{}) + default: + return nil + } +} + +// extractConfigAndDefaults 提取带有 `config` 和 `default` 标签的字段 +func extractConfigAndDefaults(v reflect.Value, result map[string]interface{}) { + t := v.Type() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + fieldType := t.Field(i) + configTag := fieldType.Tag.Get("config") + defaultTag := fieldType.Tag.Get("default") + + if configTag != "" { + var value interface{} + if defaultTag != "" { + value = convertToFieldType(fieldType, defaultTag) + } else { + value = setFieldValue(field) + } + if field.Kind() == reflect.Struct { + nestedResult := make(map[string]interface{}) + extractConfigAndDefaults(field, nestedResult) + result[configTag] = nestedResult + } else { + result[configTag] = value + } + } + } +} + +func initDefaultConfig(cfg interface{}) (string, error) { + v := reflect.ValueOf(cfg) + if v.Kind() != reflect.Struct { + return "", fmt.Errorf("expected a struct, got %s", v.Kind()) + } + + result := make(map[string]interface{}) + extractConfigAndDefaults(v, result) + + yamlData, err := yaml.Marshal(result) + if err != nil { + return "", err + } + + return string(yamlData), nil +} diff --git a/pkg/ihttp/client.go b/internal/ihttp/client.go similarity index 52% rename from pkg/ihttp/client.go rename to internal/ihttp/client.go index 39d093b..82355f9 100644 --- a/pkg/ihttp/client.go +++ b/internal/ihttp/client.go @@ -4,8 +4,14 @@ import ( "context" "crypto/tls" "fmt" + "github.com/chainreactors/logs" "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpproxy" + "golang.org/x/net/proxy" + "net" "net/http" + "net/url" + "strings" "time" ) @@ -19,27 +25,27 @@ const ( STANDARD ) -func NewClient(thread int, timeout int, clientType int) *Client { - if clientType == FAST { +func NewClient(config *ClientConfig) *Client { + if config.Type == FAST { return &Client{ fastClient: &fasthttp.Client{ TLSConfig: &tls.Config{ Renegotiation: tls.RenegotiateOnceAsClient, InsecureSkipVerify: true, }, - MaxConnsPerHost: thread * 3 / 2, - MaxIdleConnDuration: time.Duration(timeout) * time.Second, - MaxConnWaitTimeout: time.Duration(timeout) * time.Second, - ReadTimeout: time.Duration(timeout) * time.Second, - WriteTimeout: time.Duration(timeout) * time.Second, + Dial: customDialFunc(config.ProxyAddr, config.Timeout), + MaxConnsPerHost: config.Thread * 3 / 2, + MaxIdleConnDuration: config.Timeout, + //MaxConnWaitTimeout: time.Duration(timeout) * time.Second, + //ReadTimeout: time.Duration(timeout) * time.Second, + //WriteTimeout: time.Duration(timeout) * time.Second, ReadBufferSize: 16384, // 16k MaxResponseBodySize: DefaultMaxBodySize, NoDefaultUserAgentHeader: true, DisablePathNormalizing: true, DisableHeaderNamesNormalizing: true, }, - timeout: time.Duration(timeout) * time.Second, - clientType: clientType, + Config: config, } } else { return &Client{ @@ -51,27 +57,34 @@ func NewClient(thread int, timeout int, clientType int) *Client { Renegotiation: tls.RenegotiateOnceAsClient, InsecureSkipVerify: true, }, - TLSHandshakeTimeout: time.Duration(timeout) * time.Second, - MaxConnsPerHost: thread * 3 / 2, - IdleConnTimeout: time.Duration(timeout) * time.Second, - ReadBufferSize: 16384, // 16k + MaxConnsPerHost: config.Thread * 3 / 2, + IdleConnTimeout: config.Timeout, + ReadBufferSize: 16384, // 16k + Proxy: func(_ *http.Request) (*url.URL, error) { + return url.Parse(config.ProxyAddr) + }, }, - Timeout: time.Second * time.Duration(timeout), + Timeout: config.Timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, }, - timeout: time.Duration(timeout) * time.Second, - clientType: clientType, + Config: config, } } } +type ClientConfig struct { + Type int + Timeout time.Duration + Thread int + ProxyAddr string +} + type Client struct { fastClient *fasthttp.Client standardClient *http.Client - clientType int - timeout time.Duration + Config *ClientConfig } func (c *Client) TransToCheck() { @@ -103,3 +116,41 @@ func (c *Client) Do(ctx context.Context, req *Request) (*Response, error) { return nil, fmt.Errorf("not found client") } } + +func customDialFunc(proxyAddr string, timeout time.Duration) fasthttp.DialFunc { + if proxyAddr == "" { + return func(addr string) (net.Conn, error) { + return fasthttp.DialTimeout(addr, timeout) + } + } + u, err := url.Parse(proxyAddr) + if err != nil { + logs.Log.Error(err.Error()) + return nil + } + if strings.ToLower(u.Scheme) == "socks5" { + + return func(addr string) (net.Conn, error) { + dialer, err := proxy.SOCKS5("tcp", u.Host, nil, proxy.Direct) + if err != nil { + return nil, err + } + + // Set up a connection with a timeout + conn, err := dialer.Dial("tcp", addr) + if err != nil { + return nil, err + } + + // Set deadlines for the connection + deadline := time.Now().Add(timeout) + if err := conn.SetDeadline(deadline); err != nil { + conn.Close() + return nil, err + } + return conn, nil + } + } else { + return fasthttpproxy.FasthttpHTTPDialerTimeout(proxyAddr, timeout) + } +} diff --git a/pkg/ihttp/request.go b/internal/ihttp/request.go similarity index 100% rename from pkg/ihttp/request.go rename to internal/ihttp/request.go diff --git a/pkg/ihttp/response.go b/internal/ihttp/response.go similarity index 100% rename from pkg/ihttp/response.go rename to internal/ihttp/response.go diff --git a/internal/option.go b/internal/option.go index 655223a..8822f4b 100644 --- a/internal/option.go +++ b/internal/option.go @@ -1,14 +1,17 @@ package internal import ( + "bytes" + "errors" "fmt" "github.com/antonmedv/expr" "github.com/chainreactors/files" "github.com/chainreactors/logs" - "github.com/chainreactors/parsers/iutils" + "github.com/chainreactors/spray/internal/ihttp" + "github.com/chainreactors/spray/internal/pool" "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/utils" + "github.com/chainreactors/utils/iutils" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "github.com/gosuri/uiprogress" @@ -17,128 +20,132 @@ import ( "os" "strconv" "strings" + "sync" ) var ( DefaultThreads = 20 - //DefaultTimeout = 5 - //DefaultPoolSize = 5 - //DefaultRateLimit = 0 ) type Option struct { - InputOptions `group:"Input Options"` - FunctionOptions `group:"Function Options"` - OutputOptions `group:"Output Options"` - PluginOptions `group:"Plugin Options"` - RequestOptions `group:"Request Options"` - ModeOptions `group:"Modify Options"` - MiscOptions `group:"Miscellaneous Options"` + InputOptions `group:"Input Options" config:"input" default:""` + FunctionOptions `group:"Function Options" config:"functions" default:""` + OutputOptions `group:"Output Options" config:"output"` + PluginOptions `group:"Plugin Options" config:"plugins"` + RequestOptions `group:"Request Options" config:"request"` + ModeOptions `group:"Modify Options" config:"mode"` + MiscOptions `group:"Miscellaneous Options" config:"misc"` } type InputOptions struct { - ResumeFrom string `long:"resume"` - URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` - URLFile string `short:"l" long:"list" description:"File, input filename"` - PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` - CIDRs string `short:"c" long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` - Raw string `long:"raw" description:"File, input raw request filename"` - Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt"` + ResumeFrom string `long:"resume" description:"File, resume filename" ` + Config string `short:"c" long:"config" description:"File, config filename"` + URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` + URLFile string `short:"l" long:"list" description:"File, input filename"` + PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` + CIDRs string `long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` + //Raw string `long:"raw" description:"File, input raw request filename"` + Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt" config:"dictionaries"` Offset int `long:"offset" description:"Int, wordlist offset"` Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"` - Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}"` - Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt"` - AppendRule []string `long:"append-rule" description:"Files, when found valid path , use append rule generator new word with current path"` - FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'"` + Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"` + Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt" config:"rules"` + AppendRule []string `long:"append-rule" description:"Files, when found valid path , use append rule generator new word with current path" config:"append-rules"` + FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'" config:"filter-rule"` + AppendFile []string `long:"append-file" description:"Files, when found valid path , use append file new word with current path" config:"append-files"` } type FunctionOptions struct { - Extensions string `short:"e" long:"extension" description:"String, add extensions (separated by commas), e.g.: -e jsp,jspx"` - ExcludeExtensions string `long:"exclude-extension" description:"String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx"` - RemoveExtensions string `long:"remove-extension" description:"String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx"` - Uppercase bool `short:"U" long:"uppercase" description:"Bool, upper wordlist, e.g.: --uppercase"` - Lowercase bool `short:"L" long:"lowercase" description:"Bool, lower wordlist, e.g.: --lowercase"` - Prefixes []string `long:"prefix" description:"Strings, add prefix, e.g.: --prefix aaa --prefix bbb"` - Suffixes []string `long:"suffix" description:"Strings, add suffix, e.g.: --suffix aaa --suffix bbb"` - Replaces map[string]string `long:"replace" description:"Strings, replace string, e.g.: --replace aaa:bbb --replace ccc:ddd"` + Extensions string `short:"e" long:"extension" description:"String, add extensions (separated by commas), e.g.: -e jsp,jspx" config:"extension"` + ForceExtension bool `long:"force-extension" description:"Bool, force add extensions" config:"force-extension"` + ExcludeExtensions string `long:"exclude-extension" description:"String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx" config:"exclude-extension"` + RemoveExtensions string `long:"remove-extension" description:"String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx" config:"remove-extension"` + Uppercase bool `short:"U" long:"uppercase" description:"Bool, upper wordlist, e.g.: --uppercase" config:"upper"` + Lowercase bool `short:"L" long:"lowercase" description:"Bool, lower wordlist, e.g.: --lowercase" config:"lower"` + Prefixes []string `long:"prefix" description:"Strings, add prefix, e.g.: --prefix aaa --prefix bbb" config:"prefix"` + Suffixes []string `long:"suffix" description:"Strings, add suffix, e.g.: --suffix aaa --suffix bbb" config:"suffix"` + Replaces map[string]string `long:"replace" description:"Strings, replace string, e.g.: --replace aaa:bbb --replace ccc:ddd" config:"replace"` + Skips []string `long:"skip" description:"String, skip word when generate. rule, e.g.: --skip aaa" config:"skip"` + //SkipEval string `long:"skip-eval" description:"String, skip word when generate. rule, e.g.: --skip-eval 'current.Length < 4'"` } type OutputOptions struct { - Match string `long:"match" description:"String, custom match function, e.g.: --match 'current.Status != 200''" json:"match,omitempty"` - Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter 'current.Body contains \"hello\"'" json:"filter,omitempty"` - OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty"` - Format string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json"` - FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty"` - DumpFile string `long:"dump-file" description:"String, dump all request, and write to filename"` - Dump bool `long:"dump" description:"Bool, dump all request"` - AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" ` - Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output" json:"fuzzy,omitempty"` - OutputProbe string `short:"o" long:"probe" description:"String, output format" json:"output_probe,omitempty"` + Match string `long:"match" description:"String, custom match function, e.g.: --match 'current.Status != 200''" config:"match" ` + Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter 'current.Body contains \"hello\"'" config:"filter"` + Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output" config:"fuzzy"` + OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty" config:"output-file"` + FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty" config:"fuzzy-file"` + DumpFile string `long:"dump-file" description:"String, dump all request, and write to filename" config:"dump-file"` + Dump bool `long:"dump" description:"Bool, dump all request" config:"dump"` + AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" config:"auto-file"` + Format string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json" config:"format"` + OutputProbe string `short:"o" long:"probe" description:"String, output format" config:"output_probe"` } type RequestOptions struct { - Headers []string `long:"header" description:"Strings, custom headers, e.g.: --headers 'Auth: example_auth'"` - UserAgent string `long:"user-agent" description:"String, custom user-agent, e.g.: --user-agent Custom"` - RandomUserAgent bool `long:"random-agent" description:"Bool, use random with default user-agent"` - Cookie []string `long:"cookie" description:"Strings, custom cookie"` - ReadAll bool `long:"read-all" description:"Bool, read all response body"` - MaxBodyLength int `long:"max-length" default:"100" description:"Int, max response body length (kb), default 100k, e.g. -max-length 1000"` + Headers []string `long:"header" description:"Strings, custom headers, e.g.: --headers 'Auth: example_auth'" config:"headers"` + UserAgent string `long:"user-agent" description:"String, custom user-agent, e.g.: --user-agent Custom" config:"useragent"` + RandomUserAgent bool `long:"random-agent" description:"Bool, use random with default user-agent" config:"random-useragent"` + Cookie []string `long:"cookie" description:"Strings, custom cookie" config:"cookies"` + ReadAll bool `long:"read-all" description:"Bool, read all response body" config:"read-all"` + MaxBodyLength int `long:"max-length" default:"100" description:"Int, max response body length (kb), default 100k, e.g. -max-length 1000" config:"max-body-length"` } type PluginOptions struct { - Advance bool `short:"a" long:"advance" description:"Bool, enable crawl and active"` - Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)"` - Recon bool `long:"recon" description:"Bool, enable recon"` - Active bool `long:"active" description:"Bool, enable active finger detect"` - Bak bool `long:"bak" description:"Bool, enable bak found"` - FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"` - Common bool `long:"common" description:"Bool, enable common file found"` - Crawl bool `long:"crawl" description:"Bool, enable crawl"` - CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth"` + Advance bool `short:"a" long:"advance" description:"Bool, enable all plugin" config:"all" ` + Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" config:"extract"` + Recon bool `long:"recon" description:"Bool, enable recon" config:"recon"` + Finger bool `long:"finger" description:"Bool, enable active finger detect" config:"finger"` + Bak bool `long:"bak" description:"Bool, enable bak found" config:"bak"` + FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt" config:"file-bak"` + Common bool `long:"common" description:"Bool, enable common file found" config:"common"` + Crawl bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"` + CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth" config:"crawl-depth"` } type ModeOptions struct { - RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100"` - Force bool `long:"force" description:"Bool, skip error break"` - CheckOnly bool `long:"check-only" description:"Bool, check only"` - NoScope bool `long:"no-scope" description:"Bool, no scope"` - Scope []string `long:"scope" description:"String, custom scope, e.g.: --scope *.example.com"` - Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"` - Depth int `long:"depth" default:"0" description:"Int, recursive depth"` - Index string `long:"index" default:"" description:"String, custom index path"` - Random string `long:"random" default:"" description:"String, custom random path"` - CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request"` - ErrPeriod int `long:"error-period" default:"10" description:"Int, check period when error"` - BreakThreshold int `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold "` - BlackStatus string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status, "` - WhiteStatus string `long:"white-status" default:"200" description:"Strings (comma split), custom white status"` - FuzzyStatus string `long:"fuzzy-status" default:"404,403,500,501,502,503" description:"Strings (comma split), custom fuzzy status"` - UniqueStatus string `long:"unique-status" default:"403" description:"Strings (comma split), custom unique status"` - Unique bool `long:"unique" description:"Bool, unique response"` - RetryCount int `long:"retry" default:"1" description:"Int, retry count"` - SimhashDistance int `long:"distance" default:"5"` + RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100" config:"rate-limit"` + Force bool `long:"force" description:"Bool, skip error break" config:"force"` + CheckOnly bool `long:"check-only" description:"Bool, check only" config:"check-only"` + NoScope bool `long:"no-scope" description:"Bool, no scope" config:"no-scope"` + Scope []string `long:"scope" description:"String, custom scope, e.g.: --scope *.example.com" config:"scope"` + Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()" config:"recursive"` + Depth int `long:"depth" default:"0" description:"Int, recursive depth" config:"depth"` + Index string `long:"index" default:"/" description:"String, custom index path" config:"index"` + Random string `long:"random" default:"" description:"String, custom random path" config:"random"` + CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request" config:"check-period"` + ErrPeriod int `long:"error-period" default:"10" description:"Int, check period when error" config:"error-period"` + BreakThreshold int `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold" config:"error-threshold"` + BlackStatus string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status" config:"black-status"` + WhiteStatus string `long:"white-status" default:"200" description:"Strings (comma split), custom white status" config:"white-status"` + FuzzyStatus string `long:"fuzzy-status" default:"500,501,502,503" description:"Strings (comma split), custom fuzzy status" config:"fuzzy-status"` + UniqueStatus string `long:"unique-status" default:"403,200,404" description:"Strings (comma split), custom unique status" config:"unique-status"` + Unique bool `long:"unique" description:"Bool, unique response" config:"unique"` + RetryCount int `long:"retry" default:"0" description:"Int, retry count" config:"retry"` + SimhashDistance int `long:"distance" default:"5" config:"distance"` } type MiscOptions struct { - Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)"` // todo 总的超时时间,适配云函数的deadline - Timeout int `long:"timeout" default:"5" description:"Int, timeout with request (seconds)"` - PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size"` - Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool"` - Debug bool `long:"debug" description:"Bool, output debug info"` - Version bool `short:"v" long:"version" description:"Bool, show version"` - Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet"` - NoColor bool `long:"no-color" description:"Bool, no color"` - NoBar bool `long:"no-bar" description:"Bool, No progress bar"` - Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray"` - Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type"` + Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray" config:"mod"` + Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type" config:"client"` + Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)" config:"deadline"` // todo 总的超时时间,适配云函数的deadline + Timeout int `long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"` + PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size" config:"pool"` + Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool" config:"thread"` + Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"` + Version bool `long:"version" description:"Bool, show version"` + Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"` + Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet" config:"quiet"` + NoColor bool `long:"no-color" description:"Bool, no color" config:"no-color"` + NoBar bool `long:"no-bar" description:"Bool, No progress bar" config:"no-bar"` + Proxy string `long:"proxy" default:"" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"` } func (opt *Option) PrepareRunner() (*Runner, error) { - ok := opt.Validate() - if !ok { - return nil, fmt.Errorf("validate failed") + err := opt.Validate() + if err != nil { + return nil, err } - var err error r := &Runner{ Progress: uiprogress.New(), Threads: opt.Threads, @@ -151,8 +158,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Offset: opt.Offset, Total: opt.Limit, taskCh: make(chan *Task), - OutputCh: make(chan *pkg.Baseline, 100), - FuzzyCh: make(chan *pkg.Baseline, 100), + outputCh: make(chan *pkg.Baseline, 100), + outwg: &sync.WaitGroup{}, + fuzzyCh: make(chan *pkg.Baseline, 100), Fuzzy: opt.Fuzzy, Force: opt.Force, CheckOnly: opt.CheckOnly, @@ -161,28 +169,29 @@ func (opt *Option) PrepareRunner() (*Runner, error) { BreakThreshold: opt.BreakThreshold, Crawl: opt.Crawl, Scope: opt.Scope, - Active: opt.Active, + Finger: opt.Finger, Bak: opt.Bak, Common: opt.Common, RetryCount: opt.RetryCount, RandomUserAgent: opt.RandomUserAgent, Random: opt.Random, Index: opt.Index, + Proxy: opt.Proxy, } // log and bar if !opt.NoColor { - logs.Log.Color = true + logs.Log.SetColor(true) r.Color = true } if opt.Quiet { - logs.Log.Quiet = true - logs.Log.Color = false + logs.Log.SetQuiet(true) + logs.Log.SetColor(false) r.Color = false } if !(opt.Quiet || opt.NoBar) { r.Progress.Start() - logs.Log.Writer = r.Progress.Bypass() + logs.Log.SetOutput(r.Progress.Bypass()) } // configuration @@ -211,7 +220,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if opt.Advance { r.Crawl = true - r.Active = true + r.Finger = true r.Bak = true r.Common = true pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"] @@ -224,13 +233,15 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if r.Crawl { s.WriteString("crawl enable; ") } - if r.Active { + if r.Finger { + r.AppendWords = append(r.AppendWords, pkg.ActivePath...) s.WriteString("active fingerprint enable; ") } if r.Bak { s.WriteString("bak file enable; ") } if r.Common { + r.AppendWords = append(r.AppendWords, mask.SpecialWords["common_file"]...) s.WriteString("common file enable; ") } if opt.Recon { @@ -239,51 +250,57 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if len(opt.AppendRule) > 0 { s.WriteString("file bak enable; ") } - if s.Len() > 0 { - logs.Log.Important("Advance Mod: " + s.String()) + + if r.RetryCount > 0 { + s.WriteString("Retry Count: " + strconv.Itoa(r.RetryCount)) } - logs.Log.Important("Retry Count: " + strconv.Itoa(r.RetryCount)) + if s.Len() > 0 { + logs.Log.Important(s.String()) + } + if opt.NoScope { r.Scope = []string{"*"} } - BlackStatus = parseStatus(BlackStatus, opt.BlackStatus) - WhiteStatus = parseStatus(WhiteStatus, opt.WhiteStatus) + pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus) + pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus) if opt.FuzzyStatus == "all" { - enableAllFuzzy = true + pool.EnableAllFuzzy = true } else { - FuzzyStatus = parseStatus(FuzzyStatus, opt.FuzzyStatus) + pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus) } if opt.Unique { - enableAllUnique = true + pool.EnableAllUnique = true } else { - UniqueStatus = parseStatus(UniqueStatus, opt.UniqueStatus) + pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus) } // prepare word dicts := make([][]string, len(opt.Dictionaries)) - for i, f := range opt.Dictionaries { - dicts[i], err = loadFileToSlice(f) - if opt.ResumeFrom != "" { - dictCache[f] = dicts[i] + if len(opt.Dictionaries) == 0 { + dicts = append(dicts, pkg.LoadDefaultDict()) + logs.Log.Warn("not set any dictionary, use default dictionary: https://github.com/maurosoria/dirsearch/blob/master/db/dicc.txt") + } else { + for i, f := range opt.Dictionaries { + dicts[i], err = loadFileToSlice(f) + if opt.ResumeFrom != "" { + dictCache[f] = dicts[i] + } + if err != nil { + return nil, err + } + + logs.Log.Logf(pkg.LogVerbose, "Loaded %d word from %s", len(dicts[i]), f) } - if err != nil { - return nil, err - } - logs.Log.Importantf("Loaded %d word from %s", len(dicts[i]), f) } if opt.Word == "" { - if len(opt.Dictionaries) == 0 { - opt.Word = "/" - } else { - opt.Word = "{?" - for i, _ := range dicts { - opt.Word += strconv.Itoa(i) - } - opt.Word += "}" + opt.Word = "{?" + for i, _ := range dicts { + opt.Word += strconv.Itoa(i) } + opt.Word += "}" } if opt.Suffixes != nil { @@ -295,7 +312,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { opt.Word = "{@prefix}" + opt.Word } - if opt.Extensions != "" { + if opt.ForceExtension && opt.Extensions != "" { exts := strings.Split(opt.Extensions, ",") for i, e := range exts { if !strings.HasPrefix(e, ".") { @@ -311,11 +328,11 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, fmt.Errorf("%s %w", opt.Word, err) } if len(r.Wordlist) > 0 { - logs.Log.Importantf("Parsed %d words by %s", len(r.Wordlist), opt.Word) + logs.Log.Logf(pkg.LogVerbose, "Parsed %d words by %s", len(r.Wordlist), opt.Word) } if opt.Rules != nil { - rules, err := loadFileAndCombine(opt.Rules) + rules, err := loadRuleAndCombine(opt.Rules) if err != nil { return nil, err } @@ -344,13 +361,30 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } if opt.AppendRule != nil { - content, err := loadFileAndCombine(opt.AppendRule) + content, err := loadRuleAndCombine(opt.AppendRule) if err != nil { return nil, err } r.AppendRules = rule.Compile(string(content), "") } + if opt.AppendFile != nil { + var bs bytes.Buffer + for _, f := range opt.AppendFile { + content, err := ioutil.ReadFile(f) + if err != nil { + return nil, err + } + bs.Write(bytes.TrimSpace(content)) + bs.WriteString("\n") + } + lines := strings.Split(bs.String(), "\n") + for i, line := range lines { + lines[i] = strings.TrimSpace(line) + } + r.AppendWords = append(r.AppendWords, lines...) + } + ports := utils.ParsePort(opt.PortRange) // prepare task @@ -424,7 +458,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { logs.Log.Error(err.Error()) } taskfrom = opt.URLFile - } else if pkg.HasStdin() { + } else if files.HasStdin() { file = os.Stdin taskfrom = "stdin" } @@ -469,44 +503,84 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } r.Tasks = tasks - logs.Log.Importantf("Loaded %d urls from %s", len(tasks), taskfrom) + logs.Log.Logf(pkg.LogVerbose, "Loaded %d urls from %s", len(tasks), taskfrom) + + // 类似dirsearch中的 + if opt.Extensions != "" { + r.AppendFunction(func(s string) []string { + exts := strings.Split(opt.Extensions, ",") + ss := make([]string, len(exts)) + for i, e := range exts { + if strings.Contains(s, "%EXT%") { + ss[i] = strings.Replace(s, "%EXT%", e, -1) + } + } + return ss + }) + } else { + r.AppendFunction(func(s string) []string { + if strings.Contains(s, "%EXT%") { + return nil + } + return []string{s} + }) + } if opt.Uppercase { - r.Fns = append(r.Fns, strings.ToUpper) + r.AppendFunction(wrapWordsFunc(strings.ToUpper)) } if opt.Lowercase { - r.Fns = append(r.Fns, strings.ToLower) + r.AppendFunction(wrapWordsFunc(strings.ToLower)) } if opt.RemoveExtensions != "" { rexts := strings.Split(opt.ExcludeExtensions, ",") - r.Fns = append(r.Fns, func(s string) string { + r.AppendFunction(func(s string) []string { if ext := parseExtension(s); iutils.StringsContains(rexts, ext) { - return strings.TrimSuffix(s, "."+ext) + return []string{strings.TrimSuffix(s, "."+ext)} } - return s + return []string{s} }) } if opt.ExcludeExtensions != "" { exexts := strings.Split(opt.ExcludeExtensions, ",") - r.Fns = append(r.Fns, func(s string) string { + r.AppendFunction(func(s string) []string { if ext := parseExtension(s); iutils.StringsContains(exexts, ext) { - return "" + return nil } - return s + return []string{s} }) } if len(opt.Replaces) > 0 { - r.Fns = append(r.Fns, func(s string) string { + r.AppendFunction(func(s string) []string { for k, v := range opt.Replaces { s = strings.Replace(s, k, v, -1) } - return s + return []string{s} }) } - logs.Log.Importantf("Loaded %d dictionaries and %d decorators", len(opt.Dictionaries), len(r.Fns)) + + // default skip function, skip %EXT% + r.AppendFunction(func(s string) []string { + if strings.Contains(s, "%EXT%") { + return nil + } + return []string{s} + }) + if len(opt.Skips) > 0 { + r.AppendFunction(func(s string) []string { + for _, skip := range opt.Skips { + if strings.Contains(s, skip) { + return nil + } + } + return []string{s} + }) + } + + logs.Log.Logf(pkg.LogVerbose, "Loaded %d dictionaries and %d decorators", len(opt.Dictionaries), len(r.Fns)) if opt.Match != "" { exp, err := expr.Compile(opt.Match, expr.Patch(&bytesPatcher{})) @@ -528,13 +602,13 @@ func (opt *Option) PrepareRunner() (*Runner, error) { var express string if opt.Recursive != "current.IsDir()" && opt.Depth != 0 { // 默认不打开递归, 除非指定了非默认的递归表达式 - MaxRecursion = 1 + pool.MaxRecursion = 1 express = opt.Recursive } if opt.Depth != 0 { // 手动设置的depth优先级高于默认 - MaxRecursion = opt.Depth + pool.MaxRecursion = opt.Depth express = opt.Recursive } @@ -619,24 +693,26 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return r, nil } -func (opt *Option) Validate() bool { +func (opt *Option) Validate() error { if opt.Uppercase && opt.Lowercase { - logs.Log.Error("Cannot set -U and -L at the same time") - return false + return errors.New("cannot set -U and -L at the same time") } if (opt.Offset != 0 || opt.Limit != 0) && opt.Depth > 0 { // 偏移和上限与递归同时使用时也会造成混淆. - logs.Log.Error("--offset and --limit cannot be used with --depth at the same time") - return false + return errors.New("--offset and --limit cannot be used with --depth at the same time") } if opt.Depth > 0 && opt.ResumeFrom != "" { // 递归与断点续传会造成混淆, 断点续传的word与rule不是通过命令行获取的 - logs.Log.Error("--resume and --depth cannot be used at the same time") - return false + + return errors.New("--resume and --depth cannot be used at the same time") } - return true + + if opt.ResumeFrom == "" && opt.URL == nil && opt.URLFile == "" && opt.CIDRs == "" { + return fmt.Errorf("without any target, please use -u/-l/-c/--resume to set targets") + } + return nil } // Generate Tasks diff --git a/internal/pool.go b/internal/pool/brutepool.go similarity index 57% rename from internal/pool.go rename to internal/pool/brutepool.go index 0d60a4a..dcde456 100644 --- a/internal/pool.go +++ b/internal/pool/brutepool.go @@ -1,22 +1,19 @@ -package internal +package pool import ( "context" "fmt" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" - "github.com/chainreactors/parsers/iutils" + "github.com/chainreactors/spray/internal/ihttp" "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/spray/pkg/ihttp" + "github.com/chainreactors/utils/iutils" "github.com/chainreactors/words" - "github.com/chainreactors/words/mask" - "github.com/chainreactors/words/rule" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" "golang.org/x/time/rate" "math/rand" "net/url" - "path" "strings" "sync" "sync/atomic" @@ -24,39 +21,44 @@ import ( ) var ( - max = 2147483647 MaxRedirect = 3 MaxCrawl = 3 MaxRecursion = 0 - enableAllFuzzy = false - enableAllUnique = false - nilBaseline = &pkg.Baseline{} + EnableAllFuzzy = false + EnableAllUnique = false ) -func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { +func NewBrutePool(ctx context.Context, config *Config) (*BrutePool, error) { var u *url.URL var err error if u, err = url.Parse(config.BaseURL); err != nil { return nil, err } pctx, cancel := context.WithCancel(ctx) - pool := &Pool{ - Config: config, - base: u.Scheme + "://" + u.Host, - isDir: strings.HasSuffix(u.Path, "/"), - url: u, - ctx: pctx, - cancel: cancel, - client: ihttp.NewClient(config.Thread, 2, config.ClientType), - baselines: make(map[int]*pkg.Baseline), - urls: make(map[string]struct{}), + pool := &BrutePool{ + Baselines: NewBaselines(), + This: &This{ + Config: config, + ctx: pctx, + Cancel: cancel, + client: ihttp.NewClient(&ihttp.ClientConfig{ + Thread: config.Thread, + Type: config.ClientType, + Timeout: time.Duration(config.Timeout) * time.Second, + ProxyAddr: config.ProxyAddr, + }), + additionCh: make(chan *Unit, config.Thread), + closeCh: make(chan struct{}), + wg: sync.WaitGroup{}, + }, + base: u.Scheme + "://" + u.Host, + isDir: strings.HasSuffix(u.Path, "/"), + url: u, + scopeurls: make(map[string]struct{}), uniques: make(map[uint16]struct{}), - tempCh: make(chan *pkg.Baseline, 100), - checkCh: make(chan int, 100), - additionCh: make(chan *Unit, 100), - closeCh: make(chan struct{}), - waiter: sync.WaitGroup{}, + handlerCh: make(chan *pkg.Baseline, config.Thread), + checkCh: make(chan struct{}, config.Thread), initwg: sync.WaitGroup{}, limiter: rate.NewLimiter(rate.Limit(config.RateLimit), 1), failedCount: 1, @@ -68,7 +70,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { } else if pool.url.Path == "" { pool.dir = "/" } else { - pool.dir = Dir(pool.url.Path) + pool.dir = pkg.Dir(pool.url.Path) } pool.reqPool, _ = ants.NewPoolWithFunc(config.Thread, pool.Invoke) @@ -79,44 +81,32 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { return pool, nil } -type Pool struct { - *pkg.Config // read only - base string // url的根目录, 在爬虫或者redirect时, 会需要用到根目录进行拼接 - dir string - isDir bool - url *url.URL - Statistor *pkg.Statistor - client *ihttp.Client - reqPool *ants.PoolWithFunc - scopePool *ants.PoolWithFunc - bar *pkg.Bar - ctx context.Context - cancel context.CancelFunc - tempCh chan *pkg.Baseline // 待处理的baseline - checkCh chan int // 独立的check管道， 防止与redirect/crawl冲突 - additionCh chan *Unit // 插件添加的任务, 待处理管道 - closeCh chan struct{} - closed bool - wordOffset int - failedCount int32 - isFailed bool - failedBaselines []*pkg.Baseline - random *pkg.Baseline - index *pkg.Baseline - baselines map[int]*pkg.Baseline - urls map[string]struct{} - scopeurls map[string]struct{} - uniques map[uint16]struct{} - analyzeDone bool - worder *words.Worder - limiter *rate.Limiter - locker sync.Mutex - scopeLocker sync.Mutex - waiter sync.WaitGroup - initwg sync.WaitGroup // 初始化用, 之后改成锁 +type BrutePool struct { + *Baselines + *This + base string // url的根目录, 在爬虫或者redirect时, 会需要用到根目录进行拼接 + isDir bool + url *url.URL + + reqPool *ants.PoolWithFunc + scopePool *ants.PoolWithFunc + handlerCh chan *pkg.Baseline // 待处理的baseline + checkCh chan struct{} // 独立的check管道， 防止与redirect/crawl冲突 + closed bool + wordOffset int + failedCount int32 + IsFailed bool + urls sync.Map + scopeurls map[string]struct{} + uniques map[uint16]struct{} + analyzeDone bool + limiter *rate.Limiter + locker sync.Mutex + scopeLocker sync.Mutex + initwg sync.WaitGroup // 初始化用, 之后改成锁 } -func (pool *Pool) checkRedirect(redirectURL string) bool { +func (pool *BrutePool) checkRedirect(redirectURL string) bool { if pool.random.RedirectURL == "" { // 如果random的redirectURL为空, 此时该项 return true @@ -131,30 +121,31 @@ func (pool *Pool) checkRedirect(redirectURL string) bool { } } -func (pool *Pool) genReq(mod pkg.SprayMod, s string) (*ihttp.Request, error) { - if mod == pkg.HostSpray { +func (pool *BrutePool) genReq(mod SprayMod, s string) (*ihttp.Request, error) { + if mod == HostSpray { return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s) - } else if mod == pkg.PathSpray { + } else if mod == PathSpray { return ihttp.BuildPathRequest(pool.ClientType, pool.base, s) } return nil, fmt.Errorf("unknown mod") } -func (pool *Pool) Init() error { - // 分成两步是为了避免闭包的线程安全问题 +func (pool *BrutePool) Init() error { pool.initwg.Add(2) - if pool.Index != "" { - logs.Log.Importantf("custom index url: %s", BaseURL(pool.url)+FormatURL(BaseURL(pool.url), pool.Index)) - pool.reqPool.Invoke(newUnit(pool.Index, InitIndexSource)) + if pool.Index != "/" { + logs.Log.Logf(pkg.LogVerbose, "custom index url: %s", pkg.BaseURL(pool.url)+pkg.FormatURL(pkg.BaseURL(pool.url), pool.Index)) + pool.reqPool.Invoke(newUnit(pool.Index, parsers.InitIndexSource)) + //pool.urls[dir(pool.Index)] = struct{}{} } else { - pool.reqPool.Invoke(newUnit(pool.url.Path, InitIndexSource)) + pool.reqPool.Invoke(newUnit(pool.url.Path, parsers.InitIndexSource)) + //pool.urls[dir(pool.url.Path)] = struct{}{} } if pool.Random != "" { - logs.Log.Importantf("custom random url: %s", BaseURL(pool.url)+FormatURL(BaseURL(pool.url), pool.Random)) - pool.reqPool.Invoke(newUnit(pool.Random, InitRandomSource)) + logs.Log.Logf(pkg.LogVerbose, "custom random url: %s", pkg.BaseURL(pool.url)+pkg.FormatURL(pkg.BaseURL(pool.url), pool.Random)) + pool.reqPool.Invoke(newUnit(pool.Random, parsers.InitRandomSource)) } else { - pool.reqPool.Invoke(newUnit(pool.safePath(pkg.RandPath()), InitRandomSource)) + pool.reqPool.Invoke(newUnit(pool.safePath(pkg.RandPath()), parsers.InitRandomSource)) } pool.initwg.Wait() @@ -165,13 +156,13 @@ func (pool *Pool) Init() error { if pool.index.Chunked && pool.ClientType == ihttp.FAST { logs.Log.Warn("chunk encoding! buf current client FASTHTTP not support chunk decode") } - logs.Log.Info("[baseline.index] " + pool.index.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) + logs.Log.Logf(pkg.LogVerbose, "[baseline.index] "+pool.index.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) // 检测基本访问能力 if pool.random.ErrString != "" { logs.Log.Error(pool.index.String()) return fmt.Errorf(pool.index.ErrString) } - logs.Log.Info("[baseline.random] " + pool.random.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) + logs.Log.Logf(pkg.LogVerbose, "[baseline.random] "+pool.random.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) // 某些网站http会重定向到https, 如果发现随机目录出现这种情况, 则自定将baseurl升级为https if pool.url.Scheme == "http" { @@ -189,20 +180,36 @@ func (pool *Pool) Init() error { return nil } -func (pool *Pool) Run(offset, limit int) { - pool.worder.RunWithRules() +func (pool *BrutePool) Upgrade(bl *pkg.Baseline) error { + rurl, err := url.Parse(bl.RedirectURL) + if err == nil && rurl.Hostname() == bl.Url.Hostname() && bl.Url.Scheme == "http" && rurl.Scheme == "https" { + logs.Log.Infof("baseurl %s upgrade http to https, reinit", pool.BaseURL) + pool.base = strings.Replace(pool.BaseURL, "http", "https", 1) + pool.url.Scheme = "https" + // 重新初始化 + err = pool.Init() + if err != nil { + return err + } + } + + return nil +} + +func (pool *BrutePool) Run(offset, limit int) { + pool.Worder.Run() if pool.Active { - pool.waiter.Add(1) + pool.wg.Add(1) go pool.doActive() } if pool.Bak { - pool.waiter.Add(1) + pool.wg.Add(1) go pool.doBak() } if pool.Common { - pool.waiter.Add(1) + pool.wg.Add(1) go pool.doCommonFile() } @@ -211,7 +218,7 @@ func (pool *Pool) Run(offset, limit int) { go func() { for { if done { - pool.waiter.Wait() + pool.wg.Wait() close(pool.closeCh) return } @@ -222,7 +229,7 @@ func (pool *Pool) Run(offset, limit int) { Loop: for { select { - case w, ok := <-pool.worder.C: + case w, ok := <-pool.Worder.C: if !ok { done = true continue @@ -238,30 +245,30 @@ Loop: continue } - pool.waiter.Add(1) - if pool.Mod == pkg.HostSpray { - pool.reqPool.Invoke(newUnitWithNumber(w, WordSource, pool.wordOffset)) + pool.wg.Add(1) + if pool.Mod == HostSpray { + pool.reqPool.Invoke(newUnitWithNumber(w, parsers.WordSource, pool.wordOffset)) } else { // 原样的目录拼接, 输入了几个"/"就是几个, 适配/有语义的中间件 - pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(w), WordSource, pool.wordOffset)) + pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(w), parsers.WordSource, pool.wordOffset)) } - case source := <-pool.checkCh: + case <-pool.checkCh: pool.Statistor.CheckNumber++ - if pool.Mod == pkg.HostSpray { - pool.reqPool.Invoke(newUnitWithNumber(pkg.RandHost(), source, pool.wordOffset)) - } else if pool.Mod == pkg.PathSpray { - pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(pkg.RandPath()), source, pool.wordOffset)) + if pool.Mod == HostSpray { + pool.reqPool.Invoke(newUnitWithNumber(pkg.RandHost(), parsers.CheckSource, pool.wordOffset)) + } else if pool.Mod == PathSpray { + pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(pkg.RandPath()), parsers.CheckSource, pool.wordOffset)) } case unit, ok := <-pool.additionCh: if !ok || pool.closed { continue } - if _, ok := pool.urls[unit.path]; ok { - logs.Log.Debugf("[%s] duplicate path: %s, skipped", parsers.GetSpraySourceName(unit.source), pool.base+unit.path) - pool.waiter.Done() + if _, ok := pool.urls.Load(unit.path); ok { + logs.Log.Debugf("[%s] duplicate path: %s, skipped", unit.source.Name(), pool.base+unit.path) + pool.wg.Done() } else { - pool.urls[unit.path] = struct{}{} + pool.urls.Store(unit.path, nil) unit.number = pool.wordOffset pool.reqPool.Invoke(unit) } @@ -277,7 +284,7 @@ Loop: pool.Close() } -func (pool *Pool) Invoke(v interface{}) { +func (pool *BrutePool) Invoke(v interface{}) { if pool.RateLimit != 0 { pool.limiter.Wait(pool.ctx) } @@ -287,10 +294,10 @@ func (pool *Pool) Invoke(v interface{}) { var req *ihttp.Request var err error - if unit.source == WordSource { + if unit.source == parsers.WordSource { req, err = pool.genReq(pool.Mod, unit.path) } else { - req, err = pool.genReq(pkg.PathSpray, unit.path) + req, err = pool.genReq(PathSpray, unit.path) } if err != nil { @@ -299,7 +306,7 @@ func (pool *Pool) Invoke(v interface{}) { } req.SetHeaders(pool.Headers) - req.SetHeader("User-Agent", RandomUA()) + req.SetHeader("User-Agent", pkg.RandomUA()) start := time.Now() resp, reqerr := pool.client.Do(pool.ctx, req) @@ -316,17 +323,15 @@ func (pool *Pool) Invoke(v interface{}) { bl = &pkg.Baseline{ SprayResult: &parsers.SprayResult{ UrlString: pool.base + unit.path, - IsValid: false, ErrString: reqerr.Error(), Reason: pkg.ErrRequestFailed.Error(), }, } - pool.failedBaselines = append(pool.failedBaselines, bl) - // 自动重放失败请求, 默认为一次 + pool.FailedBaselines = append(pool.FailedBaselines, bl) + // 自动重放失败请求 pool.doRetry(bl) - - } else { - if unit.source <= 3 || unit.source == CrawlSource || unit.source == CommonFileSource { + } else { // 特定场景优化 + if unit.source <= 3 || unit.source == parsers.CrawlSource || unit.source == parsers.CommonFileSource { // 一些高优先级的source, 将跳过PreCompare bl = pkg.NewBaseline(req.URI(), req.Host(), resp) } else if pool.MatchExpr != nil { @@ -341,8 +346,8 @@ func (pool *Pool) Invoke(v interface{}) { } // 手动处理重定向 - if bl.IsValid && unit.source != CheckSource && bl.RedirectURL != "" { - //pool.waiter.Add(1) + if bl.IsValid && unit.source != parsers.CheckSource && bl.RedirectURL != "" { + //pool.wg.Add(1) pool.doRedirect(bl, unit.depth) } @@ -354,63 +359,63 @@ func (pool *Pool) Invoke(v interface{}) { bl.Number = unit.number bl.Spended = time.Since(start).Milliseconds() switch unit.source { - case InitRandomSource: + case parsers.InitRandomSource: bl.Collect() pool.locker.Lock() pool.random = bl pool.addFuzzyBaseline(bl) pool.locker.Unlock() pool.initwg.Done() - case InitIndexSource: + case parsers.InitIndexSource: bl.Collect() pool.locker.Lock() pool.index = bl pool.locker.Unlock() if bl.Status == 200 || (bl.Status/100) == 3 { // 保留index输出结果 - pool.waiter.Add(1) + pool.wg.Add(1) pool.doCrawl(bl) - pool.OutputCh <- bl + pool.putToOutput(bl) } pool.initwg.Done() - case CheckSource: + case parsers.CheckSource: if bl.ErrString != "" { logs.Log.Warnf("[check.error] %s maybe ip had banned, break (%d/%d), error: %s", pool.BaseURL, pool.failedCount, pool.BreakThreshold, bl.ErrString) } else if i := pool.random.Compare(bl); i < 1 { if i == 0 { if pool.Fuzzy { - logs.Log.Warn("[check.fuzzy] maybe trigger risk control, " + bl.String()) + logs.Log.Debug("[check.fuzzy] maybe trigger risk control, " + bl.String()) } } else { atomic.AddInt32(&pool.failedCount, 1) // - logs.Log.Warn("[check.failed] maybe trigger risk control, " + bl.String()) - pool.failedBaselines = append(pool.failedBaselines, bl) + logs.Log.Debug("[check.failed] maybe trigger risk control, " + bl.String()) + pool.FailedBaselines = append(pool.FailedBaselines, bl) } } else { pool.resetFailed() // 如果后续访问正常, 重置错误次数 logs.Log.Debug("[check.pass] " + bl.String()) } - case WordSource: + case parsers.WordSource: // 异步进行性能消耗较大的深度对比 - pool.tempCh <- bl + pool.handlerCh <- bl if int(pool.Statistor.ReqTotal)%pool.CheckPeriod == 0 { pool.doCheck() } else if pool.failedCount%pool.ErrPeriod == 0 { atomic.AddInt32(&pool.failedCount, 1) pool.doCheck() } - pool.bar.Done() - case RedirectSource: + pool.Bar.Done() + case parsers.RedirectSource: bl.FrontURL = unit.frontUrl - pool.tempCh <- bl + pool.handlerCh <- bl default: - pool.tempCh <- bl + pool.handlerCh <- bl } } -func (pool *Pool) NoScopeInvoke(v interface{}) { - defer pool.waiter.Done() +func (pool *BrutePool) NoScopeInvoke(v interface{}) { + defer pool.wg.Done() unit := v.(*Unit) req, err := ihttp.BuildPathRequest(pool.ClientType, unit.path, "") if err != nil { @@ -418,7 +423,7 @@ func (pool *Pool) NoScopeInvoke(v interface{}) { return } req.SetHeaders(pool.Headers) - req.SetHeader("User-Agent", RandomUA()) + req.SetHeader("User-Agent", pkg.RandomUA()) resp, reqerr := pool.client.Do(pool.ctx, req) if pool.ClientType == ihttp.FAST { defer fasthttp.ReleaseResponse(resp.FastResponse) @@ -434,14 +439,14 @@ func (pool *Pool) NoScopeInvoke(v interface{}) { bl.ReqDepth = unit.depth bl.Collect() bl.CollectURL() - pool.waiter.Add(1) + pool.wg.Add(1) pool.doScopeCrawl(bl) - pool.OutputCh <- bl + pool.putToOutput(bl) } } -func (pool *Pool) Handler() { - for bl := range pool.tempCh { +func (pool *BrutePool) Handler() { + for bl := range pool.handlerCh { if bl.IsValid { pool.addFuzzyBaseline(bl) } @@ -464,29 +469,29 @@ func (pool *Pool) Handler() { "random": pool.random, "current": bl, } - //for _, status := range FuzzyStatus { - // if bl, ok := pool.baselines[status]; ok { - // params["bl"+strconv.Itoa(status)] = bl + //for _, ok := range FuzzyStatus { + // if bl, ok := pool.baselines[ok]; ok { + // params["bl"+strconv.Itoa(ok)] = bl // } else { - // params["bl"+strconv.Itoa(status)] = nilBaseline + // params["bl"+strconv.Itoa(ok)] = nilBaseline // } //} } - var status bool + var ok bool if pool.MatchExpr != nil { - if CompareWithExpr(pool.MatchExpr, params) { - status = true + if pkg.CompareWithExpr(pool.MatchExpr, params) { + ok = true } } else { - status = pool.BaseCompare(bl) + ok = pool.BaseCompare(bl) } - if status { + if ok { pool.Statistor.FoundNumber++ // unique判断 - if enableAllUnique || iutils.IntsContains(UniqueStatus, bl.Status) { + if EnableAllUnique || iutils.IntsContains(pkg.UniqueStatus, bl.Status) { if _, ok := pool.uniques[bl.Unique]; ok { bl.IsValid = false bl.IsFuzzy = true @@ -497,7 +502,7 @@ func (pool *Pool) Handler() { } // 对通过所有对比的有效数据进行再次filter - if bl.IsValid && pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) { + if bl.IsValid && pool.FilterExpr != nil && pkg.CompareWithExpr(pool.FilterExpr, params) { pool.Statistor.FilteredNumber++ bl.Reason = pkg.ErrCustomFilter.Error() bl.IsValid = false @@ -507,14 +512,19 @@ func (pool *Pool) Handler() { } if bl.IsValid || bl.IsFuzzy { - pool.waiter.Add(2) + pool.wg.Add(2) pool.doCrawl(bl) pool.doRule(bl) + if iutils.IntsContains(pkg.WhiteStatus, bl.Status) || iutils.IntsContains(pkg.UniqueStatus, bl.Status) { + pool.wg.Add(1) + pool.doAppendWords(bl) + } } + // 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度 if bl.IsValid { if bl.RecuDepth < MaxRecursion { - if CompareWithExpr(pool.RecuExpr, params) { + if pkg.CompareWithExpr(pool.RecuExpr, params) { bl.Recu = true } } @@ -522,17 +532,17 @@ func (pool *Pool) Handler() { if !pool.closed { // 如果任务被取消, 所有还没处理的请求结果都会被丢弃 - pool.OutputCh <- bl + pool.putToOutput(bl) } - pool.waiter.Done() + pool.wg.Done() } pool.analyzeDone = true } -func (pool *Pool) PreCompare(resp *ihttp.Response) error { +func (pool *BrutePool) PreCompare(resp *ihttp.Response) error { status := resp.StatusCode() - if iutils.IntsContains(WhiteStatus, status) { + if iutils.IntsContains(pkg.WhiteStatus, status) { // 如果为白名单状态码则直接返回 return nil } @@ -540,11 +550,11 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error { return pkg.ErrSameStatus } - if iutils.IntsContains(BlackStatus, status) { + if iutils.IntsContains(pkg.BlackStatus, status) { return pkg.ErrBadStatus } - if iutils.IntsContains(WAFStatus, status) { + if iutils.IntsContains(pkg.WAFStatus, status) { return pkg.ErrWaf } @@ -555,7 +565,7 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error { return nil } -func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool { +func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { if !bl.IsValid { return false } @@ -566,7 +576,6 @@ func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool { pool.putToFuzzy(bl) return false } - // 使用与baseline相同状态码, 需要在fuzzystatus中提前配置 base, ok := pool.baselines[bl.Status] // 挑选对应状态码的baseline进行compare if !ok { @@ -611,62 +620,45 @@ func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool { return true } -func (pool *Pool) Upgrade(bl *pkg.Baseline) error { - rurl, err := url.Parse(bl.RedirectURL) - if err == nil && rurl.Hostname() == bl.Url.Hostname() && bl.Url.Scheme == "http" && rurl.Scheme == "https" { - logs.Log.Infof("baseurl %s upgrade http to https, reinit", pool.BaseURL) - pool.base = strings.Replace(pool.BaseURL, "http", "https", 1) - pool.url.Scheme = "https" - // 重新初始化 - err = pool.Init() - if err != nil { - return err - } - } - - return nil -} - -func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) { - if depth >= MaxRedirect { +func (pool *BrutePool) doCheck() { + if pool.failedCount > pool.BreakThreshold { + // 当报错次数超过上限是, 结束任务 + pool.recover() + pool.Cancel() + pool.IsFailed = true return } - reURL := FormatURL(bl.Url.Path, bl.RedirectURL) - pool.waiter.Add(1) - go func() { - defer pool.waiter.Done() - pool.addAddition(&Unit{ - path: reURL, - source: RedirectSource, - frontUrl: bl.UrlString, - depth: depth + 1, - }) - }() + + if pool.Mod == HostSpray { + pool.checkCh <- struct{}{} + } else if pool.Mod == PathSpray { + pool.checkCh <- struct{}{} + } } -func (pool *Pool) doCrawl(bl *pkg.Baseline) { +func (pool *BrutePool) doCrawl(bl *pkg.Baseline) { if !pool.Crawl || bl.ReqDepth >= MaxCrawl { - pool.waiter.Done() + pool.wg.Done() return } bl.CollectURL() if bl.URLs == nil { - pool.waiter.Done() + pool.wg.Done() return } - pool.waiter.Add(1) + pool.wg.Add(1) pool.doScopeCrawl(bl) go func() { - defer pool.waiter.Done() + defer pool.wg.Done() for _, u := range bl.URLs { - if u = FormatURL(bl.Url.Path, u); u == "" { + if u = pkg.FormatURL(bl.Url.Path, u); u == "" { continue } pool.addAddition(&Unit{ path: u, - source: CrawlSource, + source: parsers.CrawlSource, depth: bl.ReqDepth + 1, }) } @@ -674,24 +666,24 @@ func (pool *Pool) doCrawl(bl *pkg.Baseline) { } -func (pool *Pool) doScopeCrawl(bl *pkg.Baseline) { +func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) { if bl.ReqDepth >= MaxCrawl { - pool.waiter.Done() + pool.wg.Done() return } go func() { - defer pool.waiter.Done() + defer pool.wg.Done() for _, u := range bl.URLs { if strings.HasPrefix(u, "http") { - if v, _ := url.Parse(u); v == nil || !MatchWithGlobs(v.Host, pool.Scope) { + if v, _ := url.Parse(u); v == nil || !pkg.MatchWithGlobs(v.Host, pool.Scope) { continue } pool.scopeLocker.Lock() if _, ok := pool.scopeurls[u]; !ok { - pool.urls[u] = struct{}{} - pool.waiter.Add(1) - pool.scopePool.Invoke(&Unit{path: u, source: CrawlSource, depth: bl.ReqDepth + 1}) + pool.urls.Store(u, nil) + pool.wg.Add(1) + pool.scopePool.Invoke(&Unit{path: u, source: parsers.CrawlSource, depth: bl.ReqDepth + 1}) } pool.scopeLocker.Unlock() } @@ -699,54 +691,18 @@ func (pool *Pool) doScopeCrawl(bl *pkg.Baseline) { }() } -func (pool *Pool) doRule(bl *pkg.Baseline) { - if pool.AppendRule == nil { - pool.waiter.Done() - return - } - if bl.Source == RuleSource { - pool.waiter.Done() - return - } - - go func() { - defer pool.waiter.Done() - for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { - pool.addAddition(&Unit{ - path: Dir(bl.Url.Path) + u, - source: RuleSource, - }) - } - }() -} - -func (pool *Pool) doRetry(bl *pkg.Baseline) { - if bl.Retry >= pool.Retry { - return - } - pool.waiter.Add(1) - go func() { - defer pool.waiter.Done() - pool.addAddition(&Unit{ - path: bl.Path, - source: RetrySource, - retry: bl.Retry + 1, - }) - }() -} - -func (pool *Pool) doActive() { - defer pool.waiter.Done() - for _, u := range pkg.ActivePath { - pool.addAddition(&Unit{ - path: pool.dir + u[1:], - source: ActiveSource, - }) +func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) { + if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) { + bl.Collect() + pool.wg.Add(1) + pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取 + pool.baselines[bl.Status] = bl + logs.Log.Logf(pkg.LogVerbose, "[baseline.%dinit] %s", bl.Status, bl.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) } } -func (pool *Pool) doBak() { - defer pool.waiter.Done() +func (pool *BrutePool) doBak() { + defer pool.wg.Done() worder, err := words.NewWorderWithDsl("{?0}.{@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil) if err != nil { return @@ -755,7 +711,7 @@ func (pool *Pool) doBak() { for w := range worder.C { pool.addAddition(&Unit{ path: pool.dir + w, - source: BakSource, + source: parsers.BakSource, }) } @@ -767,80 +723,19 @@ func (pool *Pool) doBak() { for w := range worder.C { pool.addAddition(&Unit{ path: pool.dir + w, - source: BakSource, + source: parsers.BakSource, }) } } -func (pool *Pool) doCommonFile() { - defer pool.waiter.Done() - for _, u := range mask.SpecialWords["common_file"] { - pool.addAddition(&Unit{ - path: pool.dir + u, - source: CommonFileSource, - }) - } -} - -func (pool *Pool) doCheck() { - if pool.failedCount > pool.BreakThreshold { - // 当报错次数超过上限是, 结束任务 - pool.recover() - pool.cancel() - pool.isFailed = true - return - } - - if pool.Mod == pkg.HostSpray { - pool.checkCh <- CheckSource - } else if pool.Mod == pkg.PathSpray { - pool.checkCh <- CheckSource - } -} - -func (pool *Pool) addAddition(u *Unit) { - // 强行屏蔽报错, 防止goroutine泄露 - pool.waiter.Add(1) - defer func() { - if err := recover(); err != nil { - } - }() - pool.additionCh <- u -} - -func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) { - if _, ok := pool.baselines[bl.Status]; !ok && (enableAllFuzzy || iutils.IntsContains(FuzzyStatus, bl.Status)) { - bl.Collect() - pool.waiter.Add(1) - pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取 - pool.baselines[bl.Status] = bl - logs.Log.Infof("[baseline.%dinit] %s", bl.Status, bl.Format([]string{"status", "length", "spend", "title", "frame", "redirect"})) - } -} - -func (pool *Pool) putToInvalid(bl *pkg.Baseline, reason string) { - bl.IsValid = false - pool.OutputCh <- bl -} - -func (pool *Pool) putToFuzzy(bl *pkg.Baseline) { - bl.IsFuzzy = true - pool.FuzzyCh <- bl -} - -func (pool *Pool) resetFailed() { - pool.failedCount = 1 - pool.failedBaselines = nil -} - -func (pool *Pool) recover() { +func (pool *BrutePool) recover() { logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.wordOffset) - for i, bl := range pool.failedBaselines { + for i, bl := range pool.FailedBaselines { logs.Log.Errorf("[failed.%d] %s", i, bl.String()) } } -func (pool *Pool) Close() { +func (pool *BrutePool) Close() { for pool.analyzeDone { // 等待缓存的待处理任务完成 time.Sleep(time.Duration(100) * time.Millisecond) @@ -848,23 +743,32 @@ func (pool *Pool) Close() { close(pool.additionCh) // 关闭addition管道 close(pool.checkCh) // 关闭check管道 pool.Statistor.EndTime = time.Now().Unix() - pool.bar.Close() + pool.Bar.Close() } -func (pool *Pool) safePath(u string) string { +func (pool *BrutePool) safePath(u string) string { // 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common - hasSlash := strings.HasPrefix(u, "/") - if hasSlash { - if pool.isDir { - return pool.dir + u[1:] - } else { - return pool.url.Path + u - } + if pool.isDir { + return pkg.SafePath(pool.dir, u) } else { - if pool.isDir { - return pool.url.Path + u - } else { - return pool.url.Path + "/" + u - } + return pkg.SafePath(pool.url.Path+"/", u) } } + +func (pool *BrutePool) resetFailed() { + pool.failedCount = 1 + pool.FailedBaselines = nil +} + +func NewBaselines() *Baselines { + return &Baselines{ + baselines: map[int]*pkg.Baseline{}, + } +} + +type Baselines struct { + FailedBaselines []*pkg.Baseline + random *pkg.Baseline + index *pkg.Baseline + baselines map[int]*pkg.Baseline +} diff --git a/internal/checkpool.go b/internal/pool/checkpool.go similarity index 69% rename from internal/checkpool.go rename to internal/pool/checkpool.go index c7e8ff7..56aeb00 100644 --- a/internal/checkpool.go +++ b/internal/pool/checkpool.go @@ -1,13 +1,11 @@ -package internal +package pool import ( "context" - "fmt" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" + "github.com/chainreactors/spray/internal/ihttp" "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/spray/pkg/ihttp" - "github.com/chainreactors/words" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" "net/url" @@ -17,56 +15,37 @@ import ( ) // 类似httpx的无状态, 无scope, 无并发池的检测模式 -func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) { +func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) { pctx, cancel := context.WithCancel(ctx) pool := &CheckPool{ - Config: config, - ctx: pctx, - cancel: cancel, - client: ihttp.NewClient(config.Thread, 2, config.ClientType), - wg: sync.WaitGroup{}, - additionCh: make(chan *Unit, 100), - closeCh: make(chan struct{}), - reqCount: 1, - failedCount: 1, + &This{ + Config: config, + ctx: pctx, + Cancel: cancel, + client: ihttp.NewClient(&ihttp.ClientConfig{ + Thread: config.Thread, + Type: config.ClientType, + Timeout: time.Duration(config.Timeout) * time.Second, + ProxyAddr: config.ProxyAddr, + }), + wg: sync.WaitGroup{}, + additionCh: make(chan *Unit, 100), + closeCh: make(chan struct{}), + }, } pool.Headers = map[string]string{"Connection": "close"} p, _ := ants.NewPoolWithFunc(config.Thread, pool.Invoke) - pool.pool = p + pool.This.Pool = p return pool, nil } type CheckPool struct { - *pkg.Config - client *ihttp.Client - pool *ants.PoolWithFunc - bar *pkg.Bar - ctx context.Context - cancel context.CancelFunc - reqCount int - failedCount int - additionCh chan *Unit - closeCh chan struct{} - worder *words.Worder - wg sync.WaitGroup -} - -func (pool *CheckPool) Close() { - pool.bar.Close() -} - -func (pool *CheckPool) genReq(s string) (*ihttp.Request, error) { - if pool.Mod == pkg.HostSpray { - return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s) - } else if pool.Mod == pkg.PathSpray { - return ihttp.BuildPathRequest(pool.ClientType, pool.BaseURL, s) - } - return nil, fmt.Errorf("unknown mod") + *This } func (pool *CheckPool) Run(ctx context.Context, offset, limit int) { - pool.worder.Run() + pool.Worder.Run() var done bool // 挂起一个监控goroutine, 每100ms判断一次done, 如果已经done, 则关闭closeCh, 然后通过Loop中的select case closeCh去break, 实现退出 @@ -84,7 +63,7 @@ func (pool *CheckPool) Run(ctx context.Context, offset, limit int) { Loop: for { select { - case u, ok := <-pool.worder.C: + case u, ok := <-pool.Worder.C: if !ok { done = true continue @@ -100,12 +79,12 @@ Loop: } pool.wg.Add(1) - _ = pool.pool.Invoke(newUnit(u, CheckSource)) + _ = pool.This.Pool.Invoke(newUnit(u, parsers.CheckSource)) case u, ok := <-pool.additionCh: if !ok { continue } - _ = pool.pool.Invoke(u) + _ = pool.This.Pool.Invoke(u) case <-pool.closeCh: break Loop case <-ctx.Done(): @@ -163,23 +142,23 @@ func (pool *CheckPool) Invoke(v interface{}) { if bl.IsValid { if bl.RedirectURL != "" { pool.doRedirect(bl, unit.depth) - pool.FuzzyCh <- bl + pool.putToFuzzy(bl) } else if bl.Status == 400 { pool.doUpgrade(bl) - pool.FuzzyCh <- bl + pool.putToFuzzy(bl) } else { params := map[string]interface{}{ "current": bl, } - if pool.MatchExpr == nil || CompareWithExpr(pool.MatchExpr, params) { - pool.OutputCh <- bl + if pool.MatchExpr == nil || pkg.CompareWithExpr(pool.MatchExpr, params) { + pool.putToOutput(bl) } } } pool.reqCount++ pool.wg.Done() - pool.bar.Done() + pool.Bar.Done() } func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { @@ -194,14 +173,14 @@ func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) { } reURL = bl.RedirectURL } else { - reURL = BaseURL(bl.Url) + FormatURL(BaseURL(bl.Url), bl.RedirectURL) + reURL = pkg.BaseURL(bl.Url) + pkg.FormatURL(pkg.BaseURL(bl.Url), bl.RedirectURL) } pool.wg.Add(1) go func() { pool.additionCh <- &Unit{ path: reURL, - source: RedirectSource, + source: parsers.RedirectSource, frontUrl: bl.UrlString, depth: depth + 1, } @@ -223,7 +202,7 @@ func (pool *CheckPool) doUpgrade(bl *pkg.Baseline) { go func() { pool.additionCh <- &Unit{ path: reurl, - source: UpgradeSource, + source: parsers.UpgradeSource, depth: bl.ReqDepth + 1, } }() diff --git a/pkg/config.go b/internal/pool/config.go similarity index 81% rename from pkg/config.go rename to internal/pool/config.go index 0f34592..bbf2460 100644 --- a/pkg/config.go +++ b/internal/pool/config.go @@ -1,8 +1,10 @@ -package pkg +package pool import ( "github.com/antonmedv/expr/vm" + "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words/rule" + "sync" ) type SprayMod int @@ -21,9 +23,13 @@ var ModMap = map[string]SprayMod{ type Config struct { BaseURL string + ProxyAddr string Thread int Wordlist []string Timeout int + OutputCh chan *pkg.Baseline + FuzzyCh chan *pkg.Baseline + OutLocker *sync.WaitGroup RateLimit int CheckPeriod int ErrPeriod int32 @@ -36,8 +42,7 @@ type Config struct { FilterExpr *vm.Program RecuExpr *vm.Program AppendRule *rule.Program - OutputCh chan *Baseline - FuzzyCh chan *Baseline + AppendWords []string Fuzzy bool IgnoreWaf bool Crawl bool diff --git a/internal/pool/pool.go b/internal/pool/pool.go new file mode 100644 index 0000000..f4bb8ab --- /dev/null +++ b/internal/pool/pool.go @@ -0,0 +1,160 @@ +package pool + +import ( + "context" + "fmt" + "github.com/chainreactors/parsers" + "github.com/chainreactors/spray/internal/ihttp" + "github.com/chainreactors/spray/pkg" + "github.com/chainreactors/words" + "github.com/chainreactors/words/mask" + "github.com/chainreactors/words/rule" + "github.com/panjf2000/ants/v2" + "path" + "sync" +) + +type This struct { + *Config + Statistor *pkg.Statistor + Pool *ants.PoolWithFunc + Bar *pkg.Bar + Worder *words.Worder + client *ihttp.Client + ctx context.Context + Cancel context.CancelFunc + dir string + reqCount int + failedCount int + additionCh chan *Unit + closeCh chan struct{} + wg sync.WaitGroup +} + +func (pool *This) doRedirect(bl *pkg.Baseline, depth int) { + if depth >= MaxRedirect { + return + } + reURL := pkg.FormatURL(bl.Url.Path, bl.RedirectURL) + pool.wg.Add(1) + go func() { + defer pool.wg.Done() + pool.addAddition(&Unit{ + path: reURL, + source: parsers.RedirectSource, + frontUrl: bl.UrlString, + depth: depth + 1, + }) + }() +} + +func (pool *This) doRule(bl *pkg.Baseline) { + if pool.AppendRule == nil { + pool.wg.Done() + return + } + if bl.Source == parsers.RuleSource { + pool.wg.Done() + return + } + + go func() { + defer pool.wg.Done() + for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) { + pool.addAddition(&Unit{ + path: pkg.Dir(bl.Url.Path) + u, + source: parsers.RuleSource, + }) + } + }() +} + +func (pool *This) doAppendWords(bl *pkg.Baseline) { + if pool.AppendWords == nil { + pool.wg.Done() + return + } + if bl.Source == parsers.AppendSource { + pool.wg.Done() + return + } + + go func() { + defer pool.wg.Done() + for _, u := range pool.AppendWords { + pool.addAddition(&Unit{ + path: pkg.SafePath(bl.Path, u), + source: parsers.AppendSource, + }) + } + }() +} + +func (pool *This) doRetry(bl *pkg.Baseline) { + if bl.Retry >= pool.Retry { + return + } + pool.wg.Add(1) + go func() { + defer pool.wg.Done() + pool.addAddition(&Unit{ + path: bl.Path, + source: parsers.RetrySource, + retry: bl.Retry + 1, + }) + }() +} + +func (pool *This) doActive() { + defer pool.wg.Done() + for _, u := range pkg.ActivePath { + pool.addAddition(&Unit{ + path: pool.dir + u[1:], + source: parsers.FingerSource, + }) + } +} + +func (pool *This) doCommonFile() { + defer pool.wg.Done() + for _, u := range mask.SpecialWords["common_file"] { + pool.addAddition(&Unit{ + path: pool.dir + u, + source: parsers.CommonFileSource, + }) + } +} + +func (pool *This) addAddition(u *Unit) { + // 强行屏蔽报错, 防止goroutine泄露 + pool.wg.Add(1) + defer func() { + if err := recover(); err != nil { + } + }() + pool.additionCh <- u +} + +func (pool *This) Close() { + pool.Bar.Close() +} + +func (pool *This) genReq(s string) (*ihttp.Request, error) { + if pool.Mod == HostSpray { + return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s) + } else if pool.Mod == PathSpray { + return ihttp.BuildPathRequest(pool.ClientType, pool.BaseURL, s) + } + return nil, fmt.Errorf("unknown mod") +} + +func (pool *This) putToOutput(bl *pkg.Baseline) { + pool.OutLocker.Add(1) + pool.OutputCh <- bl +} + +func (pool *This) putToFuzzy(bl *pkg.Baseline) { + pool.OutLocker.Add(1) + bl.IsFuzzy = true + pool.FuzzyCh <- bl +} diff --git a/internal/pool/unit.go b/internal/pool/unit.go new file mode 100644 index 0000000..06ff692 --- /dev/null +++ b/internal/pool/unit.go @@ -0,0 +1,20 @@ +package pool + +import "github.com/chainreactors/parsers" + +func newUnit(path string, source parsers.SpraySource) *Unit { + return &Unit{path: path, source: source} +} + +func newUnitWithNumber(path string, source parsers.SpraySource, number int) *Unit { + return &Unit{path: path, source: source, number: number} +} + +type Unit struct { + number int + path string + source parsers.SpraySource + retry int + frontUrl string + depth int // redirect depth +} diff --git a/internal/runner.go b/internal/runner.go index ffa5dfc..50afd4f 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -6,8 +6,9 @@ import ( "github.com/antonmedv/expr/vm" "github.com/chainreactors/files" "github.com/chainreactors/logs" + "github.com/chainreactors/spray/internal/ihttp" + "github.com/chainreactors/spray/internal/pool" "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/words" "github.com/chainreactors/words/rule" "github.com/gosuri/uiprogress" @@ -17,11 +18,7 @@ import ( ) var ( - WhiteStatus = []int{200} - BlackStatus = []int{400, 410} - FuzzyStatus = []int{403, 404, 500, 501, 502, 503} - WAFStatus = []int{493, 418, 1020, 406} - UniqueStatus = []int{403} + max = 2147483647 ) var ( @@ -33,6 +30,9 @@ var ( type Runner struct { taskCh chan *Task poolwg sync.WaitGroup + outwg *sync.WaitGroup + outputCh chan *pkg.Baseline + fuzzyCh chan *pkg.Baseline bar *uiprogress.Bar finished int @@ -41,8 +41,9 @@ type Runner struct { Wordlist []string Rules *rule.Program AppendRules *rule.Program + AppendWords []string Headers map[string]string - Fns []func(string) string + Fns []func(string) []string FilterExpr *vm.Program MatchExpr *vm.Program RecursiveExpr *vm.Program @@ -55,8 +56,6 @@ type Runner struct { Timeout int Mod string Probes []string - OutputCh chan *pkg.Baseline - FuzzyCh chan *pkg.Baseline Fuzzy bool OutputFile *files.File FuzzyFile *files.File @@ -77,24 +76,26 @@ type Runner struct { IgnoreWaf bool Crawl bool Scope []string - Active bool + Finger bool Bak bool Common bool RetryCount int RandomUserAgent bool Random string Index string + Proxy string } -func (r *Runner) PrepareConfig() *pkg.Config { - config := &pkg.Config{ +func (r *Runner) PrepareConfig() *pool.Config { + config := &pool.Config{ Thread: r.Threads, Timeout: r.Timeout, RateLimit: r.RateLimit, Headers: r.Headers, - Mod: pkg.ModMap[r.Mod], - OutputCh: r.OutputCh, - FuzzyCh: r.FuzzyCh, + Mod: pool.ModMap[r.Mod], + OutputCh: r.outputCh, + FuzzyCh: r.fuzzyCh, + OutLocker: r.outwg, Fuzzy: r.Fuzzy, CheckPeriod: r.CheckPeriod, ErrPeriod: int32(r.ErrPeriod), @@ -103,10 +104,11 @@ func (r *Runner) PrepareConfig() *pkg.Config { FilterExpr: r.FilterExpr, RecuExpr: r.RecursiveExpr, AppendRule: r.AppendRules, + AppendWords: r.AppendWords, IgnoreWaf: r.IgnoreWaf, Crawl: r.Crawl, Scope: r.Scope, - Active: r.Active, + Active: r.Finger, Bak: r.Bak, Common: r.Common, Retry: r.RetryCount, @@ -114,18 +116,23 @@ func (r *Runner) PrepareConfig() *pkg.Config { RandomUserAgent: r.RandomUserAgent, Random: r.Random, Index: r.Index, + ProxyAddr: r.Proxy, } if config.ClientType == ihttp.Auto { - if config.Mod == pkg.PathSpray { + if config.Mod == pool.PathSpray { config.ClientType = ihttp.FAST - } else if config.Mod == pkg.HostSpray { + } else if config.Mod == pool.HostSpray { config.ClientType = ihttp.STANDARD } } return config } +func (r *Runner) AppendFunction(fn func(string) []string) { + r.Fns = append(r.Fns, fn) +} + func (r *Runner) Prepare(ctx context.Context) error { var err error if r.CheckOnly { @@ -133,10 +140,10 @@ func (r *Runner) Prepare(ctx context.Context) error { r.Pools, err = ants.NewPoolWithFunc(1, func(i interface{}) { config := r.PrepareConfig() - pool, err := NewCheckPool(ctx, config) + pool, err := pool.NewCheckPool(ctx, config) if err != nil { logs.Log.Error(err.Error()) - pool.cancel() + pool.Cancel() r.poolwg.Done() return } @@ -148,9 +155,9 @@ func (r *Runner) Prepare(ctx context.Context) error { } close(ch) }() - pool.worder = words.NewWorderWithChan(ch) - pool.worder.Fns = r.Fns - pool.bar = pkg.NewBar("check", r.Count-r.Offset, r.Progress) + pool.Worder = words.NewWorderWithChan(ch) + pool.Worder.Fns = r.Fns + pool.Bar = pkg.NewBar("check", r.Count-r.Offset, r.Progress) pool.Run(ctx, r.Offset, r.Count) r.poolwg.Done() }) @@ -182,17 +189,17 @@ func (r *Runner) Prepare(ctx context.Context) error { config := r.PrepareConfig() config.BaseURL = t.baseUrl - pool, err := NewPool(ctx, config) + pool, err := pool.NewBrutePool(ctx, config) if err != nil { logs.Log.Error(err.Error()) - pool.cancel() + pool.Cancel() r.Done() return } if t.origin != nil && len(r.Wordlist) == 0 { // 如果是从断点续传中恢复的任务, 则自动设置word,dict与rule, 不过优先级低于命令行参数 pool.Statistor = pkg.NewStatistorFromStat(t.origin.Statistor) - pool.worder, err = t.origin.InitWorder(r.Fns) + pool.Worder, err = t.origin.InitWorder(r.Fns) if err != nil { logs.Log.Error(err.Error()) r.Done() @@ -201,9 +208,9 @@ func (r *Runner) Prepare(ctx context.Context) error { pool.Statistor.Total = t.origin.sum } else { pool.Statistor = pkg.NewStatistor(t.baseUrl) - pool.worder = words.NewWorder(r.Wordlist) - pool.worder.Fns = r.Fns - pool.worder.Rules = r.Rules.Expressions + pool.Worder = words.NewWorder(r.Wordlist) + pool.Worder.Fns = r.Fns + pool.Worder.Rules = r.Rules.Expressions } var limit int @@ -212,7 +219,8 @@ func (r *Runner) Prepare(ctx context.Context) error { } else { limit = pool.Statistor.Total } - pool.bar = pkg.NewBar(config.BaseURL, limit-pool.Statistor.Offset, r.Progress) + pool.Bar = pkg.NewBar(config.BaseURL, limit-pool.Statistor.Offset, r.Progress) + logs.Log.Importantf("[pool] task: %s, total %d words, %d threads, proxy: %s", pool.BaseURL, limit-pool.Statistor.Offset, pool.Thread, pool.ProxyAddr) err = pool.Init() if err != nil { pool.Statistor.Error = err.Error() @@ -227,9 +235,9 @@ func (r *Runner) Prepare(ctx context.Context) error { pool.Run(pool.Statistor.Offset, limit) - if pool.isFailed && len(pool.failedBaselines) > 0 { + if pool.IsFailed && len(pool.FailedBaselines) > 0 { // 如果因为错误积累退出, end将指向第一个错误发生时, 防止resume时跳过大量目标 - pool.Statistor.End = pool.failedBaselines[0].Number + pool.Statistor.End = pool.FailedBaselines[0].Number } r.PrintStat(pool) r.Done() @@ -239,7 +247,7 @@ func (r *Runner) Prepare(ctx context.Context) error { if err != nil { return err } - r.Output() + r.OutputHandler() return nil } @@ -287,19 +295,7 @@ Loop: } r.poolwg.Wait() - time.Sleep(100 * time.Millisecond) // 延迟100ms, 等所有数据处理完毕 - for { - if len(r.OutputCh) == 0 { - break - } - } - - for { - if len(r.FuzzyCh) == 0 { - break - } - } - time.Sleep(100 * time.Millisecond) // 延迟100ms, 等所有数据处理完毕 + r.outwg.Wait() } func (r *Runner) RunWithCheck(ctx context.Context) { @@ -326,7 +322,7 @@ Loop: } for { - if len(r.OutputCh) == 0 { + if len(r.outputCh) == 0 { break } } @@ -340,18 +336,18 @@ func (r *Runner) Done() { r.poolwg.Done() } -func (r *Runner) PrintStat(pool *Pool) { +func (r *Runner) PrintStat(pool *pool.BrutePool) { if r.Color { logs.Log.Important(pool.Statistor.ColorString()) if pool.Statistor.Error == "" { - logs.Log.Important(pool.Statistor.ColorCountString()) - logs.Log.Important(pool.Statistor.ColorSourceString()) + pool.Statistor.PrintColorCount() + pool.Statistor.PrintColorSource() } } else { logs.Log.Important(pool.Statistor.String()) if pool.Statistor.Error == "" { - logs.Log.Important(pool.Statistor.CountString()) - logs.Log.Important(pool.Statistor.SourceString()) + pool.Statistor.PrintCount() + pool.Statistor.PrintSource() } } @@ -361,7 +357,7 @@ func (r *Runner) PrintStat(pool *Pool) { } } -func (r *Runner) Output() { +func (r *Runner) OutputHandler() { debugPrint := func(bl *pkg.Baseline) { if r.Color { logs.Log.Debug(bl.ColorString()) @@ -403,7 +399,7 @@ func (r *Runner) Output() { for { select { - case bl, ok := <-r.OutputCh: + case bl, ok := <-r.outputCh: if !ok { return } @@ -419,6 +415,7 @@ func (r *Runner) Output() { } else { debugPrint(bl) } + r.outwg.Done() } } }() @@ -443,15 +440,16 @@ func (r *Runner) Output() { for { select { - case bl, ok := <-r.FuzzyCh: + case bl, ok := <-r.fuzzyCh: if !ok { return } if r.Fuzzy { fuzzySaveFunc(bl) - } else { - debugPrint(bl) + //} else { + // debugPrint(bl) } + r.outwg.Done() } } }() diff --git a/internal/types.go b/internal/types.go index 6ff46a3..9f50a91 100644 --- a/internal/types.go +++ b/internal/types.go @@ -6,39 +6,6 @@ import ( "github.com/chainreactors/words/rule" ) -const ( - CheckSource = iota + 1 - InitRandomSource - InitIndexSource - RedirectSource - CrawlSource - ActiveSource - WordSource - WafSource - RuleSource - BakSource - CommonFileSource - UpgradeSource - RetrySource -) - -func newUnit(path string, source int) *Unit { - return &Unit{path: path, source: source} -} - -func newUnitWithNumber(path string, source int, number int) *Unit { - return &Unit{path: path, source: source, number: number} -} - -type Unit struct { - number int - path string - source int - retry int - frontUrl string - depth int // redirect depth -} - type Task struct { baseUrl string depth int @@ -55,7 +22,7 @@ type Origin struct { sum int } -func (o *Origin) InitWorder(fns []func(string) string) (*words.Worder, error) { +func (o *Origin) InitWorder(fns []func(string) []string) (*words.Worder, error) { var worder *words.Worder wl, err := loadWordlist(o.Word, o.Dictionaries) if err != nil { diff --git a/internal/utils.go b/internal/utils.go index 53f4925..d745f02 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -2,41 +2,15 @@ package internal import ( "bytes" - "github.com/antonmedv/expr" "github.com/antonmedv/expr/ast" - "github.com/antonmedv/expr/vm" - "github.com/chainreactors/logs" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "io/ioutil" - "math/rand" - "net/url" - "path" - "path/filepath" "strconv" "strings" ) -var ( - // from feroxbuster - randomUserAgent = []string{ - "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36", - "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254", - "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246", - "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1", - "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", - "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", - } - uacount = len(randomUserAgent) -) - func parseExtension(s string) string { if i := strings.Index(s, "."); i != -1 { return s[i+1:] @@ -97,7 +71,7 @@ func loadFileToSlice(filename string) ([]string, error) { return ss, nil } -func loadFileAndCombine(filename []string) (string, error) { +func loadRuleAndCombine(filename []string) (string, error) { var bs bytes.Buffer for _, f := range filename { if data, ok := pkg.Rules[f]; ok { @@ -171,116 +145,6 @@ func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, er return rule.Compile(rules.String(), filter).Expressions, nil } -func relaPath(base, u string) string { - // 拼接相对目录, 不使用path.join的原因是, 如果存在"////"这样的情况, 可能真的是有意义的路由, 不能随意去掉. - // "" /a /a - // "" a /a - // / "" / - // /a/ b /a/b - // /a/ /b /a/b - // /a b /b - // /a /b /b - - if u == "" { - return base - } - - pathSlash := strings.HasPrefix(u, "/") - if base == "" { - if pathSlash { - return u[1:] - } else { - return "/" + u - } - } else if strings.HasSuffix(base, "/") { - if pathSlash { - return base + u[1:] - } else { - return base + u - } - } else { - if pathSlash { - return Dir(base) + u[1:] - } else { - return Dir(base) + u - } - } -} - -func Dir(u string) string { - // 安全的获取目录, 不会额外处理多个"//", 并非用来获取上级目录 - // /a / - // /a/ /a/ - // a/ a/ - // aaa / - if strings.HasSuffix(u, "/") { - return u - } else if i := strings.LastIndex(u, "/"); i == -1 { - return "/" - } else { - return u[:i+1] - } -} - -func FormatURL(base, u string) string { - if strings.HasPrefix(u, "http") { - parsed, err := url.Parse(u) - if err != nil { - return "" - } - return parsed.Path - } else if strings.HasPrefix(u, "//") { - parsed, err := url.Parse(u) - if err != nil { - return "" - } - return parsed.Path - } else if strings.HasPrefix(u, "/") { - // 绝对目录拼接 - // 不需要进行处理, 用来跳过下面的判断 - return u - } else if strings.HasPrefix(u, "./") { - // "./"相对目录拼接 - return relaPath(base, u[2:]) - } else if strings.HasPrefix(u, "../") { - return path.Join(Dir(base), u) - } else { - // 相对目录拼接 - return relaPath(base, u) - } -} - -func BaseURL(u *url.URL) string { - return u.Scheme + "://" + u.Host -} - -func RandomUA() string { - return randomUserAgent[rand.Intn(uacount)] -} - -func CompareWithExpr(exp *vm.Program, params map[string]interface{}) bool { - res, err := expr.Run(exp, params) - if err != nil { - logs.Log.Warn(err.Error()) - } - - if res == true { - return true - } else { - return false - } -} - -func MatchWithGlobs(u string, globs []string) bool { - for _, glob := range globs { - ok, err := filepath.Match(glob, u) - if err == nil && ok { - return true - } - } - return false -} - type bytesPatcher struct{} func (p *bytesPatcher) Visit(node *ast.Node) { @@ -295,3 +159,9 @@ func (p *bytesPatcher) Visit(node *ast.Node) { }) } } + +func wrapWordsFunc(f func(string) string) func(string) []string { + return func(s string) []string { + return []string{f(s)} + } +} diff --git a/pkg/baseline.go b/pkg/baseline.go index ec27c68..d365d39 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -3,8 +3,9 @@ package pkg import ( "bytes" "github.com/chainreactors/parsers" - "github.com/chainreactors/parsers/iutils" - "github.com/chainreactors/spray/pkg/ihttp" + "github.com/chainreactors/spray/internal/ihttp" + "github.com/chainreactors/utils/encode" + "github.com/chainreactors/utils/iutils" "net/url" "strings" ) @@ -31,7 +32,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { copy(bl.Header, header) bl.HeaderLength = len(bl.Header) - if i := resp.ContentLength(); i != 0 && bl.ContentType != "bin" { + if i := resp.ContentLength(); i != 0 && i <= ihttp.DefaultMaxBodySize { body := resp.Body() bl.Body = make([]byte, len(body)) copy(bl.Body, body) @@ -101,7 +102,6 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba type Baseline struct { *parsers.SprayResult - Unique uint16 `json:"-"` Url *url.URL `json:"-"` Dir bool `json:"-"` Chunked bool `json:"-"` @@ -133,9 +133,9 @@ func (bl *Baseline) Collect() { if bl.ContentType == "html" { bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body)) } else if bl.ContentType == "ico" { - if name, ok := Md5Fingers[parsers.Md5Hash(bl.Body)]; ok { + if name, ok := Md5Fingers[encode.Md5Hash(bl.Body)]; ok { bl.Frameworks[name] = &parsers.Framework{Name: name} - } else if name, ok := Mmh3Fingers[parsers.Mmh3Hash32(bl.Body)]; ok { + } else if name, ok := Mmh3Fingers[encode.Mmh3Hash32(bl.Body)]; ok { bl.Frameworks[name] = &parsers.Framework{Name: name} } } @@ -160,8 +160,8 @@ func (bl *Baseline) CollectURL() { for _, reg := range ExtractRegexps["js"][0].CompiledRegexps { urls := reg.FindAllStringSubmatch(string(bl.Body), -1) for _, u := range urls { - u[1] = formatURL(u[1]) - if u[1] != "" && !filterJs(u[1]) { + u[1] = CleanURL(u[1]) + if u[1] != "" && !FilterJs(u[1]) { bl.URLs = append(bl.URLs, u[1]) } } @@ -170,14 +170,14 @@ func (bl *Baseline) CollectURL() { for _, reg := range ExtractRegexps["url"][0].CompiledRegexps { urls := reg.FindAllStringSubmatch(string(bl.Body), -1) for _, u := range urls { - u[1] = formatURL(u[1]) - if u[1] != "" && !filterUrl(u[1]) { + u[1] = CleanURL(u[1]) + if u[1] != "" && !FilterUrl(u[1]) { bl.URLs = append(bl.URLs, u[1]) } } } - bl.URLs = RemoveDuplication(bl.URLs) + bl.URLs = iutils.StringsUnique(bl.URLs) if bl.URLs != nil { bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{ Name: "crawl", @@ -225,7 +225,7 @@ var Distance uint8 = 5 // 数字越小越相似, 数字为0则为完全一致. func (bl *Baseline) FuzzyCompare(other *Baseline) bool { // 这里使用rawsimhash, 是为了保证一定数量的字符串, 否则超短的body会导致simhash偏差指较大 - if other.Distance = parsers.SimhashCompare(other.RawSimhash, bl.RawSimhash); other.Distance < Distance { + if other.Distance = encode.SimhashCompare(other.RawSimhash, bl.RawSimhash); other.Distance < Distance { return true } return false diff --git a/pkg/types.go b/pkg/errors.go similarity index 93% rename from pkg/types.go rename to pkg/errors.go index a5012af..1dd3575 100644 --- a/pkg/types.go +++ b/pkg/errors.go @@ -37,9 +37,3 @@ var ErrMap = map[ErrorType]string{ func (e ErrorType) Error() string { return ErrMap[e] } - -type BS []byte - -func (b BS) String() string { - return string(b) -} diff --git a/pkg/load.go b/pkg/load.go new file mode 100644 index 0000000..71448d7 --- /dev/null +++ b/pkg/load.go @@ -0,0 +1,92 @@ +package pkg + +import ( + "encoding/json" + "github.com/chainreactors/gogo/v2/pkg/fingers" + "github.com/chainreactors/parsers" + "github.com/chainreactors/utils" + "github.com/chainreactors/utils/iutils" + "github.com/chainreactors/words/mask" + "strings" +) + +func LoadTemplates() error { + var err error + // load fingers + Fingers, err = fingers.LoadFingers(LoadConfig("http")) + if err != nil { + return err + } + + for _, finger := range Fingers { + err := finger.Compile(utils.ParsePorts) + if err != nil { + return err + } + } + + for _, f := range Fingers { + for _, rule := range f.Rules { + if rule.SendDataStr != "" { + ActivePath = append(ActivePath, rule.SendDataStr) + } + if rule.Favicon != nil { + for _, mmh3 := range rule.Favicon.Mmh3 { + Mmh3Fingers[mmh3] = f.Name + } + for _, md5 := range rule.Favicon.Md5 { + Md5Fingers[md5] = f.Name + } + } + } + } + + // load rule + var data map[string]interface{} + err = json.Unmarshal(LoadConfig("spray_rule"), &data) + if err != nil { + return err + } + for k, v := range data { + Rules[k] = v.(string) + } + + // load mask + var keywords map[string]interface{} + err = json.Unmarshal(LoadConfig("spray_common"), &keywords) + if err != nil { + return err + } + + for k, v := range keywords { + t := make([]string, len(v.([]interface{}))) + for i, vv := range v.([]interface{}) { + t[i] = iutils.ToString(vv) + } + mask.SpecialWords[k] = t + } + + var extracts []*parsers.Extractor + err = json.Unmarshal(LoadConfig("extract"), &extracts) + if err != nil { + return err + } + + for _, extract := range extracts { + extract.Compile() + + ExtractRegexps[extract.Name] = []*parsers.Extractor{extract} + for _, tag := range extract.Tags { + if _, ok := ExtractRegexps[tag]; !ok { + ExtractRegexps[tag] = []*parsers.Extractor{extract} + } else { + ExtractRegexps[tag] = append(ExtractRegexps[tag], extract) + } + } + } + return nil +} + +func LoadDefaultDict() []string { + return strings.Split(strings.TrimSpace(string(LoadConfig("spray_default"))), "\n") +} diff --git a/pkg/statistor.go b/pkg/statistor.go index 7201e3c..fa5ad9e 100644 --- a/pkg/statistor.go +++ b/pkg/statistor.go @@ -18,7 +18,7 @@ func NewStatistor(url string) *Statistor { stat := DefaultStatistor stat.StartTime = time.Now().Unix() stat.Counts = make(map[int]int) - stat.Sources = make(map[int]int) + stat.Sources = make(map[parsers.SpraySource]int) stat.BaseUrl = url return &stat } @@ -32,33 +32,33 @@ func NewStatistorFromStat(origin *Statistor) *Statistor { RuleFiles: origin.RuleFiles, RuleFilter: origin.RuleFilter, Counts: make(map[int]int), - Sources: map[int]int{}, + Sources: map[parsers.SpraySource]int{}, StartTime: time.Now().Unix(), } } type Statistor struct { - BaseUrl string `json:"url"` - Error string `json:"error"` - Counts map[int]int `json:"counts"` - Sources map[int]int `json:"sources"` - FailedNumber int32 `json:"failed"` - ReqTotal int32 `json:"req_total"` - CheckNumber int `json:"check"` - FoundNumber int `json:"found"` - FilteredNumber int `json:"filtered"` - FuzzyNumber int `json:"fuzzy"` - WafedNumber int `json:"wafed"` - End int `json:"end"` - Offset int `json:"offset"` - Total int `json:"total"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - WordCount int `json:"word_count"` - Word string `json:"word"` - Dictionaries []string `json:"dictionaries"` - RuleFiles []string `json:"rule_files"` - RuleFilter string `json:"rule_filter"` + BaseUrl string `json:"url"` + Error string `json:"error"` + Counts map[int]int `json:"counts"` + Sources map[parsers.SpraySource]int `json:"sources"` + FailedNumber int32 `json:"failed"` + ReqTotal int32 `json:"req_total"` + CheckNumber int `json:"check"` + FoundNumber int `json:"found"` + FilteredNumber int `json:"filtered"` + FuzzyNumber int `json:"fuzzy"` + WafedNumber int `json:"wafed"` + End int `json:"end"` + Offset int `json:"offset"` + Total int `json:"total"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + WordCount int `json:"word_count"` + Word string `json:"word"` + Dictionaries []string `json:"dictionaries"` + RuleFiles []string `json:"rule_files"` + RuleFilter string `json:"rule_filter"` } func (stat *Statistor) ColorString() string { @@ -92,7 +92,10 @@ func (stat *Statistor) String() string { return s.String() } -func (stat *Statistor) CountString() string { +func (stat *Statistor) PrintCount() { + if len(stat.Counts) == 0 { + return + } var s strings.Builder s.WriteString("[stat] ") s.WriteString(stat.BaseUrl) @@ -102,20 +105,26 @@ func (stat *Statistor) CountString() string { } s.WriteString(fmt.Sprintf(" %d: %d,", k, v)) } - return s.String() + logs.Log.Important(s.String()) } -func (stat *Statistor) SourceString() string { +func (stat *Statistor) PrintSource() { + if len(stat.Sources) == 0 { + return + } var s strings.Builder s.WriteString("[stat] ") s.WriteString(stat.BaseUrl) for k, v := range stat.Sources { - s.WriteString(fmt.Sprintf(" %s: %d,", parsers.GetSpraySourceName(k), v)) + s.WriteString(fmt.Sprintf(" %s: %d,", k.Name(), v)) } - return s.String() + logs.Log.Important(s.String()) } -func (stat *Statistor) ColorCountString() string { +func (stat *Statistor) PrintColorCount() { + if len(stat.Counts) == 0 { + return + } var s strings.Builder s.WriteString("[stat] ") s.WriteString(stat.BaseUrl) @@ -125,17 +134,20 @@ func (stat *Statistor) ColorCountString() string { } s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(strconv.Itoa(k)), logs.YellowBold(strconv.Itoa(v)))) } - return s.String() + logs.Log.Important(s.String()) } -func (stat *Statistor) ColorSourceString() string { +func (stat *Statistor) PrintColorSource() { + if len(stat.Sources) == 0 { + return + } var s strings.Builder s.WriteString("[stat] ") s.WriteString(stat.BaseUrl) for k, v := range stat.Sources { - s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(parsers.GetSpraySourceName(k)), logs.YellowBold(strconv.Itoa(v)))) + s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(k.Name()), logs.YellowBold(strconv.Itoa(v)))) } - return s.String() + logs.Log.Important(s.String()) } func (stat *Statistor) Json() string { diff --git a/pkg/utils.go b/pkg/utils.go index b83af30..dd87205 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -1,22 +1,31 @@ package pkg import ( - "encoding/json" + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" "github.com/chainreactors/gogo/v2/pkg/fingers" - "github.com/chainreactors/ipcs" + "github.com/chainreactors/logs" "github.com/chainreactors/parsers" - "github.com/chainreactors/parsers/iutils" - "github.com/chainreactors/words/mask" + "github.com/chainreactors/utils/iutils" "math/rand" "net/url" - "os" "path" + "path/filepath" "strconv" "strings" "time" "unsafe" ) +var ( + LogVerbose = logs.Warn - 2 + LogFuzz = logs.Warn - 1 + WhiteStatus = []int{} // cmd input, 200 + BlackStatus = []int{} // cmd input, 400,410 + FuzzyStatus = []int{} // cmd input, 500,501,502,503 + WAFStatus = []int{493, 418, 1020, 406} + UniqueStatus = []int{} // 相同unique的403表示命中了同一条acl, 相同unique的200表示default页面 +) var ( Md5Fingers map[string]string = make(map[string]string) Mmh3Fingers map[string]string = make(map[string]string) @@ -52,35 +61,29 @@ var ( "video/avi": "avi", "image/x-icon": "ico", } + + // from feroxbuster + randomUserAgent = []string{ + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254", + "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246", + "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", + } + uacount = len(randomUserAgent) ) -func RemoveDuplication(arr []string) []string { - set := make(map[string]struct{}, len(arr)) - j := 0 - for _, v := range arr { - _, ok := set[v] - if ok { - continue - } - set[v] = struct{}{} - arr[j] = v - j++ - } +type BS []byte - return arr[:j] -} - -// 判断是否存在标准输入数据 -func HasStdin() bool { - stat, err := os.Stdin.Stat() - if err != nil { - return false - } - - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 - isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - - return isPipedFromChrDev || isPipedFromFIFO +func (b BS) String() string { + return string(b) } const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -133,83 +136,6 @@ func RandHost() string { return *(*string)(unsafe.Pointer(&b)) } -func LoadTemplates() error { - var err error - // load fingers - Fingers, err = fingers.LoadFingers(LoadConfig("http")) - if err != nil { - return err - } - - for _, finger := range Fingers { - err := finger.Compile(ipcs.ParsePorts) - if err != nil { - return err - } - } - - for _, f := range Fingers { - for _, rule := range f.Rules { - if rule.SendDataStr != "" { - ActivePath = append(ActivePath, rule.SendDataStr) - } - if rule.Favicon != nil { - for _, mmh3 := range rule.Favicon.Mmh3 { - Mmh3Fingers[mmh3] = f.Name - } - for _, md5 := range rule.Favicon.Md5 { - Md5Fingers[md5] = f.Name - } - } - } - } - - // load rule - var data map[string]interface{} - err = json.Unmarshal(LoadConfig("rule"), &data) - if err != nil { - return err - } - for k, v := range data { - Rules[k] = v.(string) - } - - // load mask - var keywords map[string]interface{} - err = json.Unmarshal(LoadConfig("mask"), &keywords) - if err != nil { - return err - } - - for k, v := range keywords { - t := make([]string, len(v.([]interface{}))) - for i, vv := range v.([]interface{}) { - t[i] = iutils.ToString(vv) - } - mask.SpecialWords[k] = t - } - - var extracts []*parsers.Extractor - err = json.Unmarshal(LoadConfig("extract"), &extracts) - if err != nil { - return err - } - - for _, extract := range extracts { - extract.Compile() - - ExtractRegexps[extract.Name] = []*parsers.Extractor{extract} - for _, tag := range extract.Tags { - if _, ok := ExtractRegexps[tag]; !ok { - ExtractRegexps[tag] = []*parsers.Extractor{extract} - } else { - ExtractRegexps[tag] = append(ExtractRegexps[tag], extract) - } - } - } - return nil -} - func FingerDetect(content []byte) parsers.Frameworks { frames := make(parsers.Frameworks) for _, finger := range Fingers { @@ -222,7 +148,7 @@ func FingerDetect(content []byte) parsers.Frameworks { return frames } -func filterJs(u string) bool { +func FilterJs(u string) bool { if commonFilter(u) { return true } @@ -230,7 +156,7 @@ func filterJs(u string) bool { return false } -func filterUrl(u string) bool { +func FilterUrl(u string) bool { if commonFilter(u) { return true } @@ -249,8 +175,10 @@ func filterUrl(u string) bool { return false } -func formatURL(u string) string { +func CleanURL(u string) string { // 去掉frag与params, 节约url.parse性能, 防止带参数造成意外的影响 + u = strings.Trim(u, "\"") + u = strings.Trim(u, "'") if strings.Contains(u, "2f") || strings.Contains(u, "2F") { u = strings.ReplaceAll(u, "\\u002F", "/") u = strings.ReplaceAll(u, "\\u002f", "/") @@ -341,8 +269,127 @@ func CRC16Hash(data []byte) uint16 { return crc16 } -func UniqueHash(bl *Baseline) uint16 { - // 由host+状态码+重定向url+content-type+title+length舍去个位与十位组成的hash - // body length可能会导致一些误报, 目前没有更好的解决办法 - return CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/100*100))) +func SafePath(dir, u string) string { + hasSlash := strings.HasPrefix(u, "/") + if hasSlash { + return path.Join(dir, u[1:]) + } else { + return path.Join(dir, u) + } +} + +func RelaPath(base, u string) string { + // 拼接相对目录, 不使用path.join的原因是, 如果存在"////"这样的情况, 可能真的是有意义的路由, 不能随意去掉. + // "" /a /a + // "" a /a + // / "" / + // /a/ b /a/b + // /a/ /b /a/b + // /a b /b + // /a /b /b + + if u == "" { + return base + } + + pathSlash := strings.HasPrefix(u, "/") + if base == "" { + if pathSlash { + return u[1:] + } else { + return "/" + u + } + } else if strings.HasSuffix(base, "/") { + if pathSlash { + return base + u[1:] + } else { + return base + u + } + } else { + if pathSlash { + return Dir(base) + u[1:] + } else { + return Dir(base) + u + } + } +} + +func Dir(u string) string { + // 安全的获取目录, 不会额外处理多个"//", 并非用来获取上级目录 + // /a / + // /a/ /a/ + // a/ a/ + // aaa / + if strings.HasSuffix(u, "/") { + return u + } else if i := strings.LastIndex(u, "/"); i == -1 { + return "/" + } else { + return u[:i+1] + } +} + +func UniqueHash(bl *Baseline) uint16 { + // 由host+状态码+重定向url+content-type+title+length舍去个位组成的hash + // body length可能会导致一些误报, 目前没有更好的解决办法 + return CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/10*10))) +} + +func FormatURL(base, u string) string { + if strings.HasPrefix(u, "http") { + parsed, err := url.Parse(u) + if err != nil { + return "" + } + return parsed.Path + } else if strings.HasPrefix(u, "//") { + parsed, err := url.Parse(u) + if err != nil { + return "" + } + return parsed.Path + } else if strings.HasPrefix(u, "/") { + // 绝对目录拼接 + // 不需要进行处理, 用来跳过下面的判断 + return u + } else if strings.HasPrefix(u, "./") { + // "./"相对目录拼接 + return RelaPath(base, u[2:]) + } else if strings.HasPrefix(u, "../") { + return path.Join(Dir(base), u) + } else { + // 相对目录拼接 + return RelaPath(base, u) + } +} + +func BaseURL(u *url.URL) string { + return u.Scheme + "://" + u.Host +} + +func RandomUA() string { + return randomUserAgent[rand.Intn(uacount)] +} + +func CompareWithExpr(exp *vm.Program, params map[string]interface{}) bool { + res, err := expr.Run(exp, params) + if err != nil { + logs.Log.Warn(err.Error()) + } + + if res == true { + return true + } else { + return false + } +} + +func MatchWithGlobs(u string, globs []string) bool { + for _, glob := range globs { + ok, err := filepath.Match(glob, u) + if err == nil && ok { + return true + } + } + return false } diff --git a/spray.go b/spray.go index 36358cb..71b68c7 100644 --- a/spray.go +++ b/spray.go @@ -1,7 +1,19 @@ -//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask,extract +//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need spray package main -import "github.com/chainreactors/spray/cmd" +import ( + "github.com/chainreactors/spray/cmd" + "github.com/gookit/config/v2" + "github.com/gookit/config/v2/yaml" +) + +func init() { + config.WithOptions(func(opt *config.Options) { + opt.DecoderConfig.TagName = "config" + opt.ParseDefault = true + }) + config.AddDriver(yaml.Driver) +} func main() { //f, _ := os.Create("cpu.txt") diff --git a/templates b/templates index 241a707..998cdc0 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 241a707ce2a8d32b8bc96ebb5a06bcfdecb54b24 +Subproject commit 998cdc05018e9c221e91166d10c7b2e1b62396cf