Merge pull request #67 from chainreactors/dev

merge v1.1.1

cmd/cmd.go

@@ -16,7 +16,7 @@ import (
 	"time"
 )
 
-var ver = "v1.0.1"
+var ver = "dev"
 var DefaultConfig = "config.yaml"
 
 func init() {

go.mod

@@ -2,13 +2,11 @@ module github.com/chainreactors/spray
 
 go 1.22
 
-toolchain go1.22.2
-
 require (
 	github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0
 	github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a
 	github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f
-	github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2
+	github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84
 	github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f
 	github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508
 	github.com/expr-lang/expr v1.16.9

go.sum

@@ -93,10 +93,13 @@ github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W
 github.com/chainreactors/parsers v0.0.0-20240702104902-1ce563b7ef76/go.mod h1:G/XLE5RAaUdqADkbhQ59mPrUAbsJLiQ2DN6CwtwNpBQ=
 github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2 h1:sE3SChgHLtPsEaqHo5tDSy8niDys1SO174C4eHlShSw=
 github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
+github.com/chainreactors/parsers v0.0.0-20240825154421-240ad7eba29d h1:/rtYIkD9F1Va7YbAaHEPpLSOjsEEfqow6FKL/gcO5ns=
+github.com/chainreactors/parsers v0.0.0-20240825154421-240ad7eba29d/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
+github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84 h1:F6umsdHLxKILxrH5wY2zVYwxMmb8XdCEc9apf7GWpOk=
+github.com/chainreactors/parsers v0.0.0-20240829055950-923f89a92b84/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
 github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
 github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
 github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
-github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16 h1:TCOshCp7PrWqhP/HSAM5kT3VxoOe7EoJbRseyoSX3RM=
 github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
 github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f h1:2NKmadFYP9vCwC0YrazgttFACleOhxScTPzg0i76YAY=
 github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=

internal/ihttp/client.go

@@ -48,8 +48,8 @@ func NewClient(config *ClientConfig) *Client {
 				MaxConnsPerHost:     config.Thread * 3 / 2,
 				MaxIdleConnDuration: config.Timeout,
 				//MaxConnWaitTimeout:  time.Duration(timeout) * time.Second,
-				//ReadTimeout:                   config.Timeout * time.Second,
+				ReadTimeout:                   config.Timeout * time.Second,
-				//WriteTimeout:                  config.Timeout * time.Second,
+				WriteTimeout:                  config.Timeout * time.Second,
 				ReadBufferSize:                16384, // 16k
 				MaxResponseBodySize:           int(DefaultMaxBodySize),
 				NoDefaultUserAgentHeader:      true,

internal/ihttp/request.go

@@ -5,27 +5,20 @@ import (
 	"net/http"
 )
 
-func BuildPathRequest(clientType int, base, path, method string) (*Request, error) {
+func BuildRequest(clientType int, base, path, host, method string) (*Request, error) {
 	if clientType == FAST {
 		req := fasthttp.AcquireRequest()
 		req.Header.SetMethod(method)
 		req.SetRequestURI(base + path)
+		if host != "" {
+			req.SetHost(host)
+		}
 		return &Request{FastRequest: req, ClientType: FAST}, nil
 	} else {
 		req, err := http.NewRequest(method, base+path, nil)
-		return &Request{StandardRequest: req, ClientType: STANDARD}, err
+		if host != "" {
-	}
+			req.Host = host
-}
+		}
-
-func BuildHostRequest(clientType int, base, host string) (*Request, error) {
-	if clientType == FAST {
-		req := fasthttp.AcquireRequest()
-		req.SetRequestURI(base)
-		req.SetHost(host)
-		return &Request{FastRequest: req, ClientType: FAST}, nil
-	} else {
-		req, err := http.NewRequest("GET", base, nil)
-		req.Host = host
 		return &Request{StandardRequest: req, ClientType: STANDARD}, err
 	}
 }

internal/option.go

@@ -87,7 +87,8 @@ type OutputOptions struct {
 	AutoFile    bool   `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" config:"auto-file"`
 	Format      string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json" config:"format"`
 	Json        bool   `short:"j" long:"json" description:"Bool, output json" config:"json"`
-	OutputProbe string `short:"o" long:"probe" description:"String, output format" config:"output_probe"`
+	FileOutput  string `short:"O" long:"file-output" default:"json" description:"Bool, file output format" config:"file_output"`
+	OutputProbe string `short:"o" long:"probe" description:"String, output format" config:"output"`
 	Quiet       bool   `short:"q" long:"quiet" description:"Bool, Quiet" config:"quiet"`
 	NoColor     bool   `long:"no-color" description:"Bool, no color" config:"no-color"`
 	NoBar       bool   `long:"no-bar" description:"Bool, No progress bar" config:"no-bar"`
@@ -134,7 +135,7 @@ type ModeOptions struct {
 	UniqueStatus    string   `long:"unique-status" default:"403,200,404" description:"Strings (comma split), custom unique status" config:"unique-status"`
 	Unique          bool     `long:"unique" description:"Bool, unique response" config:"unique"`
 	RetryCount      int      `long:"retry" default:"0" description:"Int, retry count" config:"retry"`
-	SimhashDistance int      `long:"sim-distance" default:"5" config:"sim-distance"`
+	SimhashDistance int      `long:"sim-distance" default:"8" config:"sim-distance"`
 }
 
 type MiscOptions struct {
@@ -164,7 +165,6 @@ func (opt *Option) Validate() error {
 
 	if opt.Depth > 0 && opt.ResumeFrom != "" {
 		// 递归与断点续传会造成混淆, 断点续传的word与rule不是通过命令行获取的
-
 		return errors.New("--resume and --depth cannot be used at the same time")
 	}
 
@@ -250,6 +250,9 @@ func (opt *Option) Prepare() error {
 	} else {
 		pkg.UniqueStatus = pkg.ParseStatus(pkg.UniqueStatus, opt.UniqueStatus)
 	}
+
+	logs.Log.Logf(pkg.LogVerbose, "Black Status: %v, WhiteStatus: %v, WAFStatus: %v", pkg.BlackStatus, pkg.WhiteStatus, pkg.WAFStatus)
+	logs.Log.Logf(pkg.LogVerbose, "Fuzzy Status: %v, Unique Status: %v", pkg.FuzzyStatus, pkg.UniqueStatus)
 	pool.MaxCrawl = opt.CrawlDepth
 
 	return nil
@@ -403,18 +406,6 @@ func (opt *Option) NewRunner() (*Runner, error) {
 		}
 	}
 
-	//if opt.FuzzyFile != "" {
-	//	r.FuzzyFile, err = files.NewFile(opt.FuzzyFile, false, false, true)
-	//	if err != nil {
-	//		return nil, err
-	//	}
-	//} else if opt.AutoFile {
-	//	r.FuzzyFile, err = files.NewFile("fuzzy.json", false, false, true)
-	//	if err != nil {
-	//		return nil, err
-	//	}
-	//}
-
 	if opt.DumpFile != "" {
 		r.DumpFile, err = files.NewFile(opt.DumpFile, false, false, true)
 		if err != nil {
@@ -696,7 +687,7 @@ func (opt *Option) BuildWords(r *Runner) error {
 		})
 	}
 
-	logs.Log.Importantf("Loaded %d dictionaries, %d rules and %d decorators", len(opt.Dictionaries), len(opt.Rules), len(r.Fns))
+	logs.Log.Importantf("%s mod, Loaded %d dictionaries, %d rules and %d decorators", opt.Mod, len(opt.Dictionaries), len(opt.Rules), len(r.Fns))
 	return nil
 }
 

internal/pool/brutepool.go

@@ -29,6 +29,7 @@ var (
 	MaxRecursion    = 0
 	EnableAllFuzzy  = false
 	EnableAllUnique = false
+	//AllowHostModSource = []parsers.SpraySource{parsers.WordSource, parsers.CheckSource, parsers.InitIndexSource, parsers.InitRandomSource}
 )
 
 func NewBrutePool(ctx context.Context, config *Config) (*BrutePool, error) {
@@ -108,46 +109,30 @@ type BrutePool struct {
 	initwg      sync.WaitGroup // 初始化用, 之后改成锁
 }
 
-func (pool *BrutePool) checkRedirect(redirectURL string) bool {
-	if pool.random.RedirectURL == "" {
-		// 如果random的redirectURL为空, 此时该项
-		return true
-	}
-
-	if redirectURL == pool.random.RedirectURL {
-		// 相同的RedirectURL将被认为是无效数据
-		return false
-	} else {
-		// path为3xx, 且与baseline中的RedirectURL不同时, 为有效数据
-		return true
-	}
-}
-
-func (pool *BrutePool) genReq(mod SprayMod, s string) (*ihttp.Request, error) {
-	if mod == HostSpray {
-		return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s)
-	} else if mod == PathSpray {
-		return ihttp.BuildPathRequest(pool.ClientType, pool.base, s, pool.Method)
-	}
-	return nil, fmt.Errorf("unknown mod")
-}
-
 func (pool *BrutePool) Init() error {
 	pool.initwg.Add(2)
 	if pool.Index != "/" {
 		logs.Log.Logf(pkg.LogVerbose, "custom index url: %s", pkg.BaseURL(pool.url)+pkg.FormatURL(pkg.BaseURL(pool.url), pool.Index))
-		pool.reqPool.Invoke(newUnit(pool.Index, parsers.InitIndexSource))
+		pool.reqPool.Invoke(&Unit{path: pool.Index, source: parsers.InitIndexSource})
 		//pool.urls[dir(pool.Index)] = struct{}{}
 	} else {
-		pool.reqPool.Invoke(newUnit(pool.url.Path, parsers.InitIndexSource))
+		pool.reqPool.Invoke(&Unit{path: pool.url.Path, source: parsers.InitIndexSource})
 		//pool.urls[dir(pool.url.Path)] = struct{}{}
 	}
 
 	if pool.Random != "" {
 		logs.Log.Logf(pkg.LogVerbose, "custom random url: %s", pkg.BaseURL(pool.url)+pkg.FormatURL(pkg.BaseURL(pool.url), pool.Random))
-		pool.reqPool.Invoke(newUnit(pool.Random, parsers.InitRandomSource))
+		if pool.Mod == PathSpray {
+			pool.reqPool.Invoke(&Unit{path: pool.Random, source: parsers.InitRandomSource})
+		} else {
+			pool.reqPool.Invoke(&Unit{host: pool.Random, source: parsers.InitRandomSource})
+		}
 	} else {
-		pool.reqPool.Invoke(newUnit(pool.safePath(pkg.RandPath()), parsers.InitRandomSource))
+		if pool.Mod == PathSpray {
+			pool.reqPool.Invoke(&Unit{path: pool.safePath(pkg.RandPath()), source: parsers.InitRandomSource})
+		} else {
+			pool.reqPool.Invoke(&Unit{host: pkg.RandHost(), source: parsers.InitRandomSource})
+		}
 	}
 
 	pool.initwg.Wait()
@@ -182,22 +167,6 @@ func (pool *BrutePool) Init() error {
 	return nil
 }
 
-func (pool *BrutePool) Upgrade(bl *pkg.Baseline) error {
-	rurl, err := url.Parse(bl.RedirectURL)
-	if err == nil && rurl.Hostname() == bl.Url.Hostname() && bl.Url.Scheme == "http" && rurl.Scheme == "https" {
-		logs.Log.Infof("baseurl %s upgrade http to https, reinit", pool.BaseURL)
-		pool.base = strings.Replace(pool.BaseURL, "http", "https", 1)
-		pool.url.Scheme = "https"
-		// 重新初始化
-		err = pool.Init()
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func (pool *BrutePool) Run(ctx context.Context, offset, limit int) {
 	pool.Worder.Run()
 	if pool.Active {
@@ -254,18 +223,18 @@ Loop:
 
 			pool.wg.Add(1)
 			if pool.Mod == HostSpray {
-				pool.reqPool.Invoke(newUnitWithNumber(w, parsers.WordSource, pool.wordOffset))
+				pool.reqPool.Invoke(&Unit{host: w, source: parsers.WordSource, number: pool.wordOffset})
 			} else {
 				// 原样的目录拼接, 输入了几个"/"就是几个, 适配/有语义的中间件
-				pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(w), parsers.WordSource, pool.wordOffset))
+				pool.reqPool.Invoke(&Unit{path: pool.safePath(w), source: parsers.WordSource, number: pool.wordOffset})
 			}
 
 		case <-pool.checkCh:
 			pool.Statistor.CheckNumber++
 			if pool.Mod == HostSpray {
-				pool.reqPool.Invoke(newUnitWithNumber(pkg.RandHost(), parsers.CheckSource, pool.wordOffset))
+				pool.reqPool.Invoke(&Unit{host: pkg.RandHost(), source: parsers.CheckSource, number: pool.wordOffset})
 			} else if pool.Mod == PathSpray {
-				pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(pkg.RandPath()), parsers.CheckSource, pool.wordOffset))
+				pool.reqPool.Invoke(&Unit{path: pool.safePath(pkg.RandPath()), source: parsers.CheckSource, number: pool.wordOffset})
 			}
 		case unit, ok := <-pool.additionCh:
 			if !ok || pool.closed {
@@ -301,12 +270,8 @@ func (pool *BrutePool) Invoke(v interface{}) {
 
 	var req *ihttp.Request
 	var err error
-	if unit.source == parsers.WordSource {
-		req, err = pool.genReq(pool.Mod, unit.path)
-	} else {
-		req, err = pool.genReq(PathSpray, unit.path)
-	}
 
+	req, err = ihttp.BuildRequest(pool.ClientType, pool.BaseURL, unit.path, unit.host, pool.Method)
 	if err != nil {
 		logs.Log.Error(err.Error())
 		return
@@ -425,7 +390,7 @@ func (pool *BrutePool) Invoke(v interface{}) {
 func (pool *BrutePool) NoScopeInvoke(v interface{}) {
 	defer pool.wg.Done()
 	unit := v.(*Unit)
-	req, err := ihttp.BuildPathRequest(pool.ClientType, unit.path, "", pool.Method)
+	req, err := ihttp.BuildRequest(pool.ClientType, unit.path, "", "", "GET")
 	if err != nil {
 		logs.Log.Error(err.Error())
 		return
@@ -554,6 +519,7 @@ func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) {
 		for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) {
 			pool.addAddition(&Unit{
 				path:   pkg.Dir(bl.Url.Path) + u,
+				host:   bl.Host,
 				source: parsers.RuleSource,
 			})
 		}
@@ -572,6 +538,7 @@ func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) {
 		for _, u := range pool.AppendWords {
 			pool.addAddition(&Unit{
 				path:   pkg.SafePath(bl.Path, u),
+				host:   bl.Host,
 				source: parsers.AppendSource,
 			})
 		}
@@ -586,6 +553,9 @@ func (pool *BrutePool) doAppend(bl *pkg.Baseline) {
 
 func (pool *BrutePool) doActive() {
 	defer pool.wg.Done()
+	if pool.Mod == HostSpray {
+		return
+	}
 	for _, u := range pkg.ActivePath {
 		pool.addAddition(&Unit{
 			path:   pool.dir + u[1:],
@@ -596,6 +566,9 @@ func (pool *BrutePool) doActive() {
 
 func (pool *BrutePool) doCommonFile() {
 	defer pool.wg.Done()
+	if pool.Mod == HostSpray {
+		return
+	}
 	for _, u := range pkg.Dicts["common"] {
 		pool.addAddition(&Unit{
 			path:   pool.dir + u,
@@ -610,6 +583,37 @@ func (pool *BrutePool) doCommonFile() {
 	}
 }
 
+func (pool *BrutePool) checkRedirect(redirectURL string) bool {
+	if pool.random.RedirectURL == "" {
+		// 如果random的redirectURL为空, 此时该项
+		return true
+	}
+
+	if redirectURL == pool.random.RedirectURL {
+		// 相同的RedirectURL将被认为是无效数据
+		return false
+	} else {
+		// path为3xx, 且与baseline中的RedirectURL不同时, 为有效数据
+		return true
+	}
+}
+
+func (pool *BrutePool) Upgrade(bl *pkg.Baseline) error {
+	rurl, err := url.Parse(bl.RedirectURL)
+	if err == nil && rurl.Hostname() == bl.Url.Hostname() && bl.Url.Scheme == "http" && rurl.Scheme == "https" {
+		logs.Log.Infof("baseurl %s upgrade http to https, reinit", pool.BaseURL)
+		pool.base = strings.Replace(pool.BaseURL, "http", "https", 1)
+		pool.url.Scheme = "https"
+		// 重新初始化
+		err = pool.Init()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (pool *BrutePool) PreCompare(resp *ihttp.Response) error {
 	status := resp.StatusCode()
 	if iutils.IntsContains(pkg.WhiteStatus, status) {
@@ -771,6 +775,7 @@ func (pool *BrutePool) doCrawl(bl *pkg.Baseline) {
 			}
 			pool.addAddition(&Unit{
 				path:   u,
+				host:   bl.Host,
 				source: parsers.CrawlSource,
 				depth:  bl.ReqDepth + 1,
 			})
@@ -806,6 +811,9 @@ func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) {
 
 func (pool *BrutePool) doBak() {
 	defer pool.wg.Done()
+	if pool.Mod == HostSpray {
+		return
+	}
 	worder, err := words.NewWorderWithDsl("{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil)
 	if err != nil {
 		return

internal/pool/checkpool.go

@@ -107,7 +107,7 @@ func (pool *CheckPool) Invoke(v interface{}) {
 	}()
 
 	unit := v.(*Unit)
-	req, err := pool.genReq(unit.path)
+	req, err := ihttp.BuildRequest(pool.ClientType, unit.path, "", "", "GET")
 	if err != nil {
 		logs.Log.Debug(err.Error())
 		bl := &pkg.Baseline{

internal/pool/pool.go

@@ -2,7 +2,6 @@ package pool
 
 import (
 	"context"
-	"fmt"
 	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/spray/internal/ihttp"
 	"github.com/chainreactors/spray/pkg"
@@ -39,6 +38,7 @@ func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) {
 		defer pool.wg.Done()
 		pool.addAddition(&Unit{
 			path:     reURL,
+			host:     bl.Host,
 			source:   parsers.RedirectSource,
 			frontUrl: bl.UrlString,
 			depth:    depth + 1,
@@ -55,6 +55,7 @@ func (pool *BasePool) doRetry(bl *pkg.Baseline) {
 		defer pool.wg.Done()
 		pool.addAddition(&Unit{
 			path:   bl.Path,
+			host:   bl.Host,
 			source: parsers.RetrySource,
 			retry:  bl.Retry + 1,
 		})
@@ -75,15 +76,6 @@ func (pool *BasePool) Close() {
 	pool.Bar.Close()
 }
 
-func (pool *BasePool) genReq(s string) (*ihttp.Request, error) {
-	if pool.Mod == HostSpray {
-		return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s)
-	} else if pool.Mod == PathSpray {
-		return ihttp.BuildPathRequest(pool.ClientType, pool.BaseURL, s, pool.Method)
-	}
-	return nil, fmt.Errorf("unknown mod")
-}
-
 func (pool *BasePool) putToOutput(bl *pkg.Baseline) {
 	if bl.IsValid || bl.IsFuzzy {
 		bl.Collect()

internal/pool/types.go

@@ -9,12 +9,9 @@ func newUnit(path string, source parsers.SpraySource) *Unit {
 	return &Unit{path: path, source: source}
 }
 
-func newUnitWithNumber(path string, source parsers.SpraySource, number int) *Unit {
-	return &Unit{path: path, source: source, number: number}
-}
-
 type Unit struct {
 	number   int
+	host     string
 	path     string
 	source   parsers.SpraySource
 	retry    int

internal/runner.go

@@ -355,7 +355,7 @@ func (r *Runner) saveStat(content string) {
 func (r *Runner) Output(bl *pkg.Baseline) {
 	var out string
 	if r.Option.Json {
-		out = bl.Jsonify()
+		out = bl.ToJson()
 	} else if len(r.Probes) > 0 {
 		out = bl.Format(r.Probes)
 	} else if r.Color {
@@ -364,14 +364,20 @@ func (r *Runner) Output(bl *pkg.Baseline) {
 		out = bl.String()
 	}
 
-	if bl.IsFuzzy {
+	if bl.IsValid {
-		logs.Log.Console("[fuzzy] " + out + "\n")
-	} else {
 		logs.Log.Console(out + "\n")
+
+	} else if r.Fuzzy && bl.IsFuzzy {
+		logs.Log.Console("[fuzzy] " + out + "\n")
 	}
 
 	if r.OutputFile != nil {
-		r.OutputFile.SafeWrite(bl.Jsonify() + "\n")
+		if r.FileOutput == "json" {
+			r.OutputFile.SafeWrite(bl.ToJson() + "\n")
+		} else if r.FileOutput == "csv" {
+			r.OutputFile.SafeWrite(bl.ToCSV() + "\n")
+		}
+
 		r.OutputFile.SafeSync()
 	}
 }
@@ -385,7 +391,7 @@ func (r *Runner) OutputHandler() {
 					return
 				}
 				if r.DumpFile != nil {
-					r.DumpFile.SafeWrite(bl.Jsonify() + "\n")
+					r.DumpFile.SafeWrite(bl.ToJson() + "\n")
 					r.DumpFile.SafeSync()
 				}
 				if bl.IsValid {