Merge pull request #9 from chainreactors/dev

merge v0.9.1

.github/workflows/gorelease.yml

@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          token: ${{secrets.TOKEN}}
+          token: ${{ secrets.GITHUB_TOKEN }}
           submodules: recursive
       -
         name: Set up Go
@@ -30,6 +30,6 @@ jobs:
           version: latest
           args: release --rm-dist --skip-validate
         env:
-          GITHUB_TOKEN: ${{ secrets.TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GOPATH: "/home/runner/go"
 

README.md

@@ -47,6 +47,10 @@
 
 `spray --resume stat.json`
 
+被动url收集
+
+参见: https://github.com/chainreactors/urlfounder
+
 ## Wiki
 
 详细用法请见[wiki](https://chainreactors.github.io/wiki/spray/)

internal/checkpool.go

@@ -194,7 +194,7 @@ func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) {
 		}
 		reURL = bl.RedirectURL
 	} else {
-		reURL = bl.BaseURL() + FormatURL(bl.BaseURL(), bl.RedirectURL)
+		reURL = BaseURL(bl.Url) + FormatURL(BaseURL(bl.Url), bl.RedirectURL)
 	}
 
 	pool.wg.Add(1)

internal/option.go

@@ -105,6 +105,8 @@ type ModeOptions struct {
 	Scope           []string `long:"scope" description:"String, custom scope, e.g.: --scope *.example.com"`
 	Recursive       string   `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"`
 	Depth           int      `long:"depth" default:"0" description:"Int, recursive depth"`
+	Index           string   `long:"index" default:"" description:"String, custom index path"`
+	Random          string   `long:"random" default:"" description:"String, custom random path"`
 	CheckPeriod     int      `long:"check-period" default:"200" description:"Int, check period when request"`
 	ErrPeriod       int      `long:"error-period" default:"10" description:"Int, check period when error"`
 	BreakThreshold  int      `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold "`
@@ -164,6 +166,8 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		Common:          opt.Common,
 		RetryCount:      opt.RetryCount,
 		RandomUserAgent: opt.RandomUserAgent,
+		Random:          opt.Random,
+		Index:           opt.Index,
 	}
 
 	// log and bar
@@ -505,7 +509,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	logs.Log.Importantf("Loaded %d dictionaries and %d decorators", len(opt.Dictionaries), len(r.Fns))
 
 	if opt.Match != "" {
-		exp, err := expr.Compile(opt.Match)
+		exp, err := expr.Compile(opt.Match, expr.Patch(&bytesPatcher{}))
 		if err != nil {
 			return nil, err
 		}
@@ -513,7 +517,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	}
 
 	if opt.Filter != "" {
-		exp, err := expr.Compile(opt.Filter)
+		exp, err := expr.Compile(opt.Filter, expr.Patch(&bytesPatcher{}))
 		if err != nil {
 			return nil, err
 		}
@@ -535,7 +539,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	}
 
 	if express != "" {
-		exp, err := expr.Compile(express)
+		exp, err := expr.Compile(express, expr.Patch(&bytesPatcher{}))
 		if err != nil {
 			return nil, err
 		}

internal/pool.go

@@ -143,8 +143,20 @@ func (pool *Pool) genReq(mod pkg.SprayMod, s string) (*ihttp.Request, error) {
 func (pool *Pool) Init() error {
 	// 分成两步是为了避免闭包的线程安全问题
 	pool.initwg.Add(2)
+	if pool.Index != "" {
+		logs.Log.Importantf("custom index url: %s", BaseURL(pool.url)+FormatURL(BaseURL(pool.url), pool.Index))
+		pool.reqPool.Invoke(newUnit(pool.Index, InitIndexSource))
+	} else {
 		pool.reqPool.Invoke(newUnit(pool.url.Path, InitIndexSource))
+	}
+
+	if pool.Random != "" {
+		logs.Log.Importantf("custom random url: %s", BaseURL(pool.url)+FormatURL(BaseURL(pool.url), pool.Random))
+		pool.reqPool.Invoke(newUnit(pool.Random, InitRandomSource))
+	} else {
 		pool.reqPool.Invoke(newUnit(pool.safePath(pkg.RandPath()), InitRandomSource))
+	}
+
 	pool.initwg.Wait()
 	if pool.index.ErrString != "" {
 		logs.Log.Error(pool.index.String())

internal/runner.go

@@ -82,6 +82,8 @@ type Runner struct {
 	Common          bool
 	RetryCount      int
 	RandomUserAgent bool
+	Random          string
+	Index           string
 }
 
 func (r *Runner) PrepareConfig() *pkg.Config {
@@ -110,6 +112,8 @@ func (r *Runner) PrepareConfig() *pkg.Config {
 		Retry:           r.RetryCount,
 		ClientType:      r.ClientType,
 		RandomUserAgent: r.RandomUserAgent,
+		Random:          r.Random,
+		Index:           r.Index,
 	}
 
 	if config.ClientType == ihttp.Auto {

internal/utils.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"bytes"
 	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/ast"
 	"github.com/antonmedv/expr/vm"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
@@ -212,7 +213,6 @@ func Dir(u string) string {
 	// /a/ 	/a/
 	// a/ 	a/
 	// aaa 	/
-
 	if strings.HasSuffix(u, "/") {
 		return u
 	} else if i := strings.LastIndex(u, "/"); i == -1 {
@@ -250,6 +250,10 @@ func FormatURL(base, u string) string {
 	}
 }
 
+func BaseURL(u *url.URL) string {
+	return u.Scheme + "://" + u.Host
+}
+
 func RandomUA() string {
 	return randomUserAgent[rand.Intn(uacount)]
 }
@@ -276,3 +280,18 @@ func MatchWithGlobs(u string, globs []string) bool {
 	}
 	return false
 }
+
+type bytesPatcher struct{}
+
+func (p *bytesPatcher) Visit(node *ast.Node) {
+	switch (*node).(type) {
+	case *ast.MemberNode:
+		ast.Patch(node, &ast.CallNode{
+			Callee: &ast.MemberNode{
+				Node:     *node,
+				Name:     "String",
+				Property: &ast.StringNode{Value: "String"},
+			},
+		})
+	}
+}

pkg/baseline.go

@@ -105,9 +105,9 @@ type Baseline struct {
 	Url       *url.URL `json:"-"`
 	Dir       bool     `json:"-"`
 	Chunked   bool     `json:"-"`
-	Body      []byte   `json:"-"`
+	Body      BS       `json:"-"`
-	Header    []byte   `json:"-"`
+	Header    BS       `json:"-"`
-	Raw       []byte   `json:"-"`
+	Raw       BS       `json:"-"`
 	Recu      bool     `json:"-"`
 	RecuDepth int      `json:"-"`
 	URLs      []string `json:"-"`
@@ -122,10 +122,6 @@ func (bl *Baseline) IsDir() bool {
 	return false
 }
 
-func (bl *Baseline) BaseURL() string {
-	return bl.Url.Scheme + "://" + bl.Url.Host
-}
-
 // Collect 深度收集信息
 func (bl *Baseline) Collect() {
 	if bl.ContentType == "html" || bl.ContentType == "json" || bl.ContentType == "txt" {

pkg/config.go

@@ -47,4 +47,6 @@ type Config struct {
 	Common          bool
 	Retry           int
 	RandomUserAgent bool
+	Random          string
+	Index           string
 }

pkg/types.go

@@ -37,3 +37,9 @@ var ErrMap = map[ErrorType]string{
 func (e ErrorType) Error() string {
 	return ErrMap[e]
 }
+
+type BS []byte
+
+func (b BS) String() string {
+	return string(b)
+}