init

.gitignore

@@ -13,3 +13,4 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+bin/

cmd/spray.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"flag"
+	"github.com/chainreactors/logs"
+	"spray/internal"
+)
+
+func main() {
+	var runner internal.Runner
+	flag.StringVar(&runner.URL, "u", "", "url")
+	flag.StringVar(&runner.URLFile, "U", "", "url filename")
+	flag.StringVar(&runner.WordFile, "w", "", "wordlist filename")
+	flag.StringVar(&runner.OutputFile, "f", "", "output filename")
+	flag.BoolVar(&runner.Debug, "debug", false, "print debug info")
+	flag.Parse()
+
+	err := runner.Prepare()
+	if err != nil {
+		logs.Log.Errorf(err.Error())
+		return
+	}
+	runner.Run()
+}

go.mod

@@ -0,0 +1,18 @@
+module spray
+
+go 1.17
+
+require (
+	github.com/chainreactors/gogo/v2 v2.8.4
+	github.com/chainreactors/logs v0.4.2
+	github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c
+	github.com/panjf2000/ants/v2 v2.5.0
+)
+
+require (
+	github.com/chainreactors/files v0.2.0 // indirect
+	github.com/chainreactors/ipcs v0.0.9 // indirect
+	github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect
+	github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect
+	github.com/twmb/murmur3 v1.1.6 // indirect
+)

internal/baseline.go

@@ -0,0 +1,133 @@
+package internal
+
+import (
+	"encoding/json"
+	"fmt"
+	gogoutil "github.com/chainreactors/gogo/v2/pkg"
+	"github.com/chainreactors/gogo/v2/pkg/dsl"
+	"github.com/chainreactors/logs"
+	"io"
+	"net/http"
+	"net/url"
+	"spray/pkg"
+	"strings"
+)
+
+func NewBaseline(u *url.URL, resp *http.Response) *baseline {
+	bl := &baseline{
+		Url:        u,
+		UrlString:  u.String(),
+		BodyLength: resp.ContentLength,
+		Status:     resp.StatusCode,
+		IsValid:    true,
+	}
+
+	var header string
+	for k, v := range resp.Header {
+		// stringbuilder
+		for _, i := range v {
+			header += fmt.Sprintf("%s: %s\r\n", k, i)
+		}
+	}
+	bl.Header = header
+	bl.HeaderLength = len(header)
+
+	redirectURL, err := resp.Location()
+	if err == nil {
+		bl.RedirectURL = redirectURL.String()
+	}
+
+	body := make([]byte, 20480)
+	if bl.BodyLength > 0 {
+		n, err := io.ReadFull(resp.Body, body)
+		if err == nil {
+			bl.Body = body
+		} else if err == io.ErrUnexpectedEOF {
+			bl.Body = body[:n]
+		} else {
+			logs.Log.Error("readfull failed" + err.Error())
+		}
+		_ = resp.Body.Close()
+	}
+
+	if len(bl.Body) > 0 {
+		bl.Md5 = dsl.Md5Hash(bl.Body)
+		bl.Mmh3 = dsl.Mmh3Hash32(bl.Body)
+		bl.Simhash = pkg.Simhash(bl.Body)
+		if strings.Contains(string(bl.Body), bl.UrlString[1:]) {
+			bl.IsDynamicUrl = true
+		}
+		// todo callback
+	}
+
+	// todo extract
+
+	// todo 指纹识别
+	return bl
+}
+
+func NewInvalidBaseline(u *url.URL, resp *http.Response) *baseline {
+	bl := &baseline{
+		Url:        u,
+		UrlString:  u.String(),
+		BodyLength: resp.ContentLength,
+		Status:     resp.StatusCode,
+		IsValid:    false,
+	}
+
+	redirectURL, err := resp.Location()
+	if err == nil {
+		bl.RedirectURL = redirectURL.String()
+	}
+
+	return bl
+}
+
+type baseline struct {
+	Url          *url.URL            `json:"-"`
+	UrlString    string              `json:"url_string"`
+	Body         []byte              `json:"-"`
+	BodyLength   int64               `json:"body_length"`
+	Header       string              `json:"-"`
+	HeaderLength int                 `json:"header_length"`
+	RedirectURL  string              `json:"redirect_url"`
+	Status       int                 `json:"status"`
+	Md5          string              `json:"md5"`
+	Mmh3         string              `json:"mmh3"`
+	Simhash      string              `json:"simhash"`
+	IsDynamicUrl bool                `json:"is_dynamic_url"` // 判断是否存在动态的url
+	Spended      int                 `json:"spended"`        // 耗时, 毫秒
+	Frameworks   gogoutil.Frameworks `json:"frameworks"`
+
+	Err     error `json:"-"`
+	IsValid bool  `json:"-"`
+}
+
+func (bl *baseline) Compare(other *baseline) bool {
+	if bl.Md5 == other.Md5 {
+		return true
+	}
+
+	if bl.RedirectURL == other.RedirectURL {
+		return true
+	}
+
+	return false
+}
+
+func (bl *baseline) FuzzyCompare() bool {
+	// todo 模糊匹配
+	return false
+}
+
+func (bl *baseline) String() string {
+	return fmt.Sprintf("%s - %d - %d [%s]", bl.UrlString, bl.Status, bl.BodyLength, bl.Frameworks.ToString())
+}
+
+func (bl *baseline) Jsonify() string {
+	bs, err := json.Marshal(bl)
+	if err != nil {
+		return ""
+	}
+	return string(bs)
+}

internal/pool.go

@@ -0,0 +1,187 @@
+package internal
+
+import (
+	"context"
+	"fmt"
+	"github.com/panjf2000/ants/v2"
+	"net/http"
+	"spray/pkg"
+	"sync"
+)
+
+var (
+	CheckStatusCode func(int) bool
+	CheckRedirect   func(*http.Response) bool
+	CheckWaf        func(*http.Response) bool
+)
+
+func NewPool(config *pkg.Config, outputCh chan *baseline) (*Pool, error) {
+	var ctx context.Context
+	err := config.Init()
+	if err != nil {
+		return nil, fmt.Errorf("pool init failed, %w", err)
+	}
+
+	//ctx, cancel := context.WithCancel(nil)
+	pool := &Pool{
+		Config: config,
+		//ctx:      ctx,
+		client: pkg.NewClient(config.Thread, 2),
+		//baseReq:  req,
+		outputCh: outputCh,
+		wg:       &sync.WaitGroup{},
+	}
+
+	switch config.Mod {
+	case pkg.PathSpray:
+		pool.genReq = func(s string) *http.Request {
+			return pkg.BuildPathRequest(s, *config.BaseReq)
+		}
+	case pkg.HostSpray:
+		pool.genReq = func(s string) *http.Request {
+			return pkg.BuildHostRequest(s, *config.BaseReq)
+		}
+	}
+
+	p, _ := ants.NewPoolWithFunc(config.Thread, func(i interface{}) {
+		var bl *baseline
+		unit := i.(*Unit)
+		req := pool.genReq(unit.path)
+		resp, err := pool.client.Do(ctx, req)
+		if err != nil {
+			//logs.Log.Debugf("%s request error, %s", strurl, err.Error())
+			pool.errorCount++
+			bl = &baseline{Err: err}
+		} else {
+			if pool.PreCompare(resp) {
+				// 通过预对比跳过一些无用数据, 减少性能消耗
+				bl = NewBaseline(req.URL, resp)
+			} else {
+				bl = NewInvalidBaseline(req.URL, resp)
+			}
+		}
+
+		switch unit.source {
+		case InitSource:
+			pool.baseline = bl
+		case WordSource:
+			// todo compare
+			pool.outputCh <- bl
+		}
+		//todo connectivity check
+		pool.wg.Done()
+	})
+
+	pool.pool = p
+
+	return pool, nil
+}
+
+type Pool struct {
+	//url    string
+	//thread int
+	*pkg.Config
+	client *pkg.Client
+	pool   *ants.PoolWithFunc
+	//ctx          context.Context
+	//baseReq      *http.Request
+	baseline   *baseline
+	outputCh   chan *baseline
+	totalCount int
+	errorCount int
+	genReq     func(string) *http.Request
+	//wordlist     []string
+	wg *sync.WaitGroup
+}
+
+func (p *Pool) Add(u *Unit) error {
+	p.wg.Add(1)
+	_ = p.pool.Invoke(u)
+	p.wg.Wait()
+
+	if p.baseline.Err != nil {
+		return p.baseline.Err
+	}
+	return nil
+}
+
+func (p *Pool) Init() error {
+	//for i := 0; i < p.baseReqCount; i++ {
+	_ = p.Add(newUnit(pkg.RandPath(), InitSource))
+	//}
+
+	// todo 分析baseline
+	// 检测基本访问能力
+
+	if p.baseline != nil && p.baseline.Err != nil {
+		return p.baseline.Err
+	}
+
+	if p.baseline.RedirectURL != "" {
+		CheckRedirect = func(resp *http.Response) bool {
+			redirectURL, err := resp.Location()
+			if err != nil {
+				// baseline 为3xx, 但path不为3xx时, 为有效数据
+				return true
+			} else if redirectURL.String() != p.baseline.RedirectURL {
+				// path为3xx, 且与baseline中的RedirectURL不同时, 为有效数据
+				return true
+			} else {
+				// 相同的RedirectURL将被认为是无效数据
+				return false
+			}
+		}
+	}
+
+	return nil
+}
+
+func (p *Pool) Run() {
+	for _, u := range p.Wordlist {
+		p.totalCount++
+		_ = p.Add(newUnit(u, WordSource))
+	}
+	p.wg.Wait()
+}
+
+func (p *Pool) PreCompare(resp *http.Response) bool {
+	if !CheckStatusCode(resp.StatusCode) {
+		return false
+	}
+
+	if CheckRedirect != nil && !CheckRedirect(resp) {
+		return false
+	}
+
+	if CheckWaf != nil && !CheckWaf(resp) {
+		return false
+	}
+
+	return true
+}
+
+func (p *Pool) RunWithWord(words []string) {
+
+}
+
+type sourceType int
+
+const (
+	InitSource sourceType = iota + 1
+	WordSource
+	WafSource
+)
+
+//var sourceMap = map[int]string{
+//
+//}
+
+func newUnit(path string, source sourceType) *Unit {
+	return &Unit{path: path, source: source}
+}
+
+type Unit struct {
+	path   string
+	source sourceType
+	//callback func()
+}

internal/runner.go

@@ -0,0 +1,156 @@
+package internal
+
+import (
+	"fmt"
+	"github.com/chainreactors/logs"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"spray/pkg"
+	"strings"
+	"sync"
+)
+
+var BlackStatus = []int{404, 410}
+var FuzzyStatus = []int{403, 500, 501, 503}
+
+type Runner struct {
+	URL        string
+	URLFile    string
+	URLList    []string
+	WordFile   string
+	Wordlist   []string
+	Headers    http.Header
+	OutputFile string
+	Offset     int
+	Limit      int
+	Threads    int
+	PoolSize   int
+	Pools      map[string]*Pool
+	Deadline   int // todo 总的超时时间,适配云函数的deadline
+	Debug      bool
+	Mod        string
+	OutputCh   chan *baseline
+}
+
+func (r *Runner) Prepare() error {
+	if r.Debug {
+		logs.Log.Level = logs.Debug
+	}
+	var file *os.File
+	var err error
+	urlfrom := r.URLFile
+	if r.URL != "" {
+		r.URLList = append(r.URLList, r.URL)
+		urlfrom = "cmd"
+	} else if r.URLFile != "" {
+		file, err = os.Open(r.URLFile)
+		if err != nil {
+			return err
+		}
+	} else if pkg.HasStdin() {
+		file = os.Stdin
+		urlfrom = "stdin"
+	}
+
+	if file != nil {
+		content, err := ioutil.ReadAll(file)
+		if err != nil {
+			return err
+		}
+		r.URLList = strings.Split(string(content), "\n")
+	}
+
+	// todo url formatter
+	for i, u := range r.URLList {
+		r.URLList[i] = strings.TrimSpace(u)
+	}
+	logs.Log.Importantf("load %d urls from %s", len(r.URLList), urlfrom)
+
+	if r.WordFile != "" {
+		content, err := ioutil.ReadFile(r.WordFile)
+		if err != nil {
+			return err
+		}
+		r.Wordlist = strings.Split(string(content), "\n")
+	} else {
+		return fmt.Errorf("not special wordlist")
+	}
+
+	if r.Wordlist != nil && len(r.Wordlist) > 0 {
+		// todo  suffix/prefix/trim/generator
+		for i, word := range r.Wordlist {
+			r.Wordlist[i] = strings.TrimSpace(word)
+		}
+		logs.Log.Importantf("load %d word from %s", len(r.Wordlist), r.WordFile)
+	} else {
+		return fmt.Errorf("no wordlist")
+	}
+
+	CheckStatusCode = func(status int) bool {
+		for _, black := range BlackStatus {
+			if black == status {
+				return false
+			}
+		}
+		return true
+	}
+
+	r.OutputCh = make(chan *baseline, 100)
+	r.Pools = make(map[string]*Pool)
+	go r.Outputting()
+	return nil
+}
+
+func (r *Runner) Run() {
+	// todo pool 结束与并发控制
+	var wg sync.WaitGroup
+	for _, u := range r.URLList {
+		wg.Add(1)
+		u := u
+		go func() {
+			config := &pkg.Config{
+				BaseURL:  u,
+				Wordlist: r.Wordlist,
+				Thread:   r.Threads,
+				Timeout:  2,
+				Headers:  r.Headers,
+			}
+			pool, err := NewPool(config, r.OutputCh)
+			if err != nil {
+				logs.Log.Error(err.Error())
+				return
+			}
+
+			err = pool.Init()
+			if err != nil {
+				logs.Log.Error(err.Error())
+				return
+			}
+			r.Pools[u] = pool
+			// todo pool 总超时时间
+			pool.Run()
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	for {
+		if len(r.OutputCh) == 0 {
+			close(r.OutputCh)
+			return
+		}
+	}
+}
+
+func (r *Runner) Outputting() {
+	for {
+		select {
+		case bl := <-r.OutputCh:
+			if bl.IsValid {
+				logs.Log.Console(bl.String() + "\n")
+			} else {
+				logs.Log.Debug(bl.String())
+			}
+		}
+	}
+}

pkg/client.go

@@ -0,0 +1,64 @@
+package pkg
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"time"
+)
+
+func NewClient(thread int, timeout int) *Client {
+	tr := &http.Transport{
+		//Proxy: Proxy,
+		//TLSHandshakeTimeout : delay * time.Second,
+		TLSClientConfig: &tls.Config{
+			Renegotiation:      tls.RenegotiateOnceAsClient,
+			InsecureSkipVerify: true,
+		},
+		DialContext: (&net.Dialer{
+			//Timeout:   time.Duration(delay) * time.Second,
+			//KeepAlive: time.Duration(delay) * time.Second,
+			//DualStack: true,
+		}).DialContext,
+		MaxIdleConnsPerHost: thread,
+		MaxIdleConns:        thread,
+		IdleConnTimeout:     time.Duration(timeout) * time.Second,
+		DisableKeepAlives:   false,
+	}
+
+	c := &Client{
+		client: &http.Client{
+			Transport:     tr,
+			Timeout:       time.Second * time.Duration(timeout),
+			CheckRedirect: checkRedirect,
+		},
+	}
+
+	//c.Method = method
+	//c.Headers = Opt.Headers
+	//c.Mod = Opt.Mod
+
+	return c
+}
+
+type Client struct {
+	client *http.Client
+}
+
+func (c Client) Do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	//if req.Header == nil {
+	//	req.Header = c.Headers
+	//}
+
+	return c.client.Do(req)
+}
+
+var MaxRedirects = 0
+var checkRedirect = func(req *http.Request, via []*http.Request) error {
+	if len(via) > MaxRedirects {
+		return http.ErrUseLastResponse
+	}
+
+	return nil
+}

pkg/config.go

@@ -0,0 +1,34 @@
+package pkg
+
+import (
+	"net/http"
+)
+
+type SprayMod int
+
+const (
+	PathSpray SprayMod = iota + 1
+	HostSpray
+	ParamSpray
+	CustomSpray
+)
+
+type Config struct {
+	BaseURL  string
+	Wordlist []string
+	Thread   int
+	Timeout  int
+	BaseReq  *http.Request
+	Method   string
+	Mod      SprayMod
+	Headers  http.Header
+}
+
+func (c *Config) Init() (err error) {
+	c.BaseReq, err = http.NewRequest(c.Method, c.BaseURL, nil)
+	if err != nil {
+		return err
+	}
+	c.BaseReq.Header = c.Headers
+	return nil
+}

pkg/requests.go

@@ -0,0 +1,15 @@
+package pkg
+
+import (
+	"net/http"
+)
+
+func BuildPathRequest(path string, req http.Request) *http.Request {
+	req.URL.Path = path
+	return &req
+}
+
+func BuildHostRequest(u string, req http.Request) *http.Request {
+	req.Host = u
+	return &req
+}

pkg/utils.go

@@ -0,0 +1,59 @@
+package pkg
+
+import (
+	"fmt"
+	"github.com/go-dedup/simhash"
+	"math/rand"
+	"os"
+	"time"
+	"unsafe"
+)
+
+func HasStdin() bool {
+	stat, err := os.Stdin.Stat()
+	if err != nil {
+		return false
+	}
+
+	isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0
+	isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0
+
+	return isPipedFromChrDev || isPipedFromFIFO
+}
+
+func Simhash(raw []byte) string {
+
+	sh := simhash.NewSimhash()
+	return fmt.Sprintf("%x", sh.GetSimhash(sh.NewWordFeatureSet(raw)))
+}
+
+const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+var src = rand.NewSource(time.Now().UnixNano())
+
+const (
+	// 6 bits to represent a letter index
+	letterIdBits = 6
+	// All 1-bits as many as letterIdBits
+	letterIdMask = 1<<letterIdBits - 1
+	letterIdMax  = 63 / letterIdBits
+)
+
+func RandPath() string {
+	n := 16
+	b := make([]byte, n)
+	b[0] = byte(0x2f)
+	// A rand.Int63() generates 63 random bits, enough for letterIdMax letters!
+	for i, cache, remain := n-1, src.Int63(), letterIdMax; i >= 1; {
+		if remain == 0 {
+			cache, remain = src.Int63(), letterIdMax
+		}
+		if idx := int(cache & letterIdMask); idx < len(letters) {
+			b[i] = letters[idx]
+			i--
+		}
+		cache >>= letterIdBits
+		remain--
+	}
+	return *(*string)(unsafe.Pointer(&b))
+}