From c91f863afdeafc87008b6551fbf1d1d23d3323a6 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 8 Sep 2022 15:57:17 +0800 Subject: [PATCH] init --- .gitignore | 1 + cmd/spray.go | 24 ++++++ go.mod | 18 +++++ internal/baseline.go | 133 ++++++++++++++++++++++++++++++ internal/pool.go | 187 +++++++++++++++++++++++++++++++++++++++++++ internal/runner.go | 156 ++++++++++++++++++++++++++++++++++++ pkg/client.go | 64 +++++++++++++++ pkg/config.go | 34 ++++++++ pkg/requests.go | 15 ++++ pkg/utils.go | 59 ++++++++++++++ 10 files changed, 691 insertions(+) create mode 100644 cmd/spray.go create mode 100644 go.mod create mode 100644 internal/baseline.go create mode 100644 internal/pool.go create mode 100644 internal/runner.go create mode 100644 pkg/client.go create mode 100644 pkg/config.go create mode 100644 pkg/requests.go create mode 100644 pkg/utils.go diff --git a/.gitignore b/.gitignore index 66fd13c..139ae32 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ # Dependency directories (remove the comment below to include it) # vendor/ +bin/ \ No newline at end of file diff --git a/cmd/spray.go b/cmd/spray.go new file mode 100644 index 0000000..d6646c5 --- /dev/null +++ b/cmd/spray.go @@ -0,0 +1,24 @@ +package main + +import ( + "flag" + "github.com/chainreactors/logs" + "spray/internal" +) + +func main() { + var runner internal.Runner + flag.StringVar(&runner.URL, "u", "", "url") + flag.StringVar(&runner.URLFile, "U", "", "url filename") + flag.StringVar(&runner.WordFile, "w", "", "wordlist filename") + flag.StringVar(&runner.OutputFile, "f", "", "output filename") + flag.BoolVar(&runner.Debug, "debug", false, "print debug info") + flag.Parse() + + err := runner.Prepare() + if err != nil { + logs.Log.Errorf(err.Error()) + return + } + runner.Run() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6e149e3 --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module spray + +go 1.17 + +require ( + github.com/chainreactors/gogo/v2 v2.8.4 + github.com/chainreactors/logs v0.4.2 + github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c + github.com/panjf2000/ants/v2 v2.5.0 +) + +require ( + github.com/chainreactors/files v0.2.0 // indirect + github.com/chainreactors/ipcs v0.0.9 // indirect + github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect + github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect + github.com/twmb/murmur3 v1.1.6 // indirect +) diff --git a/internal/baseline.go b/internal/baseline.go new file mode 100644 index 0000000..9f07e37 --- /dev/null +++ b/internal/baseline.go @@ -0,0 +1,133 @@ +package internal + +import ( + "encoding/json" + "fmt" + gogoutil "github.com/chainreactors/gogo/v2/pkg" + "github.com/chainreactors/gogo/v2/pkg/dsl" + "github.com/chainreactors/logs" + "io" + "net/http" + "net/url" + "spray/pkg" + "strings" +) + +func NewBaseline(u *url.URL, resp *http.Response) *baseline { + bl := &baseline{ + Url: u, + UrlString: u.String(), + BodyLength: resp.ContentLength, + Status: resp.StatusCode, + IsValid: true, + } + + var header string + for k, v := range resp.Header { + // stringbuilder + for _, i := range v { + header += fmt.Sprintf("%s: %s\r\n", k, i) + } + } + bl.Header = header + bl.HeaderLength = len(header) + + redirectURL, err := resp.Location() + if err == nil { + bl.RedirectURL = redirectURL.String() + } + + body := make([]byte, 20480) + if bl.BodyLength > 0 { + n, err := io.ReadFull(resp.Body, body) + if err == nil { + bl.Body = body + } else if err == io.ErrUnexpectedEOF { + bl.Body = body[:n] + } else { + logs.Log.Error("readfull failed" + err.Error()) + } + _ = resp.Body.Close() + } + + if len(bl.Body) > 0 { + bl.Md5 = dsl.Md5Hash(bl.Body) + bl.Mmh3 = dsl.Mmh3Hash32(bl.Body) + bl.Simhash = pkg.Simhash(bl.Body) + if strings.Contains(string(bl.Body), bl.UrlString[1:]) { + bl.IsDynamicUrl = true + } + // todo callback + } + + // todo extract + + // todo 指纹识别 + return bl +} + +func NewInvalidBaseline(u *url.URL, resp *http.Response) *baseline { + bl := &baseline{ + Url: u, + UrlString: u.String(), + BodyLength: resp.ContentLength, + Status: resp.StatusCode, + IsValid: false, + } + + redirectURL, err := resp.Location() + if err == nil { + bl.RedirectURL = redirectURL.String() + } + + return bl +} + +type baseline struct { + Url *url.URL `json:"-"` + UrlString string `json:"url_string"` + Body []byte `json:"-"` + BodyLength int64 `json:"body_length"` + Header string `json:"-"` + HeaderLength int `json:"header_length"` + RedirectURL string `json:"redirect_url"` + Status int `json:"status"` + Md5 string `json:"md5"` + Mmh3 string `json:"mmh3"` + Simhash string `json:"simhash"` + IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url + Spended int `json:"spended"` // 耗时, 毫秒 + Frameworks gogoutil.Frameworks `json:"frameworks"` + + Err error `json:"-"` + IsValid bool `json:"-"` +} + +func (bl *baseline) Compare(other *baseline) bool { + if bl.Md5 == other.Md5 { + return true + } + + if bl.RedirectURL == other.RedirectURL { + return true + } + + return false +} + +func (bl *baseline) FuzzyCompare() bool { + // todo 模糊匹配 + return false +} + +func (bl *baseline) String() string { + return fmt.Sprintf("%s - %d - %d [%s]", bl.UrlString, bl.Status, bl.BodyLength, bl.Frameworks.ToString()) +} + +func (bl *baseline) Jsonify() string { + bs, err := json.Marshal(bl) + if err != nil { + return "" + } + return string(bs) +} diff --git a/internal/pool.go b/internal/pool.go new file mode 100644 index 0000000..af4f6a4 --- /dev/null +++ b/internal/pool.go @@ -0,0 +1,187 @@ +package internal + +import ( + "context" + "fmt" + "github.com/panjf2000/ants/v2" + "net/http" + "spray/pkg" + "sync" +) + +var ( + CheckStatusCode func(int) bool + CheckRedirect func(*http.Response) bool + CheckWaf func(*http.Response) bool +) + +func NewPool(config *pkg.Config, outputCh chan *baseline) (*Pool, error) { + var ctx context.Context + err := config.Init() + if err != nil { + return nil, fmt.Errorf("pool init failed, %w", err) + } + + //ctx, cancel := context.WithCancel(nil) + pool := &Pool{ + Config: config, + //ctx: ctx, + client: pkg.NewClient(config.Thread, 2), + //baseReq: req, + outputCh: outputCh, + wg: &sync.WaitGroup{}, + } + + switch config.Mod { + case pkg.PathSpray: + pool.genReq = func(s string) *http.Request { + return pkg.BuildPathRequest(s, *config.BaseReq) + } + case pkg.HostSpray: + pool.genReq = func(s string) *http.Request { + return pkg.BuildHostRequest(s, *config.BaseReq) + } + } + + p, _ := ants.NewPoolWithFunc(config.Thread, func(i interface{}) { + var bl *baseline + unit := i.(*Unit) + req := pool.genReq(unit.path) + resp, err := pool.client.Do(ctx, req) + if err != nil { + //logs.Log.Debugf("%s request error, %s", strurl, err.Error()) + pool.errorCount++ + bl = &baseline{Err: err} + } else { + if pool.PreCompare(resp) { + // 通过预对比跳过一些无用数据, 减少性能消耗 + bl = NewBaseline(req.URL, resp) + } else { + bl = NewInvalidBaseline(req.URL, resp) + } + } + + switch unit.source { + case InitSource: + pool.baseline = bl + case WordSource: + // todo compare + pool.outputCh <- bl + } + //todo connectivity check + pool.wg.Done() + }) + + pool.pool = p + + return pool, nil +} + +type Pool struct { + //url string + //thread int + *pkg.Config + client *pkg.Client + pool *ants.PoolWithFunc + //ctx context.Context + //baseReq *http.Request + baseline *baseline + outputCh chan *baseline + totalCount int + errorCount int + genReq func(string) *http.Request + //wordlist []string + wg *sync.WaitGroup +} + +func (p *Pool) Add(u *Unit) error { + p.wg.Add(1) + _ = p.pool.Invoke(u) + p.wg.Wait() + + if p.baseline.Err != nil { + return p.baseline.Err + } + return nil +} + +func (p *Pool) Init() error { + //for i := 0; i < p.baseReqCount; i++ { + _ = p.Add(newUnit(pkg.RandPath(), InitSource)) + //} + + // todo 分析baseline + // 检测基本访问能力 + + if p.baseline != nil && p.baseline.Err != nil { + return p.baseline.Err + } + + if p.baseline.RedirectURL != "" { + CheckRedirect = func(resp *http.Response) bool { + redirectURL, err := resp.Location() + if err != nil { + // baseline 为3xx, 但path不为3xx时, 为有效数据 + return true + } else if redirectURL.String() != p.baseline.RedirectURL { + // path为3xx, 且与baseline中的RedirectURL不同时, 为有效数据 + return true + } else { + // 相同的RedirectURL将被认为是无效数据 + return false + } + } + } + + return nil +} + +func (p *Pool) Run() { + for _, u := range p.Wordlist { + p.totalCount++ + _ = p.Add(newUnit(u, WordSource)) + } + p.wg.Wait() +} + +func (p *Pool) PreCompare(resp *http.Response) bool { + if !CheckStatusCode(resp.StatusCode) { + return false + } + + if CheckRedirect != nil && !CheckRedirect(resp) { + return false + } + + if CheckWaf != nil && !CheckWaf(resp) { + return false + } + + return true +} + +func (p *Pool) RunWithWord(words []string) { + +} + +type sourceType int + +const ( + InitSource sourceType = iota + 1 + WordSource + WafSource +) + +//var sourceMap = map[int]string{ +// +//} + +func newUnit(path string, source sourceType) *Unit { + return &Unit{path: path, source: source} +} + +type Unit struct { + path string + source sourceType + //callback func() +} diff --git a/internal/runner.go b/internal/runner.go new file mode 100644 index 0000000..4729adf --- /dev/null +++ b/internal/runner.go @@ -0,0 +1,156 @@ +package internal + +import ( + "fmt" + "github.com/chainreactors/logs" + "io/ioutil" + "net/http" + "os" + "spray/pkg" + "strings" + "sync" +) + +var BlackStatus = []int{404, 410} +var FuzzyStatus = []int{403, 500, 501, 503} + +type Runner struct { + URL string + URLFile string + URLList []string + WordFile string + Wordlist []string + Headers http.Header + OutputFile string + Offset int + Limit int + Threads int + PoolSize int + Pools map[string]*Pool + Deadline int // todo 总的超时时间,适配云函数的deadline + Debug bool + Mod string + OutputCh chan *baseline +} + +func (r *Runner) Prepare() error { + if r.Debug { + logs.Log.Level = logs.Debug + } + var file *os.File + var err error + urlfrom := r.URLFile + if r.URL != "" { + r.URLList = append(r.URLList, r.URL) + urlfrom = "cmd" + } else if r.URLFile != "" { + file, err = os.Open(r.URLFile) + if err != nil { + return err + } + } else if pkg.HasStdin() { + file = os.Stdin + urlfrom = "stdin" + } + + if file != nil { + content, err := ioutil.ReadAll(file) + if err != nil { + return err + } + r.URLList = strings.Split(string(content), "\n") + } + + // todo url formatter + for i, u := range r.URLList { + r.URLList[i] = strings.TrimSpace(u) + } + logs.Log.Importantf("load %d urls from %s", len(r.URLList), urlfrom) + + if r.WordFile != "" { + content, err := ioutil.ReadFile(r.WordFile) + if err != nil { + return err + } + r.Wordlist = strings.Split(string(content), "\n") + } else { + return fmt.Errorf("not special wordlist") + } + + if r.Wordlist != nil && len(r.Wordlist) > 0 { + // todo suffix/prefix/trim/generator + for i, word := range r.Wordlist { + r.Wordlist[i] = strings.TrimSpace(word) + } + logs.Log.Importantf("load %d word from %s", len(r.Wordlist), r.WordFile) + } else { + return fmt.Errorf("no wordlist") + } + + CheckStatusCode = func(status int) bool { + for _, black := range BlackStatus { + if black == status { + return false + } + } + return true + } + + r.OutputCh = make(chan *baseline, 100) + r.Pools = make(map[string]*Pool) + go r.Outputting() + return nil +} + +func (r *Runner) Run() { + // todo pool 结束与并发控制 + var wg sync.WaitGroup + for _, u := range r.URLList { + wg.Add(1) + u := u + go func() { + config := &pkg.Config{ + BaseURL: u, + Wordlist: r.Wordlist, + Thread: r.Threads, + Timeout: 2, + Headers: r.Headers, + } + pool, err := NewPool(config, r.OutputCh) + if err != nil { + logs.Log.Error(err.Error()) + return + } + + err = pool.Init() + if err != nil { + logs.Log.Error(err.Error()) + return + } + r.Pools[u] = pool + // todo pool 总超时时间 + pool.Run() + wg.Done() + }() + } + wg.Wait() + for { + if len(r.OutputCh) == 0 { + close(r.OutputCh) + return + } + } +} + +func (r *Runner) Outputting() { + for { + select { + case bl := <-r.OutputCh: + if bl.IsValid { + logs.Log.Console(bl.String() + "\n") + } else { + logs.Log.Debug(bl.String()) + } + } + } +} diff --git a/pkg/client.go b/pkg/client.go new file mode 100644 index 0000000..f6a0a10 --- /dev/null +++ b/pkg/client.go @@ -0,0 +1,64 @@ +package pkg + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "time" +) + +func NewClient(thread int, timeout int) *Client { + tr := &http.Transport{ + //Proxy: Proxy, + //TLSHandshakeTimeout : delay * time.Second, + TLSClientConfig: &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + }, + DialContext: (&net.Dialer{ + //Timeout: time.Duration(delay) * time.Second, + //KeepAlive: time.Duration(delay) * time.Second, + //DualStack: true, + }).DialContext, + MaxIdleConnsPerHost: thread, + MaxIdleConns: thread, + IdleConnTimeout: time.Duration(timeout) * time.Second, + DisableKeepAlives: false, + } + + c := &Client{ + client: &http.Client{ + Transport: tr, + Timeout: time.Second * time.Duration(timeout), + CheckRedirect: checkRedirect, + }, + } + + //c.Method = method + //c.Headers = Opt.Headers + //c.Mod = Opt.Mod + + return c +} + +type Client struct { + client *http.Client +} + +func (c Client) Do(ctx context.Context, req *http.Request) (*http.Response, error) { + //if req.Header == nil { + // req.Header = c.Headers + //} + + return c.client.Do(req) +} + +var MaxRedirects = 0 +var checkRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) > MaxRedirects { + return http.ErrUseLastResponse + } + + return nil +} diff --git a/pkg/config.go b/pkg/config.go new file mode 100644 index 0000000..fccadcc --- /dev/null +++ b/pkg/config.go @@ -0,0 +1,34 @@ +package pkg + +import ( + "net/http" +) + +type SprayMod int + +const ( + PathSpray SprayMod = iota + 1 + HostSpray + ParamSpray + CustomSpray +) + +type Config struct { + BaseURL string + Wordlist []string + Thread int + Timeout int + BaseReq *http.Request + Method string + Mod SprayMod + Headers http.Header +} + +func (c *Config) Init() (err error) { + c.BaseReq, err = http.NewRequest(c.Method, c.BaseURL, nil) + if err != nil { + return err + } + c.BaseReq.Header = c.Headers + return nil +} diff --git a/pkg/requests.go b/pkg/requests.go new file mode 100644 index 0000000..18454a7 --- /dev/null +++ b/pkg/requests.go @@ -0,0 +1,15 @@ +package pkg + +import ( + "net/http" +) + +func BuildPathRequest(path string, req http.Request) *http.Request { + req.URL.Path = path + return &req +} + +func BuildHostRequest(u string, req http.Request) *http.Request { + req.Host = u + return &req +} diff --git a/pkg/utils.go b/pkg/utils.go new file mode 100644 index 0000000..2879c1b --- /dev/null +++ b/pkg/utils.go @@ -0,0 +1,59 @@ +package pkg + +import ( + "fmt" + "github.com/go-dedup/simhash" + "math/rand" + "os" + "time" + "unsafe" +) + +func HasStdin() bool { + stat, err := os.Stdin.Stat() + if err != nil { + return false + } + + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 + isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 + + return isPipedFromChrDev || isPipedFromFIFO +} + +func Simhash(raw []byte) string { + + sh := simhash.NewSimhash() + return fmt.Sprintf("%x", sh.GetSimhash(sh.NewWordFeatureSet(raw))) +} + +const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +var src = rand.NewSource(time.Now().UnixNano()) + +const ( + // 6 bits to represent a letter index + letterIdBits = 6 + // All 1-bits as many as letterIdBits + letterIdMask = 1<= 1; { + if remain == 0 { + cache, remain = src.Int63(), letterIdMax + } + if idx := int(cache & letterIdMask); idx < len(letters) { + b[i] = letters[idx] + i-- + } + cache >>= letterIdBits + remain-- + } + return *(*string)(unsafe.Pointer(&b)) +}