admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-06-22 02:40:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

实装extractor

Browse Source 
多处小优化
This commit is contained in:
M09Ic 2022-12-02 15:21:17 +08:00
parent 288d3e12c2
commit 38e654913d
8 changed files with 43 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -18,7 +18,7 @@ require (
github.com/gosuri/uiprogress v0.0.1
github.com/jessevdk/go-flags v1.5.0
github.com/panjf2000/ants/v2 v2.6.0
github.com/valyala/fasthttp v1.40.0
github.com/valyala/fasthttp v1.43.0
)
require (

11
go.sum
View File

@ -41,7 +41,7 @@ github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJS
github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo=
github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
@ -68,8 +68,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc=
github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g=
github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -77,7 +77,7 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -86,9 +86,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

11
internal/option.go
View File

@ -4,12 +4,14 @@ import (
"fmt"
"github.com/antonmedv/expr"
"github.com/chainreactors/files"
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/words/mask"
"github.com/gosuri/uiprogress"
"io/ioutil"
"os"
"regexp"
"strconv"
"strings"
)
@ -109,6 +111,15 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
return nil, err
}
if opt.Extracts != nil {
for _, e := range opt.Extracts {
if reg, ok := fingers.PresetExtracts[e]; ok {
pkg.Extractors[e] = reg
} else {
pkg.Extractors[e] = regexp.MustCompile(e)
}
}
}
// 一些全局变量初始化
if opt.Debug {
logs.Log.Level = logs.Debug

5
internal/pool.go
View File

@ -78,6 +78,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
req, err := pool.genReq(unit.path)
if err != nil {
logs.Log.Error(err.Error())
return
}
var bl *pkg.Baseline
@ -121,11 +122,9 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
pool.base = bl
pool.addFuzzyBaseline(bl)
pool.initwg.Done()
return
case InitIndexSource:
pool.index = bl
pool.initwg.Done()
return
case CheckSource:
if bl.ErrString != "" {
logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, pool.BreakThreshold, bl.ErrString)
@ -228,8 +227,8 @@ type Pool struct {
func (p *Pool) Init() error {
p.initwg.Add(2)
p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource))
p.pool.Invoke(newUnit("/", InitIndexSource))
p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource))
p.initwg.Wait()
// todo 分析baseline
// 检测基本访问能力

4
internal/types.go
View File

@ -17,9 +17,9 @@ const (
func (e ErrorType) Error() string {
switch e {
case ErrBadStatus:
return "bad status"
return "blacklist status"
case ErrSameStatus:
return "same status"
return "same status with random baseline"
case ErrRequestFailed:
return "request failed"
case ErrWaf:

15
pkg/baseline.go
View File

@ -25,7 +25,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
}
bl.Body = resp.Body()
bl.BodyLength = len(resp.Body())
bl.BodyLength = len(bl.Body)
bl.Header = resp.Header()
bl.HeaderLength = len(bl.Header)
bl.RedirectURL = resp.GetHeader("Location")
@ -53,6 +53,8 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
bl.Body = resp.Body()
bl.BodyLength = len(bl.Body)
bl.Header = resp.Header()
bl.HeaderLength = len(bl.Header)
bl.RedirectURL = string(resp.GetHeader("Location"))
return bl
@ -87,7 +89,7 @@ func (bl *Baseline) Collect() {
if len(bl.Body) > 0 {
bl.Title = parsers.MatchTitle(string(bl.Body))
}
bl.Hashes = parsers.NewHashes(bl.Raw)
bl.Hashes = parsers.NewHashes(bl.Body)
// todo extract
bl.Extracteds = Extractors.Extract(string(bl.Raw))
bl.Frameworks = FingerDetect(string(bl.Raw))
@ -164,10 +166,10 @@ func (bl *Baseline) Get(key string) string {
return strconv.Itoa(bl.Status)
case "spend":
return strconv.Itoa(bl.Spended)
//case "extract":
// return bl.Extracteds
case "extract":
return bl.Extracteds.String()
case "frame", "framework":
return bl.Frameworks.ToString()
return bl.Frameworks.String()
default:
return ""
}
@ -240,7 +242,8 @@ func (bl *Baseline) String() string {
line.WriteString(" - ")
line.WriteString(strconv.Itoa(bl.BodyLength))
line.WriteString(bl.Additional("title"))
line.WriteString(bl.Frameworks.ToString())
line.WriteString(bl.Frameworks.String())
line.WriteString(bl.Extracteds.String())
if bl.RedirectURL != "" {
line.WriteString(" --> ")
line.WriteString(bl.RedirectURL)

5
pkg/ihttp/client.go
View File

@ -26,12 +26,12 @@ func NewClient(thread int, timeout int, clientType int) *Client {
Renegotiation: tls.RenegotiateOnceAsClient,
InsecureSkipVerify: true,
},
//ReadBufferSize: 20480,
MaxConnsPerHost: thread * 2,
MaxIdleConnDuration: time.Duration(timeout) * time.Second,
MaxConnWaitTimeout: time.Duration(timeout) * time.Second,
ReadTimeout: time.Duration(timeout) * time.Second,
WriteTimeout: time.Duration(timeout) * time.Second,
ReadBufferSize: 8192,
MaxResponseBodySize: DefaultMaxBodySize,
NoDefaultUserAgentHeader: true,
DisablePathNormalizing: true,
@ -81,7 +81,8 @@ func (c *Client) TransToCheck() {
func (c *Client) FastDo(ctx context.Context, req *fasthttp.Request) (*fasthttp.Response, error) {
resp := fasthttp.AcquireResponse()
return resp, c.fastClient.Do(req, resp)
err := c.fastClient.Do(req, resp)
return resp, err
}
func (c *Client) StandardDo(ctx context.Context, req *http.Request) (*http.Response, error) {

12
pkg/types.go
View File

@ -8,14 +8,22 @@ import (
type Frameworks []*parsers.Framework
func (fs Frameworks) ToString() string {
func (fs Frameworks) String() string {
frameworkStrs := make([]string, len(fs))
for i, f := range fs {
frameworkStrs[i] = "[" + f.ToString() + "]"
}
return strings.Join(frameworkStrs, " ")
return strings.Join(frameworkStrs, " ") + " "
}
type Extracteds []*fingers.Extracted
func (es Extracteds) String() string {
var s strings.Builder
for _, e := range es {
s.WriteString("[ " + e.ToString() + " ]")
}
return s.String() + " "
}
var Extractors = make(fingers.Extractors)