From 4703be8280361f0febcffa6fc83bd9a674f7807b Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 2 Dec 2022 15:21:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E8=A3=85extractor=20=E5=A4=9A?= =?UTF-8?q?=E5=A4=84=E5=B0=8F=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- go.sum | 11 +++++------ internal/option.go | 11 +++++++++++ internal/pool.go | 5 ++--- internal/types.go | 4 ++-- pkg/baseline.go | 15 +++++++++------ pkg/ihttp/client.go | 5 +++-- pkg/types.go | 12 ++++++++++-- 8 files changed, 43 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 008dbab..d8f4bc9 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 github.com/jessevdk/go-flags v1.5.0 github.com/panjf2000/ants/v2 v2.6.0 - github.com/valyala/fasthttp v1.40.0 + github.com/valyala/fasthttp v1.43.0 ) require ( diff --git a/go.sum b/go.sum index 367fe2d..685843c 100644 --- a/go.sum +++ b/go.sum @@ -41,7 +41,7 @@ github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJS github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= @@ -68,8 +68,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc= -github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g= +github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -77,7 +77,7 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -86,9 +86,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/option.go b/internal/option.go index 2d16fb4..e84a3f7 100644 --- a/internal/option.go +++ b/internal/option.go @@ -4,12 +4,14 @@ import ( "fmt" "github.com/antonmedv/expr" "github.com/chainreactors/files" + "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/chainreactors/logs" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words/mask" "github.com/gosuri/uiprogress" "io/ioutil" "os" + "regexp" "strconv" "strings" ) @@ -109,6 +111,15 @@ func (opt *Option) PrepareRunner() (*Runner, error) { return nil, err } + if opt.Extracts != nil { + for _, e := range opt.Extracts { + if reg, ok := fingers.PresetExtracts[e]; ok { + pkg.Extractors[e] = reg + } else { + pkg.Extractors[e] = regexp.MustCompile(e) + } + } + } // 一些全局变量初始化 if opt.Debug { logs.Log.Level = logs.Debug diff --git a/internal/pool.go b/internal/pool.go index bdef116..b48f7bc 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -78,6 +78,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { req, err := pool.genReq(unit.path) if err != nil { logs.Log.Error(err.Error()) + return } var bl *pkg.Baseline @@ -121,11 +122,9 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { pool.base = bl pool.addFuzzyBaseline(bl) pool.initwg.Done() - return case InitIndexSource: pool.index = bl pool.initwg.Done() - return case CheckSource: if bl.ErrString != "" { logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, pool.BreakThreshold, bl.ErrString) @@ -228,8 +227,8 @@ type Pool struct { func (p *Pool) Init() error { p.initwg.Add(2) - p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource)) p.pool.Invoke(newUnit("/", InitIndexSource)) + p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource)) p.initwg.Wait() // todo 分析baseline // 检测基本访问能力 diff --git a/internal/types.go b/internal/types.go index 21df06c..97d4485 100644 --- a/internal/types.go +++ b/internal/types.go @@ -17,9 +17,9 @@ const ( func (e ErrorType) Error() string { switch e { case ErrBadStatus: - return "bad status" + return "blacklist status" case ErrSameStatus: - return "same status" + return "same status with random baseline" case ErrRequestFailed: return "request failed" case ErrWaf: diff --git a/pkg/baseline.go b/pkg/baseline.go index b25e127..e3abc0b 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -25,7 +25,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { } bl.Body = resp.Body() - bl.BodyLength = len(resp.Body()) + bl.BodyLength = len(bl.Body) bl.Header = resp.Header() bl.HeaderLength = len(bl.Header) bl.RedirectURL = resp.GetHeader("Location") @@ -53,6 +53,8 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba bl.Body = resp.Body() bl.BodyLength = len(bl.Body) + bl.Header = resp.Header() + bl.HeaderLength = len(bl.Header) bl.RedirectURL = string(resp.GetHeader("Location")) return bl @@ -87,7 +89,7 @@ func (bl *Baseline) Collect() { if len(bl.Body) > 0 { bl.Title = parsers.MatchTitle(string(bl.Body)) } - bl.Hashes = parsers.NewHashes(bl.Raw) + bl.Hashes = parsers.NewHashes(bl.Body) // todo extract bl.Extracteds = Extractors.Extract(string(bl.Raw)) bl.Frameworks = FingerDetect(string(bl.Raw)) @@ -164,10 +166,10 @@ func (bl *Baseline) Get(key string) string { return strconv.Itoa(bl.Status) case "spend": return strconv.Itoa(bl.Spended) - //case "extract": - // return bl.Extracteds + case "extract": + return bl.Extracteds.String() case "frame", "framework": - return bl.Frameworks.ToString() + return bl.Frameworks.String() default: return "" } @@ -240,7 +242,8 @@ func (bl *Baseline) String() string { line.WriteString(" - ") line.WriteString(strconv.Itoa(bl.BodyLength)) line.WriteString(bl.Additional("title")) - line.WriteString(bl.Frameworks.ToString()) + line.WriteString(bl.Frameworks.String()) + line.WriteString(bl.Extracteds.String()) if bl.RedirectURL != "" { line.WriteString(" --> ") line.WriteString(bl.RedirectURL) diff --git a/pkg/ihttp/client.go b/pkg/ihttp/client.go index aee4df2..10f29cb 100644 --- a/pkg/ihttp/client.go +++ b/pkg/ihttp/client.go @@ -26,12 +26,12 @@ func NewClient(thread int, timeout int, clientType int) *Client { Renegotiation: tls.RenegotiateOnceAsClient, InsecureSkipVerify: true, }, - //ReadBufferSize: 20480, MaxConnsPerHost: thread * 2, MaxIdleConnDuration: time.Duration(timeout) * time.Second, MaxConnWaitTimeout: time.Duration(timeout) * time.Second, ReadTimeout: time.Duration(timeout) * time.Second, WriteTimeout: time.Duration(timeout) * time.Second, + ReadBufferSize: 8192, MaxResponseBodySize: DefaultMaxBodySize, NoDefaultUserAgentHeader: true, DisablePathNormalizing: true, @@ -81,7 +81,8 @@ func (c *Client) TransToCheck() { func (c *Client) FastDo(ctx context.Context, req *fasthttp.Request) (*fasthttp.Response, error) { resp := fasthttp.AcquireResponse() - return resp, c.fastClient.Do(req, resp) + err := c.fastClient.Do(req, resp) + return resp, err } func (c *Client) StandardDo(ctx context.Context, req *http.Request) (*http.Response, error) { diff --git a/pkg/types.go b/pkg/types.go index cefdcdc..4fc87f6 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -8,14 +8,22 @@ import ( type Frameworks []*parsers.Framework -func (fs Frameworks) ToString() string { +func (fs Frameworks) String() string { frameworkStrs := make([]string, len(fs)) for i, f := range fs { frameworkStrs[i] = "[" + f.ToString() + "]" } - return strings.Join(frameworkStrs, " ") + return strings.Join(frameworkStrs, " ") + " " } type Extracteds []*fingers.Extracted +func (es Extracteds) String() string { + var s strings.Builder + for _, e := range es { + s.WriteString("[ " + e.ToString() + " ]") + } + return s.String() + " " +} + var Extractors = make(fingers.Extractors)