实装extractor

多处小优化

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/gosuri/uiprogress v0.0.1
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/panjf2000/ants/v2 v2.6.0
-	github.com/valyala/fasthttp v1.40.0
+	github.com/valyala/fasthttp v1.43.0
 )
 
 require (

go.sum

@@ -41,7 +41,7 @@ github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJS
 github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
-github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo=
 github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
@@ -68,8 +68,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc=
+github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g=
-github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
+github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -77,7 +77,7 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -86,9 +86,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/option.go

@@ -4,12 +4,14 @@ import (
 	"fmt"
 	"github.com/antonmedv/expr"
 	"github.com/chainreactors/files"
+	"github.com/chainreactors/gogo/v2/pkg/fingers"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/words/mask"
 	"github.com/gosuri/uiprogress"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -109,6 +111,15 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		return nil, err
 	}
 
+	if opt.Extracts != nil {
+		for _, e := range opt.Extracts {
+			if reg, ok := fingers.PresetExtracts[e]; ok {
+				pkg.Extractors[e] = reg
+			} else {
+				pkg.Extractors[e] = regexp.MustCompile(e)
+			}
+		}
+	}
 	// 一些全局变量初始化
 	if opt.Debug {
 		logs.Log.Level = logs.Debug

internal/pool.go

@@ -78,6 +78,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
 		req, err := pool.genReq(unit.path)
 		if err != nil {
 			logs.Log.Error(err.Error())
+			return
 		}
 
 		var bl *pkg.Baseline
@@ -121,11 +122,9 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
 			pool.base = bl
 			pool.addFuzzyBaseline(bl)
 			pool.initwg.Done()
-			return
 		case InitIndexSource:
 			pool.index = bl
 			pool.initwg.Done()
-			return
 		case CheckSource:
 			if bl.ErrString != "" {
 				logs.Log.Warnf("[check.error] maybe ip had banned by waf, break (%d/%d), error: %s", pool.failedCount, pool.BreakThreshold, bl.ErrString)
@@ -228,8 +227,8 @@ type Pool struct {
 
 func (p *Pool) Init() error {
 	p.initwg.Add(2)
-	p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource))
 	p.pool.Invoke(newUnit("/", InitIndexSource))
+	p.pool.Invoke(newUnit(pkg.RandPath(), InitRandomSource))
 	p.initwg.Wait()
 	// todo 分析baseline
 	// 检测基本访问能力

internal/types.go

@@ -17,9 +17,9 @@ const (
 func (e ErrorType) Error() string {
 	switch e {
 	case ErrBadStatus:
-		return "bad status"
+		return "blacklist status"
 	case ErrSameStatus:
-		return "same status"
+		return "same status with random baseline"
 	case ErrRequestFailed:
 		return "request failed"
 	case ErrWaf:

pkg/baseline.go

@@ -25,7 +25,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 	}
 
 	bl.Body = resp.Body()
-	bl.BodyLength = len(resp.Body())
+	bl.BodyLength = len(bl.Body)
 	bl.Header = resp.Header()
 	bl.HeaderLength = len(bl.Header)
 	bl.RedirectURL = resp.GetHeader("Location")
@@ -53,6 +53,8 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
 
 	bl.Body = resp.Body()
 	bl.BodyLength = len(bl.Body)
+	bl.Header = resp.Header()
+	bl.HeaderLength = len(bl.Header)
 	bl.RedirectURL = string(resp.GetHeader("Location"))
 
 	return bl
@@ -87,7 +89,7 @@ func (bl *Baseline) Collect() {
 	if len(bl.Body) > 0 {
 		bl.Title = parsers.MatchTitle(string(bl.Body))
 	}
-	bl.Hashes = parsers.NewHashes(bl.Raw)
+	bl.Hashes = parsers.NewHashes(bl.Body)
 	// todo extract
 	bl.Extracteds = Extractors.Extract(string(bl.Raw))
 	bl.Frameworks = FingerDetect(string(bl.Raw))
@@ -164,10 +166,10 @@ func (bl *Baseline) Get(key string) string {
 		return strconv.Itoa(bl.Status)
 	case "spend":
 		return strconv.Itoa(bl.Spended)
-	//case "extract":
+	case "extract":
-	//	return bl.Extracteds
+		return bl.Extracteds.String()
 	case "frame", "framework":
-		return bl.Frameworks.ToString()
+		return bl.Frameworks.String()
 	default:
 		return ""
 	}
@@ -240,7 +242,8 @@ func (bl *Baseline) String() string {
 	line.WriteString(" - ")
 	line.WriteString(strconv.Itoa(bl.BodyLength))
 	line.WriteString(bl.Additional("title"))
-	line.WriteString(bl.Frameworks.ToString())
+	line.WriteString(bl.Frameworks.String())
+	line.WriteString(bl.Extracteds.String())
 	if bl.RedirectURL != "" {
 		line.WriteString(" --> ")
 		line.WriteString(bl.RedirectURL)

pkg/ihttp/client.go

@@ -26,12 +26,12 @@ func NewClient(thread int, timeout int, clientType int) *Client {
 					Renegotiation:      tls.RenegotiateOnceAsClient,
 					InsecureSkipVerify: true,
 				},
-				//ReadBufferSize:      20480,
 				MaxConnsPerHost:               thread * 2,
 				MaxIdleConnDuration:           time.Duration(timeout) * time.Second,
 				MaxConnWaitTimeout:            time.Duration(timeout) * time.Second,
 				ReadTimeout:                   time.Duration(timeout) * time.Second,
 				WriteTimeout:                  time.Duration(timeout) * time.Second,
+				ReadBufferSize:                8192,
 				MaxResponseBodySize:           DefaultMaxBodySize,
 				NoDefaultUserAgentHeader:      true,
 				DisablePathNormalizing:        true,
@@ -81,7 +81,8 @@ func (c *Client) TransToCheck() {
 
 func (c *Client) FastDo(ctx context.Context, req *fasthttp.Request) (*fasthttp.Response, error) {
 	resp := fasthttp.AcquireResponse()
-	return resp, c.fastClient.Do(req, resp)
+	err := c.fastClient.Do(req, resp)
+	return resp, err
 }
 
 func (c *Client) StandardDo(ctx context.Context, req *http.Request) (*http.Response, error) {

pkg/types.go

@@ -8,14 +8,22 @@ import (
 
 type Frameworks []*parsers.Framework
 
-func (fs Frameworks) ToString() string {
+func (fs Frameworks) String() string {
 	frameworkStrs := make([]string, len(fs))
 	for i, f := range fs {
 		frameworkStrs[i] = "[" + f.ToString() + "]"
 	}
-	return strings.Join(frameworkStrs, " ")
+	return strings.Join(frameworkStrs, " ") + " "
 }
 
 type Extracteds []*fingers.Extracted
 
+func (es Extracteds) String() string {
+	var s strings.Builder
+	for _, e := range es {
+		s.WriteString("[ " + e.ToString() + " ]")
+	}
+	return s.String() + " "
+}
+
 var Extractors = make(fingers.Extractors)