初步实现extract与fingers

go.mod

@@ -3,27 +3,29 @@ module github.com/chainreactors/spray
 go 1.17
 
 require (
-	github.com/chainreactors/gogo/v2 v2.8.5
+	github.com/chainreactors/gogo/v2 v2.8.7
 	github.com/chainreactors/logs v0.5.0
+	github.com/chainreactors/files v0.2.0 // indirect
+	github.com/chainreactors/ipcs v0.0.9 // indirect
+	github.com/chainreactors/parsers v0.0.1 // indirect
+	github.com/chainreactors/words v0.0.1 // indirect
 	github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c
 	github.com/panjf2000/ants/v2 v2.5.0
 )
 
-require github.com/projectdiscovery/goflags v0.0.9
-
 require (
-	github.com/chainreactors/files v0.2.0 // indirect
+	github.com/M09ic/go-ntlmssp v1.2.9 // indirect
-	github.com/chainreactors/ipcs v0.0.9 // indirect
+	github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect
-	github.com/chainreactors/words v0.0.1 // indirect
+	github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect
 	github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect
 	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 // indirect
-	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
+	golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 // indirect
+	golang.org/x/net v0.0.0-20190603091049-60506f45cf65 // indirect
 	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,5 +1,8 @@
+github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew=
 github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs=
+github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c h1:k/7/05/5kPRX7HaKyVYlsGVX6XkFTyYLqkqHzceUVlU=
 github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c/go.mod h1:SI1d/2/wpSTDjHgdS9ZLy6hqvsdhzVYAc8RLztweMpA=
+github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab h1:pfx9N/EMDxIwVzGu9JLnmbOMNukW1mfPA9Ymo+S58Ng=
 github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab/go.mod h1:kEcj+iUROrUCr7AIrul5NutI2kWv0ns9BL0ezVp1h/Y=
 github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM=
 github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
@@ -7,11 +10,15 @@ github.com/chainreactors/gogo/v2 v2.8.4 h1:jwV6tawOcUn/FM91WxxOp7ZQN/Msm0losAx52
 github.com/chainreactors/gogo/v2 v2.8.4/go.mod h1:hoNawrwZtkIciwApI6kzu88LLOGOJEqtegcHBuiPyPw=
 github.com/chainreactors/gogo/v2 v2.8.5 h1:7DfAQdyupToU4f+OYG+Hz4+0phyz3UGRrsAyTmX9Fm0=
 github.com/chainreactors/gogo/v2 v2.8.5/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY=
+github.com/chainreactors/gogo/v2 v2.8.6 h1:Xc9wi8K8/3u9s0V0vlch2rKHzQ+ATkLlHExGosUvVok=
+github.com/chainreactors/gogo/v2 v2.8.6/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY=
 github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows=
 github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
 github.com/chainreactors/logs v0.4.2/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
 github.com/chainreactors/logs v0.5.0 h1:6CwTR1KaqZS0quIQRU3Ceq/gBoV0+seUF8gUJTrZvew=
 github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
+github.com/chainreactors/parsers v0.0.1 h1:GrfY4sfdv9TMuYNSKq3Jak2PO4AMJcDRAHocSb8DEx0=
+github.com/chainreactors/parsers v0.0.1/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0=
 github.com/chainreactors/words v0.0.1 h1:d0rLwuKVzwLWD1CtYeTK7u2E5tv8dmebpq6EhUtNhhg=
 github.com/chainreactors/words v0.0.1/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
@@ -57,9 +64,11 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

internal/baseline.go

@@ -3,8 +3,8 @@ package internal
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/chainreactors/gogo/v2/pkg/dsl"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/spray/pkg"
 	"io"
 	"net/http"
@@ -50,8 +50,8 @@ func NewBaseline(u *url.URL, resp *http.Response) *baseline {
 	}
 
 	if len(bl.Body) > 0 {
-		bl.Md5 = dsl.Md5Hash(bl.Body)
+		bl.Md5 = parsers.Md5Hash(bl.Body)
-		bl.Mmh3 = dsl.Mmh3Hash32(bl.Body)
+		bl.Mmh3 = parsers.Mmh3Hash32(bl.Body)
 		bl.Simhash = pkg.Simhash(bl.Body)
 		if strings.Contains(string(bl.Body), bl.UrlString[1:]) {
 			bl.IsDynamicUrl = true
@@ -60,9 +60,9 @@ func NewBaseline(u *url.URL, resp *http.Response) *baseline {
 	}
 
 	// todo extract
-
+	bl.Extracteds = pkg.Extractors.Extract(bl.Response)
 	// todo 指纹识别
-	bl.Frameworks = pkg.FingerDetect(bl.Body)
+	bl.Frameworks = pkg.FingerDetect(bl.Response)
 	return bl
 }
 
@@ -89,6 +89,7 @@ type baseline struct {
 	Body         []byte         `json:"-"`
 	BodyLength   int64          `json:"body_length"`
 	Header       string         `json:"-"`
+	Response     string         `json:"-"`
 	HeaderLength int            `json:"header_length"`
 	RedirectURL  string         `json:"redirect_url"`
 	Status       int            `json:"status"`
@@ -98,7 +99,7 @@ type baseline struct {
 	IsDynamicUrl bool           `json:"is_dynamic_url"` // 判断是否存在动态的url
 	Spended      int            `json:"spended"`        // 耗时, 毫秒
 	Frameworks   pkg.Frameworks `json:"frameworks"`
-
+	Extracteds   pkg.Extracteds `json:"extracts"`
 	Err          error          `json:"-"`
 	IsValid      bool           `json:"-"`
 }

internal/pool.go

@@ -23,11 +23,11 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 		return nil, fmt.Errorf("pool init failed, %w", err)
 	}
 
-	poolctx, cancel := context.WithCancel(ctx)
+	pctx, cancel := context.WithCancel(ctx)
 
 	pool := &Pool{
 		Config: config,
-		//ctx:      ctx,
+		ctx:    pctx,
 		client: pkg.NewClient(config.Thread, 2),
 		worder: words.NewWorder(config.Wordlist),
 		//baseReq:  req,
@@ -50,7 +50,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
 		var bl *baseline
 		unit := i.(*Unit)
 		req := pool.genReq(unit.path)
-		resp, err := pool.client.Do(poolctx, req)
+		resp, err := pool.client.Do(pctx, req)
 		if err != nil {
 			//logs.Log.Debugf("%s request error, %s", strurl, err.Error())
 			pool.errorCount++
@@ -88,7 +88,7 @@ type Pool struct {
 	*pkg.Config
 	client *pkg.Client
 	pool   *ants.PoolWithFunc
-	//ctx          context.Context
+	ctx    context.Context
 	//baseReq      *http.Request
 	baseline   *baseline
 	outputCh   chan *baseline
@@ -157,6 +157,8 @@ Loop:
 			break Loop
 		case <-ctx.Done():
 			break Loop
+		case <-p.ctx.Done():
+			break Loop
 		}
 	}
 

internal/runner.go

@@ -13,7 +13,7 @@ import (
 )
 
 var BlackStatus = []int{404, 410}
-var FuzzyStatus = []int{403, 500, 501, 503}
+var FuzzyStatus = []int{403, 500, 501, 502, 503}
 
 type Runner struct {
 	URL        string `short:"u" long:"url"`

pkg/types.go

@@ -0,0 +1,34 @@
+package pkg
+
+import (
+	"github.com/chainreactors/gogo/v2/pkg/fingers"
+	"strings"
+)
+
+var Fingers fingers.Fingers
+
+func FingerDetect(content string) Frameworks {
+	var frames Frameworks
+	//content := string(body)
+	for _, finger := range Fingers {
+		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
+		if ok {
+			frames = append(frames, frame)
+		}
+	}
+	return frames
+}
+
+type Frameworks []*fingers.Framework
+
+func (fs Frameworks) ToString() string {
+	frameworkStrs := make([]string, len(fs))
+	for i, f := range fs {
+		frameworkStrs[i] = "[" + f.ToString() + "]"
+	}
+	return strings.Join(frameworkStrs, " ")
+}
+
+type Extracteds []*fingers.Extracted
+
+var Extractors = make(fingers.Extractors)

pkg/utils.go

@@ -2,11 +2,9 @@ package pkg
 
 import (
 	"fmt"
-	"github.com/chainreactors/gogo/v2/pkg/fingers"
 	"github.com/go-dedup/simhash"
 	"math/rand"
 	"os"
-	"strings"
 	"time"
 	"unsafe"
 )
@@ -24,7 +22,6 @@ func HasStdin() bool {
 }
 
 func Simhash(raw []byte) string {
-
 	sh := simhash.NewSimhash()
 	return fmt.Sprintf("%x", sh.GetSimhash(sh.NewWordFeatureSet(raw)))
 }
@@ -59,27 +56,3 @@ func RandPath() string {
 	}
 	return *(*string)(unsafe.Pointer(&b))
 }
-
-var Fingers fingers.Fingers
-
-func FingerDetect(body []byte) Frameworks {
-	var frames Frameworks
-	content := string(body)
-	for _, finger := range Fingers {
-		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
-		if ok {
-			frames = append(frames, frame)
-		}
-	}
-	return frames
-}
-
-type Frameworks []*fingers.Framework
-
-func (fs Frameworks) ToString() string {
-	frameworkStrs := make([]string, len(fs))
-	for i, f := range fs {
-		frameworkStrs[i] = "[" + f.ToString() + "]"
-	}
-	return strings.Join(frameworkStrs, " ")
-}