实装指纹识别

go.mod

@@ -3,15 +3,16 @@ module github.com/chainreactors/spray
 go 1.17
 
 require (
-	github.com/chainreactors/files v0.2.0 // indirect
+	github.com/chainreactors/files v0.2.0
 	github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8
-	github.com/chainreactors/gogo/v2 v2.8.7
+	github.com/chainreactors/gogo/v2 v2.8.10
 	github.com/chainreactors/logs v0.6.1
-	github.com/chainreactors/parsers v0.0.1
+	github.com/chainreactors/parsers v0.0.2
 	github.com/chainreactors/words v0.1.1
 )
 
 require (
+	github.com/chainreactors/ipcs v0.0.9
 	github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c
 	github.com/gosuri/uiprogress v0.0.1
 	github.com/jessevdk/go-flags v1.5.0

go.sum

@@ -5,14 +5,15 @@ github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarC
 github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
 github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 h1:kMFr1Hj+rkp1wBPIw2pcQvelO5GnA7r7wY3h6vJ1joA=
 github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8/go.mod h1:7NDvFERNiXsujaBPD6s4WXj52uKdfnF2zVHQtKXIEV4=
-github.com/chainreactors/gogo/v2 v2.8.7 h1:pN83yw7mtLFcNV12ZwxKdTR2sDZFSiqva/JRQMMRvao=
+github.com/chainreactors/gogo/v2 v2.8.10 h1:PlPZ1lTIcVl2IfE5GgIr4+hocwrTMyUasMDbrR+7TqI=
-github.com/chainreactors/gogo/v2 v2.8.7/go.mod h1:KFMyHLZSkpgDh/17N/k9gKYQ81MO+9MLdTMHQvoBSj0=
+github.com/chainreactors/gogo/v2 v2.8.10/go.mod h1:BQy0aEZXo8nSe/bMMsFeD2KU4vHa8CZ0FHrzuVkH/hU=
+github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows=
 github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
 github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
 github.com/chainreactors/logs v0.6.1 h1:+wLVqxErUyWYayAx6x+m5LA6jQBiHcLdTAJtiOUMoZU=
 github.com/chainreactors/logs v0.6.1/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
-github.com/chainreactors/parsers v0.0.1 h1:GrfY4sfdv9TMuYNSKq3Jak2PO4AMJcDRAHocSb8DEx0=
+github.com/chainreactors/parsers v0.0.2 h1:lQ4dCWzUrVV8dUVKEAw8qlOU6O4Aju4sXtElAfIhbFI=
-github.com/chainreactors/parsers v0.0.1/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0=
+github.com/chainreactors/parsers v0.0.2/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0=
 github.com/chainreactors/words v0.1.1 h1:Zw4HKFtYcIH5SfuCV0X6kj/A5sN99jrQD2ChUonLOV8=
 github.com/chainreactors/words v0.1.1/go.mod h1:jRcFgafTKqdkd1+StzPCTJG1ESrZHluXEO2eERdHBMQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -24,24 +25,17 @@ github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c h1:mucYYQn+sMGNSx
 github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c/go.mod h1:gO3u2bjRAgUaLdQd2XK+3oooxrheOAx1BzS7WmPzw1s=
 github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 h1:11wFcswN+37U+ByjxdKzsRY5KzNqqq5Uk5ztxnLOc7w=
 github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7/go.mod h1:wSsK4VOECOSfSYTzkBFw+iGY7wj59e7X96ABtNj9aCQ=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
 github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
 github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
 github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
-github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo=
 github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc=
-github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og=
-github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU=
-github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E=
 github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q=
 github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -60,18 +54,15 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -83,9 +74,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

internal/baseline.go

@@ -140,7 +140,7 @@ func (bl *baseline) Get(key string) string {
 
 func (bl *baseline) Additional(key string) string {
 	if v := bl.Get(key); v != "" {
-		return "[" + v + "] "
+		return " [" + v + "] "
 	} else {
 		return " "
 	}

internal/option.go

@@ -37,6 +37,7 @@ type Option struct {
 }
 
 func (opt *Option) PrepareRunner() (*Runner, error) {
+	var err error
 	r := &Runner{
 		Progress: uiprogress.New(),
 		Threads:  opt.Threads,
@@ -45,6 +46,12 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		Timeout:  opt.Timeout,
 	}
 
+	err = pkg.LoadTemplates()
+	if err != nil {
+		return nil, err
+	}
+
+	// 一些全局变量初始化
 	if opt.Debug {
 		logs.Log.Level = logs.Debug
 	}
@@ -55,7 +62,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 
 	// prepare url
 	var file *os.File
-	var err error
 	urlfrom := opt.URLFile
 	if opt.URL != "" {
 		r.URLList = append(r.URLList, opt.URL)

internal/pool.go

@@ -154,7 +154,7 @@ type Pool struct {
 
 func (p *Pool) Init() error {
 	p.initwg.Add(1)
-	p.pool.Invoke(newUnit(pkg.RandHost(), InitSource))
+	p.pool.Invoke(newUnit(pkg.RandPath(), InitSource))
 	p.initwg.Wait()
 	// todo 分析baseline
 	// 检测基本访问能力

pkg/templates.go

@@ -1,4 +1,4 @@
-package internal
+package pkg
 
 import (
 	"github.com/chainreactors/files"

pkg/templates_gen.go

@@ -87,7 +87,7 @@ func recuLoadYamlFiles2JsonString(dir string, single bool) string {
 }
 
 func main() {
-	template := `package internal
+	template := `package pkg
 
 import (
 	"github.com/chainreactors/files"
@@ -105,7 +105,7 @@ func LoadConfig(typ string) []byte {
 	template = fmt.Sprintf(template,
 		recuLoadYamlFiles2JsonString("fingers/http", false),
 	)
-	f, err := os.OpenFile("internal/templates.go", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
+	f, err := os.OpenFile("pkg/templates.go", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
 	if err != nil {
 		panic(err)
 	}

pkg/types.go

@@ -5,20 +5,6 @@ import (
 	"strings"
 )
 
-var Fingers fingers.Fingers
-
-func FingerDetect(content string) Frameworks {
-	var frames Frameworks
-	//content := string(body)
-	for _, finger := range Fingers {
-		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
-		if ok {
-			frames = append(frames, frame)
-		}
-	}
-	return frames
-}
-
 type Frameworks []*fingers.Framework
 
 func (fs Frameworks) ToString() string {

pkg/utils.go

@@ -2,6 +2,9 @@ package pkg
 
 import (
 	"fmt"
+	"github.com/chainreactors/gogo/v2/pkg/fingers"
+	"github.com/chainreactors/gogo/v2/pkg/utils"
+	"github.com/chainreactors/ipcs"
 	"github.com/go-dedup/simhash"
 	"math/rand"
 	"os"
@@ -76,3 +79,51 @@ func RandHost() string {
 	b[5] = byte(0x2e)
 	return *(*string)(unsafe.Pointer(&b))
 }
+
+var (
+	Md5Fingers  map[string]string = make(map[string]string)
+	Mmh3Fingers map[string]string = make(map[string]string)
+	Fingers     fingers.Fingers
+)
+
+func LoadTemplates() error {
+	var err error
+	Fingers, err = fingers.LoadFingers(LoadConfig("http"))
+	if err != nil {
+		utils.Fatal(err.Error())
+	}
+
+	for _, finger := range Fingers {
+		err := finger.Compile(ipcs.ParsePorts)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, f := range Fingers {
+		for _, rule := range f.Rules {
+			if rule.Favicon != nil {
+				for _, mmh3 := range rule.Favicon.Mmh3 {
+					Mmh3Fingers[mmh3] = f.Name
+				}
+				for _, md5 := range rule.Favicon.Md5 {
+					Md5Fingers[md5] = f.Name
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func FingerDetect(content string) Frameworks {
+	var frames Frameworks
+	//content := string(body)
+	for _, finger := range Fingers {
+		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
+		if ok {
+			frames = append(frames, frame)
+		}
+	}
+	return frames
+}

spray.go

@@ -1,4 +1,4 @@
-//go:generate go run internal/templates_gen.go
+//go:generate go run pkg/templates_gen.go
 package main
 
 import "github.com/chainreactors/spray/cmd"