diff --git a/go.mod b/go.mod index 4e15729..be6cf3a 100644 --- a/go.mod +++ b/go.mod @@ -3,27 +3,29 @@ module github.com/chainreactors/spray go 1.17 require ( - github.com/chainreactors/gogo/v2 v2.8.5 + github.com/chainreactors/gogo/v2 v2.8.7 github.com/chainreactors/logs v0.5.0 + github.com/chainreactors/files v0.2.0 // indirect + github.com/chainreactors/ipcs v0.0.9 // indirect + github.com/chainreactors/parsers v0.0.1 // indirect + github.com/chainreactors/words v0.0.1 // indirect github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c github.com/panjf2000/ants/v2 v2.5.0 ) -require github.com/projectdiscovery/goflags v0.0.9 - require ( - github.com/chainreactors/files v0.2.0 // indirect - github.com/chainreactors/ipcs v0.0.9 // indirect - github.com/chainreactors/words v0.0.1 // indirect + github.com/M09ic/go-ntlmssp v1.2.9 // indirect + github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect + github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 // indirect - github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect github.com/twmb/murmur3 v1.1.6 // indirect + golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 // indirect + golang.org/x/net v0.0.0-20190603091049-60506f45cf65 // indirect golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 83b9f49..a443e3f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ +github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew= github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs= +github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c h1:k/7/05/5kPRX7HaKyVYlsGVX6XkFTyYLqkqHzceUVlU= github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c/go.mod h1:SI1d/2/wpSTDjHgdS9ZLy6hqvsdhzVYAc8RLztweMpA= +github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab h1:pfx9N/EMDxIwVzGu9JLnmbOMNukW1mfPA9Ymo+S58Ng= github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab/go.mod h1:kEcj+iUROrUCr7AIrul5NutI2kWv0ns9BL0ezVp1h/Y= github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM= github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= @@ -7,11 +10,15 @@ github.com/chainreactors/gogo/v2 v2.8.4 h1:jwV6tawOcUn/FM91WxxOp7ZQN/Msm0losAx52 github.com/chainreactors/gogo/v2 v2.8.4/go.mod h1:hoNawrwZtkIciwApI6kzu88LLOGOJEqtegcHBuiPyPw= github.com/chainreactors/gogo/v2 v2.8.5 h1:7DfAQdyupToU4f+OYG+Hz4+0phyz3UGRrsAyTmX9Fm0= github.com/chainreactors/gogo/v2 v2.8.5/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY= +github.com/chainreactors/gogo/v2 v2.8.6 h1:Xc9wi8K8/3u9s0V0vlch2rKHzQ+ATkLlHExGosUvVok= +github.com/chainreactors/gogo/v2 v2.8.6/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY= github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows= github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII= github.com/chainreactors/logs v0.4.2/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= github.com/chainreactors/logs v0.5.0 h1:6CwTR1KaqZS0quIQRU3Ceq/gBoV0+seUF8gUJTrZvew= github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= +github.com/chainreactors/parsers v0.0.1 h1:GrfY4sfdv9TMuYNSKq3Jak2PO4AMJcDRAHocSb8DEx0= +github.com/chainreactors/parsers v0.0.1/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0= github.com/chainreactors/words v0.0.1 h1:d0rLwuKVzwLWD1CtYeTK7u2E5tv8dmebpq6EhUtNhhg= github.com/chainreactors/words v0.0.1/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= @@ -57,9 +64,11 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/baseline.go b/internal/baseline.go index 9cf40db..69e5d86 100644 --- a/internal/baseline.go +++ b/internal/baseline.go @@ -3,8 +3,8 @@ package internal import ( "encoding/json" "fmt" - "github.com/chainreactors/gogo/v2/pkg/dsl" "github.com/chainreactors/logs" + "github.com/chainreactors/parsers" "github.com/chainreactors/spray/pkg" "io" "net/http" @@ -50,8 +50,8 @@ func NewBaseline(u *url.URL, resp *http.Response) *baseline { } if len(bl.Body) > 0 { - bl.Md5 = dsl.Md5Hash(bl.Body) - bl.Mmh3 = dsl.Mmh3Hash32(bl.Body) + bl.Md5 = parsers.Md5Hash(bl.Body) + bl.Mmh3 = parsers.Mmh3Hash32(bl.Body) bl.Simhash = pkg.Simhash(bl.Body) if strings.Contains(string(bl.Body), bl.UrlString[1:]) { bl.IsDynamicUrl = true @@ -60,9 +60,9 @@ func NewBaseline(u *url.URL, resp *http.Response) *baseline { } // todo extract - + bl.Extracteds = pkg.Extractors.Extract(bl.Response) // todo 指纹识别 - bl.Frameworks = pkg.FingerDetect(bl.Body) + bl.Frameworks = pkg.FingerDetect(bl.Response) return bl } @@ -89,6 +89,7 @@ type baseline struct { Body []byte `json:"-"` BodyLength int64 `json:"body_length"` Header string `json:"-"` + Response string `json:"-"` HeaderLength int `json:"header_length"` RedirectURL string `json:"redirect_url"` Status int `json:"status"` @@ -98,9 +99,9 @@ type baseline struct { IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url Spended int `json:"spended"` // 耗时, 毫秒 Frameworks pkg.Frameworks `json:"frameworks"` - - Err error `json:"-"` - IsValid bool `json:"-"` + Extracteds pkg.Extracteds `json:"extracts"` + Err error `json:"-"` + IsValid bool `json:"-"` } func (bl *baseline) Compare(other *baseline) bool { diff --git a/internal/pool.go b/internal/pool.go index 1089219..9e70881 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -23,11 +23,11 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( return nil, fmt.Errorf("pool init failed, %w", err) } - poolctx, cancel := context.WithCancel(ctx) + pctx, cancel := context.WithCancel(ctx) pool := &Pool{ Config: config, - //ctx: ctx, + ctx: pctx, client: pkg.NewClient(config.Thread, 2), worder: words.NewWorder(config.Wordlist), //baseReq: req, @@ -50,7 +50,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( var bl *baseline unit := i.(*Unit) req := pool.genReq(unit.path) - resp, err := pool.client.Do(poolctx, req) + resp, err := pool.client.Do(pctx, req) if err != nil { //logs.Log.Debugf("%s request error, %s", strurl, err.Error()) pool.errorCount++ @@ -88,7 +88,7 @@ type Pool struct { *pkg.Config client *pkg.Client pool *ants.PoolWithFunc - //ctx context.Context + ctx context.Context //baseReq *http.Request baseline *baseline outputCh chan *baseline @@ -157,6 +157,8 @@ Loop: break Loop case <-ctx.Done(): break Loop + case <-p.ctx.Done(): + break Loop } } diff --git a/internal/runner.go b/internal/runner.go index 946ebf0..6b5d603 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -13,7 +13,7 @@ import ( ) var BlackStatus = []int{404, 410} -var FuzzyStatus = []int{403, 500, 501, 503} +var FuzzyStatus = []int{403, 500, 501, 502, 503} type Runner struct { URL string `short:"u" long:"url"` diff --git a/pkg/types.go b/pkg/types.go new file mode 100644 index 0000000..a1d7361 --- /dev/null +++ b/pkg/types.go @@ -0,0 +1,34 @@ +package pkg + +import ( + "github.com/chainreactors/gogo/v2/pkg/fingers" + "strings" +) + +var Fingers fingers.Fingers + +func FingerDetect(content string) Frameworks { + var frames Frameworks + //content := string(body) + for _, finger := range Fingers { + frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil) + if ok { + frames = append(frames, frame) + } + } + return frames +} + +type Frameworks []*fingers.Framework + +func (fs Frameworks) ToString() string { + frameworkStrs := make([]string, len(fs)) + for i, f := range fs { + frameworkStrs[i] = "[" + f.ToString() + "]" + } + return strings.Join(frameworkStrs, " ") +} + +type Extracteds []*fingers.Extracted + +var Extractors = make(fingers.Extractors) diff --git a/pkg/utils.go b/pkg/utils.go index 85d5033..1b4f857 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -2,11 +2,9 @@ package pkg import ( "fmt" - "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/go-dedup/simhash" "math/rand" "os" - "strings" "time" "unsafe" ) @@ -24,7 +22,6 @@ func HasStdin() bool { } func Simhash(raw []byte) string { - sh := simhash.NewSimhash() return fmt.Sprintf("%x", sh.GetSimhash(sh.NewWordFeatureSet(raw))) } @@ -59,27 +56,3 @@ func RandPath() string { } return *(*string)(unsafe.Pointer(&b)) } - -var Fingers fingers.Fingers - -func FingerDetect(body []byte) Frameworks { - var frames Frameworks - content := string(body) - for _, finger := range Fingers { - frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil) - if ok { - frames = append(frames, frame) - } - } - return frames -} - -type Frameworks []*fingers.Framework - -func (fs Frameworks) ToString() string { - frameworkStrs := make([]string, len(fs)) - for i, f := range fs { - frameworkStrs[i] = "[" + f.ToString() + "]" - } - return strings.Join(frameworkStrs, " ") -}