admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-05-06 10:41:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

初步实现extract与fingers

Browse Source
This commit is contained in:
M09Ic 2022-09-19 14:42:29 +08:00
parent cb5d1aeafc
commit 26d084eb6e
7 changed files with 69 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
go.mod
View File

@ -3,27 +3,29 @@ module github.com/chainreactors/spray
go 1.17
require (
github.com/chainreactors/gogo/v2 v2.8.5
github.com/chainreactors/gogo/v2 v2.8.7
github.com/chainreactors/logs v0.5.0
github.com/chainreactors/files v0.2.0 // indirect
github.com/chainreactors/ipcs v0.0.9 // indirect
github.com/chainreactors/parsers v0.0.1 // indirect
github.com/chainreactors/words v0.0.1 // indirect
github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c
github.com/panjf2000/ants/v2 v2.5.0
)
require github.com/projectdiscovery/goflags v0.0.9
require (
github.com/chainreactors/files v0.2.0 // indirect
github.com/chainreactors/ipcs v0.0.9 // indirect
github.com/chainreactors/words v0.0.1 // indirect
github.com/M09ic/go-ntlmssp v1.2.9 // indirect
github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect
github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab // indirect
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect
github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect
github.com/jessevdk/go-flags v1.5.0 // indirect
github.com/karrick/godirwalk v1.16.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 // indirect
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect
github.com/twmb/murmur3 v1.1.6 // indirect
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 // indirect
golang.org/x/net v0.0.0-20190603091049-60506f45cf65 // indirect
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

9
go.sum
View File

@ -1,5 +1,8 @@
github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew=
github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs=
github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c h1:k/7/05/5kPRX7HaKyVYlsGVX6XkFTyYLqkqHzceUVlU=
github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c/go.mod h1:SI1d/2/wpSTDjHgdS9ZLy6hqvsdhzVYAc8RLztweMpA=
github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab h1:pfx9N/EMDxIwVzGu9JLnmbOMNukW1mfPA9Ymo+S58Ng=
github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab/go.mod h1:kEcj+iUROrUCr7AIrul5NutI2kWv0ns9BL0ezVp1h/Y=
github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM=
github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
@ -7,11 +10,15 @@ github.com/chainreactors/gogo/v2 v2.8.4 h1:jwV6tawOcUn/FM91WxxOp7ZQN/Msm0losAx52
github.com/chainreactors/gogo/v2 v2.8.4/go.mod h1:hoNawrwZtkIciwApI6kzu88LLOGOJEqtegcHBuiPyPw=
github.com/chainreactors/gogo/v2 v2.8.5 h1:7DfAQdyupToU4f+OYG+Hz4+0phyz3UGRrsAyTmX9Fm0=
github.com/chainreactors/gogo/v2 v2.8.5/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY=
github.com/chainreactors/gogo/v2 v2.8.6 h1:Xc9wi8K8/3u9s0V0vlch2rKHzQ+ATkLlHExGosUvVok=
github.com/chainreactors/gogo/v2 v2.8.6/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY=
github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows=
github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII=
github.com/chainreactors/logs v0.4.2/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
github.com/chainreactors/logs v0.5.0 h1:6CwTR1KaqZS0quIQRU3Ceq/gBoV0+seUF8gUJTrZvew=
github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
github.com/chainreactors/parsers v0.0.1 h1:GrfY4sfdv9TMuYNSKq3Jak2PO4AMJcDRAHocSb8DEx0=
github.com/chainreactors/parsers v0.0.1/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0=
github.com/chainreactors/words v0.0.1 h1:d0rLwuKVzwLWD1CtYeTK7u2E5tv8dmebpq6EhUtNhhg=
github.com/chainreactors/words v0.0.1/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
@ -57,9 +64,11 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

17
internal/baseline.go
View File

@ -3,8 +3,8 @@ package internal
import (
"encoding/json"
"fmt"
"github.com/chainreactors/gogo/v2/pkg/dsl"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
"github.com/chainreactors/spray/pkg"
"io"
"net/http"
@ -50,8 +50,8 @@ func NewBaseline(u *url.URL, resp *http.Response) *baseline {
}
if len(bl.Body) > 0 {
bl.Md5 = dsl.Md5Hash(bl.Body)
bl.Mmh3 = dsl.Mmh3Hash32(bl.Body)
bl.Md5 = parsers.Md5Hash(bl.Body)
bl.Mmh3 = parsers.Mmh3Hash32(bl.Body)
bl.Simhash = pkg.Simhash(bl.Body)
if strings.Contains(string(bl.Body), bl.UrlString[1:]) {
bl.IsDynamicUrl = true
@ -60,9 +60,9 @@ func NewBaseline(u *url.URL, resp *http.Response) *baseline {
}
// todo extract
bl.Extracteds = pkg.Extractors.Extract(bl.Response)
// todo 指纹识别
bl.Frameworks = pkg.FingerDetect(bl.Body)
bl.Frameworks = pkg.FingerDetect(bl.Response)
return bl
}
@ -89,6 +89,7 @@ type baseline struct {
Body []byte `json:"-"`
BodyLength int64 `json:"body_length"`
Header string `json:"-"`
Response string `json:"-"`
HeaderLength int `json:"header_length"`
RedirectURL string `json:"redirect_url"`
Status int `json:"status"`
@ -98,9 +99,9 @@ type baseline struct {
IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url
Spended int `json:"spended"` // 耗时, 毫秒
Frameworks pkg.Frameworks `json:"frameworks"`
Err error `json:"-"`
IsValid bool `json:"-"`
Extracteds pkg.Extracteds `json:"extracts"`
Err error `json:"-"`
IsValid bool `json:"-"`
}
func (bl *baseline) Compare(other *baseline) bool {

10
internal/pool.go
View File

@ -23,11 +23,11 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
return nil, fmt.Errorf("pool init failed, %w", err)
}
poolctx, cancel := context.WithCancel(ctx)
pctx, cancel := context.WithCancel(ctx)
pool := &Pool{
Config: config,
//ctx: ctx,
ctx: pctx,
client: pkg.NewClient(config.Thread, 2),
worder: words.NewWorder(config.Wordlist),
//baseReq: req,
@ -50,7 +50,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (
var bl *baseline
unit := i.(*Unit)
req := pool.genReq(unit.path)
resp, err := pool.client.Do(poolctx, req)
resp, err := pool.client.Do(pctx, req)
if err != nil {
//logs.Log.Debugf("%s request error, %s", strurl, err.Error())
pool.errorCount++
@ -88,7 +88,7 @@ type Pool struct {
*pkg.Config
client *pkg.Client
pool *ants.PoolWithFunc
//ctx context.Context
ctx context.Context
//baseReq *http.Request
baseline *baseline
outputCh chan *baseline
@ -157,6 +157,8 @@ Loop:
break Loop
case <-ctx.Done():
break Loop
case <-p.ctx.Done():
break Loop
}
}

2
internal/runner.go
View File

@ -13,7 +13,7 @@ import (
)
var BlackStatus = []int{404, 410}
var FuzzyStatus = []int{403, 500, 501, 503}
var FuzzyStatus = []int{403, 500, 501, 502, 503}
type Runner struct {
URL string `short:"u" long:"url"`

34
pkg/types.go Normal file
View File

@ -0,0 +1,34 @@
package pkg
import (
"github.com/chainreactors/gogo/v2/pkg/fingers"
"strings"
)
var Fingers fingers.Fingers
func FingerDetect(content string) Frameworks {
var frames Frameworks
//content := string(body)
for _, finger := range Fingers {
frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
if ok {
frames = append(frames, frame)
}
}
return frames
}
type Frameworks []*fingers.Framework
func (fs Frameworks) ToString() string {
frameworkStrs := make([]string, len(fs))
for i, f := range fs {
frameworkStrs[i] = "[" + f.ToString() + "]"
}
return strings.Join(frameworkStrs, " ")
}
type Extracteds []*fingers.Extracted
var Extractors = make(fingers.Extractors)

27
pkg/utils.go
View File

@ -2,11 +2,9 @@ package pkg
import (
"fmt"
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/go-dedup/simhash"
"math/rand"
"os"
"strings"
"time"
"unsafe"
)
@ -24,7 +22,6 @@ func HasStdin() bool {
}
func Simhash(raw []byte) string {
sh := simhash.NewSimhash()
return fmt.Sprintf("%x", sh.GetSimhash(sh.NewWordFeatureSet(raw)))
}
@ -59,27 +56,3 @@ func RandPath() string {
}
return *(*string)(unsafe.Pointer(&b))
}
var Fingers fingers.Fingers
func FingerDetect(body []byte) Frameworks {
var frames Frameworks
content := string(body)
for _, finger := range Fingers {
frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
if ok {
frames = append(frames, frame)
}
}
return frames
}
type Frameworks []*fingers.Framework
func (fs Frameworks) ToString() string {
frameworkStrs := make([]string, len(fs))
for i, f := range fs {
frameworkStrs[i] = "[" + f.ToString() + "]"
}
return strings.Join(frameworkStrs, " ")
}