From 02f601051c729c1f29bd96163c66caf876d231d3 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Wed, 1 Feb 2023 18:31:50 +0800 Subject: [PATCH] =?UTF-8?q?spray=E8=BE=93=E5=87=BA=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=E6=95=B4=E5=90=88=E5=88=B0spray?= =?UTF-8?q?=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- go.sum | 4 + internal/checkpool.go | 11 +- internal/pool.go | 12 +- pkg/baseline.go | 273 ++++-------------------------------------- pkg/statistor.go | 5 +- pkg/types.go | 45 ------- pkg/utils.go | 6 +- 8 files changed, 55 insertions(+), 303 deletions(-) delete mode 100644 pkg/types.go diff --git a/go.mod b/go.mod index ba18e35..92d02b6 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/chainreactors/gogo/v2 v2.10.4 github.com/chainreactors/ipcs v0.0.13 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 - github.com/chainreactors/parsers v0.3.0 + github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a ) diff --git a/go.sum b/go.sum index 023986f..2dc0526 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,10 @@ github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580/go.mod h1:Y0E github.com/chainreactors/parsers v0.2.9-0.20230128061921-aa83139d24f5/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= github.com/chainreactors/parsers v0.3.0 h1:OJ/K/ZRsaSWkQNf6+3yliHrps+zj0fZl2/7juhjFQsY= github.com/chainreactors/parsers v0.3.0/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= +github.com/chainreactors/parsers v0.3.1-0.20230201101112-af520af38b4d h1:gJaqiDaHDmNKV2uTaEhDPHYu37gUvBuYDcJ7FYMulfw= +github.com/chainreactors/parsers v0.3.1-0.20230201101112-af520af38b4d/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= +github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49 h1:snsLbWcScZRQV66TZxHfECkb2htZzhW2EcF1c60AA9c= +github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/checkpool.go b/internal/checkpool.go index 07debaf..f756c72 100644 --- a/internal/checkpool.go +++ b/internal/checkpool.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/chainreactors/logs" + "github.com/chainreactors/parsers" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/words" @@ -42,7 +43,15 @@ func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) { if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { pool.failedCount++ - bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} + + bl = &pkg.Baseline{ + SprayResult: &parsers.SprayResult{ + UrlString: pool.BaseURL + unit.path, + IsValid: false, + ErrString: reqerr.Error(), + Reason: ErrRequestFailed.Error(), + }, + } } else { bl = pkg.NewBaseline(req.URI(), req.Host(), resp) bl.Collect() diff --git a/internal/pool.go b/internal/pool.go index 1d887fe..27ece16 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -6,6 +6,7 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" "github.com/chainreactors/logs" + "github.com/chainreactors/parsers" "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" @@ -283,7 +284,7 @@ Loop: continue } if _, ok := pool.urls[unit.path]; ok { - logs.Log.Debugf("[%s] duplicate path: %s, skipped", pkg.GetSourceName(unit.source), pool.base+unit.path) + logs.Log.Debugf("[%s] duplicate path: %s, skipped", parsers.GetSpraySourceName(unit.source), pool.base+unit.path) pool.waiter.Done() } else { pool.urls[unit.path] = struct{}{} @@ -327,7 +328,14 @@ func (pool *Pool) Invoke(v interface{}) { if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { atomic.AddInt32(&pool.failedCount, 1) atomic.AddInt32(&pool.Statistor.FailedNumber, 1) - bl = &pkg.Baseline{UrlString: pool.base + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} + bl = &pkg.Baseline{ + SprayResult: &parsers.SprayResult{ + UrlString: pool.base + unit.path, + IsValid: false, + ErrString: reqerr.Error(), + Reason: ErrRequestFailed.Error(), + }, + } pool.failedBaselines = append(pool.failedBaselines, bl) } else { if unit.source <= 3 || unit.source == CrawlSource || unit.source == CommonFileSource { diff --git a/pkg/baseline.go b/pkg/baseline.go index 3f2842c..92a0624 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -2,21 +2,21 @@ package pkg import ( "bytes" - "encoding/json" - "github.com/chainreactors/logs" "github.com/chainreactors/parsers" "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/spray/pkg/ihttp" "net/url" - "strconv" "strings" ) func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { bl := &Baseline{ - UrlString: u, - Status: resp.StatusCode(), - IsValid: true, + SprayResult: &parsers.SprayResult{ + UrlString: u, + Status: resp.StatusCode(), + IsValid: true, + Frameworks: make(parsers.Frameworks), + }, } header := resp.Header() bl.Header = make([]byte, len(header)) @@ -60,10 +60,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline { bl := &Baseline{ - UrlString: u, - Status: resp.StatusCode(), - IsValid: false, - Reason: reason, + SprayResult: &parsers.SprayResult{ + UrlString: u, + Status: resp.StatusCode(), + IsValid: false, + Reason: reason, + }, } // 无效数据也要读取body, 否则keep-alive不生效 @@ -88,38 +90,16 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba } type Baseline struct { - Number int `json:"number"` - Url *url.URL `json:"-"` - IsValid bool `json:"valid"` - IsFuzzy bool `json:"fuzzy"` - UrlString string `json:"url"` - Path string `json:"path"` - Dir bool `json:"-"` - Chunked bool `json:"-"` - Host string `json:"host"` - Body []byte `json:"-"` - BodyLength int `json:"body_length"` - ExceedLength bool `json:"-"` - Header []byte `json:"-"` - Raw []byte `json:"-"` - HeaderLength int `json:"header_length"` - RedirectURL string `json:"redirect_url,omitempty"` - FrontURL string `json:"front_url,omitempty"` - Status int `json:"status"` - Spended int64 `json:"spend"` // 耗时, 毫秒 - ContentType string `json:"content_type"` - Title string `json:"title"` - Frameworks Frameworks `json:"frameworks"` - Extracteds parsers.Extracteds `json:"extracts"` - ErrString string `json:"error"` - Reason string `json:"reason"` - Source int `json:"source"` - ReqDepth int `json:"depth"` - Distance uint8 `json:"distance"` - Recu bool `json:"-"` - RecuDepth int `json:"-"` - URLs []string `json:"-"` - *parsers.Hashes `json:"hashes"` + *parsers.SprayResult + Url *url.URL `json:"-"` + Dir bool `json:"-"` + Chunked bool `json:"-"` + Body []byte `json:"-"` + Header []byte `json:"-"` + Raw []byte `json:"-"` + Recu bool `json:"-"` + RecuDepth int `json:"-"` + URLs []string `json:"-"` } func (bl *Baseline) IsDir() bool { @@ -137,9 +117,9 @@ func (bl *Baseline) Collect() { bl.Title = iutils.AsciiEncode(parsers.MatchTitle(string(bl.Body))) } else if bl.ContentType == "ico" { if name, ok := Md5Fingers[parsers.Md5Hash(bl.Body)]; ok { - bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name}) + bl.Frameworks[name] = &parsers.Framework{Name: name} } else if name, ok := Mmh3Fingers[parsers.Mmh3Hash32(bl.Body)]; ok { - bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name}) + bl.Frameworks[name] = &parsers.Framework{Name: name} } } } @@ -226,208 +206,3 @@ func (bl *Baseline) FuzzyCompare(other *Baseline) bool { } return false } - -func (bl *Baseline) Get(key string) string { - switch key { - case "url": - return bl.UrlString - case "host": - return bl.Host - case "content_type", "type": - return bl.ContentType - case "title": - return bl.Title - case "redirect": - return bl.RedirectURL - case "md5": - if bl.Hashes != nil { - return bl.Hashes.BodyMd5 - } else { - return "" - } - case "simhash": - if bl.Hashes != nil { - return bl.Hashes.BodySimhash - } else { - return "" - } - case "mmh3": - if bl.Hashes != nil { - return bl.Hashes.BodySimhash - } else { - return "" - } - case "stat", "status": - return strconv.Itoa(bl.Status) - case "spend": - return strconv.Itoa(int(bl.Spended)) + "ms" - case "length": - return strconv.Itoa(bl.BodyLength) - case "sim", "distance": - return "sim:" + strconv.Itoa(int(bl.Distance)) - case "source": - return GetSourceName(bl.Source) - case "extract": - return bl.Extracteds.String() - case "frame", "framework": - return bl.Frameworks.String() - case "full": - return bl.String() - default: - return "" - } -} - -func (bl *Baseline) Additional(key string) string { - if key == "frame" || key == "extract" { - return bl.Get(key) - } else if v := bl.Get(key); v != "" { - return " [" + v + "]" - } else { - return "" - } -} - -func (bl *Baseline) Format(probes []string) string { - var line strings.Builder - if bl.FrontURL != "" { - line.WriteString("\t") - line.WriteString(bl.FrontURL) - line.WriteString(" -> ") - } - line.WriteString(bl.UrlString) - if bl.Host != "" { - line.WriteString(" (" + bl.Host + ")") - } - - if bl.Reason != "" { - line.WriteString(" ,") - line.WriteString(bl.Reason) - } - if bl.ErrString != "" { - line.WriteString(" ,err: ") - line.WriteString(bl.ErrString) - return line.String() - } - - for _, p := range probes { - line.WriteString(" ") - line.WriteString(bl.Additional(p)) - } - - return line.String() -} - -func (bl *Baseline) ColorString() string { - var line strings.Builder - line.WriteString(logs.GreenLine("[" + GetSourceName(bl.Source) + "] ")) - if bl.FrontURL != "" { - line.WriteString(logs.CyanLine(bl.FrontURL)) - line.WriteString(" --> ") - } - line.WriteString(logs.GreenLine(bl.UrlString)) - if bl.Host != "" { - line.WriteString(" (" + bl.Host + ")") - } - - if bl.Reason != "" { - line.WriteString(" [reason: ") - line.WriteString(logs.YellowBold(bl.Reason)) - line.WriteString("]") - } - if bl.ErrString != "" { - line.WriteString(" [err: ") - line.WriteString(logs.RedBold(bl.ErrString)) - line.WriteString("]") - return line.String() - } - - line.WriteString(" - ") - line.WriteString(logs.GreenBold(strconv.Itoa(bl.Status))) - line.WriteString(" - ") - line.WriteString(logs.YellowBold(strconv.Itoa(bl.BodyLength))) - if bl.ExceedLength { - line.WriteString(logs.Red("(exceed)")) - } - line.WriteString(" - ") - line.WriteString(logs.YellowBold(strconv.Itoa(int(bl.Spended)) + "ms")) - line.WriteString(logs.GreenLine(bl.Additional("title"))) - if bl.Distance != 0 { - line.WriteString(logs.GreenLine(bl.Additional("sim"))) - } - line.WriteString(logs.Cyan(bl.Frameworks.String())) - line.WriteString(logs.Cyan(bl.Extracteds.String())) - if bl.RedirectURL != "" { - line.WriteString(" --> ") - line.WriteString(logs.CyanLine(bl.RedirectURL)) - line.WriteString(" ") - } - if len(bl.Extracteds) > 0 { - for _, e := range bl.Extracteds { - line.WriteString("\n " + e.Name + " (" + strconv.Itoa(len(e.ExtractResult)) + ") items : \n\t") - line.WriteString(logs.GreenLine(strings.Join(e.ExtractResult, "\n\t"))) - } - } - return line.String() -} - -func (bl *Baseline) String() string { - var line strings.Builder - line.WriteString(logs.GreenLine("[" + GetSourceName(bl.Source) + "] ")) - if bl.FrontURL != "" { - line.WriteString(bl.FrontURL) - line.WriteString(" --> ") - } - line.WriteString(bl.UrlString) - if bl.Host != "" { - line.WriteString(" (" + bl.Host + ")") - } - - if bl.Reason != "" { - line.WriteString(" [reason: ") - line.WriteString(bl.Reason) - line.WriteString("]") - } - if bl.ErrString != "" { - line.WriteString(" [err: ") - line.WriteString(bl.ErrString) - line.WriteString("]") - return line.String() - } - - line.WriteString(" - ") - line.WriteString(strconv.Itoa(bl.Status)) - line.WriteString(" - ") - line.WriteString(strconv.Itoa(bl.BodyLength)) - if bl.ExceedLength { - line.WriteString("(exceed)") - } - line.WriteString(" - ") - line.WriteString(strconv.Itoa(int(bl.Spended)) + "ms") - line.WriteString(bl.Additional("title")) - if bl.Distance != 0 { - line.WriteString(logs.GreenLine(bl.Additional("sim"))) - } - line.WriteString(bl.Frameworks.String()) - line.WriteString(bl.Extracteds.String()) - if bl.RedirectURL != "" { - line.WriteString(" --> ") - line.WriteString(bl.RedirectURL) - line.WriteString(" ") - } - if len(bl.Extracteds) > 0 { - for _, e := range bl.Extracteds { - line.WriteString("\n " + e.Name + " (" + strconv.Itoa(len(e.ExtractResult)) + ") items : \n\t") - line.WriteString(strings.Join(e.ExtractResult, "\n\t")) - } - } - return line.String() -} - -func (bl *Baseline) Jsonify() string { - bs, err := json.Marshal(bl) - if err != nil { - return "" - } - return string(bs) -} diff --git a/pkg/statistor.go b/pkg/statistor.go index aab7480..7201e3c 100644 --- a/pkg/statistor.go +++ b/pkg/statistor.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/chainreactors/logs" + "github.com/chainreactors/parsers" "io/ioutil" "strconv" "strings" @@ -109,7 +110,7 @@ func (stat *Statistor) SourceString() string { s.WriteString("[stat] ") s.WriteString(stat.BaseUrl) for k, v := range stat.Sources { - s.WriteString(fmt.Sprintf(" %s: %d,", GetSourceName(k), v)) + s.WriteString(fmt.Sprintf(" %s: %d,", parsers.GetSpraySourceName(k), v)) } return s.String() } @@ -132,7 +133,7 @@ func (stat *Statistor) ColorSourceString() string { s.WriteString("[stat] ") s.WriteString(stat.BaseUrl) for k, v := range stat.Sources { - s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(GetSourceName(k)), logs.YellowBold(strconv.Itoa(v)))) + s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(parsers.GetSpraySourceName(k)), logs.YellowBold(strconv.Itoa(v)))) } return s.String() } diff --git a/pkg/types.go b/pkg/types.go deleted file mode 100644 index f277c67..0000000 --- a/pkg/types.go +++ /dev/null @@ -1,45 +0,0 @@ -package pkg - -import ( - "github.com/chainreactors/parsers" - "strings" -) - -type Frameworks []*parsers.Framework - -func (fs Frameworks) String() string { - frameworkStrs := make([]string, len(fs)) - for i, f := range fs { - frameworkStrs[i] = " [" + f.String() + "]" - } - return strings.Join(frameworkStrs, " ") + " " -} - -func GetSourceName(s int) string { - switch s { - case 1: - return "check" - case 2: - return "random" - case 3: - return "index" - case 4: - return "redirect" - case 5: - return "crawl" - case 6: - return "active" - case 7: - return "word" - case 8: - return "waf" - case 9: - return "rule" - case 10: - return "bak" - case 11: - return "common" - default: - return "unknown" - } -} diff --git a/pkg/utils.go b/pkg/utils.go index c18895f..afa13e4 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -221,13 +221,13 @@ func LoadTemplates() error { return nil } -func FingerDetect(content string) Frameworks { - var frames Frameworks +func FingerDetect(content string) parsers.Frameworks { + frames := make(parsers.Frameworks) for _, finger := range Fingers { // sender置空, 所有的发包交给spray的pool frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil) if ok { - frames = append(frames, frame) + frames[frame.Name] = frame } } return frames