spray输出相关的代码整合到spray中

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/chainreactors/gogo/v2 v2.10.4
 	github.com/chainreactors/ipcs v0.0.13
 	github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580
-	github.com/chainreactors/parsers v0.3.0
+	github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49
 	github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a
 )
 

go.sum

@@ -22,6 +22,10 @@ github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580/go.mod h1:Y0E
 github.com/chainreactors/parsers v0.2.9-0.20230128061921-aa83139d24f5/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
 github.com/chainreactors/parsers v0.3.0 h1:OJ/K/ZRsaSWkQNf6+3yliHrps+zj0fZl2/7juhjFQsY=
 github.com/chainreactors/parsers v0.3.0/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
+github.com/chainreactors/parsers v0.3.1-0.20230201101112-af520af38b4d h1:gJaqiDaHDmNKV2uTaEhDPHYu37gUvBuYDcJ7FYMulfw=
+github.com/chainreactors/parsers v0.3.1-0.20230201101112-af520af38b4d/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA=
+github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49 h1:snsLbWcScZRQV66TZxHfECkb2htZzhW2EcF1c60AA9c=
+github.com/chainreactors/parsers v0.3.1-0.20230201103008-e20167926b49/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA=
 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U=
 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
 github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/checkpool.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
 	"github.com/chainreactors/words"
@@ -42,7 +43,15 @@ func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) {
 
 		if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
 			pool.failedCount++
-			bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()}
+
+			bl = &pkg.Baseline{
+				SprayResult: &parsers.SprayResult{
+					UrlString: pool.BaseURL + unit.path,
+					IsValid:   false,
+					ErrString: reqerr.Error(),
+					Reason:    ErrRequestFailed.Error(),
+				},
+			}
 		} else {
 			bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
 			bl.Collect()

internal/pool.go

@@ -6,6 +6,7 @@ import (
 	"github.com/antonmedv/expr"
 	"github.com/antonmedv/expr/vm"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/parsers/iutils"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
@@ -283,7 +284,7 @@ Loop:
 				continue
 			}
 			if _, ok := pool.urls[unit.path]; ok {
-				logs.Log.Debugf("[%s] duplicate path: %s, skipped", pkg.GetSourceName(unit.source), pool.base+unit.path)
+				logs.Log.Debugf("[%s] duplicate path: %s, skipped", parsers.GetSpraySourceName(unit.source), pool.base+unit.path)
 				pool.waiter.Done()
 			} else {
 				pool.urls[unit.path] = struct{}{}
@@ -327,7 +328,14 @@ func (pool *Pool) Invoke(v interface{}) {
 	if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
 		atomic.AddInt32(&pool.failedCount, 1)
 		atomic.AddInt32(&pool.Statistor.FailedNumber, 1)
-		bl = &pkg.Baseline{UrlString: pool.base + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()}
+		bl = &pkg.Baseline{
+			SprayResult: &parsers.SprayResult{
+				UrlString: pool.base + unit.path,
+				IsValid:   false,
+				ErrString: reqerr.Error(),
+				Reason:    ErrRequestFailed.Error(),
+			},
+		}
 		pool.failedBaselines = append(pool.failedBaselines, bl)
 	} else {
 		if unit.source <= 3 || unit.source == CrawlSource || unit.source == CommonFileSource {

pkg/baseline.go

@@ -2,21 +2,21 @@ package pkg
 
 import (
 	"bytes"
-	"encoding/json"
-	"github.com/chainreactors/logs"
 	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/parsers/iutils"
 	"github.com/chainreactors/spray/pkg/ihttp"
 	"net/url"
-	"strconv"
 	"strings"
 )
 
 func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 	bl := &Baseline{
+		SprayResult: &parsers.SprayResult{
 			UrlString:  u,
 			Status:     resp.StatusCode(),
 			IsValid:    true,
+			Frameworks: make(parsers.Frameworks),
+		},
 	}
 	header := resp.Header()
 	bl.Header = make([]byte, len(header))
@@ -60,10 +60,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 
 func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline {
 	bl := &Baseline{
+		SprayResult: &parsers.SprayResult{
 			UrlString: u,
 			Status:    resp.StatusCode(),
 			IsValid:   false,
 			Reason:    reason,
+		},
 	}
 
 	// 无效数据也要读取body, 否则keep-alive不生效
@@ -88,38 +90,16 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
 }
 
 type Baseline struct {
-	Number          int                `json:"number"`
+	*parsers.SprayResult
 	Url       *url.URL `json:"-"`
-	IsValid         bool               `json:"valid"`
-	IsFuzzy         bool               `json:"fuzzy"`
-	UrlString       string             `json:"url"`
-	Path            string             `json:"path"`
 	Dir       bool     `json:"-"`
 	Chunked   bool     `json:"-"`
-	Host            string             `json:"host"`
 	Body      []byte   `json:"-"`
-	BodyLength      int                `json:"body_length"`
-	ExceedLength    bool               `json:"-"`
 	Header    []byte   `json:"-"`
 	Raw       []byte   `json:"-"`
-	HeaderLength    int                `json:"header_length"`
-	RedirectURL     string             `json:"redirect_url,omitempty"`
-	FrontURL        string             `json:"front_url,omitempty"`
-	Status          int                `json:"status"`
-	Spended         int64              `json:"spend"` // 耗时, 毫秒
-	ContentType     string             `json:"content_type"`
-	Title           string             `json:"title"`
-	Frameworks      Frameworks         `json:"frameworks"`
-	Extracteds      parsers.Extracteds `json:"extracts"`
-	ErrString       string             `json:"error"`
-	Reason          string             `json:"reason"`
-	Source          int                `json:"source"`
-	ReqDepth        int                `json:"depth"`
-	Distance        uint8              `json:"distance"`
 	Recu      bool     `json:"-"`
 	RecuDepth int      `json:"-"`
 	URLs      []string `json:"-"`
-	*parsers.Hashes `json:"hashes"`
 }
 
 func (bl *Baseline) IsDir() bool {
@@ -137,9 +117,9 @@ func (bl *Baseline) Collect() {
 			bl.Title = iutils.AsciiEncode(parsers.MatchTitle(string(bl.Body)))
 		} else if bl.ContentType == "ico" {
 			if name, ok := Md5Fingers[parsers.Md5Hash(bl.Body)]; ok {
-				bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name})
+				bl.Frameworks[name] = &parsers.Framework{Name: name}
 			} else if name, ok := Mmh3Fingers[parsers.Mmh3Hash32(bl.Body)]; ok {
-				bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name})
+				bl.Frameworks[name] = &parsers.Framework{Name: name}
 			}
 		}
 	}
@@ -226,208 +206,3 @@ func (bl *Baseline) FuzzyCompare(other *Baseline) bool {
 	}
 	return false
 }
-
-func (bl *Baseline) Get(key string) string {
-	switch key {
-	case "url":
-		return bl.UrlString
-	case "host":
-		return bl.Host
-	case "content_type", "type":
-		return bl.ContentType
-	case "title":
-		return bl.Title
-	case "redirect":
-		return bl.RedirectURL
-	case "md5":
-		if bl.Hashes != nil {
-			return bl.Hashes.BodyMd5
-		} else {
-			return ""
-		}
-	case "simhash":
-		if bl.Hashes != nil {
-			return bl.Hashes.BodySimhash
-		} else {
-			return ""
-		}
-	case "mmh3":
-		if bl.Hashes != nil {
-			return bl.Hashes.BodySimhash
-		} else {
-			return ""
-		}
-	case "stat", "status":
-		return strconv.Itoa(bl.Status)
-	case "spend":
-		return strconv.Itoa(int(bl.Spended)) + "ms"
-	case "length":
-		return strconv.Itoa(bl.BodyLength)
-	case "sim", "distance":
-		return "sim:" + strconv.Itoa(int(bl.Distance))
-	case "source":
-		return GetSourceName(bl.Source)
-	case "extract":
-		return bl.Extracteds.String()
-	case "frame", "framework":
-		return bl.Frameworks.String()
-	case "full":
-		return bl.String()
-	default:
-		return ""
-	}
-}
-
-func (bl *Baseline) Additional(key string) string {
-	if key == "frame" || key == "extract" {
-		return bl.Get(key)
-	} else if v := bl.Get(key); v != "" {
-		return " [" + v + "]"
-	} else {
-		return ""
-	}
-}
-
-func (bl *Baseline) Format(probes []string) string {
-	var line strings.Builder
-	if bl.FrontURL != "" {
-		line.WriteString("\t")
-		line.WriteString(bl.FrontURL)
-		line.WriteString(" -> ")
-	}
-	line.WriteString(bl.UrlString)
-	if bl.Host != "" {
-		line.WriteString(" (" + bl.Host + ")")
-	}
-
-	if bl.Reason != "" {
-		line.WriteString(" ,")
-		line.WriteString(bl.Reason)
-	}
-	if bl.ErrString != "" {
-		line.WriteString(" ,err: ")
-		line.WriteString(bl.ErrString)
-		return line.String()
-	}
-
-	for _, p := range probes {
-		line.WriteString(" ")
-		line.WriteString(bl.Additional(p))
-	}
-
-	return line.String()
-}
-
-func (bl *Baseline) ColorString() string {
-	var line strings.Builder
-	line.WriteString(logs.GreenLine("[" + GetSourceName(bl.Source) + "] "))
-	if bl.FrontURL != "" {
-		line.WriteString(logs.CyanLine(bl.FrontURL))
-		line.WriteString(" --> ")
-	}
-	line.WriteString(logs.GreenLine(bl.UrlString))
-	if bl.Host != "" {
-		line.WriteString(" (" + bl.Host + ")")
-	}
-
-	if bl.Reason != "" {
-		line.WriteString(" [reason: ")
-		line.WriteString(logs.YellowBold(bl.Reason))
-		line.WriteString("]")
-	}
-	if bl.ErrString != "" {
-		line.WriteString(" [err: ")
-		line.WriteString(logs.RedBold(bl.ErrString))
-		line.WriteString("]")
-		return line.String()
-	}
-
-	line.WriteString(" - ")
-	line.WriteString(logs.GreenBold(strconv.Itoa(bl.Status)))
-	line.WriteString(" - ")
-	line.WriteString(logs.YellowBold(strconv.Itoa(bl.BodyLength)))
-	if bl.ExceedLength {
-		line.WriteString(logs.Red("(exceed)"))
-	}
-	line.WriteString(" - ")
-	line.WriteString(logs.YellowBold(strconv.Itoa(int(bl.Spended)) + "ms"))
-	line.WriteString(logs.GreenLine(bl.Additional("title")))
-	if bl.Distance != 0 {
-		line.WriteString(logs.GreenLine(bl.Additional("sim")))
-	}
-	line.WriteString(logs.Cyan(bl.Frameworks.String()))
-	line.WriteString(logs.Cyan(bl.Extracteds.String()))
-	if bl.RedirectURL != "" {
-		line.WriteString(" --> ")
-		line.WriteString(logs.CyanLine(bl.RedirectURL))
-		line.WriteString(" ")
-	}
-	if len(bl.Extracteds) > 0 {
-		for _, e := range bl.Extracteds {
-			line.WriteString("\n  " + e.Name + " (" + strconv.Itoa(len(e.ExtractResult)) + ") items : \n\t")
-			line.WriteString(logs.GreenLine(strings.Join(e.ExtractResult, "\n\t")))
-		}
-	}
-	return line.String()
-}
-
-func (bl *Baseline) String() string {
-	var line strings.Builder
-	line.WriteString(logs.GreenLine("[" + GetSourceName(bl.Source) + "] "))
-	if bl.FrontURL != "" {
-		line.WriteString(bl.FrontURL)
-		line.WriteString(" --> ")
-	}
-	line.WriteString(bl.UrlString)
-	if bl.Host != "" {
-		line.WriteString(" (" + bl.Host + ")")
-	}
-
-	if bl.Reason != "" {
-		line.WriteString(" [reason: ")
-		line.WriteString(bl.Reason)
-		line.WriteString("]")
-	}
-	if bl.ErrString != "" {
-		line.WriteString(" [err: ")
-		line.WriteString(bl.ErrString)
-		line.WriteString("]")
-		return line.String()
-	}
-
-	line.WriteString(" - ")
-	line.WriteString(strconv.Itoa(bl.Status))
-	line.WriteString(" - ")
-	line.WriteString(strconv.Itoa(bl.BodyLength))
-	if bl.ExceedLength {
-		line.WriteString("(exceed)")
-	}
-	line.WriteString(" - ")
-	line.WriteString(strconv.Itoa(int(bl.Spended)) + "ms")
-	line.WriteString(bl.Additional("title"))
-	if bl.Distance != 0 {
-		line.WriteString(logs.GreenLine(bl.Additional("sim")))
-	}
-	line.WriteString(bl.Frameworks.String())
-	line.WriteString(bl.Extracteds.String())
-	if bl.RedirectURL != "" {
-		line.WriteString(" --> ")
-		line.WriteString(bl.RedirectURL)
-		line.WriteString(" ")
-	}
-	if len(bl.Extracteds) > 0 {
-		for _, e := range bl.Extracteds {
-			line.WriteString("\n  " + e.Name + " (" + strconv.Itoa(len(e.ExtractResult)) + ") items : \n\t")
-			line.WriteString(strings.Join(e.ExtractResult, "\n\t"))
-		}
-	}
-	return line.String()
-}
-
-func (bl *Baseline) Jsonify() string {
-	bs, err := json.Marshal(bl)
-	if err != nil {
-		return ""
-	}
-	return string(bs)
-}

pkg/statistor.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers"
 	"io/ioutil"
 	"strconv"
 	"strings"
@@ -109,7 +110,7 @@ func (stat *Statistor) SourceString() string {
 	s.WriteString("[stat] ")
 	s.WriteString(stat.BaseUrl)
 	for k, v := range stat.Sources {
-		s.WriteString(fmt.Sprintf(" %s: %d,", GetSourceName(k), v))
+		s.WriteString(fmt.Sprintf(" %s: %d,", parsers.GetSpraySourceName(k), v))
 	}
 	return s.String()
 }
@@ -132,7 +133,7 @@ func (stat *Statistor) ColorSourceString() string {
 	s.WriteString("[stat] ")
 	s.WriteString(stat.BaseUrl)
 	for k, v := range stat.Sources {
-		s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(GetSourceName(k)), logs.YellowBold(strconv.Itoa(v))))
+		s.WriteString(fmt.Sprintf(" %s: %s,", logs.Cyan(parsers.GetSpraySourceName(k)), logs.YellowBold(strconv.Itoa(v))))
 	}
 	return s.String()
 }

pkg/types.go

@@ -1,45 +0,0 @@
-package pkg
-
-import (
-	"github.com/chainreactors/parsers"
-	"strings"
-)
-
-type Frameworks []*parsers.Framework
-
-func (fs Frameworks) String() string {
-	frameworkStrs := make([]string, len(fs))
-	for i, f := range fs {
-		frameworkStrs[i] = " [" + f.String() + "]"
-	}
-	return strings.Join(frameworkStrs, " ") + " "
-}
-
-func GetSourceName(s int) string {
-	switch s {
-	case 1:
-		return "check"
-	case 2:
-		return "random"
-	case 3:
-		return "index"
-	case 4:
-		return "redirect"
-	case 5:
-		return "crawl"
-	case 6:
-		return "active"
-	case 7:
-		return "word"
-	case 8:
-		return "waf"
-	case 9:
-		return "rule"
-	case 10:
-		return "bak"
-	case 11:
-		return "common"
-	default:
-		return "unknown"
-	}
-}

pkg/utils.go

@@ -221,13 +221,13 @@ func LoadTemplates() error {
 	return nil
 }
 
-func FingerDetect(content string) Frameworks {
+func FingerDetect(content string) parsers.Frameworks {
-	var frames Frameworks
+	frames := make(parsers.Frameworks)
 	for _, finger := range Fingers {
 		// sender置空, 所有的发包交给spray的pool
 		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
 		if ok {
-			frames = append(frames, frame)
+			frames[frame.Name] = frame
 		}
 	}
 	return frames