完成对extract相关功能的重构

cmd/cmd.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"github.com/chainreactors/gogo/v2/pkg/fingers"
 	"github.com/chainreactors/gogo/v2/pkg/utils"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/internal"
@@ -61,10 +60,10 @@ func Spray() {
 
 	if option.Extracts != nil {
 		for _, e := range option.Extracts {
-			if reg, ok := fingers.PresetExtracts[e]; ok {
+			if reg, ok := pkg.ExtractRegexps[e]; ok {
 				pkg.Extractors[e] = reg
 			} else {
-				pkg.Extractors[e] = regexp.MustCompile(e)
+				pkg.Extractors[e] = []*regexp.Regexp{regexp.MustCompile(e)}
 			}
 		}
 	}

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/chainreactors/gogo/v2 v2.10.1
 	github.com/chainreactors/ipcs v0.0.13
 	github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580
-	github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410
+	github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3
 	github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a
 )
 

go.sum

@@ -21,6 +21,12 @@ github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 h1:28gbL1t+Mm
 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
 github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410 h1:K7EV0wtUuN6Rvh/MgqaBXyElD3guPsgNR5kF8nrV7iw=
 github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
+github.com/chainreactors/parsers v0.2.9-0.20230117063421-a463730bca48 h1:zuCauWp/Em+hQoHkwymOsHjHm7sp9ydxhcymQVuY/+o=
+github.com/chainreactors/parsers v0.2.9-0.20230117063421-a463730bca48/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
+github.com/chainreactors/parsers v0.2.9-0.20230128044108-411f7c3b1bb7 h1:gUTcvvEaixo0Fom4i+hEc9ZuJwTmgp+TGlb+kYoJ8pI=
+github.com/chainreactors/parsers v0.2.9-0.20230128044108-411f7c3b1bb7/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
+github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3 h1:Hovp8f/5UK+cu+LnKECKSr2lNAakoyhcoybsGwQeg+8=
+github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U=
 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
 github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/option.go

@@ -5,6 +5,7 @@ import (
 	"github.com/antonmedv/expr"
 	"github.com/chainreactors/files"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers/iutils"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
 	"github.com/chainreactors/words/mask"
@@ -55,7 +56,6 @@ type FunctionOptions struct {
 type OutputOptions struct {
 	Match       string `long:"match" description:"String, custom match function, e.g.: --match current.Status != 200" json:"match,omitempty"`
 	Filter      string `long:"filter" description:"String, custom filter function, e.g.: --filter current.Body contains 'hello'" json:"filter,omitempty"`
-	Extracts    []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" json:"extracts,omitempty"`
 	OutputFile  string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty"`
 	Format      string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json"`
 	FuzzyFile   string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty"`
@@ -77,6 +77,7 @@ type RequestOptions struct {
 
 type PluginOptions struct {
 	Advance    bool     `short:"a" long:"advance" description:"Bool, enable crawl and active"`
+	Extracts   []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)"`
 	Active     bool     `long:"active" description:"Bool, enable active finger detect"`
 	Bak        bool     `long:"bak" description:"Bool, enable bak found"`
 	FileBak    bool     `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"`
@@ -369,7 +370,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	if opt.RemoveExtensions != "" {
 		rexts := strings.Split(opt.ExcludeExtensions, ",")
 		r.Fns = append(r.Fns, func(s string) string {
-			if ext := parseExtension(s); pkg.StringsContains(rexts, ext) {
+			if ext := parseExtension(s); iutils.StringsContains(rexts, ext) {
 				return strings.TrimSuffix(s, "."+ext)
 			}
 			return s
@@ -379,7 +380,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	if opt.ExcludeExtensions != "" {
 		exexts := strings.Split(opt.ExcludeExtensions, ",")
 		r.Fns = append(r.Fns, func(s string) string {
-			if ext := parseExtension(s); pkg.StringsContains(exexts, ext) {
+			if ext := parseExtension(s); iutils.StringsContains(exexts, ext) {
 				return ""
 			}
 			return s

internal/pool.go

@@ -6,6 +6,7 @@ import (
 	"github.com/antonmedv/expr"
 	"github.com/antonmedv/expr/vm"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers/iutils"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
 	"github.com/chainreactors/words"
@@ -439,7 +440,7 @@ func (pool *Pool) genReq(s string) (*ihttp.Request, error) {
 
 func (pool *Pool) PreCompare(resp *ihttp.Response) error {
 	status := resp.StatusCode()
-	if pkg.IntsContains(WhiteStatus, status) {
+	if iutils.IntsContains(WhiteStatus, status) {
 		// 如果为白名单状态码则直接返回
 		return nil
 	}
@@ -447,11 +448,11 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error {
 		return ErrSameStatus
 	}
 
-	if pkg.IntsContains(BlackStatus, status) {
+	if iutils.IntsContains(BlackStatus, status) {
 		return ErrBadStatus
 	}
 
-	if pkg.IntsContains(WAFStatus, status) {
+	if iutils.IntsContains(WAFStatus, status) {
 		return ErrWaf
 	}
 
@@ -683,7 +684,7 @@ func (pool *Pool) addAddition(u *Unit) {
 }
 
 func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) {
-	if _, ok := pool.baselines[bl.Status]; !ok && pkg.IntsContains(FuzzyStatus, bl.Status) {
+	if _, ok := pool.baselines[bl.Status]; !ok && iutils.IntsContains(FuzzyStatus, bl.Status) {
 		bl.Collect()
 		pool.waiter.Add(1)
 		pool.doCrawl(bl)

internal/utils.go

@@ -232,21 +232,3 @@ func FormatURL(base, u string) string {
 		return relaPath(base, u)
 	}
 }
-
-//func Join(base, u string) string {
-//	// //././ ../../../a
-//	base = Dir(base)
-//	for strings.HasPrefix(u, "../") {
-//		u = u[3:]
-//		for strings.HasSuffix(base, "/") {
-//			// 去掉多余的"/"
-//			base = base[:len(base)-2]
-//		}
-//		if i := strings.LastIndex(base, "/"); i == -1 {
-//			return "/"
-//		} else {
-//			return base[:i+1]
-//		}
-//	}
-//	return base + u
-//}

pkg/baseline.go

@@ -3,7 +3,6 @@ package pkg
 import (
 	"bytes"
 	"encoding/json"
-	"github.com/chainreactors/gogo/v2/pkg/fingers"
 	"github.com/chainreactors/gogo/v2/pkg/utils"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/parsers"
@@ -111,7 +110,7 @@ type Baseline struct {
 	ContentType     string             `json:"content_type"`
 	Title           string             `json:"title"`
 	Frameworks      Frameworks         `json:"frameworks"`
-	Extracteds      Extracteds `json:"extracts"`
+	Extracteds      parsers.Extracteds `json:"extracts"`
 	ErrString       string             `json:"error"`
 	Reason          string             `json:"reason"`
 	Source          int                `json:"source"`
@@ -154,7 +153,7 @@ func (bl *Baseline) CollectURL() {
 	if len(bl.Body) == 0 {
 		return
 	}
-	for _, reg := range JSRegexps {
+	for _, reg := range ExtractRegexps["js"] {
 		urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
 		for _, u := range urls {
 			u[1] = formatURL(u[1])
@@ -164,7 +163,7 @@ func (bl *Baseline) CollectURL() {
 		}
 	}
 
-	for _, reg := range URLRegexps {
+	for _, reg := range ExtractRegexps["url"] {
 		urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
 		for _, u := range urls {
 			u[1] = formatURL(u[1])
@@ -176,7 +175,7 @@ func (bl *Baseline) CollectURL() {
 
 	bl.URLs = RemoveDuplication(bl.URLs)
 	if bl.URLs != nil {
-		bl.Extracteds = append(bl.Extracteds, &fingers.Extracted{
+		bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{
 			Name:          "crawl",
 			ExtractResult: bl.URLs,
 		})

pkg/types.go

@@ -1,7 +1,6 @@
 package pkg
 
 import (
-	"github.com/chainreactors/gogo/v2/pkg/fingers"
 	"github.com/chainreactors/parsers"
 	"strings"
 )
@@ -16,18 +15,6 @@ func (fs Frameworks) String() string {
 	return strings.Join(frameworkStrs, " ") + " "
 }
 
-type Extracteds []*fingers.Extracted
-
-func (es Extracteds) String() string {
-	var s strings.Builder
-	for _, e := range es {
-		s.WriteString("[ " + e.ToString() + " ]")
-	}
-	return s.String() + " "
-}
-
-var Extractors = make(fingers.Extractors)
-
 func GetSourceName(s int) string {
 	switch s {
 	case 1:

pkg/utils.go

@@ -5,6 +5,8 @@ import (
 	"github.com/chainreactors/gogo/v2/pkg/fingers"
 	"github.com/chainreactors/gogo/v2/pkg/utils"
 	"github.com/chainreactors/ipcs"
+	"github.com/chainreactors/parsers"
+	"github.com/chainreactors/parsers/iutils"
 	"github.com/chainreactors/words/mask"
 	"math/rand"
 	"net/url"
@@ -22,18 +24,21 @@ var (
 	Rules       map[string]string = make(map[string]string)
 	ActivePath  []string
 	Fingers     fingers.Fingers
-	JSRegexps   []*regexp.Regexp = []*regexp.Regexp{
+	//JSRegexps   []*regexp.Regexp = []*regexp.Regexp{
-		regexp.MustCompile(`.(https{0,1}:[^\s'’"”><()|*\[]{2,250}?[^=*\s'’><:;|()[]{3}\[]\.js)`),
+	//	regexp.MustCompile(`.(https{0,1}:[^\s'’"”><()|*\[]{2,250}?[^=*\s'’><:;|()[]{3}\[]\.js)`),
-		regexp.MustCompile(`["']([^\s',’"”><;()|*:\[]{2,250}?[^=*\s'’|"”><^:;()\[]{3}\.js)`),
+	//	regexp.MustCompile(`["']([^\s',’"”><;()|*:\[]{2,250}?[^=*\s'’|"”><^:;()\[]{3}\.js)`),
-		regexp.MustCompile(`=\s{0,6}["']{0,1}\s{0,6}([^\s^'’,+><;()|*\[]{2,250}?[^=,\s'’"”>|<:;*()\[]{3}\.js)`),
+	//	regexp.MustCompile(`=\s{0,6}["']{0,1}\s{0,6}([^\s^'’,+><;()|*\[]{2,250}?[^=,\s'’"”>|<:;*()\[]{3}\.js)`),
-	}
+	//}
-	URLRegexps []*regexp.Regexp = []*regexp.Regexp{
+	//URLRegexps []*regexp.Regexp = []*regexp.Regexp{
-		regexp.MustCompile(`=\s{0,6}(https{0,1}:[^\s'"><()|*\[]{2,250})`),
+	//	regexp.MustCompile(`=\s{0,6}(https{0,1}:[^\s'"><()|*\[]{2,250})`),
-		regexp.MustCompile(`["']([^\s',’"”><.@$;:()|*\[]{2,250}\.[a-zA-Z]\w{1,4})["']`),
+	//	regexp.MustCompile(`["']([^\s',’"”><.@$;:()|*\[]{2,250}\.[a-zA-Z]\w{1,4})["']`),
-		regexp.MustCompile(`["'](https?:[^\s'"><()@|*\[]{2,250}?\.[^\s',’"”><;()|*\[]{2,250}?)["']`),
+	//	regexp.MustCompile(`["'](https?:[^\s'"><()@|*\[]{2,250}?\.[^\s',’"”><;()|*\[]{2,250}?)["']`),
-		regexp.MustCompile(`["']\s{0,6}([#,.]{0,2}/[^\s'",><;@$()|*\[]{2,250}?)\s{0,6}["']`),
+	//	regexp.MustCompile(`["']\s{0,6}([#,.]{0,2}/[^\s'",><;@$()|*\[]{2,250}?)\s{0,6}["']`),
-		regexp.MustCompile(`href\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s',’"”><$@;()|*\[]{2,250})|action\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s'’"“><)(]{2,250})`),
+	//	regexp.MustCompile(`href\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s',’"”><$@;()|*\[]{2,250})|action\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s'’"“><)(]{2,250})`),
-	}
+	//}
+	ExtractRegexps map[string][]*regexp.Regexp = map[string][]*regexp.Regexp{}
+	Extractors                                 = make(parsers.Extractors)
+
 	BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4"}
 	BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"}
 
@@ -62,24 +67,6 @@ var (
 	}
 )
 
-func StringsContains(s []string, e string) bool {
-	for _, v := range s {
-		if v == e {
-			return true
-		}
-	}
-	return false
-}
-
-func IntsContains(s []int, e int) bool {
-	for _, v := range s {
-		if v == e {
-			return true
-		}
-	}
-	return false
-}
-
 func RemoveDuplication(arr []string) []string {
 	set := make(map[string]struct{}, len(arr))
 	j := 0
@@ -213,12 +200,32 @@ func LoadTemplates() error {
 		}
 		mask.SpecialWords[k] = t
 	}
+
+	var extracts []*parsers.Extractor
+	err = json.Unmarshal(LoadConfig("extract"), &extracts)
+	if err != nil {
+		return err
+	}
+
+	for _, extract := range extracts {
+		extract.Compile()
+
+		ExtractRegexps[extract.Name] = extract.CompiledRegexps
+		for _, tag := range extract.Tags {
+			if _, ok := ExtractRegexps[tag]; !ok {
+				ExtractRegexps[tag] = extract.CompiledRegexps
+			} else {
+				ExtractRegexps[tag] = append(ExtractRegexps[tag], extract.CompiledRegexps...)
+			}
+		}
+	}
 	return nil
 }
 
 func FingerDetect(content string) Frameworks {
 	var frames Frameworks
 	for _, finger := range Fingers {
+		// sender置空, 所有的发包交给spray的pool
 		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
 		if ok {
 			frames = append(frames, frame)
@@ -317,7 +324,7 @@ func BakGenerator(domain string) []string {
 	for first, _ := range domain {
 		for last, _ := range domain[first:] {
 			p := domain[first : first+last+1]
-			if !StringsContains(possibilities, p) {
+			if !iutils.StringsContains(possibilities, p) {
 				possibilities = append(possibilities, p)
 			}
 		}

spray.go

@@ -1,4 +1,4 @@
-//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask
+//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask,extract
 package main
 
 import "github.com/chainreactors/spray/cmd"