issues19, add custom extract config

cmd/cmd.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"github.com/chainreactors/logs"
-	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/spray/internal"
 	"github.com/chainreactors/spray/internal/ihttp"
 	"github.com/chainreactors/spray/internal/pool"
@@ -13,7 +12,6 @@ import (
 	"github.com/jessevdk/go-flags"
 	"os"
 	"os/signal"
-	"regexp"
 	"syscall"
 	"time"
 )
@@ -93,20 +91,6 @@ func Spray() {
 	if err != nil {
 		iutils.Fatal(err.Error())
 	}
-	if option.Extracts != nil {
-		for _, e := range option.Extracts {
-			if reg, ok := pkg.ExtractRegexps[e]; ok {
-				pkg.Extractors[e] = reg
-			} else {
-				pkg.Extractors[e] = []*parsers.Extractor{
-					&parsers.Extractor{
-						Name:            e,
-						CompiledRegexps: []*regexp.Regexp{regexp.MustCompile(e)},
-					},
-				}
-			}
-		}
-	}
 
 	// 初始化全局变量
 	pkg.Distance = uint8(option.SimhashDistance)

internal/option.go

@@ -7,6 +7,7 @@ import (
 	"github.com/antonmedv/expr"
 	"github.com/chainreactors/files"
 	"github.com/chainreactors/logs"
+	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/spray/internal/ihttp"
 	"github.com/chainreactors/spray/internal/pool"
 	"github.com/chainreactors/spray/pkg"
@@ -18,6 +19,7 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -95,6 +97,7 @@ type RequestOptions struct {
 type PluginOptions struct {
 	Advance       bool     `short:"a" long:"advance" description:"Bool, enable all plugin" config:"all" `
 	Extracts      []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" config:"extract"`
+	ExtractConfig string   `long:"extract-config" description:"String, extract config filename" config:"extract-config"`
 	Recon         bool     `long:"recon" description:"Bool, enable recon" config:"recon"`
 	Finger        bool     `long:"finger" description:"Bool, enable active finger detect" config:"finger"`
 	Bak           bool     `long:"bak" description:"Bool, enable bak found" config:"bak"`
@@ -215,6 +218,29 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 	if opt.Threads == DefaultThreads && opt.CheckOnly {
 		r.Threads = 1000
 	}
+
+	if opt.Extracts != nil {
+		for _, e := range opt.Extracts {
+			if reg, ok := pkg.ExtractRegexps[e]; ok {
+				pkg.Extractors[e] = reg
+			} else {
+				pkg.Extractors[e] = []*parsers.Extractor{
+					&parsers.Extractor{
+						Name:            e,
+						CompiledRegexps: []*regexp.Regexp{regexp.MustCompile(e)},
+					},
+				}
+			}
+		}
+	}
+	if opt.ExtractConfig != "" {
+		extracts, err := pkg.LoadExtractorConfig(opt.ExtractConfig)
+		if err != nil {
+			return nil, err
+		}
+		pkg.Extractors[opt.ExtractConfig] = extracts
+	}
+
 	if opt.Recon {
 		pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"]
 	}

pkg/load.go

@@ -7,12 +7,16 @@ import (
 	"github.com/chainreactors/utils"
 	"github.com/chainreactors/utils/iutils"
 	"github.com/chainreactors/words/mask"
+	"os"
+	yaml "sigs.k8s.io/yaml/goyaml.v3"
 	"strings"
 )
 
 var (
 	Md5Fingers      map[string]string = make(map[string]string)
 	Mmh3Fingers     map[string]string = make(map[string]string)
+	ExtractRegexps                    = make(parsers.Extractors)
+	Extractors                        = make(parsers.Extractors)
 	Fingers         fingers.Fingers
 	ActivePath      []string
 	FingerPrintHubs []FingerPrintHub
@@ -95,6 +99,25 @@ func LoadTemplates() error {
 	return nil
 }
 
+func LoadExtractorConfig(filename string) ([]*parsers.Extractor, error) {
+	var extracts []*parsers.Extractor
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(content, &extracts)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, extract := range extracts {
+		extract.Compile()
+	}
+
+	return extracts, nil
+}
+
 func LoadFingerPrintHub() error {
 	content := LoadConfig("fingerprinthub")
 	err := json.Unmarshal(content, &FingerPrintHubs)

pkg/utils.go

@@ -4,7 +4,6 @@ import (
 	"github.com/antonmedv/expr"
 	"github.com/antonmedv/expr/vm"
 	"github.com/chainreactors/logs"
-	"github.com/chainreactors/parsers"
 	"github.com/chainreactors/utils/iutils"
 	"math/rand"
 	"net/url"
@@ -30,8 +29,6 @@ var (
 )
 var (
 	Rules map[string]string = make(map[string]string)
-	ExtractRegexps                   = map[string][]*parsers.Extractor{}
-	Extractors                       = make(parsers.Extractors)
 
 	BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4", ".zip", ".rar"}
 	BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"}