From 26cc384de0455b5f3b7942a90827b9c2ae201f71 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 5 Jan 2023 23:26:14 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E--file-bak=E5=8F=82=E6=95=B0,?= =?UTF-8?q?=20=E5=BC=80=E5=90=AF=E6=9C=89=E6=95=88=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E7=9A=84=E5=A4=87=E4=BB=BD=E6=96=87=E4=BB=B6=E7=88=86=E7=A0=B4?= =?UTF-8?q?,=20=E5=8F=AF=E4=BB=A5=E4=BD=BF=E7=94=A8-a=E5=90=8C=E6=97=B6?= =?UTF-8?q?=E5=BC=80=E5=90=AF=E4=B8=89=E4=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/cmd.go | 12 ++++++++++++ internal/option.go | 38 +++++++++++++++++--------------------- internal/utils.go | 19 +++++++++++++++++++ pkg/utils.go | 18 ++++++++++++++++-- rule/filebak.txt | 12 ------------ spray.go | 2 +- 6 files changed, 65 insertions(+), 36 deletions(-) delete mode 100644 rule/filebak.txt diff --git a/cmd/cmd.go b/cmd/cmd.go index 91727b8..f238b1e 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -3,8 +3,10 @@ package cmd import ( "context" "fmt" + "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/logs" "github.com/chainreactors/spray/internal" + "github.com/chainreactors/spray/pkg" "github.com/jessevdk/go-flags" "os" "os/signal" @@ -48,6 +50,16 @@ func Spray() { internal.Format(option.Format) os.Exit(0) } + + err = pkg.LoadTemplates() + if err != nil { + utils.Fatal(err.Error()) + } + err = pkg.LoadRules() + if err != nil { + utils.Fatal(err.Error()) + } + var runner *internal.Runner if option.ResumeFrom != "" { runner, err = option.PrepareRunner() diff --git a/internal/option.go b/internal/option.go index 7e54e63..824a5a6 100644 --- a/internal/option.go +++ b/internal/option.go @@ -1,7 +1,6 @@ package internal import ( - "bytes" "fmt" "github.com/antonmedv/expr" "github.com/chainreactors/files" @@ -39,7 +38,7 @@ type InputOptions struct { Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt"` Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}"` Rules []string `short:"r" long:"rules" description:"Files, Multi, rule files, e.g.: -r rule1.txt -r rule2.txt"` - AppendRule string `long:"append-rule" description:"File, when found valid path , use append rule generator new word with current path"` + AppendRule []string `long:"append-rule" description:"File, when found valid path , use append rule generator new word with current path"` FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'"` } @@ -84,6 +83,7 @@ type ModeOptions struct { Depth int `long:"depth" default:"0" description:"Int, recursive depth"` Active bool `long:"active" description:"Bool, enable active finger detect"` Crawl bool `long:"crawl" description:"Bool, enable crawl"` + FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"` CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth"` CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request"` ErrPeriod int `long:"error-period" default:"10" description:"Int, check period when error"` @@ -135,14 +135,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Crawl: opt.Crawl, Active: opt.Active, } - if opt.Advance { - r.Crawl = true - r.Active = true - } - err = pkg.LoadTemplates() - if err != nil { - return nil, err - } if opt.Extracts != nil { for _, e := range opt.Extracts { @@ -175,6 +167,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { pkg.Distance = uint8(opt.SimhashDistance) ihttp.DefaultMaxBodySize = opt.MaxBodyLength * 1024 + // configuration if opt.Force { // 如果开启了force模式, 将关闭check机制, err积累到一定数量自动退出机制 r.BreakThreshold = max @@ -182,6 +175,14 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.ErrPeriod = max } + if opt.Advance { + r.Crawl = true + r.Active = true + opt.AppendRule = append(opt.AppendRule, "filebak") + } else if opt.FileBak { + opt.AppendRule = append(opt.AppendRule, "filebak") + } + if opt.BlackStatus != "" { for _, s := range strings.Split(opt.BlackStatus, ",") { si, err := strconv.Atoi(s) @@ -262,16 +263,11 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } if opt.Rules != nil { - var rules bytes.Buffer - for _, rule := range opt.Rules { - content, err := ioutil.ReadFile(rule) - if err != nil { - return nil, err - } - rules.Write(content) - rules.WriteString("\n") + rules, err := loadFileAndCombine(opt.Rules) + if err != nil { + return nil, err } - r.Rules = rule.Compile(rules.String(), opt.FilterRule) + r.Rules = rule.Compile(rules, opt.FilterRule) } else if opt.FilterRule != "" { // if filter rule is not empty, set rules to ":", force to open filter mode r.Rules = rule.Compile(":", opt.FilterRule) @@ -295,8 +291,8 @@ func (opt *Option) PrepareRunner() (*Runner, error) { Total: r.Total, } - if opt.AppendRule != "" { - content, err := ioutil.ReadFile(opt.AppendRule) + if opt.AppendRule != nil { + content, err := loadFileAndCombine(opt.AppendRule) if err != nil { return nil, err } diff --git a/internal/utils.go b/internal/utils.go index 329aca9..d383023 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -2,6 +2,7 @@ package internal import ( "bytes" + "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "io/ioutil" @@ -50,6 +51,24 @@ func loadFileToSlice(filename string) ([]string, error) { return ss, nil } +func loadFileAndCombine(filename []string) (string, error) { + var bs bytes.Buffer + for _, f := range filename { + if data, ok := pkg.Rules[f]; ok { + bs.WriteString(strings.TrimSpace(data)) + bs.WriteString("\n") + } else { + content, err := ioutil.ReadFile(f) + if err != nil { + return "", err + } + bs.Write(bytes.TrimSpace(content)) + bs.WriteString("\n") + } + } + return bs.String(), nil +} + func loadFileWithCache(filename string) ([]string, error) { if dict, ok := dictCache[filename]; ok { return dict, nil diff --git a/pkg/utils.go b/pkg/utils.go index aa6b06e..2dc9dec 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -1,8 +1,8 @@ package pkg import ( + "encoding/json" "github.com/chainreactors/gogo/v2/pkg/fingers" - "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/ipcs" "math/rand" "net/url" @@ -17,6 +17,7 @@ import ( var ( Md5Fingers map[string]string = make(map[string]string) Mmh3Fingers map[string]string = make(map[string]string) + Rules map[string]string = make(map[string]string) ActivePath []string Fingers fingers.Fingers JSRegexps []*regexp.Regexp = []*regexp.Regexp{ @@ -99,7 +100,7 @@ func LoadTemplates() error { var err error Fingers, err = fingers.LoadFingers(LoadConfig("http")) if err != nil { - utils.Fatal(err.Error()) + return err } for _, finger := range Fingers { @@ -128,6 +129,18 @@ func LoadTemplates() error { return nil } +func LoadRules() error { + var data map[string]interface{} + err := json.Unmarshal(LoadConfig("rule"), &data) + if err != nil { + return err + } + for k, v := range data { + Rules[k] = v.(string) + } + return nil +} + func FingerDetect(content string) Frameworks { var frames Frameworks //content := string(body) @@ -174,6 +187,7 @@ func filterUrl(u string) bool { } return false } + func URLJoin(base, uri string) string { baseSlash := strings.HasSuffix(base, "/") uriSlash := strings.HasPrefix(uri, "/") diff --git a/rule/filebak.txt b/rule/filebak.txt deleted file mode 100644 index 881f5b7..0000000 --- a/rule/filebak.txt +++ /dev/null @@ -1,12 +0,0 @@ -$~ -$b $a $k -$. $b $a $k -$. $b $a $k $2 -$. $o $l $d -$. $1 -$. $2 -$. $z $i $p -$. $t $a $r -$. $g $z -^. -^. $. $s $w $p \ No newline at end of file diff --git a/spray.go b/spray.go index 73ab69b..8972553 100644 --- a/spray.go +++ b/spray.go @@ -1,4 +1,4 @@ -//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http +//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule package main import "github.com/chainreactors/spray/cmd"