diff --git a/cmd/cmd.go b/cmd/cmd.go index 0c70573..ef862fd 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/logs" "github.com/chainreactors/spray/internal" @@ -61,10 +60,10 @@ func Spray() { if option.Extracts != nil { for _, e := range option.Extracts { - if reg, ok := fingers.PresetExtracts[e]; ok { + if reg, ok := pkg.ExtractRegexps[e]; ok { pkg.Extractors[e] = reg } else { - pkg.Extractors[e] = regexp.MustCompile(e) + pkg.Extractors[e] = []*regexp.Regexp{regexp.MustCompile(e)} } } } diff --git a/go.mod b/go.mod index 05790b4..78a89d9 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/chainreactors/gogo/v2 v2.10.1 github.com/chainreactors/ipcs v0.0.13 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 - github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410 + github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a ) diff --git a/go.sum b/go.sum index d9793b2..c3446b1 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,12 @@ github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 h1:28gbL1t+Mm github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410 h1:K7EV0wtUuN6Rvh/MgqaBXyElD3guPsgNR5kF8nrV7iw= github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= +github.com/chainreactors/parsers v0.2.9-0.20230117063421-a463730bca48 h1:zuCauWp/Em+hQoHkwymOsHjHm7sp9ydxhcymQVuY/+o= +github.com/chainreactors/parsers v0.2.9-0.20230117063421-a463730bca48/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= +github.com/chainreactors/parsers v0.2.9-0.20230128044108-411f7c3b1bb7 h1:gUTcvvEaixo0Fom4i+hEc9ZuJwTmgp+TGlb+kYoJ8pI= +github.com/chainreactors/parsers v0.2.9-0.20230128044108-411f7c3b1bb7/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= +github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3 h1:Hovp8f/5UK+cu+LnKECKSr2lNAakoyhcoybsGwQeg+8= +github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/option.go b/internal/option.go index aae4de8..7b6e263 100644 --- a/internal/option.go +++ b/internal/option.go @@ -5,6 +5,7 @@ import ( "github.com/antonmedv/expr" "github.com/chainreactors/files" "github.com/chainreactors/logs" + "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/words/mask" @@ -53,17 +54,16 @@ type FunctionOptions struct { } type OutputOptions struct { - Match string `long:"match" description:"String, custom match function, e.g.: --match current.Status != 200" json:"match,omitempty"` - Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter current.Body contains 'hello'" json:"filter,omitempty"` - Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" json:"extracts,omitempty"` - OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty"` - Format string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json"` - FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty"` - DumpFile string `long:"dump-file" description:"String, dump all request, and write to filename"` - Dump bool `long:"dump" description:"Bool, dump all request"` - AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" ` - Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output" json:"fuzzy,omitempty"` - OutputProbe string `short:"o" long:"probe" description:"String, output format" json:"output_probe,omitempty"` + Match string `long:"match" description:"String, custom match function, e.g.: --match current.Status != 200" json:"match,omitempty"` + Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter current.Body contains 'hello'" json:"filter,omitempty"` + OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty"` + Format string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json"` + FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty"` + DumpFile string `long:"dump-file" description:"String, dump all request, and write to filename"` + Dump bool `long:"dump" description:"Bool, dump all request"` + AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" ` + Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output" json:"fuzzy,omitempty"` + OutputProbe string `short:"o" long:"probe" description:"String, output format" json:"output_probe,omitempty"` } type RequestOptions struct { @@ -76,14 +76,15 @@ type RequestOptions struct { } type PluginOptions struct { - Advance bool `short:"a" long:"advance" description:"Bool, enable crawl and active"` - Active bool `long:"active" description:"Bool, enable active finger detect"` - Bak bool `long:"bak" description:"Bool, enable bak found"` - FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"` - Common bool `long:"common" description:"Bool, enable common file found"` - Crawl bool `long:"crawl" description:"Bool, enable crawl"` - CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth"` - CrawlScope string `long:"crawl-scope" description:"Int, crawl scope (todo)"` + Advance bool `short:"a" long:"advance" description:"Bool, enable crawl and active"` + Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)"` + Active bool `long:"active" description:"Bool, enable active finger detect"` + Bak bool `long:"bak" description:"Bool, enable bak found"` + FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"` + Common bool `long:"common" description:"Bool, enable common file found"` + Crawl bool `long:"crawl" description:"Bool, enable crawl"` + CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth"` + CrawlScope string `long:"crawl-scope" description:"Int, crawl scope (todo)"` } type ModeOptions struct { @@ -369,7 +370,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if opt.RemoveExtensions != "" { rexts := strings.Split(opt.ExcludeExtensions, ",") r.Fns = append(r.Fns, func(s string) string { - if ext := parseExtension(s); pkg.StringsContains(rexts, ext) { + if ext := parseExtension(s); iutils.StringsContains(rexts, ext) { return strings.TrimSuffix(s, "."+ext) } return s @@ -379,7 +380,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if opt.ExcludeExtensions != "" { exexts := strings.Split(opt.ExcludeExtensions, ",") r.Fns = append(r.Fns, func(s string) string { - if ext := parseExtension(s); pkg.StringsContains(exexts, ext) { + if ext := parseExtension(s); iutils.StringsContains(exexts, ext) { return "" } return s diff --git a/internal/pool.go b/internal/pool.go index 6e55b70..2ab9e37 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -6,6 +6,7 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" "github.com/chainreactors/logs" + "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" "github.com/chainreactors/words" @@ -439,7 +440,7 @@ func (pool *Pool) genReq(s string) (*ihttp.Request, error) { func (pool *Pool) PreCompare(resp *ihttp.Response) error { status := resp.StatusCode() - if pkg.IntsContains(WhiteStatus, status) { + if iutils.IntsContains(WhiteStatus, status) { // 如果为白名单状态码则直接返回 return nil } @@ -447,11 +448,11 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error { return ErrSameStatus } - if pkg.IntsContains(BlackStatus, status) { + if iutils.IntsContains(BlackStatus, status) { return ErrBadStatus } - if pkg.IntsContains(WAFStatus, status) { + if iutils.IntsContains(WAFStatus, status) { return ErrWaf } @@ -683,7 +684,7 @@ func (pool *Pool) addAddition(u *Unit) { } func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) { - if _, ok := pool.baselines[bl.Status]; !ok && pkg.IntsContains(FuzzyStatus, bl.Status) { + if _, ok := pool.baselines[bl.Status]; !ok && iutils.IntsContains(FuzzyStatus, bl.Status) { bl.Collect() pool.waiter.Add(1) pool.doCrawl(bl) diff --git a/internal/utils.go b/internal/utils.go index c92ce14..b0bd2f6 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -232,21 +232,3 @@ func FormatURL(base, u string) string { return relaPath(base, u) } } - -//func Join(base, u string) string { -// // //././ ../../../a -// base = Dir(base) -// for strings.HasPrefix(u, "../") { -// u = u[3:] -// for strings.HasSuffix(base, "/") { -// // 去掉多余的"/" -// base = base[:len(base)-2] -// } -// if i := strings.LastIndex(base, "/"); i == -1 { -// return "/" -// } else { -// return base[:i+1] -// } -// } -// return base + u -//} diff --git a/pkg/baseline.go b/pkg/baseline.go index 27dc262..34fa449 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -3,7 +3,6 @@ package pkg import ( "bytes" "encoding/json" - "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/logs" "github.com/chainreactors/parsers" @@ -89,37 +88,37 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba } type Baseline struct { - Number int `json:"number"` - Url *url.URL `json:"-"` - IsValid bool `json:"valid"` - IsFuzzy bool `json:"fuzzy"` - UrlString string `json:"url"` - Path string `json:"path"` - Dir bool `json:"-"` - Chunked bool `json:"-"` - Host string `json:"host"` - Body []byte `json:"-"` - BodyLength int `json:"body_length"` - ExceedLength bool `json:"-"` - Header []byte `json:"-"` - Raw []byte `json:"-"` - HeaderLength int `json:"header_length"` - RedirectURL string `json:"redirect_url,omitempty"` - FrontURL string `json:"front_url,omitempty"` - Status int `json:"status"` - Spended int64 `json:"spend"` // 耗时, 毫秒 - ContentType string `json:"content_type"` - Title string `json:"title"` - Frameworks Frameworks `json:"frameworks"` - Extracteds Extracteds `json:"extracts"` - ErrString string `json:"error"` - Reason string `json:"reason"` - Source int `json:"source"` - ReqDepth int `json:"depth"` - Distance uint8 `json:"distance"` - Recu bool `json:"-"` - RecuDepth int `json:"-"` - URLs []string `json:"-"` + Number int `json:"number"` + Url *url.URL `json:"-"` + IsValid bool `json:"valid"` + IsFuzzy bool `json:"fuzzy"` + UrlString string `json:"url"` + Path string `json:"path"` + Dir bool `json:"-"` + Chunked bool `json:"-"` + Host string `json:"host"` + Body []byte `json:"-"` + BodyLength int `json:"body_length"` + ExceedLength bool `json:"-"` + Header []byte `json:"-"` + Raw []byte `json:"-"` + HeaderLength int `json:"header_length"` + RedirectURL string `json:"redirect_url,omitempty"` + FrontURL string `json:"front_url,omitempty"` + Status int `json:"status"` + Spended int64 `json:"spend"` // 耗时, 毫秒 + ContentType string `json:"content_type"` + Title string `json:"title"` + Frameworks Frameworks `json:"frameworks"` + Extracteds parsers.Extracteds `json:"extracts"` + ErrString string `json:"error"` + Reason string `json:"reason"` + Source int `json:"source"` + ReqDepth int `json:"depth"` + Distance uint8 `json:"distance"` + Recu bool `json:"-"` + RecuDepth int `json:"-"` + URLs []string `json:"-"` *parsers.Hashes `json:"hashes"` } @@ -154,7 +153,7 @@ func (bl *Baseline) CollectURL() { if len(bl.Body) == 0 { return } - for _, reg := range JSRegexps { + for _, reg := range ExtractRegexps["js"] { urls := reg.FindAllStringSubmatch(string(bl.Body), -1) for _, u := range urls { u[1] = formatURL(u[1]) @@ -164,7 +163,7 @@ func (bl *Baseline) CollectURL() { } } - for _, reg := range URLRegexps { + for _, reg := range ExtractRegexps["url"] { urls := reg.FindAllStringSubmatch(string(bl.Body), -1) for _, u := range urls { u[1] = formatURL(u[1]) @@ -176,7 +175,7 @@ func (bl *Baseline) CollectURL() { bl.URLs = RemoveDuplication(bl.URLs) if bl.URLs != nil { - bl.Extracteds = append(bl.Extracteds, &fingers.Extracted{ + bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{ Name: "crawl", ExtractResult: bl.URLs, }) diff --git a/pkg/types.go b/pkg/types.go index 31d3994..f277c67 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -1,7 +1,6 @@ package pkg import ( - "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/chainreactors/parsers" "strings" ) @@ -16,18 +15,6 @@ func (fs Frameworks) String() string { return strings.Join(frameworkStrs, " ") + " " } -type Extracteds []*fingers.Extracted - -func (es Extracteds) String() string { - var s strings.Builder - for _, e := range es { - s.WriteString("[ " + e.ToString() + " ]") - } - return s.String() + " " -} - -var Extractors = make(fingers.Extractors) - func GetSourceName(s int) string { switch s { case 1: diff --git a/pkg/utils.go b/pkg/utils.go index 1ff0d7a..24672e5 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -5,6 +5,8 @@ import ( "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/chainreactors/gogo/v2/pkg/utils" "github.com/chainreactors/ipcs" + "github.com/chainreactors/parsers" + "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/words/mask" "math/rand" "net/url" @@ -22,18 +24,21 @@ var ( Rules map[string]string = make(map[string]string) ActivePath []string Fingers fingers.Fingers - JSRegexps []*regexp.Regexp = []*regexp.Regexp{ - regexp.MustCompile(`.(https{0,1}:[^\s'’"”><()|*\[]{2,250}?[^=*\s'’><:;|()[]{3}\[]\.js)`), - regexp.MustCompile(`["']([^\s',’"”><;()|*:\[]{2,250}?[^=*\s'’|"”><^:;()\[]{3}\.js)`), - regexp.MustCompile(`=\s{0,6}["']{0,1}\s{0,6}([^\s^'’,+><;()|*\[]{2,250}?[^=,\s'’"”>|<:;*()\[]{3}\.js)`), - } - URLRegexps []*regexp.Regexp = []*regexp.Regexp{ - regexp.MustCompile(`=\s{0,6}(https{0,1}:[^\s'"><()|*\[]{2,250})`), - regexp.MustCompile(`["']([^\s',’"”><.@$;:()|*\[]{2,250}\.[a-zA-Z]\w{1,4})["']`), - regexp.MustCompile(`["'](https?:[^\s'"><()@|*\[]{2,250}?\.[^\s',’"”><;()|*\[]{2,250}?)["']`), - regexp.MustCompile(`["']\s{0,6}([#,.]{0,2}/[^\s'",><;@$()|*\[]{2,250}?)\s{0,6}["']`), - regexp.MustCompile(`href\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s',’"”><$@;()|*\[]{2,250})|action\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s'’"“><)(]{2,250})`), - } + //JSRegexps []*regexp.Regexp = []*regexp.Regexp{ + // regexp.MustCompile(`.(https{0,1}:[^\s'’"”><()|*\[]{2,250}?[^=*\s'’><:;|()[]{3}\[]\.js)`), + // regexp.MustCompile(`["']([^\s',’"”><;()|*:\[]{2,250}?[^=*\s'’|"”><^:;()\[]{3}\.js)`), + // regexp.MustCompile(`=\s{0,6}["']{0,1}\s{0,6}([^\s^'’,+><;()|*\[]{2,250}?[^=,\s'’"”>|<:;*()\[]{3}\.js)`), + //} + //URLRegexps []*regexp.Regexp = []*regexp.Regexp{ + // regexp.MustCompile(`=\s{0,6}(https{0,1}:[^\s'"><()|*\[]{2,250})`), + // regexp.MustCompile(`["']([^\s',’"”><.@$;:()|*\[]{2,250}\.[a-zA-Z]\w{1,4})["']`), + // regexp.MustCompile(`["'](https?:[^\s'"><()@|*\[]{2,250}?\.[^\s',’"”><;()|*\[]{2,250}?)["']`), + // regexp.MustCompile(`["']\s{0,6}([#,.]{0,2}/[^\s'",><;@$()|*\[]{2,250}?)\s{0,6}["']`), + // regexp.MustCompile(`href\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s',’"”><$@;()|*\[]{2,250})|action\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s'’"“><)(]{2,250})`), + //} + ExtractRegexps map[string][]*regexp.Regexp = map[string][]*regexp.Regexp{} + Extractors = make(parsers.Extractors) + BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4"} BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"} @@ -62,24 +67,6 @@ var ( } ) -func StringsContains(s []string, e string) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} - -func IntsContains(s []int, e int) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} - func RemoveDuplication(arr []string) []string { set := make(map[string]struct{}, len(arr)) j := 0 @@ -213,12 +200,32 @@ func LoadTemplates() error { } mask.SpecialWords[k] = t } + + var extracts []*parsers.Extractor + err = json.Unmarshal(LoadConfig("extract"), &extracts) + if err != nil { + return err + } + + for _, extract := range extracts { + extract.Compile() + + ExtractRegexps[extract.Name] = extract.CompiledRegexps + for _, tag := range extract.Tags { + if _, ok := ExtractRegexps[tag]; !ok { + ExtractRegexps[tag] = extract.CompiledRegexps + } else { + ExtractRegexps[tag] = append(ExtractRegexps[tag], extract.CompiledRegexps...) + } + } + } return nil } func FingerDetect(content string) Frameworks { var frames Frameworks for _, finger := range Fingers { + // sender置空, 所有的发包交给spray的pool frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil) if ok { frames = append(frames, frame) @@ -317,7 +324,7 @@ func BakGenerator(domain string) []string { for first, _ := range domain { for last, _ := range domain[first:] { p := domain[first : first+last+1] - if !StringsContains(possibilities, p) { + if !iutils.StringsContains(possibilities, p) { possibilities = append(possibilities, p) } } diff --git a/spray.go b/spray.go index a279e05..36358cb 100644 --- a/spray.go +++ b/spray.go @@ -1,4 +1,4 @@ -//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask +//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask,extract package main import "github.com/chainreactors/spray/cmd"