admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-09-15 11:40:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

完成对extract相关功能的重构

Browse Source
This commit is contained in:
M09Ic 2023-01-28 13:15:49 +08:00
parent 8756b7503e
commit 3943943405
10 changed files with 109 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/cmd.go
View File

@ -3,7 +3,6 @@ package cmd
import (
"context"
"fmt"
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/gogo/v2/pkg/utils"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/internal"
@ -61,10 +60,10 @@ func Spray() {
if option.Extracts != nil {
for _, e := range option.Extracts {
if reg, ok := fingers.PresetExtracts[e]; ok {
if reg, ok := pkg.ExtractRegexps[e]; ok {
pkg.Extractors[e] = reg
} else {
pkg.Extractors[e] = regexp.MustCompile(e)
pkg.Extractors[e] = []*regexp.Regexp{regexp.MustCompile(e)}
}
}
}

2
go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/chainreactors/gogo/v2 v2.10.1
github.com/chainreactors/ipcs v0.0.13
github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580
github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410
github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3
github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a
)

6
go.sum
View File

@ -21,6 +21,12 @@ github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 h1:28gbL1t+Mm
github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA=
github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410 h1:K7EV0wtUuN6Rvh/MgqaBXyElD3guPsgNR5kF8nrV7iw=
github.com/chainreactors/parsers v0.2.9-0.20221210155102-cc0814762410/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
github.com/chainreactors/parsers v0.2.9-0.20230117063421-a463730bca48 h1:zuCauWp/Em+hQoHkwymOsHjHm7sp9ydxhcymQVuY/+o=
github.com/chainreactors/parsers v0.2.9-0.20230117063421-a463730bca48/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
github.com/chainreactors/parsers v0.2.9-0.20230128044108-411f7c3b1bb7 h1:gUTcvvEaixo0Fom4i+hEc9ZuJwTmgp+TGlb+kYoJ8pI=
github.com/chainreactors/parsers v0.2.9-0.20230128044108-411f7c3b1bb7/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3 h1:Hovp8f/5UK+cu+LnKECKSr2lNAakoyhcoybsGwQeg+8=
github.com/chainreactors/parsers v0.2.9-0.20230128045953-708a73b0e3b3/go.mod h1:Z9weht+lnFCk7UcwqFu6lXpS7u5vttiy0AJYOAyCCLA=
github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U=
github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

7
internal/option.go
View File

@ -5,6 +5,7 @@ import (
"github.com/antonmedv/expr"
"github.com/chainreactors/files"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers/iutils"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/words/mask"
@ -55,7 +56,6 @@ type FunctionOptions struct {
type OutputOptions struct {
Match string `long:"match" description:"String, custom match function, e.g.: --match current.Status != 200" json:"match,omitempty"`
Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter current.Body contains 'hello'" json:"filter,omitempty"`
Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" json:"extracts,omitempty"`
OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty"`
Format string `short:"F" long:"format" description:"String, output format, e.g.: --format 1.json"`
FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty"`
@ -77,6 +77,7 @@ type RequestOptions struct {
type PluginOptions struct {
Advance bool `short:"a" long:"advance" description:"Bool, enable crawl and active"`
Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)"`
Active bool `long:"active" description:"Bool, enable active finger detect"`
Bak bool `long:"bak" description:"Bool, enable bak found"`
FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt"`
@ -369,7 +370,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
if opt.RemoveExtensions != "" {
rexts := strings.Split(opt.ExcludeExtensions, ",")
r.Fns = append(r.Fns, func(s string) string {
if ext := parseExtension(s); pkg.StringsContains(rexts, ext) {
if ext := parseExtension(s); iutils.StringsContains(rexts, ext) {
return strings.TrimSuffix(s, "."+ext)
}
return s
@ -379,7 +380,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
if opt.ExcludeExtensions != "" {
exexts := strings.Split(opt.ExcludeExtensions, ",")
r.Fns = append(r.Fns, func(s string) string {
if ext := parseExtension(s); pkg.StringsContains(exexts, ext) {
if ext := parseExtension(s); iutils.StringsContains(exexts, ext) {
return ""
}
return s

9
internal/pool.go
View File

@ -6,6 +6,7 @@ import (
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers/iutils"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/words"
@ -439,7 +440,7 @@ func (pool *Pool) genReq(s string) (*ihttp.Request, error) {
func (pool *Pool) PreCompare(resp *ihttp.Response) error {
status := resp.StatusCode()
if pkg.IntsContains(WhiteStatus, status) {
if iutils.IntsContains(WhiteStatus, status) {
// 如果为白名单状态码则直接返回
return nil
}
@ -447,11 +448,11 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error {
return ErrSameStatus
}
if pkg.IntsContains(BlackStatus, status) {
if iutils.IntsContains(BlackStatus, status) {
return ErrBadStatus
}
if pkg.IntsContains(WAFStatus, status) {
if iutils.IntsContains(WAFStatus, status) {
return ErrWaf
}
@ -683,7 +684,7 @@ func (pool *Pool) addAddition(u *Unit) {
}
func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) {
if _, ok := pool.baselines[bl.Status]; !ok && pkg.IntsContains(FuzzyStatus, bl.Status) {
if _, ok := pool.baselines[bl.Status]; !ok && iutils.IntsContains(FuzzyStatus, bl.Status) {
bl.Collect()
pool.waiter.Add(1)
pool.doCrawl(bl)

18
internal/utils.go
View File

@ -232,21 +232,3 @@ func FormatURL(base, u string) string {
return relaPath(base, u)
}
}
//func Join(base, u string) string {
// // //././ ../../../a
// base = Dir(base)
// for strings.HasPrefix(u, "../") {
// u = u[3:]
// for strings.HasSuffix(base, "/") {
// // 去掉多余的"/"
// base = base[:len(base)-2]
// }
// if i := strings.LastIndex(base, "/"); i == -1 {
// return "/"
// } else {
// return base[:i+1]
// }
// }
// return base + u
//}

9
pkg/baseline.go
View File

@ -3,7 +3,6 @@ package pkg
import (
"bytes"
"encoding/json"
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/gogo/v2/pkg/utils"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
@ -111,7 +110,7 @@ type Baseline struct {
ContentType string `json:"content_type"`
Title string `json:"title"`
Frameworks Frameworks `json:"frameworks"`
Extracteds Extracteds `json:"extracts"`
Extracteds parsers.Extracteds `json:"extracts"`
ErrString string `json:"error"`
Reason string `json:"reason"`
Source int `json:"source"`
@ -154,7 +153,7 @@ func (bl *Baseline) CollectURL() {
if len(bl.Body) == 0 {
return
}
for _, reg := range JSRegexps {
for _, reg := range ExtractRegexps["js"] {
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
for _, u := range urls {
u[1] = formatURL(u[1])
@ -164,7 +163,7 @@ func (bl *Baseline) CollectURL() {
}
}
for _, reg := range URLRegexps {
for _, reg := range ExtractRegexps["url"] {
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
for _, u := range urls {
u[1] = formatURL(u[1])
@ -176,7 +175,7 @@ func (bl *Baseline) CollectURL() {
bl.URLs = RemoveDuplication(bl.URLs)
if bl.URLs != nil {
bl.Extracteds = append(bl.Extracteds, &fingers.Extracted{
bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{
Name: "crawl",
ExtractResult: bl.URLs,
})

13
pkg/types.go
View File

@ -1,7 +1,6 @@
package pkg
import (
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/parsers"
"strings"
)
@ -16,18 +15,6 @@ func (fs Frameworks) String() string {
return strings.Join(frameworkStrs, " ") + " "
}
type Extracteds []*fingers.Extracted
func (es Extracteds) String() string {
var s strings.Builder
for _, e := range es {
s.WriteString("[ " + e.ToString() + " ]")
}
return s.String() + " "
}
var Extractors = make(fingers.Extractors)
func GetSourceName(s int) string {
switch s {
case 1:

69
pkg/utils.go
View File

@ -5,6 +5,8 @@ import (
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/gogo/v2/pkg/utils"
"github.com/chainreactors/ipcs"
"github.com/chainreactors/parsers"
"github.com/chainreactors/parsers/iutils"
"github.com/chainreactors/words/mask"
"math/rand"
"net/url"
@ -22,18 +24,21 @@ var (
Rules map[string]string = make(map[string]string)
ActivePath []string
Fingers fingers.Fingers
JSRegexps []*regexp.Regexp = []*regexp.Regexp{
regexp.MustCompile(`.(https{0,1}:[^\s'’"”><()|*\[]{2,250}?[^=*\s'’><:;|()[]{3}\[]\.js)`),
regexp.MustCompile(`["']([^\s',’"”><;()|*:\[]{2,250}?[^=*\s'’|"”><^:;()\[]{3}\.js)`),
regexp.MustCompile(`=\s{0,6}["']{0,1}\s{0,6}([^\s^'’,+><;()|*\[]{2,250}?[^=,\s'’"”>|<:;*()\[]{3}\.js)`),
}
URLRegexps []*regexp.Regexp = []*regexp.Regexp{
regexp.MustCompile(`=\s{0,6}(https{0,1}:[^\s'"><()|*\[]{2,250})`),
regexp.MustCompile(`["']([^\s',’"”><.@$;:()|*\[]{2,250}\.[a-zA-Z]\w{1,4})["']`),
regexp.MustCompile(`["'](https?:[^\s'"><()@|*\[]{2,250}?\.[^\s',’"”><;()|*\[]{2,250}?)["']`),
regexp.MustCompile(`["']\s{0,6}([#,.]{0,2}/[^\s'",><;@$()|*\[]{2,250}?)\s{0,6}["']`),
regexp.MustCompile(`href\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s',’"”><$@;()|*\[]{2,250})|action\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s'’"“><)(]{2,250})`),
}
//JSRegexps []*regexp.Regexp = []*regexp.Regexp{
// regexp.MustCompile(`.(https{0,1}:[^\s'’"”><()|*\[]{2,250}?[^=*\s'’><:;|()[]{3}\[]\.js)`),
// regexp.MustCompile(`["']([^\s',’"”><;()|*:\[]{2,250}?[^=*\s'’|"”><^:;()\[]{3}\.js)`),
// regexp.MustCompile(`=\s{0,6}["']{0,1}\s{0,6}([^\s^'’,+><;()|*\[]{2,250}?[^=,\s'’"”>|<:;*()\[]{3}\.js)`),
//}
//URLRegexps []*regexp.Regexp = []*regexp.Regexp{
// regexp.MustCompile(`=\s{0,6}(https{0,1}:[^\s'"><()|*\[]{2,250})`),
// regexp.MustCompile(`["']([^\s',’"”><.@$;:()|*\[]{2,250}\.[a-zA-Z]\w{1,4})["']`),
// regexp.MustCompile(`["'](https?:[^\s'"><()@|*\[]{2,250}?\.[^\s',’"”><;()|*\[]{2,250}?)["']`),
// regexp.MustCompile(`["']\s{0,6}([#,.]{0,2}/[^\s'",><;@$()|*\[]{2,250}?)\s{0,6}["']`),
// regexp.MustCompile(`href\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s',’"”><$@;()|*\[]{2,250})|action\s{0,6}=\s{0,6}["'‘“]{0,1}\s{0,6}([^\s'’"“><)(]{2,250})`),
//}
ExtractRegexps map[string][]*regexp.Regexp = map[string][]*regexp.Regexp{}
Extractors = make(parsers.Extractors)
BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4"}
BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"}
@ -62,24 +67,6 @@ var (
}
)
func StringsContains(s []string, e string) bool {
for _, v := range s {
if v == e {
return true
}
}
return false
}
func IntsContains(s []int, e int) bool {
for _, v := range s {
if v == e {
return true
}
}
return false
}
func RemoveDuplication(arr []string) []string {
set := make(map[string]struct{}, len(arr))
j := 0
@ -213,12 +200,32 @@ func LoadTemplates() error {
}
mask.SpecialWords[k] = t
}
var extracts []*parsers.Extractor
err = json.Unmarshal(LoadConfig("extract"), &extracts)
if err != nil {
return err
}
for _, extract := range extracts {
extract.Compile()
ExtractRegexps[extract.Name] = extract.CompiledRegexps
for _, tag := range extract.Tags {
if _, ok := ExtractRegexps[tag]; !ok {
ExtractRegexps[tag] = extract.CompiledRegexps
} else {
ExtractRegexps[tag] = append(ExtractRegexps[tag], extract.CompiledRegexps...)
}
}
}
return nil
}
func FingerDetect(content string) Frameworks {
var frames Frameworks
for _, finger := range Fingers {
// sender置空, 所有的发包交给spray的pool
frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
if ok {
frames = append(frames, frame)
@ -317,7 +324,7 @@ func BakGenerator(domain string) []string {
for first, _ := range domain {
for last, _ := range domain[first:] {
p := domain[first : first+last+1]
if !StringsContains(possibilities, p) {
if !iutils.StringsContains(possibilities, p) {
possibilities = append(possibilities, p)
}
}

2
spray.go
View File

@ -1,4 +1,4 @@
//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask
//go:generate go run templates/templates_gen.go -t templates -o pkg/templates.go -need http,rule,mask,extract
package main
import "github.com/chainreactors/spray/cmd"