admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-11-05 10:27:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor package dependency

Browse Source
This commit is contained in:
M09Ic 2024-02-08 15:26:01 +08:00
parent 007ff96478
commit 09c2a86a18
14 changed files with 179 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/cmd.go
View File

@ -6,8 +6,8 @@ import (
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
"github.com/chainreactors/spray/internal"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/utils/iutils"
"github.com/jessevdk/go-flags"
"os"
@ -92,7 +92,7 @@ func Spray() {
})
// 初始化全局变量
pkg.Distance = uint8(option.SimhashDistance)
internal.Distance = uint8(option.SimhashDistance)
ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024
internal.MaxCrawl = option.CrawlDepth

35
pkg/baseline.go → internal/baseline.go
View File

@ -1,9 +1,10 @@
package pkg
package internal
import (
"bytes"
"github.com/chainreactors/parsers"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/utils/encode"
"github.com/chainreactors/utils/iutils"
"net/url"
@ -20,7 +21,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
},
}
if t, ok := ContentTypeMap[resp.ContentType()]; ok {
if t, ok := pkg.ContentTypeMap[resp.ContentType()]; ok {
bl.ContentType = t
bl.Title = t + " data"
} else {
@ -106,9 +107,9 @@ type Baseline struct {
Url *url.URL `json:"-"`
Dir bool `json:"-"`
Chunked bool `json:"-"`
Body BS `json:"-"`
Header BS `json:"-"`
Raw BS `json:"-"`
Body pkg.BS `json:"-"`
Header pkg.BS `json:"-"`
Raw pkg.BS `json:"-"`
Recu bool `json:"-"`
RecuDepth int `json:"-"`
URLs []string `json:"-"`
@ -127,23 +128,23 @@ func (bl *Baseline) IsDir() bool {
func (bl *Baseline) Collect() {
if bl.ContentType == "html" || bl.ContentType == "json" || bl.ContentType == "txt" {
// 指纹库设计的时候没考虑js,css文件的指纹, 跳过非必要的指纹收集减少误报提高性能
bl.Frameworks = FingerDetect(bl.Raw)
bl.Frameworks = pkg.FingerDetect(bl.Raw)
}
if len(bl.Body) > 0 {
if bl.ContentType == "html" {
bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
} else if bl.ContentType == "ico" {
if name, ok := Md5Fingers[encode.Md5Hash(bl.Body)]; ok {
if name, ok := pkg.Md5Fingers[encode.Md5Hash(bl.Body)]; ok {
bl.Frameworks[name] = &parsers.Framework{Name: name}
} else if name, ok := Mmh3Fingers[encode.Mmh3Hash32(bl.Body)]; ok {
} else if name, ok := pkg.Mmh3Fingers[encode.Mmh3Hash32(bl.Body)]; ok {
bl.Frameworks[name] = &parsers.Framework{Name: name}
}
}
}
bl.Hashes = parsers.NewHashes(bl.Raw)
bl.Extracteds = Extractors.Extract(string(bl.Raw))
bl.Extracteds = pkg.Extractors.Extract(string(bl.Raw))
bl.Unique = UniqueHash(bl)
}
@ -158,27 +159,27 @@ func (bl *Baseline) CollectURL() {
if len(bl.Body) == 0 {
return
}
for _, reg := range ExtractRegexps["js"][0].CompiledRegexps {
for _, reg := range pkg.ExtractRegexps["js"][0].CompiledRegexps {
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
for _, u := range urls {
u[1] = formatURL(u[1])
if u[1] != "" && !filterJs(u[1]) {
u[1] = pkg.CleanURL(u[1])
if u[1] != "" && !pkg.FilterJs(u[1]) {
bl.URLs = append(bl.URLs, u[1])
}
}
}
for _, reg := range ExtractRegexps["url"][0].CompiledRegexps {
for _, reg := range pkg.ExtractRegexps["url"][0].CompiledRegexps {
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
for _, u := range urls {
u[1] = formatURL(u[1])
if u[1] != "" && !filterUrl(u[1]) {
u[1] = pkg.CleanURL(u[1])
if u[1] != "" && !pkg.FilterUrl(u[1]) {
bl.URLs = append(bl.URLs, u[1])
}
}
}
bl.URLs = RemoveDuplication(bl.URLs)
bl.URLs = iutils.StringsUnique(bl.URLs)
if bl.URLs != nil {
bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{
Name: "crawl",

34
internal/checkpool.go
View File

@ -5,8 +5,8 @@ import (
"fmt"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
ihttp2 "github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/words"
"github.com/panjf2000/ants/v2"
"github.com/valyala/fasthttp"
@ -17,13 +17,13 @@ import (
)
// 类似httpx的无状态, 无scope, 无并发池的检测模式
func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) {
func NewCheckPool(ctx context.Context, config *Config) (*CheckPool, error) {
pctx, cancel := context.WithCancel(ctx)
pool := &CheckPool{
Config: config,
ctx: pctx,
cancel: cancel,
client: ihttp.NewClient(&ihttp.ClientConfig{
client: ihttp2.NewClient(&ihttp2.ClientConfig{
Thread: config.Thread,
Type: config.ClientType,
Timeout: time.Duration(config.Timeout) * time.Second,
@ -43,8 +43,8 @@ func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) {
}
type CheckPool struct {
*pkg.Config
client *ihttp.Client
*Config
client *ihttp2.Client
pool *ants.PoolWithFunc
bar *pkg.Bar
ctx context.Context
@ -61,11 +61,11 @@ func (pool *CheckPool) Close() {
pool.bar.Close()
}
func (pool *CheckPool) genReq(s string) (*ihttp.Request, error) {
if pool.Mod == pkg.HostSpray {
return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s)
} else if pool.Mod == pkg.PathSpray {
return ihttp.BuildPathRequest(pool.ClientType, pool.BaseURL, s)
func (pool *CheckPool) genReq(s string) (*ihttp2.Request, error) {
if pool.Mod == HostSpray {
return ihttp2.BuildHostRequest(pool.ClientType, pool.BaseURL, s)
} else if pool.Mod == PathSpray {
return ihttp2.BuildPathRequest(pool.ClientType, pool.BaseURL, s)
}
return nil, fmt.Errorf("unknown mod")
}
@ -131,21 +131,21 @@ func (pool *CheckPool) Invoke(v interface{}) {
}
req.SetHeaders(pool.Headers)
start := time.Now()
var bl *pkg.Baseline
var bl *Baseline
resp, reqerr := pool.client.Do(pool.ctx, req)
if pool.ClientType == ihttp.FAST {
if pool.ClientType == ihttp2.FAST {
defer fasthttp.ReleaseResponse(resp.FastResponse)
defer fasthttp.ReleaseRequest(req.FastRequest)
}
if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
pool.failedCount++
bl = &pkg.Baseline{
bl = &Baseline{
SprayResult: &parsers.SprayResult{
UrlString: unit.path,
IsValid: false,
ErrString: reqerr.Error(),
Reason: pkg.ErrRequestFailed.Error(),
Reason: ErrRequestFailed.Error(),
ReqDepth: unit.depth,
},
}
@ -157,7 +157,7 @@ func (pool *CheckPool) Invoke(v interface{}) {
}
} else {
bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
bl = NewBaseline(req.URI(), req.Host(), resp)
bl.Collect()
}
bl.ReqDepth = unit.depth
@ -187,7 +187,7 @@ func (pool *CheckPool) Invoke(v interface{}) {
pool.bar.Done()
}
func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) {
func (pool *CheckPool) doRedirect(bl *Baseline, depth int) {
if depth >= MaxRedirect {
return
}
@ -214,7 +214,7 @@ func (pool *CheckPool) doRedirect(bl *pkg.Baseline, depth int) {
}
// tcp与400进行协议转换
func (pool *CheckPool) doUpgrade(bl *pkg.Baseline) {
func (pool *CheckPool) doUpgrade(bl *Baseline) {
if bl.ReqDepth >= 1 {
return
}

2
pkg/config.go → internal/config.go
View File

@ -1,4 +1,4 @@
package pkg
package internal
import (
"github.com/antonmedv/expr/vm"

8
pkg/types.go → internal/errors.go
View File

@ -1,4 +1,4 @@
package pkg
package internal
type ErrorType uint
@ -37,9 +37,3 @@ var ErrMap = map[ErrorType]string{
func (e ErrorType) Error() string {
return ErrMap[e]
}
type BS []byte
func (b BS) String() string {
return string(b)
}

5
internal/format.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/pkg"
"io/ioutil"
"os"
)
@ -21,9 +20,9 @@ func Format(filename string, color bool) {
if err != nil {
return
}
var results []*pkg.Baseline
var results []*Baseline
for _, line := range bytes.Split(bytes.TrimSpace(content), []byte("\n")) {
var result pkg.Baseline
var result Baseline
err := json.Unmarshal(line, &result)
if err != nil {
logs.Log.Error(err.Error())

0
pkg/ihttp/client.go → internal/ihttp/client.go
View File

0
pkg/ihttp/request.go → internal/ihttp/request.go
View File

0
pkg/ihttp/response.go → internal/ihttp/response.go
View File

6
internal/option.go
View File

@ -6,8 +6,8 @@ import (
"github.com/antonmedv/expr"
"github.com/chainreactors/files"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/utils"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words/mask"
@ -154,8 +154,8 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
Offset: opt.Offset,
Total: opt.Limit,
taskCh: make(chan *Task),
OutputCh: make(chan *pkg.Baseline, 100),
FuzzyCh: make(chan *pkg.Baseline, 100),
OutputCh: make(chan *Baseline, 100),
FuzzyCh: make(chan *Baseline, 100),
Fuzzy: opt.Fuzzy,
Force: opt.Force,
CheckOnly: opt.CheckOnly,

197
internal/pool.go
View File

@ -5,8 +5,8 @@ import (
"fmt"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
ihttp2 "github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words"
"github.com/chainreactors/words/mask"
@ -30,10 +30,10 @@ var (
MaxRecursion = 0
enableAllFuzzy = false
enableAllUnique = false
nilBaseline = &pkg.Baseline{}
nilBaseline = &Baseline{}
)
func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
func NewPool(ctx context.Context, config *Config) (*Pool, error) {
var u *url.URL
var err error
if u, err = url.Parse(config.BaseURL); err != nil {
@ -41,23 +41,23 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
}
pctx, cancel := context.WithCancel(ctx)
pool := &Pool{
Config: config,
base: u.Scheme + "://" + u.Host,
isDir: strings.HasSuffix(u.Path, "/"),
url: u,
ctx: pctx,
cancel: cancel,
client: ihttp.NewClient(&ihttp.ClientConfig{
Config: config,
Baselines: NewBaselines(),
base: u.Scheme + "://" + u.Host,
isDir: strings.HasSuffix(u.Path, "/"),
url: u,
ctx: pctx,
cancel: cancel,
client: ihttp2.NewClient(&ihttp2.ClientConfig{
Thread: config.Thread,
Type: config.ClientType,
Timeout: time.Duration(config.Timeout) * time.Second,
ProxyAddr: config.ProxyAddr,
}),
baselines: make(map[int]*pkg.Baseline),
urls: make(map[string]struct{}),
scopeurls: make(map[string]struct{}),
uniques: make(map[uint16]struct{}),
tempCh: make(chan *pkg.Baseline, 100),
tempCh: make(chan *Baseline, 100),
checkCh: make(chan int, 100),
additionCh: make(chan *Unit, 100),
closeCh: make(chan struct{}),
@ -85,40 +85,37 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
}
type Pool struct {
*pkg.Config // read only
base string // url的根目录, 在爬虫或者redirect时, 会需要用到根目录进行拼接
dir string
isDir bool
url *url.URL
Statistor *pkg.Statistor
client *ihttp.Client
reqPool *ants.PoolWithFunc
scopePool *ants.PoolWithFunc
bar *pkg.Bar
ctx context.Context
cancel context.CancelFunc
tempCh chan *pkg.Baseline // 待处理的baseline
checkCh chan int // 独立的check管道， 防止与redirect/crawl冲突
additionCh chan *Unit // 插件添加的任务, 待处理管道
closeCh chan struct{}
closed bool
wordOffset int
failedCount int32
isFailed bool
failedBaselines []*pkg.Baseline
random *pkg.Baseline
index *pkg.Baseline
baselines map[int]*pkg.Baseline
urls map[string]struct{}
scopeurls map[string]struct{}
uniques map[uint16]struct{}
analyzeDone bool
worder *words.Worder
limiter *rate.Limiter
locker sync.Mutex
scopeLocker sync.Mutex
waiter sync.WaitGroup
initwg sync.WaitGroup // 初始化用, 之后改成锁
*Config // read only
*Baselines
base string // url的根目录, 在爬虫或者redirect时, 会需要用到根目录进行拼接
dir string
isDir bool
url *url.URL
Statistor *pkg.Statistor
client *ihttp2.Client
reqPool *ants.PoolWithFunc
scopePool *ants.PoolWithFunc
bar *pkg.Bar
ctx context.Context
cancel context.CancelFunc
tempCh chan *Baseline // 待处理的baseline
checkCh chan int // 独立的check管道， 防止与redirect/crawl冲突
additionCh chan *Unit // 插件添加的任务, 待处理管道
closeCh chan struct{}
closed bool
wordOffset int
failedCount int32
isFailed bool
urls map[string]struct{}
scopeurls map[string]struct{}
uniques map[uint16]struct{}
analyzeDone bool
worder *words.Worder
limiter *rate.Limiter
locker sync.Mutex
scopeLocker sync.Mutex
waiter sync.WaitGroup
initwg sync.WaitGroup // 初始化用, 之后改成锁
}
func (pool *Pool) checkRedirect(redirectURL string) bool {
@ -136,11 +133,11 @@ func (pool *Pool) checkRedirect(redirectURL string) bool {
}
}
func (pool *Pool) genReq(mod pkg.SprayMod, s string) (*ihttp.Request, error) {
if mod == pkg.HostSpray {
return ihttp.BuildHostRequest(pool.ClientType, pool.BaseURL, s)
} else if mod == pkg.PathSpray {
return ihttp.BuildPathRequest(pool.ClientType, pool.base, s)
func (pool *Pool) genReq(mod SprayMod, s string) (*ihttp2.Request, error) {
if mod == HostSpray {
return ihttp2.BuildHostRequest(pool.ClientType, pool.BaseURL, s)
} else if mod == PathSpray {
return ihttp2.BuildPathRequest(pool.ClientType, pool.base, s)
}
return nil, fmt.Errorf("unknown mod")
}
@ -166,7 +163,7 @@ func (pool *Pool) Init() error {
logs.Log.Error(pool.index.String())
return fmt.Errorf(pool.index.ErrString)
}
if pool.index.Chunked && pool.ClientType == ihttp.FAST {
if pool.index.Chunked && pool.ClientType == ihttp2.FAST {
logs.Log.Warn("chunk encoding! buf current client FASTHTTP not support chunk decode")
}
logs.Log.Logf(LogVerbose, "[baseline.index] "+pool.index.Format([]string{"status", "length", "spend", "title", "frame", "redirect"}))
@ -243,7 +240,7 @@ Loop:
}
pool.waiter.Add(1)
if pool.Mod == pkg.HostSpray {
if pool.Mod == HostSpray {
pool.reqPool.Invoke(newUnitWithNumber(w, WordSource, pool.wordOffset))
} else {
// 原样的目录拼接, 输入了几个"/"就是几个, 适配/有语义的中间件
@ -252,9 +249,9 @@ Loop:
case source := <-pool.checkCh:
pool.Statistor.CheckNumber++
if pool.Mod == pkg.HostSpray {
if pool.Mod == HostSpray {
pool.reqPool.Invoke(newUnitWithNumber(pkg.RandHost(), source, pool.wordOffset))
} else if pool.Mod == pkg.PathSpray {
} else if pool.Mod == PathSpray {
pool.reqPool.Invoke(newUnitWithNumber(pool.safePath(pkg.RandPath()), source, pool.wordOffset))
}
case unit, ok := <-pool.additionCh:
@ -289,12 +286,12 @@ func (pool *Pool) Invoke(v interface{}) {
atomic.AddInt32(&pool.Statistor.ReqTotal, 1)
unit := v.(*Unit)
var req *ihttp.Request
var req *ihttp2.Request
var err error
if unit.source == WordSource {
req, err = pool.genReq(pool.Mod, unit.path)
} else {
req, err = pool.genReq(pkg.PathSpray, unit.path)
req, err = pool.genReq(PathSpray, unit.path)
}
if err != nil {
@ -307,40 +304,39 @@ func (pool *Pool) Invoke(v interface{}) {
start := time.Now()
resp, reqerr := pool.client.Do(pool.ctx, req)
if pool.ClientType == ihttp.FAST {
if pool.ClientType == ihttp2.FAST {
defer fasthttp.ReleaseResponse(resp.FastResponse)
defer fasthttp.ReleaseRequest(req.FastRequest)
}
// compare与各种错误处理
var bl *pkg.Baseline
var bl *Baseline
if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
atomic.AddInt32(&pool.failedCount, 1)
atomic.AddInt32(&pool.Statistor.FailedNumber, 1)
bl = &pkg.Baseline{
bl = &Baseline{
SprayResult: &parsers.SprayResult{
UrlString: pool.base + unit.path,
IsValid: false,
ErrString: reqerr.Error(),
Reason: pkg.ErrRequestFailed.Error(),
Reason: ErrRequestFailed.Error(),
},
}
pool.failedBaselines = append(pool.failedBaselines, bl)
// 自动重放失败请求, 默认为一次
// 自动重放失败请求
pool.doRetry(bl)
} else {
} else { // 特定场景优化
if unit.source <= 3 || unit.source == CrawlSource || unit.source == CommonFileSource {
// 一些高优先级的source, 将跳过PreCompare
bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
bl = NewBaseline(req.URI(), req.Host(), resp)
} else if pool.MatchExpr != nil {
// 如果自定义了match函数, 则所有数据送入tempch中
bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
bl = NewBaseline(req.URI(), req.Host(), resp)
} else if err = pool.PreCompare(resp); err == nil {
// 通过预对比跳过一些无用数据, 减少性能消耗
bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
bl = NewBaseline(req.URI(), req.Host(), resp)
} else {
bl = pkg.NewInvalidBaseline(req.URI(), req.Host(), resp, err.Error())
bl = NewInvalidBaseline(req.URI(), req.Host(), resp, err.Error())
}
}
@ -350,7 +346,7 @@ func (pool *Pool) Invoke(v interface{}) {
pool.doRedirect(bl, unit.depth)
}
if ihttp.DefaultMaxBodySize != 0 && bl.BodyLength > ihttp.DefaultMaxBodySize {
if ihttp2.DefaultMaxBodySize != 0 && bl.BodyLength > ihttp2.DefaultMaxBodySize {
bl.ExceedLength = true
}
bl.Source = unit.source
@ -416,7 +412,7 @@ func (pool *Pool) Invoke(v interface{}) {
func (pool *Pool) NoScopeInvoke(v interface{}) {
defer pool.waiter.Done()
unit := v.(*Unit)
req, err := ihttp.BuildPathRequest(pool.ClientType, unit.path, "")
req, err := ihttp2.BuildPathRequest(pool.ClientType, unit.path, "")
if err != nil {
logs.Log.Error(err.Error())
return
@ -424,7 +420,7 @@ func (pool *Pool) NoScopeInvoke(v interface{}) {
req.SetHeaders(pool.Headers)
req.SetHeader("User-Agent", RandomUA())
resp, reqerr := pool.client.Do(pool.ctx, req)
if pool.ClientType == ihttp.FAST {
if pool.ClientType == ihttp2.FAST {
defer fasthttp.ReleaseResponse(resp.FastResponse)
defer fasthttp.ReleaseRequest(req.FastRequest)
}
@ -433,7 +429,7 @@ func (pool *Pool) NoScopeInvoke(v interface{}) {
return
}
if resp.StatusCode() == 200 {
bl := pkg.NewBaseline(req.URI(), req.Host(), resp)
bl := NewBaseline(req.URI(), req.Host(), resp)
bl.Source = unit.source
bl.ReqDepth = unit.depth
bl.Collect()
@ -494,7 +490,7 @@ func (pool *Pool) Handler() {
if _, ok := pool.uniques[bl.Unique]; ok {
bl.IsValid = false
bl.IsFuzzy = true
bl.Reason = pkg.ErrFuzzyNotUnique.Error()
bl.Reason = ErrFuzzyNotUnique.Error()
} else {
pool.uniques[bl.Unique] = struct{}{}
}
@ -503,7 +499,7 @@ func (pool *Pool) Handler() {
// 对通过所有对比的有效数据进行再次filter
if bl.IsValid && pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) {
pool.Statistor.FilteredNumber++
bl.Reason = pkg.ErrCustomFilter.Error()
bl.Reason = ErrCustomFilter.Error()
bl.IsValid = false
}
} else {
@ -534,39 +530,39 @@ func (pool *Pool) Handler() {
pool.analyzeDone = true
}
func (pool *Pool) PreCompare(resp *ihttp.Response) error {
func (pool *Pool) PreCompare(resp *ihttp2.Response) error {
status := resp.StatusCode()
if iutils.IntsContains(WhiteStatus, status) {
// 如果为白名单状态码则直接返回
return nil
}
if pool.random.Status != 200 && pool.random.Status == status {
return pkg.ErrSameStatus
return ErrSameStatus
}
if iutils.IntsContains(BlackStatus, status) {
return pkg.ErrBadStatus
return ErrBadStatus
}
if iutils.IntsContains(WAFStatus, status) {
return pkg.ErrWaf
return ErrWaf
}
if !pool.checkRedirect(resp.GetHeader("Location")) {
return pkg.ErrRedirect
return ErrRedirect
}
return nil
}
func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool {
func (pool *Pool) BaseCompare(bl *Baseline) bool {
if !bl.IsValid {
return false
}
var status = -1
// 30x状态码的特殊处理
if bl.RedirectURL != "" && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") {
bl.Reason = pkg.ErrFuzzyRedirect.Error()
bl.Reason = ErrFuzzyRedirect.Error()
pool.putToFuzzy(bl)
return false
}
@ -587,7 +583,7 @@ func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool {
if ok {
if status = base.Compare(bl); status == 1 {
bl.Reason = pkg.ErrCompareFailed.Error()
bl.Reason = ErrCompareFailed.Error()
return false
}
}
@ -607,7 +603,7 @@ func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool {
if ok && status == 0 && base.FuzzyCompare(bl) {
pool.Statistor.FuzzyNumber++
bl.Reason = pkg.ErrFuzzyCompareFailed.Error()
bl.Reason = ErrFuzzyCompareFailed.Error()
pool.putToFuzzy(bl)
return false
}
@ -615,7 +611,7 @@ func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool {
return true
}
func (pool *Pool) Upgrade(bl *pkg.Baseline) error {
func (pool *Pool) Upgrade(bl *Baseline) error {
rurl, err := url.Parse(bl.RedirectURL)
if err == nil && rurl.Hostname() == bl.Url.Hostname() && bl.Url.Scheme == "http" && rurl.Scheme == "https" {
logs.Log.Infof("baseurl %s upgrade http to https, reinit", pool.BaseURL)
@ -631,7 +627,7 @@ func (pool *Pool) Upgrade(bl *pkg.Baseline) error {
return nil
}
func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) {
func (pool *Pool) doRedirect(bl *Baseline, depth int) {
if depth >= MaxRedirect {
return
}
@ -648,7 +644,7 @@ func (pool *Pool) doRedirect(bl *pkg.Baseline, depth int) {
}()
}
func (pool *Pool) doCrawl(bl *pkg.Baseline) {
func (pool *Pool) doCrawl(bl *Baseline) {
if !pool.Crawl || bl.ReqDepth >= MaxCrawl {
pool.waiter.Done()
return
@ -678,7 +674,7 @@ func (pool *Pool) doCrawl(bl *pkg.Baseline) {
}
func (pool *Pool) doScopeCrawl(bl *pkg.Baseline) {
func (pool *Pool) doScopeCrawl(bl *Baseline) {
if bl.ReqDepth >= MaxCrawl {
pool.waiter.Done()
return
@ -703,7 +699,7 @@ func (pool *Pool) doScopeCrawl(bl *pkg.Baseline) {
}()
}
func (pool *Pool) doRule(bl *pkg.Baseline) {
func (pool *Pool) doRule(bl *Baseline) {
if pool.AppendRule == nil {
pool.waiter.Done()
return
@ -724,7 +720,7 @@ func (pool *Pool) doRule(bl *pkg.Baseline) {
}()
}
func (pool *Pool) doRetry(bl *pkg.Baseline) {
func (pool *Pool) doRetry(bl *Baseline) {
if bl.Retry >= pool.Retry {
return
}
@ -795,9 +791,9 @@ func (pool *Pool) doCheck() {
return
}
if pool.Mod == pkg.HostSpray {
if pool.Mod == HostSpray {
pool.checkCh <- CheckSource
} else if pool.Mod == pkg.PathSpray {
} else if pool.Mod == PathSpray {
pool.checkCh <- CheckSource
}
}
@ -812,7 +808,7 @@ func (pool *Pool) addAddition(u *Unit) {
pool.additionCh <- u
}
func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) {
func (pool *Pool) addFuzzyBaseline(bl *Baseline) {
if _, ok := pool.baselines[bl.Status]; !ok && (enableAllFuzzy || iutils.IntsContains(FuzzyStatus, bl.Status)) {
bl.Collect()
pool.waiter.Add(1)
@ -822,12 +818,12 @@ func (pool *Pool) addFuzzyBaseline(bl *pkg.Baseline) {
}
}
func (pool *Pool) putToInvalid(bl *pkg.Baseline, reason string) {
func (pool *Pool) putToInvalid(bl *Baseline, reason string) {
bl.IsValid = false
pool.OutputCh <- bl
}
func (pool *Pool) putToFuzzy(bl *pkg.Baseline) {
func (pool *Pool) putToFuzzy(bl *Baseline) {
bl.IsFuzzy = true
pool.FuzzyCh <- bl
}
@ -872,3 +868,16 @@ func (pool *Pool) safePath(u string) string {
}
}
}
func NewBaselines() *Baselines {
return &Baselines{
baselines: map[int]*Baseline{},
}
}
type Baselines struct {
failedBaselines []*Baseline
random *Baseline
index *Baseline
baselines map[int]*Baseline
}

40
internal/runner.go
View File

@ -6,8 +6,8 @@ import (
"github.com/antonmedv/expr/vm"
"github.com/chainreactors/files"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/spray/pkg/ihttp"
"github.com/chainreactors/words"
"github.com/chainreactors/words/rule"
"github.com/gosuri/uiprogress"
@ -55,8 +55,8 @@ type Runner struct {
Timeout int
Mod string
Probes []string
OutputCh chan *pkg.Baseline
FuzzyCh chan *pkg.Baseline
OutputCh chan *Baseline
FuzzyCh chan *Baseline
Fuzzy bool
OutputFile *files.File
FuzzyFile *files.File
@ -87,13 +87,13 @@ type Runner struct {
Proxy string
}
func (r *Runner) PrepareConfig() *pkg.Config {
config := &pkg.Config{
func (r *Runner) PrepareConfig() *Config {
config := &Config{
Thread: r.Threads,
Timeout: r.Timeout,
RateLimit: r.RateLimit,
Headers: r.Headers,
Mod: pkg.ModMap[r.Mod],
Mod: ModMap[r.Mod],
OutputCh: r.OutputCh,
FuzzyCh: r.FuzzyCh,
Fuzzy: r.Fuzzy,
@ -119,9 +119,9 @@ func (r *Runner) PrepareConfig() *pkg.Config {
}
if config.ClientType == ihttp.Auto {
if config.Mod == pkg.PathSpray {
if config.Mod == PathSpray {
config.ClientType = ihttp.FAST
} else if config.Mod == pkg.HostSpray {
} else if config.Mod == HostSpray {
config.ClientType = ihttp.STANDARD
}
}
@ -250,7 +250,7 @@ func (r *Runner) Prepare(ctx context.Context) error {
return nil
}
func (r *Runner) AddRecursive(bl *pkg.Baseline) {
func (r *Runner) AddRecursive(bl *Baseline) {
// 递归新任务
task := &Task{
baseUrl: bl.UrlString,
@ -369,7 +369,7 @@ func (r *Runner) PrintStat(pool *Pool) {
}
func (r *Runner) Output() {
debugPrint := func(bl *pkg.Baseline) {
debugPrint := func(bl *Baseline) {
if r.Color {
logs.Log.Debug(bl.ColorString())
} else {
@ -377,31 +377,31 @@ func (r *Runner) Output() {
}
}
go func() {
var saveFunc func(*pkg.Baseline)
var saveFunc func(*Baseline)
if r.OutputFile != nil {
saveFunc = func(bl *pkg.Baseline) {
saveFunc = func(bl *Baseline) {
r.OutputFile.SafeWrite(bl.Jsonify() + "\n")
r.OutputFile.SafeSync()
}
} else {
if len(r.Probes) > 0 {
if r.Color {
saveFunc = func(bl *pkg.Baseline) {
saveFunc = func(bl *Baseline) {
logs.Log.Console(logs.GreenBold("[+] " + bl.Format(r.Probes) + "\n"))
}
} else {
saveFunc = func(bl *pkg.Baseline) {
saveFunc = func(bl *Baseline) {
logs.Log.Console("[+] " + bl.Format(r.Probes) + "\n")
}
}
} else {
if r.Color {
saveFunc = func(bl *pkg.Baseline) {
saveFunc = func(bl *Baseline) {
logs.Log.Console(logs.GreenBold("[+] " + bl.ColorString() + "\n"))
}
} else {
saveFunc = func(bl *pkg.Baseline) {
saveFunc = func(bl *Baseline) {
logs.Log.Console("[+] " + bl.String() + "\n")
}
}
@ -431,18 +431,18 @@ func (r *Runner) Output() {
}()
go func() {
var fuzzySaveFunc func(*pkg.Baseline)
var fuzzySaveFunc func(*Baseline)
if r.FuzzyFile != nil {
fuzzySaveFunc = func(bl *pkg.Baseline) {
fuzzySaveFunc = func(bl *Baseline) {
r.FuzzyFile.SafeWrite(bl.Jsonify() + "\n")
}
} else {
if r.Color {
fuzzySaveFunc = func(bl *pkg.Baseline) {
fuzzySaveFunc = func(bl *Baseline) {
logs.Log.Console(logs.GreenBold("[fuzzy] " + bl.ColorString() + "\n"))
}
} else {
fuzzySaveFunc = func(bl *pkg.Baseline) {
fuzzySaveFunc = func(bl *Baseline) {
logs.Log.Console("[fuzzy] " + bl.String() + "\n")
}
}

6
internal/utils.go
View File

@ -301,3 +301,9 @@ func wrapWordsFunc(f func(string) string) func(string) []string {
return []string{f(s)}
}
}
func UniqueHash(bl *Baseline) uint16 {
// 由host+状态码+重定向url+content-type+title+length舍去个位与十位组成的hash
// body length可能会导致一些误报, 目前没有更好的解决办法
return pkg.CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/100*100)))
}

29
pkg/utils.go
View File

@ -7,7 +7,6 @@ import (
"math/rand"
"net/url"
"path"
"strconv"
"strings"
"time"
"unsafe"
@ -50,20 +49,10 @@ var (
}
)
func RemoveDuplication(arr []string) []string {
set := make(map[string]struct{}, len(arr))
j := 0
for _, v := range arr {
_, ok := set[v]
if ok {
continue
}
set[v] = struct{}{}
arr[j] = v
j++
}
type BS []byte
return arr[:j]
func (b BS) String() string {
return string(b)
}
const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
@ -128,7 +117,7 @@ func FingerDetect(content []byte) parsers.Frameworks {
return frames
}
func filterJs(u string) bool {
func FilterJs(u string) bool {
if commonFilter(u) {
return true
}
@ -136,7 +125,7 @@ func filterJs(u string) bool {
return false
}
func filterUrl(u string) bool {
func FilterUrl(u string) bool {
if commonFilter(u) {
return true
}
@ -155,7 +144,7 @@ func filterUrl(u string) bool {
return false
}
func formatURL(u string) string {
func CleanURL(u string) string {
// 去掉frag与params, 节约url.parse性能, 防止带参数造成意外的影响
u = strings.Trim(u, "\"")
u = strings.Trim(u, "'")
@ -248,9 +237,3 @@ func CRC16Hash(data []byte) uint16 {
}
return crc16
}
func UniqueHash(bl *Baseline) uint16 {
// 由host+状态码+重定向url+content-type+title+length舍去个位与十位组成的hash
// body length可能会导致一些误报, 目前没有更好的解决办法
return CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/100*100)))
}