diff --git a/internal/option.go b/internal/option.go index 025bb03..6229c48 100644 --- a/internal/option.go +++ b/internal/option.go @@ -89,18 +89,21 @@ type PluginOptions struct { } type ModeOptions struct { - RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100"` - Force bool `long:"force" description:"Bool, skip error break"` - CheckOnly bool `long:"check-only" description:"Bool, check only"` - Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"` - Depth int `long:"depth" default:"0" description:"Int, recursive depth"` - CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request"` - ErrPeriod int `long:"error-period" default:"10" description:"Int, check period when error"` - BreakThreshold int `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold "` - BlackStatus string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status, "` - WhiteStatus string `long:"white-status" default:"200" description:"Strings (comma split), custom white status"` - FuzzyStatus string `long:"fuzzy-status" default:"404,403,500,501,502,503" description:"Strings (comma split), custom fuzzy status"` - SimhashDistance int `long:"distance" default:"5"` + RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100"` + Force bool `long:"force" description:"Bool, skip error break"` + CheckOnly bool `long:"check-only" description:"Bool, check only"` + Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"` + Depth int `long:"depth" default:"0" description:"Int, recursive depth"` + CheckPeriod int `long:"check-period" default:"200" description:"Int, check period when request"` + ErrPeriod int `long:"error-period" default:"10" description:"Int, check period when error"` + BreakThreshold int `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold "` + BlackStatus string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status, "` + WhiteStatus string `long:"white-status" default:"200" description:"Strings (comma split), custom white status"` + FuzzyStatus string `long:"fuzzy-status" default:"404,403,500,501,502,503" description:"Strings (comma split), custom fuzzy status"` + UniqueStatus string `long:"unique-status" default:"403" description:"Strings (comma split), custom unique status"` + Unique bool `long:"unique" description:"Bool, unique response"` + + SimhashDistance int `long:"distance" default:"5"` } type MiscOptions struct { @@ -225,6 +228,12 @@ func (opt *Option) PrepareRunner() (*Runner, error) { FuzzyStatus = parseStatus(FuzzyStatus, opt.FuzzyStatus) } + if opt.Unique { + enableAllUnique = true + } else { + UniqueStatus = parseStatus(UniqueStatus, opt.UniqueStatus) + } + // prepare word dicts := make([][]string, len(opt.Dictionaries)) for i, f := range opt.Dictionaries { diff --git a/internal/pool.go b/internal/pool.go index ece906b..9e4c533 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -25,12 +25,13 @@ import ( ) var ( - max = 2147483647 - MaxRedirect = 3 - MaxCrawl = 3 - MaxRecursion = 0 - enableAllFuzzy = false - nilBaseline = &pkg.Baseline{} + max = 2147483647 + MaxRedirect = 3 + MaxCrawl = 3 + MaxRecursion = 0 + enableAllFuzzy = false + enableAllUnique = false + nilBaseline = &pkg.Baseline{} ) func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { @@ -50,6 +51,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { client: ihttp.NewClient(config.Thread, 2, config.ClientType), baselines: make(map[int]*pkg.Baseline), urls: make(map[string]struct{}), + uniques: make(map[uint16]struct{}), tempCh: make(chan *pkg.Baseline, 100), checkCh: make(chan int, 100), additionCh: make(chan *Unit, 100), @@ -102,6 +104,7 @@ type Pool struct { index *pkg.Baseline baselines map[int]*pkg.Baseline urls map[string]struct{} + uniques map[uint16]struct{} analyzeDone bool worder *words.Worder limiter *rate.Limiter @@ -389,7 +392,20 @@ func (pool *Pool) Handler() { if status { pool.Statistor.FoundNumber++ - if pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) { + + // unique判断 + if enableAllUnique || iutils.IntsContains(UniqueStatus, bl.Status) { + if _, ok := pool.uniques[bl.Unique]; ok { + bl.IsValid = false + bl.IsFuzzy = true + bl.Reason = ErrFuzzyNotUnique.Error() + } else { + pool.uniques[bl.Unique] = struct{}{} + } + } + + // 对通过所有对比的有效数据进行再次filter + if bl.IsValid && pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) { pool.Statistor.FilteredNumber++ bl.Reason = ErrCustomFilter.Error() bl.IsValid = false @@ -472,10 +488,12 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error { } func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool { + if !bl.IsValid { + return false + } var status = -1 - // 30x状态码的特殊处理 - if strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { + if bl.RedirectURL != "" && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { bl.Reason = ErrFuzzyRedirect.Error() pool.putToFuzzy(bl) return false diff --git a/internal/runner.go b/internal/runner.go index 0b4fad2..30160cc 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -17,10 +17,11 @@ import ( ) var ( - WhiteStatus = []int{200} - BlackStatus = []int{400, 410} - FuzzyStatus = []int{403, 404, 500, 501, 502, 503} - WAFStatus = []int{493, 418, 1020, 406} + WhiteStatus = []int{200} + BlackStatus = []int{400, 410} + FuzzyStatus = []int{403, 404, 500, 501, 502, 503} + WAFStatus = []int{493, 418, 1020, 406} + UniqueStatus = []int{403} ) var ( diff --git a/internal/types.go b/internal/types.go index 4633113..ea49baf 100644 --- a/internal/types.go +++ b/internal/types.go @@ -8,43 +8,37 @@ import ( type ErrorType uint const ( - ErrBadStatus ErrorType = iota + NoErr ErrorType = iota + ErrBadStatus ErrSameStatus ErrRequestFailed ErrWaf ErrRedirect ErrCompareFailed - ErrFuzzyCompareFailed - ErrFuzzyRedirect ErrCustomCompareFailed ErrCustomFilter + ErrFuzzyCompareFailed + ErrFuzzyRedirect + ErrFuzzyNotUnique ) +var ErrMap = map[ErrorType]string{ + NoErr: "", + ErrBadStatus: "blacklist status", + ErrSameStatus: "same status with random baseline", + ErrRequestFailed: "request failed", + ErrWaf: "maybe banned by waf", + ErrRedirect: "duplicate redirect url", + ErrCompareFailed: "compare failed", + ErrCustomCompareFailed: "custom compare failed", + ErrCustomFilter: "custom filtered", + ErrFuzzyCompareFailed: "fuzzy compare failed", + ErrFuzzyRedirect: "fuzzy redirect", + ErrFuzzyNotUnique: "not unique", +} + func (e ErrorType) Error() string { - switch e { - case ErrBadStatus: - return "blacklist status" - case ErrSameStatus: - return "same status with random baseline" - case ErrRequestFailed: - return "request failed" - case ErrWaf: - return "maybe banned by waf" - case ErrRedirect: - return "duplicate redirect url" - case ErrCompareFailed: - return "compare failed" - case ErrFuzzyCompareFailed: - return "fuzzy compare failed" - case ErrFuzzyRedirect: - return "fuzzy redirect" - case ErrCustomCompareFailed: - return "custom compare failed" - case ErrCustomFilter: - return "custom filtered" - default: - return "unknown error" - } + return ErrMap[e] } const ( diff --git a/pkg/baseline.go b/pkg/baseline.go index 30c7eda..550e8df 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -59,6 +59,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { if bl.Url.Host != host { bl.Host = host } + bl.Unique = UniqueHash(bl) return bl } @@ -95,6 +96,7 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba type Baseline struct { *parsers.SprayResult + Unique uint16 `json:"-"` Url *url.URL `json:"-"` Dir bool `json:"-"` Chunked bool `json:"-"` @@ -131,7 +133,7 @@ func (bl *Baseline) Collect() { bl.Hashes = parsers.NewHashes(bl.Raw) bl.Extracteds = Extractors.Extract(string(bl.Raw)) - + bl.Unique = UniqueHash(bl) } func (bl *Baseline) CollectURL() { diff --git a/pkg/utils.go b/pkg/utils.go index afa13e4..03f60c6 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -12,6 +12,7 @@ import ( "os" "path" "regexp" + "strconv" "strings" "time" "unsafe" @@ -306,3 +307,53 @@ func BakGenerator(domain string) []string { } return possibilities } + +var MbTable = []uint16{ + 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, + 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, + 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, + 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, + 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, + 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, + 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, + 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, + 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, + 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, + 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, + 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, + 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, + 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, + 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, + 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, + 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, + 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, + 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, + 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, + 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, + 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, + 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, + 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, + 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, + 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, + 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, + 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, + 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, + 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, + 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, + 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040} + +func CRC16Hash(data []byte) uint16 { + var crc16 uint16 + crc16 = 0xffff + for _, v := range data { + n := uint8(uint16(v) ^ crc16) + crc16 >>= 8 + crc16 ^= MbTable[n] + } + return crc16 +} + +func UniqueHash(bl *Baseline) uint16 { + // 由host+状态码+重定向url+content-type+title+length舍去个位与十位组成的hash, 没有body length, 因为可能存在随机值 + return CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/100*100))) +}