新增一种特殊的过滤模式--unique 或指定状态码的--unique-status 200

internal/option.go

@@ -89,18 +89,21 @@ type PluginOptions struct {
 }
 
 type ModeOptions struct {
-	RateLimit       int    `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100"`
-	Force           bool   `long:"force" description:"Bool, skip error break"`
-	CheckOnly       bool   `long:"check-only" description:"Bool, check only"`
-	Recursive       string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"`
-	Depth           int    `long:"depth" default:"0" description:"Int, recursive depth"`
-	CheckPeriod     int    `long:"check-period" default:"200" description:"Int, check period when request"`
-	ErrPeriod       int    `long:"error-period" default:"10" description:"Int, check period when error"`
-	BreakThreshold  int    `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold "`
-	BlackStatus     string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status, "`
-	WhiteStatus     string `long:"white-status" default:"200" description:"Strings (comma split), custom white status"`
-	FuzzyStatus     string `long:"fuzzy-status" default:"404,403,500,501,502,503" description:"Strings (comma split), custom fuzzy status"`
-	SimhashDistance int    `long:"distance" default:"5"`
+	RateLimit      int    `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100"`
+	Force          bool   `long:"force" description:"Bool, skip error break"`
+	CheckOnly      bool   `long:"check-only" description:"Bool, check only"`
+	Recursive      string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()"`
+	Depth          int    `long:"depth" default:"0" description:"Int, recursive depth"`
+	CheckPeriod    int    `long:"check-period" default:"200" description:"Int, check period when request"`
+	ErrPeriod      int    `long:"error-period" default:"10" description:"Int, check period when error"`
+	BreakThreshold int    `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold "`
+	BlackStatus    string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status, "`
+	WhiteStatus    string `long:"white-status" default:"200" description:"Strings (comma split), custom white status"`
+	FuzzyStatus    string `long:"fuzzy-status" default:"404,403,500,501,502,503" description:"Strings (comma split), custom fuzzy status"`
+	UniqueStatus   string `long:"unique-status" default:"403" description:"Strings (comma split), custom unique status"`
+	Unique         bool   `long:"unique" description:"Bool, unique response"`
+
+	SimhashDistance int `long:"distance" default:"5"`
 }
 
 type MiscOptions struct {
@@ -225,6 +228,12 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		FuzzyStatus = parseStatus(FuzzyStatus, opt.FuzzyStatus)
 	}
 
+	if opt.Unique {
+		enableAllUnique = true
+	} else {
+		UniqueStatus = parseStatus(UniqueStatus, opt.UniqueStatus)
+	}
+
 	// prepare word
 	dicts := make([][]string, len(opt.Dictionaries))
 	for i, f := range opt.Dictionaries {

internal/pool.go

@@ -25,12 +25,13 @@ import (
 )
 
 var (
-	max            = 2147483647
-	MaxRedirect    = 3
-	MaxCrawl       = 3
-	MaxRecursion   = 0
-	enableAllFuzzy = false
-	nilBaseline    = &pkg.Baseline{}
+	max             = 2147483647
+	MaxRedirect     = 3
+	MaxCrawl        = 3
+	MaxRecursion    = 0
+	enableAllFuzzy  = false
+	enableAllUnique = false
+	nilBaseline     = &pkg.Baseline{}
 )
 
 func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
@@ -50,6 +51,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
 		client:      ihttp.NewClient(config.Thread, 2, config.ClientType),
 		baselines:   make(map[int]*pkg.Baseline),
 		urls:        make(map[string]struct{}),
+		uniques:     make(map[uint16]struct{}),
 		tempCh:      make(chan *pkg.Baseline, 100),
 		checkCh:     make(chan int, 100),
 		additionCh:  make(chan *Unit, 100),
@@ -102,6 +104,7 @@ type Pool struct {
 	index           *pkg.Baseline
 	baselines       map[int]*pkg.Baseline
 	urls            map[string]struct{}
+	uniques         map[uint16]struct{}
 	analyzeDone     bool
 	worder          *words.Worder
 	limiter         *rate.Limiter
@@ -389,7 +392,20 @@ func (pool *Pool) Handler() {
 
 		if status {
 			pool.Statistor.FoundNumber++
-			if pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) {
+
+			// unique判断
+			if enableAllUnique || iutils.IntsContains(UniqueStatus, bl.Status) {
+				if _, ok := pool.uniques[bl.Unique]; ok {
+					bl.IsValid = false
+					bl.IsFuzzy = true
+					bl.Reason = ErrFuzzyNotUnique.Error()
+				} else {
+					pool.uniques[bl.Unique] = struct{}{}
+				}
+			}
+
+			// 对通过所有对比的有效数据进行再次filter
+			if bl.IsValid && pool.FilterExpr != nil && CompareWithExpr(pool.FilterExpr, params) {
 				pool.Statistor.FilteredNumber++
 				bl.Reason = ErrCustomFilter.Error()
 				bl.IsValid = false
@@ -472,10 +488,12 @@ func (pool *Pool) PreCompare(resp *ihttp.Response) error {
 }
 
 func (pool *Pool) BaseCompare(bl *pkg.Baseline) bool {
+	if !bl.IsValid {
+		return false
+	}
 	var status = -1
-
 	// 30x状态码的特殊处理
-	if strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") {
+	if bl.RedirectURL != "" && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") {
 		bl.Reason = ErrFuzzyRedirect.Error()
 		pool.putToFuzzy(bl)
 		return false

internal/runner.go

@@ -17,10 +17,11 @@ import (
 )
 
 var (
-	WhiteStatus = []int{200}
-	BlackStatus = []int{400, 410}
-	FuzzyStatus = []int{403, 404, 500, 501, 502, 503}
-	WAFStatus   = []int{493, 418, 1020, 406}
+	WhiteStatus  = []int{200}
+	BlackStatus  = []int{400, 410}
+	FuzzyStatus  = []int{403, 404, 500, 501, 502, 503}
+	WAFStatus    = []int{493, 418, 1020, 406}
+	UniqueStatus = []int{403}
 )
 
 var (

internal/types.go

@@ -8,43 +8,37 @@ import (
 type ErrorType uint
 
 const (
-	ErrBadStatus ErrorType = iota
+	NoErr ErrorType = iota
+	ErrBadStatus
 	ErrSameStatus
 	ErrRequestFailed
 	ErrWaf
 	ErrRedirect
 	ErrCompareFailed
-	ErrFuzzyCompareFailed
-	ErrFuzzyRedirect
 	ErrCustomCompareFailed
 	ErrCustomFilter
+	ErrFuzzyCompareFailed
+	ErrFuzzyRedirect
+	ErrFuzzyNotUnique
 )
 
+var ErrMap = map[ErrorType]string{
+	NoErr:                  "",
+	ErrBadStatus:           "blacklist status",
+	ErrSameStatus:          "same status with random baseline",
+	ErrRequestFailed:       "request failed",
+	ErrWaf:                 "maybe banned by waf",
+	ErrRedirect:            "duplicate redirect url",
+	ErrCompareFailed:       "compare failed",
+	ErrCustomCompareFailed: "custom compare failed",
+	ErrCustomFilter:        "custom filtered",
+	ErrFuzzyCompareFailed:  "fuzzy compare failed",
+	ErrFuzzyRedirect:       "fuzzy redirect",
+	ErrFuzzyNotUnique:      "not unique",
+}
+
 func (e ErrorType) Error() string {
-	switch e {
-	case ErrBadStatus:
-		return "blacklist status"
-	case ErrSameStatus:
-		return "same status with random baseline"
-	case ErrRequestFailed:
-		return "request failed"
-	case ErrWaf:
-		return "maybe banned by waf"
-	case ErrRedirect:
-		return "duplicate redirect url"
-	case ErrCompareFailed:
-		return "compare failed"
-	case ErrFuzzyCompareFailed:
-		return "fuzzy compare failed"
-	case ErrFuzzyRedirect:
-		return "fuzzy redirect"
-	case ErrCustomCompareFailed:
-		return "custom compare failed"
-	case ErrCustomFilter:
-		return "custom filtered"
-	default:
-		return "unknown error"
-	}
+	return ErrMap[e]
 }
 
 const (

pkg/baseline.go

@@ -59,6 +59,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 	if bl.Url.Host != host {
 		bl.Host = host
 	}
+	bl.Unique = UniqueHash(bl)
 	return bl
 }
 
@@ -95,6 +96,7 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
 
 type Baseline struct {
 	*parsers.SprayResult
+	Unique    uint16   `json:"-"`
 	Url       *url.URL `json:"-"`
 	Dir       bool     `json:"-"`
 	Chunked   bool     `json:"-"`
@@ -131,7 +133,7 @@ func (bl *Baseline) Collect() {
 
 	bl.Hashes = parsers.NewHashes(bl.Raw)
 	bl.Extracteds = Extractors.Extract(string(bl.Raw))
-
+	bl.Unique = UniqueHash(bl)
 }
 
 func (bl *Baseline) CollectURL() {

pkg/utils.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 	"unsafe"
@@ -306,3 +307,53 @@ func BakGenerator(domain string) []string {
 	}
 	return possibilities
 }
+
+var MbTable = []uint16{
+	0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
+	0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
+	0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
+	0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
+	0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
+	0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
+	0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
+	0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
+	0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
+	0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
+	0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
+	0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
+	0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
+	0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
+	0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
+	0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
+	0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
+	0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
+	0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
+	0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
+	0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
+	0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
+	0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
+	0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
+	0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
+	0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
+	0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
+	0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
+	0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
+	0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
+	0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
+	0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040}
+
+func CRC16Hash(data []byte) uint16 {
+	var crc16 uint16
+	crc16 = 0xffff
+	for _, v := range data {
+		n := uint8(uint16(v) ^ crc16)
+		crc16 >>= 8
+		crc16 ^= MbTable[n]
+	}
+	return crc16
+}
+
+func UniqueHash(bl *Baseline) uint16 {
+	// 由host+状态码+重定向url+content-type+title+length舍去个位与十位组成的hash, 没有body length, 因为可能存在随机值
+	return CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/100*100)))
+}