自动判断协议升级

internal/checkpool.go

@@ -51,7 +51,7 @@ func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) {
 
 		if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
 			pool.failedCount++
-			bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()}
+			bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()}
 		} else {
 			bl = pkg.NewBaseline(req.URI(), req.Host(), resp)
 			bl.Collect()

internal/option.go

@@ -117,7 +117,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		r.Progress.Start()
 		logs.Log.Writer = r.Progress.Bypass()
 	} else {
-		logs.Log.Level = 100
+		logs.Log.Quiet = true
 	}
 
 	if opt.SimhashDistance != 0 {
@@ -315,10 +315,12 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		}
 	}
 
+	if opt.FuzzyFile != "" {
 		r.FuzzyFile, err = files.NewFile(opt.FuzzyFile, false, false, true)
 		if err != nil {
 			return nil, err
 		}
+	}
 
 	r.StatFile, err = files.NewFile("stat.json", false, false, true)
 	if err != nil {

internal/pool.go

@@ -11,7 +11,9 @@ import (
 	"github.com/chainreactors/words"
 	"github.com/panjf2000/ants/v2"
 	"github.com/valyala/fasthttp"
+	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -87,7 +89,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) {
 		if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge {
 			pool.failedCount++
 			pool.Statistor.FailedNumber++
-			bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()}
+			bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()}
 			pool.failedBaselines = append(pool.failedBaselines, bl)
 		} else {
 			if unit.source != WordSource {
@@ -228,6 +230,15 @@ func (p *Pool) Init() error {
 		return fmt.Errorf(p.index.String())
 	}
 
+	if p.base.RedirectURL != "" {
+		// 自定协议升级
+		// 某些网站http会重定向到https, 如果发现随机目录出现这种情况, 则自定将baseurl升级为https
+		rurl, err := url.Parse(p.base.RedirectURL)
+		if err == nil && rurl.Host == p.base.Url.Host && p.base.Url.Scheme == "http" && rurl.Scheme == "https" {
+			logs.Log.Importantf("baseurl %s upgrade http to https", p.BaseURL)
+			p.BaseURL = strings.Replace(p.BaseURL, "http", "https", 1)
+		}
+	}
 	p.base.Collect()
 	p.index.Collect()
 
@@ -341,7 +352,7 @@ func (p *Pool) BaseCompare(bl *pkg.Baseline) bool {
 
 	bl.Collect()
 	for _, f := range bl.Frameworks {
-		if f.Tag == "waf/cdn" {
+		if f.Tag == "waf" || f.Tag == "cdn" {
 			p.Statistor.WafedNumber++
 			bl.Reason = ErrWaf.Error()
 			return false

pkg/baseline.go

@@ -11,13 +11,14 @@ import (
 
 func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 	bl := &Baseline{
-		Url:     u,
+		UrlString: u,
 		Status:    resp.StatusCode(),
 		IsValid:   true,
 	}
 	uu, err := url.Parse(u)
 	if err == nil {
 		bl.Path = uu.Path
+		bl.Url = uu
 	}
 	if resp.ClientType == ihttp.STANDARD {
 		bl.Host = host
@@ -34,7 +35,7 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 
 func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline {
 	bl := &Baseline{
-		Url:     u,
+		UrlString: u,
 		Status:    resp.StatusCode(),
 		IsValid:   false,
 		Reason:    reason,
@@ -43,6 +44,7 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
 	uu, err := url.Parse(u)
 	if err == nil {
 		bl.Path = uu.Path
+		bl.Url = uu
 	}
 
 	if resp.ClientType == ihttp.STANDARD {
@@ -57,7 +59,8 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
 }
 
 type Baseline struct {
-	Url          string     `json:"url"`
+	Url          *url.URL   `json:"-"`
+	UrlString    string     `json:"url"`
 	Path         string     `json:"path"`
 	Host         string     `json:"host"`
 	Body         []byte     `json:"-"`
@@ -132,7 +135,7 @@ func (bl *Baseline) FuzzyCompare(other *Baseline) bool {
 func (bl *Baseline) Get(key string) string {
 	switch key {
 	case "url":
-		return bl.Url
+		return bl.UrlString
 	case "host":
 		return bl.Host
 	case "title":
@@ -180,7 +183,7 @@ func (bl *Baseline) Additional(key string) string {
 
 func (bl *Baseline) Format(probes []string) string {
 	var line strings.Builder
-	line.WriteString(bl.Url)
+	line.WriteString(bl.UrlString)
 	if bl.Host != "" {
 		line.WriteString(" (" + bl.Host + ")")
 	}
@@ -206,7 +209,7 @@ func (bl *Baseline) Format(probes []string) string {
 func (bl *Baseline) String() string {
 	var line strings.Builder
 	//line.WriteString("[+] ")
-	line.WriteString(bl.Url)
+	line.WriteString(bl.UrlString)
 	if bl.Host != "" {
 		line.WriteString(" (" + bl.Host + ")")
 	}