diff --git a/internal/checkpool.go b/internal/checkpool.go index a1ced30..74e8b9a 100644 --- a/internal/checkpool.go +++ b/internal/checkpool.go @@ -51,7 +51,7 @@ func NewCheckPool(ctx context.Context, config *pkg.Config) (*CheckPool, error) { if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { pool.failedCount++ - bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} + bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} } else { bl = pkg.NewBaseline(req.URI(), req.Host(), resp) bl.Collect() diff --git a/internal/option.go b/internal/option.go index 1e5c54d..052866c 100644 --- a/internal/option.go +++ b/internal/option.go @@ -117,7 +117,7 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.Progress.Start() logs.Log.Writer = r.Progress.Bypass() } else { - logs.Log.Level = 100 + logs.Log.Quiet = true } if opt.SimhashDistance != 0 { @@ -315,9 +315,11 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } } - r.FuzzyFile, err = files.NewFile(opt.FuzzyFile, false, false, true) - if err != nil { - return nil, err + if opt.FuzzyFile != "" { + r.FuzzyFile, err = files.NewFile(opt.FuzzyFile, false, false, true) + if err != nil { + return nil, err + } } r.StatFile, err = files.NewFile("stat.json", false, false, true) diff --git a/internal/pool.go b/internal/pool.go index 074c56a..8e556d1 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -11,7 +11,9 @@ import ( "github.com/chainreactors/words" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" + "net/url" "strconv" + "strings" "sync" "time" ) @@ -87,7 +89,7 @@ func NewPool(ctx context.Context, config *pkg.Config) (*Pool, error) { if reqerr != nil && reqerr != fasthttp.ErrBodyTooLarge { pool.failedCount++ pool.Statistor.FailedNumber++ - bl = &pkg.Baseline{Url: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} + bl = &pkg.Baseline{UrlString: pool.BaseURL + unit.path, IsValid: false, ErrString: reqerr.Error(), Reason: ErrRequestFailed.Error()} pool.failedBaselines = append(pool.failedBaselines, bl) } else { if unit.source != WordSource { @@ -228,6 +230,15 @@ func (p *Pool) Init() error { return fmt.Errorf(p.index.String()) } + if p.base.RedirectURL != "" { + // 自定协议升级 + // 某些网站http会重定向到https, 如果发现随机目录出现这种情况, 则自定将baseurl升级为https + rurl, err := url.Parse(p.base.RedirectURL) + if err == nil && rurl.Host == p.base.Url.Host && p.base.Url.Scheme == "http" && rurl.Scheme == "https" { + logs.Log.Importantf("baseurl %s upgrade http to https", p.BaseURL) + p.BaseURL = strings.Replace(p.BaseURL, "http", "https", 1) + } + } p.base.Collect() p.index.Collect() @@ -341,7 +352,7 @@ func (p *Pool) BaseCompare(bl *pkg.Baseline) bool { bl.Collect() for _, f := range bl.Frameworks { - if f.Tag == "waf/cdn" { + if f.Tag == "waf" || f.Tag == "cdn" { p.Statistor.WafedNumber++ bl.Reason = ErrWaf.Error() return false diff --git a/pkg/baseline.go b/pkg/baseline.go index b86eba1..5f1fd31 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -11,13 +11,14 @@ import ( func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { bl := &Baseline{ - Url: u, - Status: resp.StatusCode(), - IsValid: true, + UrlString: u, + Status: resp.StatusCode(), + IsValid: true, } uu, err := url.Parse(u) if err == nil { bl.Path = uu.Path + bl.Url = uu } if resp.ClientType == ihttp.STANDARD { bl.Host = host @@ -34,15 +35,16 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline { bl := &Baseline{ - Url: u, - Status: resp.StatusCode(), - IsValid: false, - Reason: reason, + UrlString: u, + Status: resp.StatusCode(), + IsValid: false, + Reason: reason, } uu, err := url.Parse(u) if err == nil { bl.Path = uu.Path + bl.Url = uu } if resp.ClientType == ihttp.STANDARD { @@ -57,7 +59,8 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba } type Baseline struct { - Url string `json:"url"` + Url *url.URL `json:"-"` + UrlString string `json:"url"` Path string `json:"path"` Host string `json:"host"` Body []byte `json:"-"` @@ -132,7 +135,7 @@ func (bl *Baseline) FuzzyCompare(other *Baseline) bool { func (bl *Baseline) Get(key string) string { switch key { case "url": - return bl.Url + return bl.UrlString case "host": return bl.Host case "title": @@ -180,7 +183,7 @@ func (bl *Baseline) Additional(key string) string { func (bl *Baseline) Format(probes []string) string { var line strings.Builder - line.WriteString(bl.Url) + line.WriteString(bl.UrlString) if bl.Host != "" { line.WriteString(" (" + bl.Host + ")") } @@ -206,7 +209,7 @@ func (bl *Baseline) Format(probes []string) string { func (bl *Baseline) String() string { var line strings.Builder //line.WriteString("[+] ") - line.WriteString(bl.Url) + line.WriteString(bl.UrlString) if bl.Host != "" { line.WriteString(" (" + bl.Host + ")") }