适配了favicon的指纹识别

添加了基于contenttype的一些优化
index现在发送的是空数据, 而非添加了"/"之后的

internal/pool.go

@@ -148,7 +148,7 @@ type Pool struct {
 func (pool *Pool) Init() error {
 	// 分成两步是为了避免闭包的线程安全问题
 	pool.initwg.Add(2)
-	pool.reqPool.Invoke(newUnit("/", InitIndexSource))
+	pool.reqPool.Invoke(newUnit("", InitIndexSource))
 	pool.reqPool.Invoke(newUnit(pkg.RandPath(), InitRandomSource))
 	pool.initwg.Wait()
 	if pool.index.ErrString != "" {
@@ -175,6 +175,11 @@ func (pool *Pool) Init() error {
 }
 
 func (pool *Pool) checkRedirect(redirectURL string) bool {
+	if pool.random.RedirectURL == "" {
+		// 如果random的redirectURL为空, 此时该项
+		return true
+	}
+
 	if redirectURL == pool.random.RedirectURL {
 		// 相同的RedirectURL将被认为是无效数据
 		return false

pkg/baseline.go

@@ -70,6 +70,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
 		bl.BodyLength = len(bl.Body)
 	}
 
+	if t, ok := ContentTypeMap[resp.ContentType()]; ok {
+		bl.ContentType = t
+		bl.Title = t + " data"
+	} else {
+		bl.ContentType = "other"
+	}
 	bl.Raw = append(bl.Header, bl.Body...)
 	bl.RedirectURL = resp.GetHeader("Location")
 	return bl
@@ -119,6 +125,7 @@ type Baseline struct {
 	FrontURL        string     `json:"front_url,omitempty"`
 	Status          int        `json:"status"`
 	Spended         int64      `json:"spend"` // 耗时, 毫秒
+	ContentType     string     `json:"content_type"`
 	Title           string     `json:"title"`
 	Frameworks      Frameworks `json:"frameworks"`
 	Extracteds      Extracteds `json:"extracts"`
@@ -143,12 +150,22 @@ func (bl *Baseline) IsDir() bool {
 
 // Collect 深度收集信息
 func (bl *Baseline) Collect() {
+	bl.Frameworks = FingerDetect(string(bl.Raw))
 	if len(bl.Body) > 0 {
-		bl.Title = utils.AsciiEncode(parsers.MatchTitle(string(bl.Body)))
+		if bl.ContentType == "html" {
+			bl.Title = utils.AsciiEncode(parsers.MatchTitle(string(bl.Body)))
+		} else if bl.ContentType == "ico" {
+			if name, ok := Md5Fingers[parsers.Md5Hash(bl.Body)]; ok {
+				bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name})
+			} else if name, ok := Mmh3Fingers[parsers.Mmh3Hash32(bl.Body)]; ok {
+				bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name})
+			}
+		}
 	}
+
 	bl.Hashes = parsers.NewHashes(bl.Raw)
 	bl.Extracteds = Extractors.Extract(string(bl.Raw))
-	bl.Frameworks = FingerDetect(string(bl.Raw))
+
 }
 
 func (bl *Baseline) CollectURL() {
@@ -231,6 +248,8 @@ func (bl *Baseline) Get(key string) string {
 		return bl.UrlString
 	case "host":
 		return bl.Host
+	case "content_type", "type":
+		return bl.ContentType
 	case "title":
 		return bl.Title
 	case "redirect":

pkg/ihttp/request.go

@@ -77,6 +77,10 @@ func (r *Request) Host() string {
 }
 
 func safeUrlJoin(base, uri string) string {
+	if uri == "" {
+		// 如果url为空, 则直接对原样的url请求
+		return base
+	}
 	if !strings.HasSuffix(base, "/") && !strings.HasPrefix(uri, "/") {
 		return base + "/" + uri
 	} else {

pkg/ihttp/response.go

@@ -6,6 +6,7 @@ import (
 	"github.com/valyala/fasthttp"
 	"io"
 	"net/http"
+	"strings"
 )
 
 type Response struct {
@@ -57,6 +58,23 @@ func (r *Response) ContentLength() int {
 	}
 }
 
+func (r *Response) ContentType() string {
+	var t string
+	if r.FastResponse != nil {
+		t = string(r.FastResponse.Header.ContentType())
+	} else if r.StandardResponse != nil {
+		t = r.StandardResponse.Header.Get("Content-Type")
+	} else {
+		return ""
+	}
+
+	if i := strings.Index(t, ";"); i > 0 {
+		return t[:i]
+	} else {
+		return t
+	}
+}
+
 func (r *Response) Header() []byte {
 	if r.FastResponse != nil {
 		return r.FastResponse.Header.Header()

pkg/utils.go

@@ -33,6 +33,30 @@ var (
 		regexp.MustCompile("[\",',‘,“]\\s{0,6}([#,.]{0,2}/[^\\s,^',^’,^\",^”,^>,^<,^:,^),^(]{2,250}?)\\s{0,6}[\",',‘,“]"),
 		regexp.MustCompile("href\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([^\\s,^',^’,^\",^“,^>,^<,^,^+),^(]{2,250})|action\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([^\\s,^',^’,^\",^“,^>,^<,^,^+),^(]{2,250})"),
 	}
+
+	ContentTypeMap = map[string]string{
+		"application/javascript":   "js",
+		"application/json":         "json",
+		"application/xml":          "xml",
+		"application/octet-stream": "bin",
+		"application/atom+xml":     "atom",
+		"application/msword":       "doc",
+		"application/pdf":          "pdf",
+		"image/gif":                "gif",
+		"image/jpeg":               "jpg",
+		"image/png":                "png",
+		"image/svg+xml":            "svg",
+		"text/css":                 "css",
+		"text/plain":               "txt",
+		"text/html":                "html",
+		"audio/mpeg":               "mp3",
+		"video/mp4":                "mp4",
+		"video/ogg":                "ogg",
+		"video/webm":               "webm",
+		"video/x-ms-wmv":           "wmv",
+		"video/avi":                "avi",
+		"image/x-icon":             "ico",
+	}
 )
 
 func StringsContains(s []string, e string) bool {
@@ -176,7 +200,6 @@ func LoadTemplates() error {
 
 func FingerDetect(content string) Frameworks {
 	var frames Frameworks
-	//content := string(body)
 	for _, finger := range Fingers {
 		frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil)
 		if ok {