diff --git a/internal/pool.go b/internal/pool.go index 0ba9505..a5ff4df 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -148,7 +148,7 @@ type Pool struct { func (pool *Pool) Init() error { // 分成两步是为了避免闭包的线程安全问题 pool.initwg.Add(2) - pool.reqPool.Invoke(newUnit("/", InitIndexSource)) + pool.reqPool.Invoke(newUnit("", InitIndexSource)) pool.reqPool.Invoke(newUnit(pkg.RandPath(), InitRandomSource)) pool.initwg.Wait() if pool.index.ErrString != "" { @@ -175,6 +175,11 @@ func (pool *Pool) Init() error { } func (pool *Pool) checkRedirect(redirectURL string) bool { + if pool.random.RedirectURL == "" { + // 如果random的redirectURL为空, 此时该项 + return true + } + if redirectURL == pool.random.RedirectURL { // 相同的RedirectURL将被认为是无效数据 return false diff --git a/pkg/baseline.go b/pkg/baseline.go index 44a6cbd..17ca9cf 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -70,6 +70,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline { bl.BodyLength = len(bl.Body) } + if t, ok := ContentTypeMap[resp.ContentType()]; ok { + bl.ContentType = t + bl.Title = t + " data" + } else { + bl.ContentType = "other" + } bl.Raw = append(bl.Header, bl.Body...) bl.RedirectURL = resp.GetHeader("Location") return bl @@ -119,6 +125,7 @@ type Baseline struct { FrontURL string `json:"front_url,omitempty"` Status int `json:"status"` Spended int64 `json:"spend"` // 耗时, 毫秒 + ContentType string `json:"content_type"` Title string `json:"title"` Frameworks Frameworks `json:"frameworks"` Extracteds Extracteds `json:"extracts"` @@ -143,12 +150,22 @@ func (bl *Baseline) IsDir() bool { // Collect 深度收集信息 func (bl *Baseline) Collect() { + bl.Frameworks = FingerDetect(string(bl.Raw)) if len(bl.Body) > 0 { - bl.Title = utils.AsciiEncode(parsers.MatchTitle(string(bl.Body))) + if bl.ContentType == "html" { + bl.Title = utils.AsciiEncode(parsers.MatchTitle(string(bl.Body))) + } else if bl.ContentType == "ico" { + if name, ok := Md5Fingers[parsers.Md5Hash(bl.Body)]; ok { + bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name}) + } else if name, ok := Mmh3Fingers[parsers.Mmh3Hash32(bl.Body)]; ok { + bl.Frameworks = append(bl.Frameworks, &parsers.Framework{Name: name}) + } + } } + bl.Hashes = parsers.NewHashes(bl.Raw) bl.Extracteds = Extractors.Extract(string(bl.Raw)) - bl.Frameworks = FingerDetect(string(bl.Raw)) + } func (bl *Baseline) CollectURL() { @@ -231,6 +248,8 @@ func (bl *Baseline) Get(key string) string { return bl.UrlString case "host": return bl.Host + case "content_type", "type": + return bl.ContentType case "title": return bl.Title case "redirect": diff --git a/pkg/ihttp/request.go b/pkg/ihttp/request.go index 8e908bc..20a530a 100644 --- a/pkg/ihttp/request.go +++ b/pkg/ihttp/request.go @@ -77,6 +77,10 @@ func (r *Request) Host() string { } func safeUrlJoin(base, uri string) string { + if uri == "" { + // 如果url为空, 则直接对原样的url请求 + return base + } if !strings.HasSuffix(base, "/") && !strings.HasPrefix(uri, "/") { return base + "/" + uri } else { diff --git a/pkg/ihttp/response.go b/pkg/ihttp/response.go index 420c37c..c835657 100644 --- a/pkg/ihttp/response.go +++ b/pkg/ihttp/response.go @@ -6,6 +6,7 @@ import ( "github.com/valyala/fasthttp" "io" "net/http" + "strings" ) type Response struct { @@ -57,6 +58,23 @@ func (r *Response) ContentLength() int { } } +func (r *Response) ContentType() string { + var t string + if r.FastResponse != nil { + t = string(r.FastResponse.Header.ContentType()) + } else if r.StandardResponse != nil { + t = r.StandardResponse.Header.Get("Content-Type") + } else { + return "" + } + + if i := strings.Index(t, ";"); i > 0 { + return t[:i] + } else { + return t + } +} + func (r *Response) Header() []byte { if r.FastResponse != nil { return r.FastResponse.Header.Header() diff --git a/pkg/utils.go b/pkg/utils.go index a3ea522..ae786f4 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -33,6 +33,30 @@ var ( regexp.MustCompile("[\",',‘,“]\\s{0,6}([#,.]{0,2}/[^\\s,^',^’,^\",^”,^>,^<,^:,^),^(]{2,250}?)\\s{0,6}[\",',‘,“]"), regexp.MustCompile("href\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([^\\s,^',^’,^\",^“,^>,^<,^,^+),^(]{2,250})|action\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([^\\s,^',^’,^\",^“,^>,^<,^,^+),^(]{2,250})"), } + + ContentTypeMap = map[string]string{ + "application/javascript": "js", + "application/json": "json", + "application/xml": "xml", + "application/octet-stream": "bin", + "application/atom+xml": "atom", + "application/msword": "doc", + "application/pdf": "pdf", + "image/gif": "gif", + "image/jpeg": "jpg", + "image/png": "png", + "image/svg+xml": "svg", + "text/css": "css", + "text/plain": "txt", + "text/html": "html", + "audio/mpeg": "mp3", + "video/mp4": "mp4", + "video/ogg": "ogg", + "video/webm": "webm", + "video/x-ms-wmv": "wmv", + "video/avi": "avi", + "image/x-icon": "ico", + } ) func StringsContains(s []string, e string) bool { @@ -176,7 +200,6 @@ func LoadTemplates() error { func FingerDetect(content string) Frameworks { var frames Frameworks - //content := string(body) for _, finger := range Fingers { frame, _, ok := fingers.FingerMatcher(finger, content, 0, nil) if ok {