package pkg import ( "encoding/json" "github.com/chainreactors/gogo/v2/pkg/fingers" "github.com/chainreactors/ipcs" "github.com/chainreactors/parsers" "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/words/mask" "math/rand" "net/url" "os" "path" "strconv" "strings" "time" "unsafe" ) var ( Md5Fingers map[string]string = make(map[string]string) Mmh3Fingers map[string]string = make(map[string]string) Rules map[string]string = make(map[string]string) ActivePath []string Fingers fingers.Fingers ExtractRegexps = map[string][]*parsers.Extractor{} Extractors = make(parsers.Extractors) BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4", ".zip", ".rar"} BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"} ContentTypeMap = map[string]string{ "application/javascript": "js", "application/json": "json", "application/xml": "xml", "application/octet-stream": "bin", "application/atom+xml": "atom", "application/msword": "doc", "application/pdf": "pdf", "image/gif": "gif", "image/jpeg": "jpg", "image/png": "png", "image/svg+xml": "svg", "text/css": "css", "text/plain": "txt", "text/html": "html", "audio/mpeg": "mp3", "video/mp4": "mp4", "video/ogg": "ogg", "video/webm": "webm", "video/x-ms-wmv": "wmv", "video/avi": "avi", "image/x-icon": "ico", } ) func RemoveDuplication(arr []string) []string { set := make(map[string]struct{}, len(arr)) j := 0 for _, v := range arr { _, ok := set[v] if ok { continue } set[v] = struct{}{} arr[j] = v j++ } return arr[:j] } // 判断是否存在标准输入数据 func HasStdin() bool { stat, err := os.Stdin.Stat() if err != nil { return false } isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 return isPipedFromChrDev || isPipedFromFIFO } const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" var src = rand.NewSource(time.Now().UnixNano()) const ( // 6 bits to represent a letter index letterIdBits = 6 // All 1-bits as many as letterIdBits letterIdMask = 1<= 0; { if remain == 0 { cache, remain = src.Int63(), letterIdMax } if idx := int(cache & letterIdMask); idx < len(letters) { b[i] = letters[idx] i-- } cache >>= letterIdBits remain-- } return *(*string)(unsafe.Pointer(&b)) } func RandHost() string { n := 8 b := make([]byte, n) // A rand.Int63() generates 63 random bits, enough for letterIdMax letters! for i, cache, remain := n-1, src.Int63(), letterIdMax; i >= 1; { if remain == 0 { cache, remain = src.Int63(), letterIdMax } if idx := int(cache & letterIdMask); idx < len(letters) { b[i] = letters[idx] i-- } cache >>= letterIdBits remain-- } b[5] = byte(0x2e) return *(*string)(unsafe.Pointer(&b)) } func LoadTemplates() error { var err error // load fingers Fingers, err = fingers.LoadFingers(LoadConfig("http")) if err != nil { return err } for _, finger := range Fingers { err := finger.Compile(ipcs.ParsePorts) if err != nil { return err } } for _, f := range Fingers { for _, rule := range f.Rules { if rule.SendDataStr != "" { ActivePath = append(ActivePath, rule.SendDataStr) } if rule.Favicon != nil { for _, mmh3 := range rule.Favicon.Mmh3 { Mmh3Fingers[mmh3] = f.Name } for _, md5 := range rule.Favicon.Md5 { Md5Fingers[md5] = f.Name } } } } // load rule var data map[string]interface{} err = json.Unmarshal(LoadConfig("rule"), &data) if err != nil { return err } for k, v := range data { Rules[k] = v.(string) } // load mask var keywords map[string]interface{} err = json.Unmarshal(LoadConfig("mask"), &keywords) if err != nil { return err } for k, v := range keywords { t := make([]string, len(v.([]interface{}))) for i, vv := range v.([]interface{}) { t[i] = iutils.ToString(vv) } mask.SpecialWords[k] = t } var extracts []*parsers.Extractor err = json.Unmarshal(LoadConfig("extract"), &extracts) if err != nil { return err } for _, extract := range extracts { extract.Compile() ExtractRegexps[extract.Name] = []*parsers.Extractor{extract} for _, tag := range extract.Tags { if _, ok := ExtractRegexps[tag]; !ok { ExtractRegexps[tag] = []*parsers.Extractor{extract} } else { ExtractRegexps[tag] = append(ExtractRegexps[tag], extract) } } } return nil } func FingerDetect(content []byte) parsers.Frameworks { frames := make(parsers.Frameworks) for _, finger := range Fingers { // sender置空, 所有的发包交给spray的pool frame, _, ok := fingers.FingerMatcher(finger, map[string]interface{}{"content": content}, 0, nil) if ok { frames[frame.Name] = frame } } return frames } func filterJs(u string) bool { if commonFilter(u) { return true } return false } func filterUrl(u string) bool { if commonFilter(u) { return true } parsed, err := url.Parse(u) if err != nil { return true } else { ext := path.Ext(parsed.Path) for _, e := range BadExt { if strings.EqualFold(e, ext) { return true } } } return false } func formatURL(u string) string { // 去掉frag与params, 节约url.parse性能, 防止带参数造成意外的影响 if strings.Contains(u, "2f") || strings.Contains(u, "2F") { u = strings.ReplaceAll(u, "\\u002F", "/") u = strings.ReplaceAll(u, "\\u002f", "/") u = strings.ReplaceAll(u, "%252F", "/") u = strings.ReplaceAll(u, "%252f", "/") u = strings.ReplaceAll(u, "%2f", "/") u = strings.ReplaceAll(u, "%2F", "/") } u = strings.TrimRight(u, "\\") if i := strings.Index(u, "?"); i != -1 { return u[:i] } if i := strings.Index(u, "#"); i != -1 { return u[:i] } return u } func commonFilter(u string) bool { if strings.HasPrefix(u, "http") && len(u) < 15 { return true } for _, bad := range BadURL { if strings.Contains(u, bad) { return true } } return false } func BakGenerator(domain string) []string { var possibilities []string for first, _ := range domain { for last, _ := range domain[first:] { p := domain[first : first+last+1] if !iutils.StringsContains(possibilities, p) { possibilities = append(possibilities, p) } } } return possibilities } var MbTable = []uint16{ 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040} func CRC16Hash(data []byte) uint16 { var crc16 uint16 crc16 = 0xffff for _, v := range data { n := uint8(uint16(v) ^ crc16) crc16 >>= 8 crc16 ^= MbTable[n] } return crc16 } func UniqueHash(bl *Baseline) uint16 { // 由host+状态码+重定向url+content-type+title+length舍去个位与十位组成的hash // body length可能会导致一些误报, 目前没有更好的解决办法 return CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/100*100))) }