admin/spray
1
0 Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2026-10-29 03:50:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor fingers

Browse Source
This commit is contained in:
M09Ic committed 2024-03-04 20:03:06 +08:00
1 parent 424fd7b5d3
commit 34bcc60ddf
11 files changed
+115 -205
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/baseline.go
+26 -19
View File
@@ -2,21 +2,24 @@ package pkg
import (
"bytes"
"github.com/chainreactors/fingers/common"
"github.com/chainreactors/parsers"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/utils/encode"
"github.com/chainreactors/utils/iutils"
"net/http"
"net/url"
"strings"
)
func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
var err error
bl := &Baseline{
SprayResult: &parsers.SprayResult{
UrlString: u,
Status: resp.StatusCode(),
IsValid: true,
Frameworks: make(parsers.Frameworks),
Frameworks: make(common.Frameworks),
},
}
@@ -46,10 +49,12 @@ func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
}
bl.Raw = append(bl.Header, bl.Body...)
if r := resp.GetHeader("Location"); r != "" {
bl.Response, err = ParseRawResponse(bl.Raw)
if r := bl.Response.Header.Get("Location"); r != "" {
bl.RedirectURL = r
} else {
bl.RedirectURL = resp.GetHeader("location")
bl.RedirectURL = bl.Response.Header.Get("location")
}
bl.Dir = bl.IsDir()
@@ -102,17 +107,18 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba
type Baseline struct {
*parsers.SprayResult
Url *url.URL `json:"-"`
Dir bool `json:"-"`
Chunked bool `json:"-"`
Body BS `json:"-"`
Header BS `json:"-"`
Raw BS `json:"-"`
Recu bool `json:"-"`
RecuDepth int `json:"-"`
URLs []string `json:"-"`
Collected bool `json:"-"`
Retry int `json:"-"`
Url *url.URL `json:"-"`
Dir bool `json:"-"`
Chunked bool `json:"-"`
Body BS `json:"-"`
Header BS `json:"-"`
Raw BS `json:"-"`
Response *http.Response `json:"-"`
Recu bool `json:"-"`
RecuDepth int `json:"-"`
URLs []string `json:"-"`
Collected bool `json:"-"`
Retry int `json:"-"`
}
func (bl *Baseline) IsDir() bool {
@@ -126,9 +132,10 @@ func (bl *Baseline) IsDir() bool {
func (bl *Baseline) Collect() {
if bl.ContentType == "html" || bl.ContentType == "json" || bl.ContentType == "txt" {
// 指纹库设计的时候没考虑js,css文件的指纹, 跳过非必要的指纹收集减少误报提高性能
bl.Frameworks = FingerDetect(bl.Raw)
if EnableFingerPrintHub {
bl.Frameworks.Merge(FingerPrintHubDetect(string(bl.Header), string(bl.Body)))
bl.Frameworks = FingersDetect(bl.Raw)
if EnableAllFingerEngine {
bl.Frameworks.Merge(FingerPrintHubDetect(bl.Response.Header, string(bl.Body)))
bl.Frameworks.Merge(WappalyzerDetect(bl.Response.Header, bl.Body))
}
}
@@ -137,9 +144,9 @@ func (bl *Baseline) Collect() {
bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
} else if bl.ContentType == "ico" {
if name, ok := Md5Fingers[encode.Md5Hash(bl.Body)]; ok {
bl.Frameworks[name] = &parsers.Framework{Name: name}
bl.Frameworks[name] = &common.Framework{Name: name}
} else if name, ok := Mmh3Fingers[encode.Mmh3Hash32(bl.Body)]; ok {
bl.Frameworks[name] = &parsers.Framework{Name: name}
bl.Frameworks[name] = &common.Framework{Name: name}
}
}
}
pkg/fingerprinthub.go
-68
View File
@@ -1,68 +0,0 @@
package pkg
import (
"github.com/chainreactors/parsers"
"strings"
)
type FingerPrintHub struct {
Name string `json:"name"`
FaviconHash []string `json:"favicon_hash,omitempty"`
Keyword []string `json:"keyword,omitempty"`
Path string `json:"path"`
Headers map[string]string `json:"headers,omitempty"`
}
func FingerPrintHubDetect(header, body string) parsers.Frameworks {
frames := make(parsers.Frameworks)
for _, finger := range FingerPrintHubs {
status := false
if fingerPrintHubMatchHeader(finger, header) && fingerPrintHubMatchBody(finger, body) {
status = true
}
if status {
frames.Add(&parsers.Framework{
Name: finger.Name,
From: parsers.FrameFromDefault,
Tags: []string{"fingerprinthub"},
})
}
}
return frames
}
func fingerPrintHubMatchHeader(finger *FingerPrintHub, header string) bool {
if len(finger.Headers) == 0 {
return true
}
status := true
for k, v := range finger.Headers {
if v == "*" && strings.Contains(header, k) {
status = true
} else if strings.Contains(header, k) && strings.Contains(header, v) {
status = true
} else {
return false
}
}
return status
}
func fingerPrintHubMatchBody(finger *FingerPrintHub, body string) bool {
if len(finger.Keyword) == 0 {
return true
}
if body == "" {
return false
}
status := true
for _, key := range finger.Keyword {
if strings.Contains(body, key) {
status = true
} else {
return false
}
}
return status
}
pkg/fingers.go
+14 -11
View File
@@ -2,19 +2,22 @@ package pkg
import (
"bytes"
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/parsers"
"github.com/chainreactors/fingers/common"
"net/http"
)
// gogo fingers engine
func FingerDetect(content []byte) parsers.Frameworks {
frames := make(parsers.Frameworks)
for _, finger := range Fingers {
// sender置空, 所有的发包交给spray的pool
frame, _, ok := fingers.FingerMatcher(finger, map[string]interface{}{"content": bytes.ToLower(content)}, 0, nil)
if ok {
frames.Add(frame)
}
}
func FingersDetect(content []byte) common.Frameworks {
frames, _ := FingerEngine.FingersEngine.Match(bytes.ToLower(content), "")
return frames
}
func FingerPrintHubDetect(header http.Header, body string) common.Frameworks {
frames := FingerEngine.FingerPrintEngine.Match(header, body)
return frames
}
func WappalyzerDetect(header http.Header, body []byte) common.Frameworks {
frames := FingerEngine.WappalyzerEngine.Fingerprint(header, body)
return frames
}
pkg/load.go
+21 -36
View File
@@ -2,9 +2,8 @@ package pkg
import (
"encoding/json"
"github.com/chainreactors/gogo/v2/pkg/fingers"
"github.com/chainreactors/fingers"
"github.com/chainreactors/parsers"
"github.com/chainreactors/utils"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words/mask"
"os"
@@ -13,31 +12,23 @@ import (
)
var (
Md5Fingers map[string]string = make(map[string]string)
Mmh3Fingers map[string]string = make(map[string]string)
ExtractRegexps = make(parsers.Extractors)
Extractors = make(parsers.Extractors)
Fingers fingers.Fingers
ActivePath []string
FingerPrintHubs []*FingerPrintHub
Md5Fingers map[string]string = make(map[string]string)
Mmh3Fingers map[string]string = make(map[string]string)
ExtractRegexps = make(parsers.Extractors)
Extractors = make(parsers.Extractors)
FingerEngine *fingers.Engine
ActivePath []string
)
func LoadTemplates() error {
var err error
// load fingers
Fingers, err = fingers.LoadFingers(LoadConfig("http"))
FingerEngine, err = fingers.NewEngine()
if err != nil {
return err
}
for _, finger := range Fingers {
err := finger.Compile(utils.ParsePorts)
if err != nil {
return err
}
}
for _, f := range Fingers {
for _, f := range FingerEngine.FingersEngine.Fingers {
for _, rule := range f.Rules {
if rule.SendDataStr != "" {
ActivePath = append(ActivePath, rule.SendDataStr)
@@ -52,6 +43,14 @@ func LoadTemplates() error {
}
}
}
for _, f := range FingerEngine.FingerPrintEngine {
if f.Path != "/" {
ActivePath = append(ActivePath, f.Path)
}
for _, ico := range f.FaviconHash {
Md5Fingers[ico] = f.Name
}
}
// load rule
var data map[string]interface{}
@@ -118,26 +117,12 @@ func LoadExtractorConfig(filename string) ([]*parsers.Extractor, error) {
return extracts, nil
}
func LoadFingerPrintHub() error {
content := LoadConfig("fingerprinthub")
err := json.Unmarshal(content, &FingerPrintHubs)
func Load() error {
// load fingers
err := LoadTemplates()
if err != nil {
return err
}
var fingers []*FingerPrintHub
for _, f := range FingerPrintHubs {
if f.Path != "/" {
ActivePath = append(ActivePath, f.Path)
}
for _, ico := range f.FaviconHash {
Md5Fingers[ico] = f.Name
}
if len(f.Keyword) > 0 || len(f.Headers) > 0 {
fingers = append(fingers, f)
}
}
FingerPrintHubs = fingers
return nil
}
pkg/utils.go
+16 -1
View File
@@ -1,11 +1,14 @@
package pkg
import (
"bufio"
"bytes"
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/chainreactors/logs"
"github.com/chainreactors/utils/iutils"
"math/rand"
"net/http"
"net/url"
"path"
"path/filepath"
@@ -25,7 +28,7 @@ var (
UniqueStatus = []int{} // 相同unique的403表示命中了同一条acl, 相同unique的200表示default页面
// plugins
EnableFingerPrintHub = false
EnableAllFingerEngine = false
)
var (
Rules map[string]string = make(map[string]string)
@@ -376,3 +379,15 @@ func MatchWithGlobs(u string, globs []string) bool {
}
return false
}
func ParseRawResponse(raw []byte) (*http.Response, error) {
reader := bytes.NewReader(raw)
// 使用http.ReadResponse解析HTTP响应
resp, err := http.ReadResponse(bufio.NewReader(reader), nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return resp, nil
}