From 04944cf829eeb7a1cc004085a00093f28ca42b04 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 1 Jul 2024 19:51:16 +0800 Subject: [PATCH] adapt fingers --- go.mod | 6 ++++-- go.sum | 7 +++++++ internal/pool/brutepool.go | 2 +- internal/pool/pool.go | 3 +++ pkg/baseline.go | 14 +++++++------- pkg/fingers.go | 2 +- pkg/load.go | 4 ++-- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index ad5287b..f1fb5c2 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ toolchain go1.22.2 require ( github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 - github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da + github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f - github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8 + github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29 github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 github.com/expr-lang/expr v1.16.9 @@ -43,6 +43,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/twmb/murmur3 v1.1.8 // indirect @@ -55,4 +56,5 @@ require ( golang.org/x/text v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index fb454ad..c13fca5 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,10 @@ github.com/chainreactors/fingers v0.0.0-20240628165313-8c7e41bf9ab5 h1:tmr3wbiwZ github.com/chainreactors/fingers v0.0.0-20240628165313-8c7e41bf9ab5/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8= github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da h1:5kX/KxHFFjKzpZ7H8Ofdu7ukyrj7IXURhToWpeL3ID0= github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8= +github.com/chainreactors/fingers v0.0.0-20240701103336-582e82977506 h1:hH7PXBn+nMU0uBFn7PYVqJagZM7EsP3X3RwAcqaTQ8U= +github.com/chainreactors/fingers v0.0.0-20240701103336-582e82977506/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4= +github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673 h1:ezIijqDJmoKFVmXxnUufUzKo/HBxQxp6QnQZvHxaBaY= +github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4= github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= @@ -105,6 +109,8 @@ github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412 h1:GobKh1fzt github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412/go.mod h1:0zPZn7glHadCyL2lvThINGU57UrLS7Mf+gf/kFW+k28= github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8 h1:e0RDG17t6kcwOg4CLmwygn9Fh0alYa1GjxtRAcgm+so= github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8/go.mod h1:0zPZn7glHadCyL2lvThINGU57UrLS7Mf+gf/kFW+k28= +github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29 h1:0RJnI9nDGao3LUSnqWhcg0ELex4RUlVvGY+SfAMuDSQ= +github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29/go.mod h1:91mj0+pHkKsXd1SGD6+jVW1tl6W25TuXgoESwK5dJh8= github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 h1:+awuysRKLmdLQbVK+HPSOGvO3dFGdNSbM2jyLh+VYOA= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= @@ -375,6 +381,7 @@ github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 110cc17..146e629 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -606,7 +606,7 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { } } - bl.Collect() + bl.Hashes = parsers.NewHashes(bl.Raw) //if !pool.IgnoreWaf { // // 部分情况下waf的特征可能是全局, 指定了--ignore-waf则不会进行waf的指纹检测 diff --git a/internal/pool/pool.go b/internal/pool/pool.go index ebbf385..ea7def9 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -150,6 +150,9 @@ func (pool *BasePool) genReq(s string) (*ihttp.Request, error) { } func (pool *BasePool) putToOutput(bl *pkg.Baseline) { + if bl.IsValid || bl.IsFuzzy { + bl.Collect() + } pool.OutLocker.Add(1) pool.OutputCh <- bl } diff --git a/pkg/baseline.go b/pkg/baseline.go index 4718f18..dded082 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -135,8 +135,15 @@ func (bl *Baseline) IsDir() bool { // Collect 深度收集信息 func (bl *Baseline) Collect() { + if bl.Collected { // 防止重复收集 + return + } else { + bl.Collected = true + } + if bl.ContentType == "html" || bl.ContentType == "json" || bl.ContentType == "txt" { // 指纹库设计的时候没考虑js,css文件的指纹, 跳过非必要的指纹收集减少误报提高性能 + //fmt.Println(bl.Source, bl.Url.String()+bl.Path, bl.RedirectURL, "call fingersengine") if EnableAllFingerEngine { bl.Frameworks = EngineDetect(bl.Raw) } else { @@ -160,13 +167,6 @@ func (bl *Baseline) Collect() { } func (bl *Baseline) CollectURL() { - if bl.Collected { - // 防止重复收集 - return - } else { - bl.Collected = true - } - if len(bl.Body) == 0 { return } diff --git a/pkg/fingers.go b/pkg/fingers.go index 9c298b2..25d0f22 100644 --- a/pkg/fingers.go +++ b/pkg/fingers.go @@ -7,7 +7,7 @@ import ( // gogo fingers engine func FingersDetect(content []byte) common.Frameworks { - frames, _ := FingerEngine.FingersEngine.HTTPMatch(bytes.ToLower(content), "") + frames, _ := FingerEngine.Fingers().HTTPMatch(bytes.ToLower(content), "") return frames } diff --git a/pkg/load.go b/pkg/load.go index 3c40ecf..49310ea 100644 --- a/pkg/load.go +++ b/pkg/load.go @@ -26,14 +26,14 @@ func LoadTemplates() error { if err != nil { return err } - for _, f := range FingerEngine.FingersEngine.HTTPFingers { + for _, f := range FingerEngine.Fingers().HTTPFingers { for _, rule := range f.Rules { if rule.SendDataStr != "" { ActivePath = append(ActivePath, rule.SendDataStr) } } } - for _, f := range FingerEngine.FingerPrintEngine.FingerPrints { + for _, f := range FingerEngine.FingerPrintHub().FingerPrints { if f.Path != "/" { ActivePath = append(ActivePath, f.Path) }