support multi fingers features

cmd/cmd.go

@@ -7,9 +7,7 @@ import (
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/internal"
 	"github.com/chainreactors/spray/internal/ihttp"
-	"github.com/chainreactors/spray/internal/pool"
 	"github.com/chainreactors/spray/pkg"
-	"github.com/chainreactors/utils/iutils"
 	"github.com/jessevdk/go-flags"
 	"os"
 	"os/signal"
@@ -113,27 +111,13 @@ func Spray() {
 		return
 	}
 
-	err = pkg.Load()
+	err = option.PreCompare()
 	if err != nil {
-		iutils.Fatal(err.Error())
+		logs.Log.Errorf(err.Error())
+		return
 	}
 
-	// 初始化全局变量
+	runner, err := option.Compare()
-	pkg.Distance = uint8(option.SimhashDistance)
-	if option.MaxBodyLength == -1 {
-		ihttp.DefaultMaxBodySize = -1
-	} else {
-		ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024
-	}
-
-	pool.MaxCrawl = option.CrawlDepth
-
-	var runner *internal.Runner
-	if option.ResumeFrom != "" {
-		runner, err = option.PrepareRunner()
-	} else {
-		runner, err = option.PrepareRunner()
-	}
 	if err != nil {
 		logs.Log.Errorf(err.Error())
 		return

go.mod

@@ -6,10 +6,10 @@ toolchain go1.22.2
 
 require (
 	github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8
-	github.com/chainreactors/fingers v0.0.0-20240711193807-8c1518dc63fc
+	github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a
 	github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f
 	github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2
-	github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed
+	github.com/chainreactors/utils v0.0.0-20240715093949-e1faa388e281
 	github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508
 	github.com/expr-lang/expr v1.16.9
 	github.com/gookit/config/v2 v2.2.5
@@ -27,6 +27,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/chainreactors/neutron v0.0.0-20240715194532-a6cc6c007916 // indirect
 	github.com/facebookincubator/nvdtools v0.1.5 // indirect
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect

go.sum

@@ -52,6 +52,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
@@ -70,6 +71,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -106,9 +108,25 @@ github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b h1:vqkkXIyeD
 github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b/go.mod h1:vQ/LJzHnMdxbK6n1PwqZmvgPudfNpoQsyFAPdt3IlBo=
 github.com/chainreactors/fingers v0.0.0-20240711193807-8c1518dc63fc h1:vMEF9hlKqQ1U6rHG+ZD4wAR3ieSzDNH2PH5PTFcYyFk=
 github.com/chainreactors/fingers v0.0.0-20240711193807-8c1518dc63fc/go.mod h1:/F4FibGLZHIuhLM1F6ZZnqtGysED0IGFUSp6KQs/SjY=
+github.com/chainreactors/fingers v0.0.0-20240716060852-2fda69a0d7fa h1:+V0Rz+to7K+VsdZlM0OggwPS4BM4awQp8+bGHRVknIc=
+github.com/chainreactors/fingers v0.0.0-20240716060852-2fda69a0d7fa/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
+github.com/chainreactors/fingers v0.0.0-20240716072513-7aa252d49efe h1:SakSTrrgP+aeo2bFWxGR1RFYzEDLrJj7W7gGbA6h4zM=
+github.com/chainreactors/fingers v0.0.0-20240716072513-7aa252d49efe/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
+github.com/chainreactors/fingers v0.0.0-20240716075824-ed43a10c267e h1:MiHDZyrp5SqNaCJ4Q84TNufcosD0LHxG1QWeqHw3+ng=
+github.com/chainreactors/fingers v0.0.0-20240716075824-ed43a10c267e/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
+github.com/chainreactors/fingers v0.0.0-20240716094502-2a7e5440aca4 h1:QpTAsrBmfS0Rf6c1/KgXB42EPQ/7OpsGfbGne28vUIk=
+github.com/chainreactors/fingers v0.0.0-20240716094502-2a7e5440aca4/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
+github.com/chainreactors/fingers v0.0.0-20240716105450-071ba392f9c8 h1:YFylx8JO4DVoMdGI2yf5wr1O7blmg6Vh3kQ2JbjBqQA=
+github.com/chainreactors/fingers v0.0.0-20240716105450-071ba392f9c8/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
+github.com/chainreactors/fingers v0.0.0-20240716170452-17af47087a82 h1:an3tjLQtKyFYH86Xg4bPejlfGzwtIpzfAFQ6haj4Wp0=
+github.com/chainreactors/fingers v0.0.0-20240716170452-17af47087a82/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
+github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a h1:5l4i8TdHRlz088J5xZM30yvTUMLVcWJ6iXiO/VyD3ro=
+github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
 github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ=
 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k=
 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o=
+github.com/chainreactors/neutron v0.0.0-20240715194532-a6cc6c007916 h1:rCd0XVTAdRAYKh2h+9QfFJWvRY0Gye+nMwXju4lgQbI=
+github.com/chainreactors/neutron v0.0.0-20240715194532-a6cc6c007916/go.mod h1:A6oRL1+WyEBCWwUeVcx0W5xONNS6lc6HJ0Y2R/tuXSc=
 github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1FGWD7gVZuF0cufxd7i9HJ2gltUewxXCfvs4=
 github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg=
 github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412 h1:GobKh1fztHQvslp8Ya6AgduYQshc1+CaomLYAf37yK8=
@@ -138,6 +156,8 @@ github.com/chainreactors/utils v0.0.0-20240711195624-66696b0879b2 h1:0Bwlb6eDCcR
 github.com/chainreactors/utils v0.0.0-20240711195624-66696b0879b2/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
 github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed h1:5zxxMweHgo2e7r4ic7LHGibGBnOiVmCqbzbcbgoPn7U=
 github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
+github.com/chainreactors/utils v0.0.0-20240715093949-e1faa388e281 h1:9tuSG+4PlhrdAGekRI1daxotFtV9RDUacBujWLt0DUc=
+github.com/chainreactors/utils v0.0.0-20240715093949-e1faa388e281/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw=
 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -164,6 +184,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
+github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -185,6 +207,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
 github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
@@ -247,6 +270,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -261,6 +285,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -298,6 +323,7 @@ github.com/gookit/goutil v0.6.15/go.mod h1:qdKdYEHQdEtyH+4fNdQNZfJHhI0jUZzHxQVAV
 github.com/gookit/ini/v2 v2.2.3 h1:nSbN+x9OfQPcMObTFP+XuHt8ev6ndv/fWWqxFhPMu2E=
 github.com/gookit/ini/v2 v2.2.3/go.mod h1:Vu6p7P7xcfmb8KYu3L0ek8bqu/Im63N81q208SCCZY4=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
 github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -317,6 +343,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
@@ -343,8 +370,10 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -380,6 +409,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -397,11 +427,13 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/panjf2000/ants/v2 v2.9.1 h1:Q5vh5xohbsZXGcD6hhszzGqB7jSSc2/CRr3QKIga8Kw=
 github.com/panjf2000/ants/v2 v2.9.1/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -440,6 +472,7 @@ github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -468,12 +501,16 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
 github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4zc=
 github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
 github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0=
 github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM=
+github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -507,6 +544,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
@@ -520,6 +558,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
@@ -550,6 +589,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -594,6 +635,7 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
@@ -628,6 +670,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -701,13 +744,16 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
@@ -722,6 +768,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
@@ -786,6 +833,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/finger.go

@@ -0,0 +1,162 @@
+package internal
+
+import (
+	"fmt"
+	"github.com/chainreactors/files"
+	"github.com/chainreactors/fingers"
+	"github.com/chainreactors/fingers/resources"
+	"github.com/chainreactors/logs"
+	"github.com/chainreactors/utils/encode"
+	"github.com/chainreactors/utils/iutils"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+)
+
+var (
+	DefaultFingerPath     = "fingers"
+	DefaultFingerTemplate = "fingers/templates"
+
+	FingerConfigs = map[string]string{
+		fingers.FingersEngine:     "fingers_http.json",
+		fingers.FingerPrintEngine: "fingerprinthub_v3.json",
+		fingers.WappalyzerEngine:  "wappalyzer.json",
+		fingers.EHoleEngine:       "ehole.json",
+		fingers.GobyEngine:        "goby.json",
+	}
+	baseURL = "https://raw.githubusercontent.com/chainreactors/fingers/master/resources/"
+)
+
+type FingerOptions struct {
+	Finger               bool   `long:"finger" description:"Bool, enable active finger detect" config:"finger"`
+	FingerUpdate         bool   `long:"update" description:"Bool, update finger database" config:"update"`
+	FingerPath           string `long:"finger-path" default:"fingers" description:"String, 3rd finger config path" config:"finger-path"`
+	FingersTemplatesPath string `long:"finger-template" default:"fingers/templates" description:"Bool, use finger templates path" config:"finger-template"`
+	FingerEngines        string `long:"finger-engine" default:"all" description:"String, custom finger engine, e.g. --finger-engine ehole,goby" config:"finger-engine"`
+}
+
+func (opt *FingerOptions) Validate() error {
+	var err error
+	if opt.FingerUpdate {
+		if opt.FingerPath != DefaultFingerPath && !files.IsExist(opt.FingerPath) {
+			err = os.MkdirAll(opt.FingerPath, 0755)
+			if err != nil {
+				return err
+			}
+		} else if !files.IsExist(DefaultFingerPath) {
+			opt.FingerPath = DefaultFingerPath
+			err = os.MkdirAll(DefaultFingerPath, 0755)
+			if err != nil {
+				return err
+			}
+		}
+
+		if opt.FingersTemplatesPath != DefaultFingerTemplate && !files.IsExist(opt.FingersTemplatesPath) {
+			err = os.MkdirAll(opt.FingersTemplatesPath, 0755)
+			if err != nil {
+				return err
+			}
+		} else if !files.IsExist(DefaultFingerTemplate) {
+			err = os.MkdirAll(DefaultFingerTemplate, 0755)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if opt.FingerEngines != "all" {
+		for _, name := range strings.Split(opt.FingerEngines, ",") {
+			if !iutils.StringsContains(fingers.AllEngines, name) {
+				return fmt.Errorf("invalid finger engine: %s, please input one of %v", name, fingers.FingersEngine)
+			}
+		}
+	}
+	return nil
+}
+
+func (opt *FingerOptions) LoadLocalFingerConfig() error {
+	for name, fingerPath := range FingerConfigs {
+		if content, err := os.ReadFile(fingerPath); err == nil {
+			if encode.Md5Hash(content) != resources.CheckSum[name] {
+				logs.Log.Importantf("found %s difference, use %s replace embed", name, fingerPath)
+				switch name {
+				case fingers.FingersEngine:
+					resources.FingersHTTPData = content
+				case fingers.FingerPrintEngine:
+					resources.Fingerprinthubdata = content
+				case fingers.EHoleEngine:
+					resources.EholeData = content
+				case fingers.GobyEngine:
+					resources.GobyData = content
+				case fingers.WappalyzerEngine:
+					resources.WappalyzerData = content
+				default:
+					return fmt.Errorf("unknown engine name")
+				}
+			} else {
+				logs.Log.Infof("%s config is up to date", name)
+			}
+		}
+	}
+	return nil
+}
+
+func (opt *FingerOptions) UpdateFinger() error {
+	modified := false
+	for name, _ := range FingerConfigs {
+		if ok, err := opt.downloadConfig(name); err != nil {
+			return err
+		} else {
+			if ok {
+				modified = ok
+			}
+		}
+	}
+	if !modified {
+		logs.Log.Importantf("everything is up to date")
+	}
+	return nil
+}
+
+func (opt *FingerOptions) downloadConfig(name string) (bool, error) {
+	fingerPath, ok := FingerConfigs[name]
+	if !ok {
+		return false, fmt.Errorf("unknown engine name")
+	}
+	url := baseURL + fingerPath
+	resp, err := http.Get(url)
+	if err != nil {
+		return false, fmt.Errorf("error fetching file: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return false, fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	filePath := path.Join(opt.FingerPath, fingerPath)
+	out, err := os.Create(filePath)
+	if err != nil {
+		return false, fmt.Errorf("error creating file: %w", err)
+	}
+	defer out.Close()
+	content, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return false, fmt.Errorf("error writing to file: %w", err)
+	}
+
+	if origin, err := os.ReadFile(filePath); err == nil {
+		if encode.Md5Hash(content) != encode.Md5Hash(origin) {
+			logs.Log.Infof("download %s config from %s save to %s", name, url, fingerPath)
+			err = os.WriteFile(filePath, content, 0644)
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

internal/format.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
-	"io/ioutil"
+	"io"
 	"os"
 )
 
@@ -13,9 +13,9 @@ func Format(filename string, color bool) {
 	var content []byte
 	var err error
 	if filename == "stdin" {
-		content, err = ioutil.ReadAll(os.Stdin)
+		content, err = io.ReadAll(os.Stdin)
 	} else {
-		content, err = ioutil.ReadFile(filename)
+		content, err = os.ReadFile(filename)
 	}
 
 	if err != nil {

internal/option.go

@@ -39,6 +39,7 @@ type Option struct {
 	FunctionOptions `group:"Function Options" config:"functions" `
 	OutputOptions   `group:"Output Options" config:"output"`
 	PluginOptions   `group:"Plugin Options" config:"plugins"`
+	FingerOptions   `group:"Finger Options" config:"finger"`
 	RequestOptions  `group:"Request Options" config:"request"`
 	ModeOptions     `group:"Modify Options" config:"mode"`
 	MiscOptions     `group:"Miscellaneous Options" config:"misc"`
@@ -111,7 +112,6 @@ type PluginOptions struct {
 	Extracts      []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" config:"extract"`
 	ExtractConfig string   `long:"extract-config" description:"String, extract config filename" config:"extract-config"`
 	Recon         bool     `long:"recon" description:"Bool, enable recon" config:"recon"`
-	Finger        bool     `long:"finger" description:"Bool, enable active finger detect" config:"finger"`
 	Bak           bool     `long:"bak" description:"Bool, enable bak found" config:"bak"`
 	FileBak       bool     `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt" config:"file-bak"`
 	Common        bool     `long:"common" description:"Bool, enable common file found" config:"common"`
@@ -155,11 +155,88 @@ type MiscOptions struct {
 	InitConfig bool   `long:"init" description:"Bool, init config file"`
 }
 
-func (opt *Option) PrepareRunner() (*Runner, error) {
+func (opt *Option) PreCompare() error {
-	err := opt.Validate()
+	var err error
-	if err != nil {
+	logs.Log.SetColor(true)
-		return nil, err
+	if err = opt.FingerOptions.Validate(); err != nil {
+		return err
 	}
+
+	if opt.FingerUpdate {
+		err = opt.UpdateFinger()
+		if err != nil {
+			return err
+		}
+	}
+	err = opt.LoadLocalFingerConfig()
+	if err != nil {
+		return err
+	}
+
+	err = opt.Validate()
+	if err != nil {
+		return err
+	}
+	err = pkg.LoadFingers()
+	if err != nil {
+		return err
+	}
+
+	if opt.Extracts != nil {
+		for _, e := range opt.Extracts {
+			if reg, ok := pkg.ExtractRegexps[e]; ok {
+				pkg.Extractors[e] = reg
+			} else {
+				pkg.Extractors[e] = []*parsers.Extractor{
+					&parsers.Extractor{
+						Name:            e,
+						CompiledRegexps: []*regexp.Regexp{regexp.MustCompile(e)},
+					},
+				}
+			}
+		}
+	}
+	if opt.ExtractConfig != "" {
+		extracts, err := pkg.LoadExtractorConfig(opt.ExtractConfig)
+		if err != nil {
+			return err
+		}
+		pkg.Extractors[opt.ExtractConfig] = extracts
+	}
+
+	err = pkg.Load()
+	if err != nil {
+		iutils.Fatal(err.Error())
+	}
+
+	// 初始化全局变量
+	pkg.Distance = uint8(opt.SimhashDistance)
+	if opt.MaxBodyLength == -1 {
+		ihttp.DefaultMaxBodySize = -1
+	} else {
+		ihttp.DefaultMaxBodySize = opt.MaxBodyLength * 1024
+	}
+
+	pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus)
+	pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus)
+	if opt.FuzzyStatus == "all" {
+		pool.EnableAllFuzzy = true
+	} else {
+		pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus)
+	}
+
+	if opt.Unique {
+		pool.EnableAllUnique = true
+	} else {
+		pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus)
+	}
+	pool.MaxCrawl = opt.CrawlDepth
+
+	return nil
+}
+
+func (opt *Option) Compare() (*Runner, error) {
+	var err error
 	r := &Runner{
 		Option:   opt,
 		taskCh:   make(chan *Task),
@@ -168,12 +245,13 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		fuzzyCh:  make(chan *pkg.Baseline, 256),
 		Headers:  make(map[string]string),
 		Total:    opt.Limit,
+		Color:    true,
 	}
 
 	// log and bar
-	if !opt.NoColor {
+	if opt.NoColor {
-		logs.Log.SetColor(true)
+		logs.Log.SetColor(false)
-		r.Color = true
+		r.Color = false
 	}
 	if opt.Quiet {
 		logs.Log.SetQuiet(true)
@@ -206,28 +284,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		r.Threads = 1000
 	}
 
-	if opt.Extracts != nil {
-		for _, e := range opt.Extracts {
-			if reg, ok := pkg.ExtractRegexps[e]; ok {
-				pkg.Extractors[e] = reg
-			} else {
-				pkg.Extractors[e] = []*parsers.Extractor{
-					&parsers.Extractor{
-						Name:            e,
-						CompiledRegexps: []*regexp.Regexp{regexp.MustCompile(e)},
-					},
-				}
-			}
-		}
-	}
-	if opt.ExtractConfig != "" {
-		extracts, err := pkg.LoadExtractorConfig(opt.ExtractConfig)
-		if err != nil {
-			return nil, err
-		}
-		pkg.Extractors[opt.ExtractConfig] = extracts
-	}
-
 	if opt.Recon {
 		pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"]
 	}
@@ -278,20 +334,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		r.Scope = []string{"*"}
 	}
 
-	pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus)
-	pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus)
-	if opt.FuzzyStatus == "all" {
-		pool.EnableAllFuzzy = true
-	} else {
-		pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus)
-	}
-
-	if opt.Unique {
-		pool.EnableAllUnique = true
-	} else {
-		pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus)
-	}
-
 	// prepare word
 	var dicts [][]string
 	if opt.DefaultDict {
@@ -745,6 +787,7 @@ func (opt *Option) Validate() error {
 	if opt.ResumeFrom == "" && len(opt.URL) == 0 && opt.URLFile == "" && len(opt.CIDRs) == 0 && opt.RawFile == "" {
 		return fmt.Errorf("without any target, please use -u/-l/-c/--resume to set targets")
 	}
+
 	return nil
 }
 

pkg/baseline.go

@@ -155,8 +155,8 @@ func (bl *Baseline) Collect() {
 		if bl.ContentType == "html" {
 			bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
 		} else if bl.ContentType == "ico" {
-			if frame := FingerEngine.HashContentMatch(bl.Body); frame != nil {
+			if frame := FingerEngine.Favicon().Match(bl.Body); frame != nil {
-				bl.Frameworks.Add(frame)
+				bl.Frameworks.Merge(frame)
 			}
 		}
 	}

pkg/load.go

@@ -20,9 +20,19 @@ var (
 	ActivePath   []string
 )
 
-func LoadTemplates() error {
+func LoadPorts() error {
+	var err error
+	var ports []*utils.PortConfig
+	err = json.Unmarshal(LoadConfig("port"), &ports)
+	if err != nil {
+		return err
+	}
+	utils.PrePort = utils.NewPortPreset(ports)
+	return nil
+}
+
+func LoadFingers() error {
 	var err error
-	// load fingers
 	FingerEngine, err = fingers.NewEngine()
 	if err != nil {
 		return err
@@ -39,7 +49,11 @@ func LoadTemplates() error {
 			ActivePath = append(ActivePath, f.Path)
 		}
 	}
+	return nil
+}
 
+func LoadTemplates() error {
+	var err error
 	// load rule
 	var data map[string]interface{}
 	err = json.Unmarshal(LoadConfig("spray_rule"), &data)
@@ -65,14 +79,6 @@ func LoadTemplates() error {
 		mask.SpecialWords[k] = t
 	}
 
-	// load ports
-	var ports []*utils.PortConfig
-	err = json.Unmarshal(LoadConfig("port"), &ports)
-	if err != nil {
-		return err
-	}
-	utils.PrePort = utils.NewPortPreset(ports)
-
 	var extracts []*parsers.Extractor
 	err = json.Unmarshal(LoadConfig("extract"), &extracts)
 	if err != nil {
@@ -114,11 +120,15 @@ func LoadExtractorConfig(filename string) ([]*parsers.Extractor, error) {
 }
 
 func Load() error {
-	// load fingers
+	err := LoadPorts()
-	err := LoadTemplates()
 	if err != nil {
 		return err
 	}
+	err = LoadTemplates()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }