diff --git a/cmd/cmd.go b/cmd/cmd.go index 457080f..21a64fc 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -7,9 +7,7 @@ import ( "github.com/chainreactors/logs" "github.com/chainreactors/spray/internal" "github.com/chainreactors/spray/internal/ihttp" - "github.com/chainreactors/spray/internal/pool" "github.com/chainreactors/spray/pkg" - "github.com/chainreactors/utils/iutils" "github.com/jessevdk/go-flags" "os" "os/signal" @@ -113,27 +111,13 @@ func Spray() { return } - err = pkg.Load() + err = option.PreCompare() if err != nil { - iutils.Fatal(err.Error()) + logs.Log.Errorf(err.Error()) + return } - // 初始化全局变量 - pkg.Distance = uint8(option.SimhashDistance) - if option.MaxBodyLength == -1 { - ihttp.DefaultMaxBodySize = -1 - } else { - ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024 - } - - pool.MaxCrawl = option.CrawlDepth - - var runner *internal.Runner - if option.ResumeFrom != "" { - runner, err = option.PrepareRunner() - } else { - runner, err = option.PrepareRunner() - } + runner, err := option.Compare() if err != nil { logs.Log.Errorf(err.Error()) return diff --git a/go.mod b/go.mod index aeaecae..f3badf6 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ toolchain go1.22.2 require ( github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 - github.com/chainreactors/fingers v0.0.0-20240711193807-8c1518dc63fc + github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2 - github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed + github.com/chainreactors/utils v0.0.0-20240715093949-e1faa388e281 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 github.com/expr-lang/expr v1.16.9 github.com/gookit/config/v2 v2.2.5 @@ -27,6 +27,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.1.0 // indirect + github.com/chainreactors/neutron v0.0.0-20240715194532-a6cc6c007916 // indirect github.com/facebookincubator/nvdtools v0.1.5 // indirect github.com/fatih/color v1.17.0 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect diff --git a/go.sum b/go.sum index 88ec85d..cfc97f6 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -70,6 +71,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -106,9 +108,25 @@ github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b h1:vqkkXIyeD github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b/go.mod h1:vQ/LJzHnMdxbK6n1PwqZmvgPudfNpoQsyFAPdt3IlBo= github.com/chainreactors/fingers v0.0.0-20240711193807-8c1518dc63fc h1:vMEF9hlKqQ1U6rHG+ZD4wAR3ieSzDNH2PH5PTFcYyFk= github.com/chainreactors/fingers v0.0.0-20240711193807-8c1518dc63fc/go.mod h1:/F4FibGLZHIuhLM1F6ZZnqtGysED0IGFUSp6KQs/SjY= +github.com/chainreactors/fingers v0.0.0-20240716060852-2fda69a0d7fa h1:+V0Rz+to7K+VsdZlM0OggwPS4BM4awQp8+bGHRVknIc= +github.com/chainreactors/fingers v0.0.0-20240716060852-2fda69a0d7fa/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= +github.com/chainreactors/fingers v0.0.0-20240716072513-7aa252d49efe h1:SakSTrrgP+aeo2bFWxGR1RFYzEDLrJj7W7gGbA6h4zM= +github.com/chainreactors/fingers v0.0.0-20240716072513-7aa252d49efe/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= +github.com/chainreactors/fingers v0.0.0-20240716075824-ed43a10c267e h1:MiHDZyrp5SqNaCJ4Q84TNufcosD0LHxG1QWeqHw3+ng= +github.com/chainreactors/fingers v0.0.0-20240716075824-ed43a10c267e/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= +github.com/chainreactors/fingers v0.0.0-20240716094502-2a7e5440aca4 h1:QpTAsrBmfS0Rf6c1/KgXB42EPQ/7OpsGfbGne28vUIk= +github.com/chainreactors/fingers v0.0.0-20240716094502-2a7e5440aca4/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= +github.com/chainreactors/fingers v0.0.0-20240716105450-071ba392f9c8 h1:YFylx8JO4DVoMdGI2yf5wr1O7blmg6Vh3kQ2JbjBqQA= +github.com/chainreactors/fingers v0.0.0-20240716105450-071ba392f9c8/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= +github.com/chainreactors/fingers v0.0.0-20240716170452-17af47087a82 h1:an3tjLQtKyFYH86Xg4bPejlfGzwtIpzfAFQ6haj4Wp0= +github.com/chainreactors/fingers v0.0.0-20240716170452-17af47087a82/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= +github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a h1:5l4i8TdHRlz088J5xZM30yvTUMLVcWJ6iXiO/VyD3ro= +github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4= github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k= github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o= +github.com/chainreactors/neutron v0.0.0-20240715194532-a6cc6c007916 h1:rCd0XVTAdRAYKh2h+9QfFJWvRY0Gye+nMwXju4lgQbI= +github.com/chainreactors/neutron v0.0.0-20240715194532-a6cc6c007916/go.mod h1:A6oRL1+WyEBCWwUeVcx0W5xONNS6lc6HJ0Y2R/tuXSc= github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1FGWD7gVZuF0cufxd7i9HJ2gltUewxXCfvs4= github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg= github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412 h1:GobKh1fztHQvslp8Ya6AgduYQshc1+CaomLYAf37yK8= @@ -138,6 +156,8 @@ github.com/chainreactors/utils v0.0.0-20240711195624-66696b0879b2 h1:0Bwlb6eDCcR github.com/chainreactors/utils v0.0.0-20240711195624-66696b0879b2/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed h1:5zxxMweHgo2e7r4ic7LHGibGBnOiVmCqbzbcbgoPn7U= github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= +github.com/chainreactors/utils v0.0.0-20240715093949-e1faa388e281 h1:9tuSG+4PlhrdAGekRI1daxotFtV9RDUacBujWLt0DUc= +github.com/chainreactors/utils v0.0.0-20240715093949-e1faa388e281/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw= github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -164,6 +184,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -185,6 +207,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= @@ -247,6 +270,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -261,6 +285,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -298,6 +323,7 @@ github.com/gookit/goutil v0.6.15/go.mod h1:qdKdYEHQdEtyH+4fNdQNZfJHhI0jUZzHxQVAV github.com/gookit/ini/v2 v2.2.3 h1:nSbN+x9OfQPcMObTFP+XuHt8ev6ndv/fWWqxFhPMu2E= github.com/gookit/ini/v2 v2.2.3/go.mod h1:Vu6p7P7xcfmb8KYu3L0ek8bqu/Im63N81q208SCCZY4= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -317,6 +343,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -343,8 +370,10 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -380,6 +409,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -397,11 +427,13 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/panjf2000/ants/v2 v2.9.1 h1:Q5vh5xohbsZXGcD6hhszzGqB7jSSc2/CRr3QKIga8Kw= github.com/panjf2000/ants/v2 v2.9.1/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -440,6 +472,7 @@ github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -468,12 +501,16 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4zc= github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0= github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM= +github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -507,6 +544,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= @@ -520,6 +558,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -550,6 +589,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -594,6 +635,7 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= @@ -628,6 +670,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -701,13 +744,16 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= @@ -722,6 +768,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -786,6 +833,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/finger.go b/internal/finger.go new file mode 100644 index 0000000..fc080b7 --- /dev/null +++ b/internal/finger.go @@ -0,0 +1,162 @@ +package internal + +import ( + "fmt" + "github.com/chainreactors/files" + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/resources" + "github.com/chainreactors/logs" + "github.com/chainreactors/utils/encode" + "github.com/chainreactors/utils/iutils" + "io" + "net/http" + "os" + "path" + "strings" +) + +var ( + DefaultFingerPath = "fingers" + DefaultFingerTemplate = "fingers/templates" + + FingerConfigs = map[string]string{ + fingers.FingersEngine: "fingers_http.json", + fingers.FingerPrintEngine: "fingerprinthub_v3.json", + fingers.WappalyzerEngine: "wappalyzer.json", + fingers.EHoleEngine: "ehole.json", + fingers.GobyEngine: "goby.json", + } + baseURL = "https://raw.githubusercontent.com/chainreactors/fingers/master/resources/" +) + +type FingerOptions struct { + Finger bool `long:"finger" description:"Bool, enable active finger detect" config:"finger"` + FingerUpdate bool `long:"update" description:"Bool, update finger database" config:"update"` + FingerPath string `long:"finger-path" default:"fingers" description:"String, 3rd finger config path" config:"finger-path"` + FingersTemplatesPath string `long:"finger-template" default:"fingers/templates" description:"Bool, use finger templates path" config:"finger-template"` + FingerEngines string `long:"finger-engine" default:"all" description:"String, custom finger engine, e.g. --finger-engine ehole,goby" config:"finger-engine"` +} + +func (opt *FingerOptions) Validate() error { + var err error + if opt.FingerUpdate { + if opt.FingerPath != DefaultFingerPath && !files.IsExist(opt.FingerPath) { + err = os.MkdirAll(opt.FingerPath, 0755) + if err != nil { + return err + } + } else if !files.IsExist(DefaultFingerPath) { + opt.FingerPath = DefaultFingerPath + err = os.MkdirAll(DefaultFingerPath, 0755) + if err != nil { + return err + } + } + + if opt.FingersTemplatesPath != DefaultFingerTemplate && !files.IsExist(opt.FingersTemplatesPath) { + err = os.MkdirAll(opt.FingersTemplatesPath, 0755) + if err != nil { + return err + } + } else if !files.IsExist(DefaultFingerTemplate) { + err = os.MkdirAll(DefaultFingerTemplate, 0755) + if err != nil { + return err + } + } + } + + if opt.FingerEngines != "all" { + for _, name := range strings.Split(opt.FingerEngines, ",") { + if !iutils.StringsContains(fingers.AllEngines, name) { + return fmt.Errorf("invalid finger engine: %s, please input one of %v", name, fingers.FingersEngine) + } + } + } + return nil +} + +func (opt *FingerOptions) LoadLocalFingerConfig() error { + for name, fingerPath := range FingerConfigs { + if content, err := os.ReadFile(fingerPath); err == nil { + if encode.Md5Hash(content) != resources.CheckSum[name] { + logs.Log.Importantf("found %s difference, use %s replace embed", name, fingerPath) + switch name { + case fingers.FingersEngine: + resources.FingersHTTPData = content + case fingers.FingerPrintEngine: + resources.Fingerprinthubdata = content + case fingers.EHoleEngine: + resources.EholeData = content + case fingers.GobyEngine: + resources.GobyData = content + case fingers.WappalyzerEngine: + resources.WappalyzerData = content + default: + return fmt.Errorf("unknown engine name") + } + } else { + logs.Log.Infof("%s config is up to date", name) + } + } + } + return nil +} + +func (opt *FingerOptions) UpdateFinger() error { + modified := false + for name, _ := range FingerConfigs { + if ok, err := opt.downloadConfig(name); err != nil { + return err + } else { + if ok { + modified = ok + } + } + } + if !modified { + logs.Log.Importantf("everything is up to date") + } + return nil +} + +func (opt *FingerOptions) downloadConfig(name string) (bool, error) { + fingerPath, ok := FingerConfigs[name] + if !ok { + return false, fmt.Errorf("unknown engine name") + } + url := baseURL + fingerPath + resp, err := http.Get(url) + if err != nil { + return false, fmt.Errorf("error fetching file: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("bad status: %s", resp.Status) + } + + filePath := path.Join(opt.FingerPath, fingerPath) + out, err := os.Create(filePath) + if err != nil { + return false, fmt.Errorf("error creating file: %w", err) + } + defer out.Close() + content, err := io.ReadAll(resp.Body) + if err != nil { + return false, fmt.Errorf("error writing to file: %w", err) + } + + if origin, err := os.ReadFile(filePath); err == nil { + if encode.Md5Hash(content) != encode.Md5Hash(origin) { + logs.Log.Infof("download %s config from %s save to %s", name, url, fingerPath) + err = os.WriteFile(filePath, content, 0644) + if err != nil { + return false, err + } + return true, nil + } + } + + return false, nil +} diff --git a/internal/format.go b/internal/format.go index 40b1d3b..59bde37 100644 --- a/internal/format.go +++ b/internal/format.go @@ -5,7 +5,7 @@ import ( "encoding/json" "github.com/chainreactors/logs" "github.com/chainreactors/spray/pkg" - "io/ioutil" + "io" "os" ) @@ -13,9 +13,9 @@ func Format(filename string, color bool) { var content []byte var err error if filename == "stdin" { - content, err = ioutil.ReadAll(os.Stdin) + content, err = io.ReadAll(os.Stdin) } else { - content, err = ioutil.ReadFile(filename) + content, err = os.ReadFile(filename) } if err != nil { diff --git a/internal/option.go b/internal/option.go index be916f0..5c7a4ed 100644 --- a/internal/option.go +++ b/internal/option.go @@ -39,6 +39,7 @@ type Option struct { FunctionOptions `group:"Function Options" config:"functions" ` OutputOptions `group:"Output Options" config:"output"` PluginOptions `group:"Plugin Options" config:"plugins"` + FingerOptions `group:"Finger Options" config:"finger"` RequestOptions `group:"Request Options" config:"request"` ModeOptions `group:"Modify Options" config:"mode"` MiscOptions `group:"Miscellaneous Options" config:"misc"` @@ -111,7 +112,6 @@ type PluginOptions struct { Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" config:"extract"` ExtractConfig string `long:"extract-config" description:"String, extract config filename" config:"extract-config"` Recon bool `long:"recon" description:"Bool, enable recon" config:"recon"` - Finger bool `long:"finger" description:"Bool, enable active finger detect" config:"finger"` Bak bool `long:"bak" description:"Bool, enable bak found" config:"bak"` FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt" config:"file-bak"` Common bool `long:"common" description:"Bool, enable common file found" config:"common"` @@ -155,11 +155,88 @@ type MiscOptions struct { InitConfig bool `long:"init" description:"Bool, init config file"` } -func (opt *Option) PrepareRunner() (*Runner, error) { - err := opt.Validate() - if err != nil { - return nil, err +func (opt *Option) PreCompare() error { + var err error + logs.Log.SetColor(true) + if err = opt.FingerOptions.Validate(); err != nil { + return err } + + if opt.FingerUpdate { + err = opt.UpdateFinger() + if err != nil { + return err + } + } + err = opt.LoadLocalFingerConfig() + if err != nil { + return err + } + + err = opt.Validate() + if err != nil { + return err + } + err = pkg.LoadFingers() + if err != nil { + return err + } + + if opt.Extracts != nil { + for _, e := range opt.Extracts { + if reg, ok := pkg.ExtractRegexps[e]; ok { + pkg.Extractors[e] = reg + } else { + pkg.Extractors[e] = []*parsers.Extractor{ + &parsers.Extractor{ + Name: e, + CompiledRegexps: []*regexp.Regexp{regexp.MustCompile(e)}, + }, + } + } + } + } + if opt.ExtractConfig != "" { + extracts, err := pkg.LoadExtractorConfig(opt.ExtractConfig) + if err != nil { + return err + } + pkg.Extractors[opt.ExtractConfig] = extracts + } + + err = pkg.Load() + if err != nil { + iutils.Fatal(err.Error()) + } + + // 初始化全局变量 + pkg.Distance = uint8(opt.SimhashDistance) + if opt.MaxBodyLength == -1 { + ihttp.DefaultMaxBodySize = -1 + } else { + ihttp.DefaultMaxBodySize = opt.MaxBodyLength * 1024 + } + + pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus) + pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus) + if opt.FuzzyStatus == "all" { + pool.EnableAllFuzzy = true + } else { + pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus) + } + + if opt.Unique { + pool.EnableAllUnique = true + } else { + pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus) + } + pool.MaxCrawl = opt.CrawlDepth + + return nil +} + +func (opt *Option) Compare() (*Runner, error) { + var err error r := &Runner{ Option: opt, taskCh: make(chan *Task), @@ -168,12 +245,13 @@ func (opt *Option) PrepareRunner() (*Runner, error) { fuzzyCh: make(chan *pkg.Baseline, 256), Headers: make(map[string]string), Total: opt.Limit, + Color: true, } // log and bar - if !opt.NoColor { - logs.Log.SetColor(true) - r.Color = true + if opt.NoColor { + logs.Log.SetColor(false) + r.Color = false } if opt.Quiet { logs.Log.SetQuiet(true) @@ -206,28 +284,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.Threads = 1000 } - if opt.Extracts != nil { - for _, e := range opt.Extracts { - if reg, ok := pkg.ExtractRegexps[e]; ok { - pkg.Extractors[e] = reg - } else { - pkg.Extractors[e] = []*parsers.Extractor{ - &parsers.Extractor{ - Name: e, - CompiledRegexps: []*regexp.Regexp{regexp.MustCompile(e)}, - }, - } - } - } - } - if opt.ExtractConfig != "" { - extracts, err := pkg.LoadExtractorConfig(opt.ExtractConfig) - if err != nil { - return nil, err - } - pkg.Extractors[opt.ExtractConfig] = extracts - } - if opt.Recon { pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"] } @@ -278,20 +334,6 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.Scope = []string{"*"} } - pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus) - pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus) - if opt.FuzzyStatus == "all" { - pool.EnableAllFuzzy = true - } else { - pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus) - } - - if opt.Unique { - pool.EnableAllUnique = true - } else { - pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus) - } - // prepare word var dicts [][]string if opt.DefaultDict { @@ -745,6 +787,7 @@ func (opt *Option) Validate() error { if opt.ResumeFrom == "" && len(opt.URL) == 0 && opt.URLFile == "" && len(opt.CIDRs) == 0 && opt.RawFile == "" { return fmt.Errorf("without any target, please use -u/-l/-c/--resume to set targets") } + return nil } diff --git a/pkg/baseline.go b/pkg/baseline.go index dded082..350e066 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -155,8 +155,8 @@ func (bl *Baseline) Collect() { if bl.ContentType == "html" { bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body)) } else if bl.ContentType == "ico" { - if frame := FingerEngine.HashContentMatch(bl.Body); frame != nil { - bl.Frameworks.Add(frame) + if frame := FingerEngine.Favicon().Match(bl.Body); frame != nil { + bl.Frameworks.Merge(frame) } } } diff --git a/pkg/load.go b/pkg/load.go index d7d55ee..790d2b4 100644 --- a/pkg/load.go +++ b/pkg/load.go @@ -20,9 +20,19 @@ var ( ActivePath []string ) -func LoadTemplates() error { +func LoadPorts() error { + var err error + var ports []*utils.PortConfig + err = json.Unmarshal(LoadConfig("port"), &ports) + if err != nil { + return err + } + utils.PrePort = utils.NewPortPreset(ports) + return nil +} + +func LoadFingers() error { var err error - // load fingers FingerEngine, err = fingers.NewEngine() if err != nil { return err @@ -39,7 +49,11 @@ func LoadTemplates() error { ActivePath = append(ActivePath, f.Path) } } + return nil +} +func LoadTemplates() error { + var err error // load rule var data map[string]interface{} err = json.Unmarshal(LoadConfig("spray_rule"), &data) @@ -65,14 +79,6 @@ func LoadTemplates() error { mask.SpecialWords[k] = t } - // load ports - var ports []*utils.PortConfig - err = json.Unmarshal(LoadConfig("port"), &ports) - if err != nil { - return err - } - utils.PrePort = utils.NewPortPreset(ports) - var extracts []*parsers.Extractor err = json.Unmarshal(LoadConfig("extract"), &extracts) if err != nil { @@ -114,11 +120,15 @@ func LoadExtractorConfig(filename string) ([]*parsers.Extractor, error) { } func Load() error { - // load fingers - err := LoadTemplates() + err := LoadPorts() if err != nil { return err } + err = LoadTemplates() + if err != nil { + return err + } + return nil }