From fd299db8c4faab6a2905363b990deb26aedaa2d6 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 23 Sep 2022 01:20:01 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=86net/http=E6=9B=BF=E6=8D=A2=E4=B8=BAfas?= =?UTF-8?q?thttp,=20=E5=B9=B6=E5=88=9D=E6=AD=A5=E5=AE=9E=E8=A3=85compare?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 26 +++++---- go.sum | 62 +++++++++------------- internal/baseline.go | 123 +++++++++++++++++-------------------------- internal/pool.go | 99 +++++++++++++++++++--------------- pkg/client.go | 54 +++++++++++-------- pkg/config.go | 1 + 6 files changed, 177 insertions(+), 188 deletions(-) diff --git a/go.mod b/go.mod index ed12bf0..9f16206 100644 --- a/go.mod +++ b/go.mod @@ -5,30 +5,28 @@ go 1.17 require ( github.com/chainreactors/files v0.2.0 // indirect github.com/chainreactors/gogo/v2 v2.8.7 - github.com/chainreactors/ipcs v0.0.9 // indirect - github.com/chainreactors/logs v0.5.0 - github.com/chainreactors/parsers v0.0.1 // indirect - github.com/chainreactors/words v0.0.1 // indirect + github.com/chainreactors/logs v0.6.1 + github.com/chainreactors/parsers v0.0.1 + github.com/chainreactors/words v0.0.1 github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c github.com/panjf2000/ants/v2 v2.5.0 ) require ( - github.com/M09ic/go-ntlmssp v1.2.9 // indirect - github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect - github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab // indirect - github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/gosuri/uiprogress v0.0.1 + github.com/jessevdk/go-flags v1.5.0 +) + +require ( + github.com/andybalholm/brotli v1.0.4 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect - github.com/jessevdk/go-flags v1.5.0 // indirect - github.com/karrick/godirwalk v1.16.1 // indirect + github.com/klauspost/compress v1.15.10 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/twmb/murmur3 v1.1.6 // indirect - golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 // indirect - golang.org/x/net v0.0.0-20190603091049-60506f45cf65 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.40.0 // indirect golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 805ee6b..3470c34 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,18 @@ -github.com/M09ic/go-ntlmssp v1.2.9 h1:VVpoldqbRlmyhxrVsJBy0qjAF1RSq89kSLEie/IWRew= github.com/M09ic/go-ntlmssp v1.2.9/go.mod h1:yMNEF6ulbFipt3CakMhcmcNVACshPRG4Ap4l00V+mMs= -github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c h1:k/7/05/5kPRX7HaKyVYlsGVX6XkFTyYLqkqHzceUVlU= -github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c/go.mod h1:SI1d/2/wpSTDjHgdS9ZLy6hqvsdhzVYAc8RLztweMpA= -github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab h1:pfx9N/EMDxIwVzGu9JLnmbOMNukW1mfPA9Ymo+S58Ng= -github.com/alouca/gosnmp v0.0.0-20170620005048-04d83944c9ab/go.mod h1:kEcj+iUROrUCr7AIrul5NutI2kWv0ns9BL0ezVp1h/Y= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/chainreactors/files v0.2.0 h1:LeN97o4VxIvK9ZACjXfdRTR+N7puXuWyQO5GarCkMLM= github.com/chainreactors/files v0.2.0/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A= -github.com/chainreactors/gogo/v2 v2.8.4 h1:jwV6tawOcUn/FM91WxxOp7ZQN/Msm0losAx5230QgL4= -github.com/chainreactors/gogo/v2 v2.8.4/go.mod h1:hoNawrwZtkIciwApI6kzu88LLOGOJEqtegcHBuiPyPw= -github.com/chainreactors/gogo/v2 v2.8.5 h1:7DfAQdyupToU4f+OYG+Hz4+0phyz3UGRrsAyTmX9Fm0= -github.com/chainreactors/gogo/v2 v2.8.5/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY= -github.com/chainreactors/gogo/v2 v2.8.6 h1:Xc9wi8K8/3u9s0V0vlch2rKHzQ+ATkLlHExGosUvVok= -github.com/chainreactors/gogo/v2 v2.8.6/go.mod h1:SqlLDXXN/WgjVHpRIjB8OytRmHnj8l4Ic+h7pe/4wsY= github.com/chainreactors/gogo/v2 v2.8.7 h1:pN83yw7mtLFcNV12ZwxKdTR2sDZFSiqva/JRQMMRvao= github.com/chainreactors/gogo/v2 v2.8.7/go.mod h1:KFMyHLZSkpgDh/17N/k9gKYQ81MO+9MLdTMHQvoBSj0= -github.com/chainreactors/ipcs v0.0.9 h1:4Onroq7gXLG5SLCCgNDx3JmtLxB4XgepGdHCtLp1Ows= github.com/chainreactors/ipcs v0.0.9/go.mod h1:E9M3Ohyq0TYQLlV4i2dbM9ThBZB1Nnd7Oexoie2xLII= -github.com/chainreactors/logs v0.4.2/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= -github.com/chainreactors/logs v0.5.0 h1:6CwTR1KaqZS0quIQRU3Ceq/gBoV0+seUF8gUJTrZvew= github.com/chainreactors/logs v0.5.0/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= +github.com/chainreactors/logs v0.6.1 h1:+wLVqxErUyWYayAx6x+m5LA6jQBiHcLdTAJtiOUMoZU= +github.com/chainreactors/logs v0.6.1/go.mod h1:Y0EtAnoF0kiASIJUnXN0pcOt420iRpHOAnOhEphzRHA= github.com/chainreactors/parsers v0.0.1 h1:GrfY4sfdv9TMuYNSKq3Jak2PO4AMJcDRAHocSb8DEx0= github.com/chainreactors/parsers v0.0.1/go.mod h1:pJ7As+BbXlC9SHBPCE4edc9xYPh5xik+YtKGHRclKx0= github.com/chainreactors/words v0.0.1 h1:d0rLwuKVzwLWD1CtYeTK7u2E5tv8dmebpq6EhUtNhhg= github.com/chainreactors/words v0.0.1/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= -github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= -github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -43,8 +31,9 @@ github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= +github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc= @@ -53,47 +42,48 @@ github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jm github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q= github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= -github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI= -github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= -github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q= -github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc= +github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/baseline.go b/internal/baseline.go index b1fe27a..6414ca7 100644 --- a/internal/baseline.go +++ b/internal/baseline.go @@ -3,120 +3,92 @@ package internal import ( "encoding/json" "fmt" - "github.com/chainreactors/logs" "github.com/chainreactors/parsers" "github.com/chainreactors/spray/pkg" - "io" - "net/http" - "net/url" + "github.com/valyala/fasthttp" "strings" ) -func NewBaseline(u *url.URL, resp *http.Response) *baseline { +func NewBaseline(u *fasthttp.URI, resp *fasthttp.Response) (*baseline, error) { bl := &baseline{ - Url: u, - UrlString: u.String(), - BodyLength: resp.ContentLength, - Status: resp.StatusCode, - IsValid: true, + Url: u, + UrlString: u.String(), + Status: resp.StatusCode(), + IsValid: true, + Body: make([]byte, 20480), } - var header strings.Builder - for k, v := range resp.Header { - for _, i := range v { - header.WriteString(k) - header.WriteString(": ") - header.WriteString(i) - header.WriteString("\r\n") - } - } - bl.Header = header.String() - bl.HeaderLength = header.Len() - - redirectURL, err := resp.Location() - if err == nil { - bl.RedirectURL = redirectURL.String() - } - - body := make([]byte, 20480) - if bl.BodyLength > 0 { - n, err := io.ReadFull(resp.Body, body) - if err == nil { - bl.Body = body - } else if err == io.ErrUnexpectedEOF { - bl.Body = body[:n] - } else { - logs.Log.Error("readfull failed" + err.Error()) - } - } - if len(bl.Body) > 0 { - bl.Md5 = parsers.Md5Hash(bl.Body) - bl.Mmh3 = parsers.Mmh3Hash32(bl.Body) - bl.Simhash = pkg.Simhash(bl.Body) - if strings.Contains(string(bl.Body), bl.UrlString[1:]) { - bl.IsDynamicUrl = true - } - // todo callback - } - - // todo extract - bl.Extracteds = pkg.Extractors.Extract(bl.Response) - // todo 指纹识别 - bl.Frameworks = pkg.FingerDetect(bl.Response) - return bl + bl.Body = resp.Body() + bl.BodyLength = resp.Header.ContentLength() + bl.Header = resp.Header.Header() + bl.HeaderLength = resp.Header.Len() + bl.RedirectURL = string(resp.Header.Peek("Location")) + bl.Raw = append(bl.Header, bl.Body...) + return bl, nil } -func NewInvalidBaseline(u *url.URL, resp *http.Response) *baseline { +func NewInvalidBaseline(u *fasthttp.URI, resp *fasthttp.Response) *baseline { bl := &baseline{ - Url: u, - UrlString: u.String(), - BodyLength: resp.ContentLength, - Status: resp.StatusCode, - IsValid: false, + Url: u, + UrlString: u.String(), + //BodyLength: resp.ContentLength, + Status: resp.StatusCode(), + IsValid: false, } - redirectURL, err := resp.Location() - if err == nil { - bl.RedirectURL = redirectURL.String() - } + bl.RedirectURL = string(resp.Header.Peek("Location")) return bl } type baseline struct { - Url *url.URL `json:"-"` + Url *fasthttp.URI `json:"-"` UrlString string `json:"url_string"` Body []byte `json:"-"` - BodyLength int64 `json:"body_length"` - Header string `json:"-"` - Response string `json:"-"` + BodyLength int `json:"body_length"` + Header []byte `json:"-"` + Raw []byte `json:"-"` HeaderLength int `json:"header_length"` RedirectURL string `json:"redirect_url"` Status int `json:"status"` - Md5 string `json:"md5"` - Mmh3 string `json:"mmh3"` - Simhash string `json:"simhash"` IsDynamicUrl bool `json:"is_dynamic_url"` // 判断是否存在动态的url Spended int `json:"spended"` // 耗时, 毫秒 Frameworks pkg.Frameworks `json:"frameworks"` Extracteds pkg.Extracteds `json:"extracts"` Err error `json:"-"` IsValid bool `json:"-"` + *parsers.Hashes } -func (bl *baseline) Compare(other *baseline) bool { - if bl.Md5 == other.Md5 { +// Collect 深度收集信息 +func (bl *baseline) Collect() { + bl.Hashes = parsers.NewHashes(bl.Raw) + // todo extract + bl.Extracteds = pkg.Extractors.Extract(string(bl.Raw)) + // todo 指纹识别 + bl.Frameworks = pkg.FingerDetect(string(bl.Raw)) +} + +// Equal if this equal other return true +func (bl *baseline) Equal(other *baseline) bool { + if other.RedirectURL != "" && bl.RedirectURL == other.RedirectURL { + // 如果重定向url不为空, 且与bl不相同, 则说明不是同一个页面 return true } - if bl.RedirectURL == other.RedirectURL { - return true + if bl.BodyLength == other.BodyLength { + // 如果body length相等且md5相等, 则说明是同一个页面 + if bl.BodyMd5 == parsers.Md5Hash(other.Raw) { + return true + } else { + return true + } } return false } -func (bl *baseline) FuzzyCompare() bool { +func (bl *baseline) FuzzyEqual(other *baseline) bool { // todo 模糊匹配 return false } @@ -132,6 +104,7 @@ func (bl *baseline) String() string { } line.WriteString(bl.Frameworks.ToString()) //line.WriteString(bl.Extracteds) + line.WriteString("\n") return line.String() } diff --git a/internal/pool.go b/internal/pool.go index b2db7c3..ce0c54f 100644 --- a/internal/pool.go +++ b/internal/pool.go @@ -6,6 +6,7 @@ import ( "github.com/chainreactors/spray/pkg" "github.com/chainreactors/words" "github.com/panjf2000/ants/v2" + "github.com/valyala/fasthttp" "net/http" "sync" "time" @@ -13,55 +14,58 @@ import ( var ( CheckStatusCode func(int) bool - CheckRedirect func(*http.Response) bool + CheckRedirect func(string) bool CheckWaf func(*http.Response) bool ) func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) (*Pool, error) { pctx, cancel := context.WithCancel(ctx) pool := &Pool{ - Config: config, - ctx: pctx, - client: pkg.NewClient(config.Thread, 2), - worder: words.NewWorder(config.Wordlist), - //baseReq: req, + Config: config, + ctx: pctx, + client: pkg.NewClient(config.Thread, 2), + worder: words.NewWorder(config.Wordlist), outputCh: outputCh, + tempCh: make(chan *baseline, config.Thread), wg: &sync.WaitGroup{}, } switch config.Mod { case pkg.PathSpray: - pool.genReq = func(s string) (*http.Request, error) { + pool.genReq = func(s string) (*fasthttp.Request, error) { return pool.BuildPathRequest(s) } case pkg.HostSpray: - pool.genReq = func(s string) (*http.Request, error) { + pool.genReq = func(s string) (*fasthttp.Request, error) { return pool.BuildHostRequest(s) } } p, _ := ants.NewPoolWithFunc(config.Thread, func(i interface{}) { - var bl *baseline unit := i.(*Unit) req, err := pool.genReq(unit.path) if err != nil { logs.Log.Error(err.Error()) return } + + var bl *baseline resp, err := pool.client.Do(pctx, req) if err != nil { //logs.Log.Debugf("%s request error, %s", strurl, err.Error()) pool.errorCount++ bl = &baseline{Err: err} } else { - defer resp.Body.Close() // 必须要关闭body ,否则keep-alive无法生效 + defer fasthttp.ReleaseResponse(resp) + defer fasthttp.ReleaseRequest(req) + //defer resp.Body.Close() // 必须要关闭body ,否则keep-alive无法生效 if err = pool.PreCompare(resp); err == nil || unit.source == CheckSource { // 通过预对比跳过一些无用数据, 减少性能消耗 - bl = NewBaseline(req.URL, resp) + bl, err = NewBaseline(req.URI(), resp) } else if err == ErrWaf { cancel() } else { - bl = NewInvalidBaseline(req.URL, resp) + bl = NewInvalidBaseline(req.URI(), resp) } } @@ -69,8 +73,8 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( case CheckSource: pool.baseline = bl case WordSource: - // todo compare - pool.outputCh <- bl + // 异步进行性能消耗较大的深度对比 + pool.tempCh <- bl } //todo connectivity check pool.bar.Done() @@ -78,7 +82,7 @@ func NewPool(ctx context.Context, config *pkg.Config, outputCh chan *baseline) ( }) pool.pool = p - + go pool.Comparing() return pool, nil } @@ -93,9 +97,10 @@ type Pool struct { //baseReq *http.Request baseline *baseline outputCh chan *baseline + tempCh chan *baseline totalCount int errorCount int - genReq func(s string) (*http.Request, error) + genReq func(s string) (*fasthttp.Request, error) //wordlist []string worder *words.Worder wg *sync.WaitGroup @@ -115,17 +120,13 @@ func (p *Pool) Init() error { } if p.baseline.RedirectURL != "" { - CheckRedirect = func(resp *http.Response) bool { - redirectURL, err := resp.Location() - if err != nil { - // baseline 为3xx, 但path不为3xx时, 为有效数据 - return true - } else if redirectURL.String() != p.baseline.RedirectURL { - // path为3xx, 且与baseline中的RedirectURL不同时, 为有效数据 - return true - } else { + CheckRedirect = func(redirectURL string) bool { + if redirectURL == p.baseline.RedirectURL { // 相同的RedirectURL将被认为是无效数据 return false + } else { + // path为3xx, 且与baseline中的RedirectURL不同时, 为有效数据 + return true } } } @@ -157,18 +158,18 @@ Loop: p.wg.Wait() } -func (p *Pool) PreCompare(resp *http.Response) error { - if !CheckStatusCode(resp.StatusCode) { +func (p *Pool) PreCompare(resp *fasthttp.Response) error { + if !CheckStatusCode(resp.StatusCode()) { return ErrBadStatus } - if CheckRedirect != nil && !CheckRedirect(resp) { + if CheckRedirect != nil && !CheckRedirect(string(resp.Header.Peek("Location"))) { return ErrRedirect } - if CheckWaf != nil && !CheckWaf(resp) { - return ErrWaf - } + //if CheckWaf != nil && !CheckWaf(resp) { + // return ErrWaf + //} return nil } @@ -177,20 +178,36 @@ func (p *Pool) RunWithWord(words []string) { } -func (p *Pool) BuildPathRequest(path string) (*http.Request, error) { - req, err := http.NewRequest("GET", p.BaseURL+path, nil) - if err != nil { - return nil, err +func (p *Pool) Comparing() { + for bl := range p.tempCh { + if p.baseline.Equal(bl) { + // 如果是同一个包则设置为无效包 + bl.IsValid = false + p.outputCh <- bl + continue + } + + bl.Collect() + if p.EnableFuzzy && p.baseline.FuzzyEqual(bl) { + bl.IsValid = false + p.outputCh <- bl + continue + } + + p.outputCh <- bl } +} + +func (p *Pool) BuildPathRequest(path string) (*fasthttp.Request, error) { + req := fasthttp.AcquireRequest() + req.SetRequestURI(p.BaseURL + path) return req, nil } -func (p *Pool) BuildHostRequest(host string) (*http.Request, error) { - req, err := http.NewRequest("GET", p.BaseURL, nil) - req.Host = host - if err != nil { - return nil, err - } +func (p *Pool) BuildHostRequest(host string) (*fasthttp.Request, error) { + req := fasthttp.AcquireRequest() + req.SetRequestURI(p.BaseURL) + req.SetHost(host) return req, nil } diff --git a/pkg/client.go b/pkg/client.go index 7851a10..6a3fb61 100644 --- a/pkg/client.go +++ b/pkg/client.go @@ -3,36 +3,45 @@ package pkg import ( "context" "crypto/tls" + "github.com/valyala/fasthttp" "net/http" "time" ) func NewClient(thread int, timeout int) *Client { - tr := &http.Transport{ - //Proxy: Proxy, - //TLSHandshakeTimeout : delay * time.Second, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, - MaxConnsPerHost: thread, - IdleConnTimeout: time.Duration(timeout) * time.Second, - } - //c := &fasthttp.Client{ - // TLSConfig: &tls.Config{ + //tr := &http.Transport{ + // //Proxy: Proxy, + // //TLSHandshakeTimeout : delay * time.Second, + // TLSClientConfig: &tls.Config{ // Renegotiation: tls.RenegotiateOnceAsClient, // InsecureSkipVerify: true, // }, - // MaxConnsPerHost: thread, - // MaxIdleConnDuration: time.Duration(timeout) * time.Second, + // MaxConnsPerHost: thread, + // IdleConnTimeout: time.Duration(timeout) * time.Second, //} c := &Client{ - client: &http.Client{ - Transport: tr, - Timeout: time.Second * time.Duration(timeout), - CheckRedirect: checkRedirect, + client: &fasthttp.Client{ + TLSConfig: &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + }, + //ReadBufferSize: 20480, + MaxConnsPerHost: thread, + MaxIdleConnDuration: time.Duration(timeout) * time.Second, + MaxConnWaitTimeout: time.Duration(timeout) * time.Second, + ReadTimeout: time.Duration(timeout) * time.Second, + WriteTimeout: time.Duration(timeout) * time.Second, + MaxResponseBodySize: 20480, }, + timeout: time.Duration(timeout) * time.Second, } + //c := &Client{ + // client: &http.Client{ + // Transport: tr, + // Timeout: time.Second * time.Duration(timeout), + // CheckRedirect: checkRedirect, + // }, + //} //c.Method = method //c.Headers = Opt.Headers @@ -42,15 +51,16 @@ func NewClient(thread int, timeout int) *Client { } type Client struct { - client *http.Client + client *fasthttp.Client + timeout time.Duration } -func (c Client) Do(ctx context.Context, req *http.Request) (*http.Response, error) { +func (c *Client) Do(ctx context.Context, req *fasthttp.Request) (*fasthttp.Response, error) { //if req.Header == nil { // req.Header = c.Headers //} - - return c.client.Do(req) + resp := fasthttp.AcquireResponse() + return resp, c.client.Do(req, resp) } var MaxRedirects = 0 diff --git a/pkg/config.go b/pkg/config.go index e966946..517d3c8 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -28,4 +28,5 @@ type Config struct { Mod SprayMod Headers http.Header DeadlineTime int + EnableFuzzy bool }