admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-05-31 02:31:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #52 from chainreactors/dev

Browse Source 
merge v1.0.0
This commit is contained in:
M09Ic 2024-07-29 17:08:42 +08:00 committed by GitHub
commit 3791b765ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 866 additions and 593 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
cmd/cmd.go
View File

@ -7,9 +7,7 @@ import (
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/internal"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/internal/pool"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/utils/iutils"
"github.com/jessevdk/go-flags"
"os"
"os/signal"
@ -17,7 +15,7 @@ import (
"time"
)
var ver = "v0.9.6"
var ver = "v1.0.0"
var DefaultConfig = "config.yaml"
func init() {
@ -113,27 +111,13 @@ func Spray() {
return
}
err = pkg.Load()
err = option.Prepare()
if err != nil {
iutils.Fatal(err.Error())
logs.Log.Errorf(err.Error())
return
}
// 初始化全局变量
pkg.Distance = uint8(option.SimhashDistance)
if option.MaxBodyLength == -1 {
ihttp.DefaultMaxBodySize = -1
} else {
ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024
}
pool.MaxCrawl = option.CrawlDepth
var runner *internal.Runner
if option.ResumeFrom != "" {
runner, err = option.PrepareRunner()
} else {
runner, err = option.PrepareRunner()
}
runner, err := option.NewRunner()
if err != nil {
logs.Log.Errorf(err.Error())
return
@ -151,18 +135,29 @@ func Spray() {
}
go func() {
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
exitChan := make(chan os.Signal, 2)
signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
logs.Log.Important("exit signal, save stat and exit")
canceler()
sigCount := 0
for {
<-exitChan
sigCount++
if sigCount == 1 {
logs.Log.Infof("Exit signal received, saving task and exiting...")
canceler()
} else if sigCount == 2 {
logs.Log.Infof("forcing exit...")
os.Exit(1)
}
}
}()
}()
if runner.CheckOnly {
if runner.IsCheck {
runner.RunWithCheck(ctx)
} else {
runner.Run(ctx)
}
time.Sleep(1 * time.Second)
}

4
config.yaml
View File

@ -101,7 +101,7 @@ mode:
# Bool, skip error break
force: false
# Bool, check only
check-only: false
default: false
# Bool, no scope
no-scope: false
# String, custom scope, e.g.: --scope *.example.com
@ -132,7 +132,7 @@ mode:
unique: false
# Int, retry count
retry: 0
distance: 5
sim-distance: 5
misc:
# String, path/host spray
mod: path

8
go.mod
View File

@ -5,11 +5,11 @@ go 1.22
toolchain go1.22.2
require (
github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8
github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b
github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0
github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a
github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f
github.com/chainreactors/parsers v0.0.0-20240704071623-9d0ee90230a6
github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4
github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2
github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16
github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508
github.com/expr-lang/expr v1.16.9
github.com/gookit/config/v2 v2.2.5

48
go.sum
View File

@ -81,53 +81,23 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chainreactors/files v0.0.0-20230731174853-acee21c8c45a/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
github.com/chainreactors/files v0.0.0-20231102192550-a652458cee26/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 h1:8Plpi6haQbU8NzH+JtU6bkGDWF/OeC+GFj8DIDuY5yk=
github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE=
github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541 h1:aDY5A+G53En6t3Pr4tbl+vxJle2p1VJsqHVlyhnWU8s=
github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541/go.mod h1:s3lvNYcSW7NfM1inpgyn/wY3UEqQIvp6gE6BthFaOVo=
github.com/chainreactors/fingers v0.0.0-20240628163007-c08576f96117 h1:EJqE2nST/CigTcfexGLdoovYP46LFsJrcYknT8DD5qA=
github.com/chainreactors/fingers v0.0.0-20240628163007-c08576f96117/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
github.com/chainreactors/fingers v0.0.0-20240628164555-606f64752b5c h1:DWcMYWDzHWJ8kgQoAV6Z6/gN047o3BeKttFDPMM1xok=
github.com/chainreactors/fingers v0.0.0-20240628164555-606f64752b5c/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
github.com/chainreactors/fingers v0.0.0-20240628165118-3f8dcb553f8b h1:1KeC7d+Pc5GPerODPPtT0C2mnGI/y+SC+7wYQGLGNsI=
github.com/chainreactors/fingers v0.0.0-20240628165118-3f8dcb553f8b/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
github.com/chainreactors/fingers v0.0.0-20240628165313-8c7e41bf9ab5 h1:tmr3wbiwZZ+d8pgmD7LdjRY5Qp8bDm1STU0u3ADazRI=
github.com/chainreactors/fingers v0.0.0-20240628165313-8c7e41bf9ab5/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da h1:5kX/KxHFFjKzpZ7H8Ofdu7ukyrj7IXURhToWpeL3ID0=
github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
github.com/chainreactors/fingers v0.0.0-20240701103336-582e82977506 h1:hH7PXBn+nMU0uBFn7PYVqJagZM7EsP3X3RwAcqaTQ8U=
github.com/chainreactors/fingers v0.0.0-20240701103336-582e82977506/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4=
github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673 h1:ezIijqDJmoKFVmXxnUufUzKo/HBxQxp6QnQZvHxaBaY=
github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4=
github.com/chainreactors/fingers v0.0.0-20240702104653-a66e34aa41df h1:1SrOHwdlU+X3/hxViU1ZCcwO+KRvSmJGtTstSph5xL0=
github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 h1:cU3sGEODXZsUZGBXfnz0nyxF6+37vA+ZGDx6L/FKN4o=
github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0/go.mod h1:NSxGNMRWryAyrDzZpVwmujI22wbGw6c52bQOd5zEvyU=
github.com/chainreactors/fingers v0.0.0-20240702104653-a66e34aa41df/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4=
github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b h1:vqkkXIyeD0PGvWdJuKxinjEerfh8Rov/PqoPlnLVt0E=
github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b/go.mod h1:vQ/LJzHnMdxbK6n1PwqZmvgPudfNpoQsyFAPdt3IlBo=
github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a h1:5l4i8TdHRlz088J5xZM30yvTUMLVcWJ6iXiO/VyD3ro=
github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ=
github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k=
github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o=
github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1FGWD7gVZuF0cufxd7i9HJ2gltUewxXCfvs4=
github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg=
github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412 h1:GobKh1fztHQvslp8Ya6AgduYQshc1+CaomLYAf37yK8=
github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412/go.mod h1:0zPZn7glHadCyL2lvThINGU57UrLS7Mf+gf/kFW+k28=
github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8 h1:e0RDG17t6kcwOg4CLmwygn9Fh0alYa1GjxtRAcgm+so=
github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8/go.mod h1:0zPZn7glHadCyL2lvThINGU57UrLS7Mf+gf/kFW+k28=
github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29 h1:0RJnI9nDGao3LUSnqWhcg0ELex4RUlVvGY+SfAMuDSQ=
github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29/go.mod h1:91mj0+pHkKsXd1SGD6+jVW1tl6W25TuXgoESwK5dJh8=
github.com/chainreactors/parsers v0.0.0-20240702104902-1ce563b7ef76 h1:i4sHuonM50X/Tbgl6kNd7CYJqfRxgDUORIphcSuj4Bs=
github.com/chainreactors/parsers v0.0.0-20240702104902-1ce563b7ef76/go.mod h1:G/XLE5RAaUdqADkbhQ59mPrUAbsJLiQ2DN6CwtwNpBQ=
github.com/chainreactors/parsers v0.0.0-20240704062910-decf861def9e h1:42ILX5kS76M1D9IQvXgfelpgUJDi/K+4/egE0tLzuSE=
github.com/chainreactors/parsers v0.0.0-20240704062910-decf861def9e/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
github.com/chainreactors/parsers v0.0.0-20240704071443-8a8558f34cf9 h1:XxPUVhP29vnbLuhxFt8VT3eyBR8d/GfHR7YK44zyDVo=
github.com/chainreactors/parsers v0.0.0-20240704071443-8a8558f34cf9/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
github.com/chainreactors/parsers v0.0.0-20240704071623-9d0ee90230a6 h1:jUxPo0RJ/f+/4x3ydeXqCeMq5VbvYBjtmpBePWFfNc8=
github.com/chainreactors/parsers v0.0.0-20240704071623-9d0ee90230a6/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 h1:+awuysRKLmdLQbVK+HPSOGvO3dFGdNSbM2jyLh+VYOA=
github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2 h1:sE3SChgHLtPsEaqHo5tDSy8niDys1SO174C4eHlShSw=
github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4 h1:pW7yzxGD19ykkWnKkuJ8oq+hLt1WuTq9HP+cJZibip8=
github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16 h1:TCOshCp7PrWqhP/HSAM5kT3VxoOe7EoJbRseyoSX3RM=
github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw=
github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

181
internal/finger.go Normal file
View File

@ -0,0 +1,181 @@
package internal
import (
"fmt"
"github.com/chainreactors/files"
"github.com/chainreactors/fingers"
"github.com/chainreactors/fingers/resources"
"github.com/chainreactors/logs"
"github.com/chainreactors/utils/encode"
"github.com/chainreactors/utils/iutils"
"io"
"net/http"
"os"
"path/filepath"
"strings"
)
var (
DefaultFingerPath = "fingers"
DefaultFingerTemplate = "fingers/templates"
FingerConfigs = map[string]string{
fingers.FingersEngine: "fingers_http.json.gz",
fingers.FingerPrintEngine: "fingerprinthub_v3.json.gz",
fingers.WappalyzerEngine: "wappalyzer.json.gz",
fingers.EHoleEngine: "ehole.json.gz",
fingers.GobyEngine: "goby.json.gz",
}
baseURL = "https://raw.githubusercontent.com/chainreactors/fingers/master/resources/"
)
type FingerOptions struct {
Finger bool `long:"finger" description:"Bool, enable active finger detect" config:"finger"`
FingerUpdate bool `long:"update" description:"Bool, update finger database" config:"update"`
FingerPath string `long:"finger-path" default:"fingers" description:"String, 3rd finger config path" config:"finger-path"`
//FingersTemplatesPath string `long:"finger-template" default:"fingers/templates" description:"Bool, use finger templates path" config:"finger-template"`
FingerEngines string `long:"finger-engine" default:"all" description:"String, custom finger engine, e.g. --finger-engine ehole,goby" config:"finger-engine"`
}
func (opt *FingerOptions) Validate() error {
var err error
if opt.FingerUpdate {
if opt.FingerPath != DefaultFingerPath && !files.IsExist(opt.FingerPath) {
err = os.MkdirAll(opt.FingerPath, 0755)
if err != nil {
return err
}
} else if !files.IsExist(DefaultFingerPath) {
opt.FingerPath = DefaultFingerPath
err = os.MkdirAll(DefaultFingerPath, 0755)
if err != nil {
return err
}
}
//if opt.FingersTemplatesPath != DefaultFingerTemplate && !files.IsExist(opt.FingersTemplatesPath) {
// err = os.MkdirAll(opt.FingersTemplatesPath, 0755)
// if err != nil {
// return err
// }
//} else if !files.IsExist(DefaultFingerTemplate) {
// err = os.MkdirAll(DefaultFingerTemplate, 0755)
// if err != nil {
// return err
// }
//}
}
if opt.FingerEngines != "all" {
for _, name := range strings.Split(opt.FingerEngines, ",") {
if !iutils.StringsContains(fingers.AllEngines, name) {
return fmt.Errorf("invalid finger engine: %s, please input one of %v", name, fingers.FingersEngine)
}
}
}
return nil
}
func (opt *FingerOptions) LoadLocalFingerConfig() error {
for name, fingerPath := range FingerConfigs {
if content, err := os.ReadFile(fingerPath); err == nil {
if encode.Md5Hash(content) != resources.CheckSum[name] {
logs.Log.Importantf("found %s difference, use %s replace embed", name, fingerPath)
switch name {
case fingers.FingersEngine:
resources.FingersHTTPData = content
case fingers.FingerPrintEngine:
resources.Fingerprinthubdata = content
case fingers.EHoleEngine:
resources.EholeData = content
case fingers.GobyEngine:
resources.GobyData = content
case fingers.WappalyzerEngine:
resources.WappalyzerData = content
default:
return fmt.Errorf("unknown engine name")
}
} else {
logs.Log.Infof("%s config is up to date", name)
}
}
}
return nil
}
func (opt *FingerOptions) UpdateFinger() error {
modified := false
for name, _ := range FingerConfigs {
if ok, err := opt.downloadConfig(name); err != nil {
return err
} else {
if ok {
modified = ok
}
}
}
if !modified {
logs.Log.Importantf("everything is up to date")
}
return nil
}
func (opt *FingerOptions) downloadConfig(name string) (bool, error) {
fingerFile, ok := FingerConfigs[name]
if !ok {
return false, fmt.Errorf("unknown engine name")
}
url := baseURL + fingerFile
resp, err := http.Get(url)
if err != nil {
return false, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, fmt.Errorf("bad status: %s", resp.Status)
}
content, err := io.ReadAll(resp.Body)
filePath := filepath.Join(files.GetExcPath(), opt.FingerPath, fingerFile)
if files.IsExist(filePath) {
origin, err := os.ReadFile(filePath)
if err != nil {
return false, err
}
if resources.CheckSum[name] != encode.Md5Hash(origin) {
logs.Log.Importantf("update %s config from %s save to %s", name, url, fingerFile)
err = os.WriteFile(filePath, content, 0644)
if err != nil {
return false, err
}
return true, nil
}
} else {
out, err := os.Create(filePath)
if err != nil {
return false, err
}
defer out.Close()
logs.Log.Importantf("download %s config from %s save to %s", name, url, fingerFile)
err = os.WriteFile(filePath, content, 0644)
if err != nil {
return false, err
}
}
if err != nil {
return false, err
}
if origin, err := os.ReadFile(filePath); err == nil {
if encode.Md5Hash(content) != encode.Md5Hash(origin) {
logs.Log.Infof("download %s config from %s save to %s", name, url, fingerFile)
err = os.WriteFile(filePath, content, 0644)
if err != nil {
return false, err
}
return true, nil
}
}
return false, nil
}

6
internal/format.go
View File

@ -5,7 +5,7 @@ import (
"encoding/json"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/pkg"
"io/ioutil"
"io"
"os"
)
@ -13,9 +13,9 @@ func Format(filename string, color bool) {
var content []byte
var err error
if filename == "stdin" {
content, err = ioutil.ReadAll(os.Stdin)
content, err = io.ReadAll(os.Stdin)
} else {
content, err = ioutil.ReadFile(filename)
content, err = os.ReadFile(filename)
}
if err != nil {

893
internal/option.go
View File

File diff suppressed because it is too large Load Diff

4
internal/pool/checkpool.go
View File

@ -154,10 +154,10 @@ func (pool *CheckPool) Handler() {
if bl.IsValid {
if bl.RedirectURL != "" {
pool.doRedirect(bl, bl.ReqDepth)
pool.putToFuzzy(bl)
pool.putToOutput(bl)
} else if bl.Status == 400 {
pool.doUpgrade(bl)
pool.putToFuzzy(bl)
pool.putToOutput(bl)
} else {
params := map[string]interface{}{
"current": bl,

2
internal/pool/config.go
View File

@ -30,7 +30,7 @@ type Config struct {
ProcessCh chan *pkg.Baseline
OutputCh chan *pkg.Baseline
FuzzyCh chan *pkg.Baseline
OutLocker *sync.WaitGroup
Outwg *sync.WaitGroup
RateLimit int
CheckPeriod int
ErrPeriod int32

4
internal/pool/pool.go
View File

@ -153,12 +153,12 @@ func (pool *BasePool) putToOutput(bl *pkg.Baseline) {
if bl.IsValid || bl.IsFuzzy {
bl.Collect()
}
pool.OutLocker.Add(1)
pool.Outwg.Add(1)
pool.OutputCh <- bl
}
func (pool *BasePool) putToFuzzy(bl *pkg.Baseline) {
pool.OutLocker.Add(1)
pool.Outwg.Add(1)
bl.IsFuzzy = true
pool.FuzzyCh <- bl
}

138
internal/runner.go
View File

@ -14,7 +14,6 @@ import (
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
"sync"
"time"
)
var (
@ -36,10 +35,10 @@ type Runner struct {
outputCh chan *pkg.Baseline
fuzzyCh chan *pkg.Baseline
bar *mpb.Bar
finished int
IsCheck bool
Pools *ants.PoolWithFunc
PoolName map[string]bool
Tasks chan *Task
Tasks *TaskGenerator
Rules *rule.Program
AppendRules *rule.Program
Headers map[string]string
@ -73,7 +72,7 @@ func (r *Runner) PrepareConfig() *pool.Config {
Mod: pool.ModMap[r.Mod],
OutputCh: r.outputCh,
FuzzyCh: r.fuzzyCh,
OutLocker: r.outwg,
Outwg: r.outwg,
Fuzzy: r.Fuzzy,
CheckPeriod: r.CheckPeriod,
ErrPeriod: int32(r.ErrPeriod),
@ -113,106 +112,106 @@ func (r *Runner) AppendFunction(fn func(string) []string) {
func (r *Runner) Prepare(ctx context.Context) error {
var err error
if r.CheckOnly {
if r.IsCheck {
// 仅check, 类似httpx
r.Pools, err = ants.NewPoolWithFunc(1, func(i interface{}) {
config := r.PrepareConfig()
pool, err := pool.NewCheckPool(ctx, config)
checkPool, err := pool.NewCheckPool(ctx, config)
if err != nil {
logs.Log.Error(err.Error())
pool.Cancel()
checkPool.Cancel()
r.poolwg.Done()
return
}
ch := make(chan string)
go func() {
for t := range r.Tasks {
for t := range r.Tasks.tasks {
ch <- t.baseUrl
}
close(ch)
}()
pool.Worder = words.NewWorderWithChan(ch)
pool.Worder.Fns = r.Fns
pool.Bar = pkg.NewBar("check", r.Count-r.Offset, pool.Statistor, r.Progress)
pool.Run(ctx, r.Offset, r.Count)
checkPool.Worder = words.NewWorderWithChan(ch)
checkPool.Worder.Fns = r.Fns
checkPool.Bar = pkg.NewBar("check", r.Count-r.Offset, checkPool.Statistor, r.Progress)
checkPool.Run(ctx, r.Offset, r.Count)
r.poolwg.Done()
})
} else {
// 完整探测模式
go func() {
for t := range r.Tasks {
for t := range r.Tasks.tasks {
r.taskCh <- t
}
close(r.taskCh)
}()
if r.Count > 0 {
r.addBar(r.Count)
r.newBar(r.Count)
}
r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) {
t := i.(*Task)
if t.origin != nil && t.origin.End == t.origin.Total {
r.StatFile.SafeWrite(t.origin.Json())
r.saveStat(t.origin.Json())
r.Done()
return
}
config := r.PrepareConfig()
config.BaseURL = t.baseUrl
pool, err := pool.NewBrutePool(ctx, config)
brutePool, err := pool.NewBrutePool(ctx, config)
if err != nil {
logs.Log.Error(err.Error())
pool.Cancel()
brutePool.Cancel()
r.Done()
return
}
if t.origin != nil && len(r.Wordlist) == 0 {
// 如果是从断点续传中恢复的任务, 则自动设置word,dict与rule, 不过优先级低于命令行参数
pool.Statistor = pkg.NewStatistorFromStat(t.origin.Statistor)
pool.Worder, err = t.origin.InitWorder(r.Fns)
brutePool.Statistor = pkg.NewStatistorFromStat(t.origin.Statistor)
brutePool.Worder, err = t.origin.InitWorder(r.Fns)
if err != nil {
logs.Log.Error(err.Error())
r.Done()
return
}
pool.Statistor.Total = t.origin.sum
brutePool.Statistor.Total = t.origin.sum
} else {
pool.Statistor = pkg.NewStatistor(t.baseUrl)
pool.Worder = words.NewWorder(r.Wordlist)
pool.Worder.Fns = r.Fns
pool.Worder.Rules = r.Rules.Expressions
brutePool.Statistor = pkg.NewStatistor(t.baseUrl)
brutePool.Worder = words.NewWorder(r.Wordlist)
brutePool.Worder.Fns = r.Fns
brutePool.Worder.Rules = r.Rules.Expressions
}
var limit int
if pool.Statistor.Total > r.Limit && r.Limit != 0 {
if brutePool.Statistor.Total > r.Limit && r.Limit != 0 {
limit = r.Limit
} else {
limit = pool.Statistor.Total
limit = brutePool.Statistor.Total
}
pool.Bar = pkg.NewBar(config.BaseURL, limit-pool.Statistor.Offset, pool.Statistor, r.Progress)
logs.Log.Importantf("[pool] task: %s, total %d words, %d threads, proxy: %s", pool.BaseURL, limit-pool.Statistor.Offset, pool.Thread, pool.ProxyAddr)
err = pool.Init()
brutePool.Bar = pkg.NewBar(config.BaseURL, limit-brutePool.Statistor.Offset, brutePool.Statistor, r.Progress)
logs.Log.Importantf("[pool] task: %s, total %d words, %d threads, proxy: %s", brutePool.BaseURL, limit-brutePool.Statistor.Offset, brutePool.Thread, brutePool.ProxyAddr)
err = brutePool.Init()
if err != nil {
pool.Statistor.Error = err.Error()
brutePool.Statistor.Error = err.Error()
if !r.Force {
// 如果没开启force, init失败将会关闭pool
pool.Close()
r.PrintStat(pool)
brutePool.Close()
r.PrintStat(brutePool)
r.Done()
return
}
}
pool.Run(pool.Statistor.Offset, limit)
brutePool.Run(brutePool.Statistor.Offset, limit)
if pool.IsFailed && len(pool.FailedBaselines) > 0 {
if brutePool.IsFailed && len(brutePool.FailedBaselines) > 0 {
// 如果因为错误积累退出, end将指向第一个错误发生时, 防止resume时跳过大量目标
pool.Statistor.End = pool.FailedBaselines[0].Number
brutePool.Statistor.End = brutePool.FailedBaselines[0].Number
}
r.PrintStat(pool)
r.PrintStat(brutePool)
r.Done()
})
}
@ -224,28 +223,6 @@ func (r *Runner) Prepare(ctx context.Context) error {
return nil
}
func (r *Runner) AddRecursive(bl *pkg.Baseline) {
// 递归新任务
task := &Task{
baseUrl: bl.UrlString,
depth: bl.RecuDepth + 1,
origin: NewOrigin(pkg.NewStatistor(bl.UrlString)),
}
r.AddPool(task)
}
func (r *Runner) AddPool(task *Task) {
// 递归新任务
if _, ok := r.PoolName[task.baseUrl]; ok {
logs.Log.Importantf("already added pool, skip %s", task.baseUrl)
return
}
task.depth++
r.poolwg.Add(1)
r.Pools.Invoke(task)
}
func (r *Runner) Run(ctx context.Context) {
Loop:
for {
@ -254,10 +231,12 @@ Loop:
if len(r.taskCh) > 0 {
for t := range r.taskCh {
stat := pkg.NewStatistor(t.baseUrl)
r.StatFile.SafeWrite(stat.Json())
r.saveStat(stat.Json())
}
}
logs.Log.Importantf("already save all stat to %s", r.StatFile.Filename)
if r.StatFile != nil {
logs.Log.Importantf("already save all stat to %s", r.StatFile.Filename)
}
break Loop
case t, ok := <-r.taskCh:
if !ok {
@ -294,16 +273,32 @@ Loop:
}
}
for {
if len(r.outputCh) == 0 {
break
}
}
time.Sleep(100 * time.Millisecond) // 延迟100ms, 等所有数据处理完毕
r.outwg.Wait()
}
func (r *Runner) addBar(total int) {
func (r *Runner) AddRecursive(bl *pkg.Baseline) {
// 递归新任务
task := &Task{
baseUrl: bl.UrlString,
depth: bl.RecuDepth + 1,
origin: NewOrigin(pkg.NewStatistor(bl.UrlString)),
}
r.AddPool(task)
}
func (r *Runner) AddPool(task *Task) {
// 递归新任务
if _, ok := r.PoolName[task.baseUrl]; ok {
logs.Log.Importantf("already added pool, skip %s", task.baseUrl)
return
}
task.depth++
r.poolwg.Add(1)
r.Pools.Invoke(task)
}
func (r *Runner) newBar(total int) {
if r.Progress == nil {
return
}
@ -329,7 +324,6 @@ func (r *Runner) Done() {
if r.bar != nil {
r.bar.Increment()
}
r.finished++
r.poolwg.Done()
}
@ -348,8 +342,12 @@ func (r *Runner) PrintStat(pool *pool.BrutePool) {
}
}
r.saveStat(pool.Statistor.Json())
}
func (r *Runner) saveStat(content string) {
if r.StatFile != nil {
r.StatFile.SafeWrite(pool.Statistor.Json())
r.StatFile.SafeWrite(content)
r.StatFile.SafeSync()
}
}

73
internal/task.go Normal file
View File

@ -0,0 +1,73 @@
package internal
import (
"fmt"
"github.com/chainreactors/logs"
"github.com/chainreactors/utils"
"github.com/chainreactors/words/rule"
"net/url"
)
type Task struct {
baseUrl string
depth int
rule []rule.Expression
origin *Origin
}
func NewTaskGenerator(port string) *TaskGenerator {
gen := &TaskGenerator{
ports: utils.ParsePortsString(port),
tasks: make(chan *Task),
In: make(chan *Task),
}
go func() {
for task := range gen.In {
gen.tasks <- task
}
close(gen.tasks)
}()
return gen
}
type TaskGenerator struct {
Name string
ports []string
tasks chan *Task
In chan *Task
}
func (gen *TaskGenerator) Run(baseurl string) {
parsed, err := url.Parse(baseurl)
if err != nil {
logs.Log.Warnf("parse %s, %s ", baseurl, err.Error())
return
}
if parsed.Scheme == "" {
if parsed.Port() == "443" {
parsed.Scheme = "https"
} else {
parsed.Scheme = "http"
}
}
if len(gen.ports) == 0 {
gen.In <- &Task{baseUrl: parsed.String()}
return
}
for _, p := range gen.ports {
if parsed.Host == "" {
gen.In <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s", parsed.Scheme, parsed.Path, p)}
} else {
gen.In <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s/%s", parsed.Scheme, parsed.Host, p, parsed.Path)}
}
}
}
func (gen *TaskGenerator) Close() {
close(gen.tasks)
}

8
internal/types.go
View File

@ -3,16 +3,8 @@ package internal
import (
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/words"
"github.com/chainreactors/words/rule"
)
type Task struct {
baseUrl string
depth int
rule []rule.Expression
origin *Origin
}
func NewOrigin(stat *pkg.Statistor) *Origin {
return &Origin{Statistor: stat}
}

8
internal/utils.go
View File

@ -164,3 +164,11 @@ func wrapWordsFunc(f func(string) string) func(string) []string {
return []string{f(s)}
}
}
func safeFilename(filename string) string {
filename = strings.ReplaceAll(filename, "http://", "")
filename = strings.ReplaceAll(filename, "https://", "")
filename = strings.ReplaceAll(filename, ":", "_")
filename = strings.ReplaceAll(filename, "/", "_")
return filename
}

4
pkg/baseline.go
View File

@ -155,8 +155,8 @@ func (bl *Baseline) Collect() {
if bl.ContentType == "html" {
bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
} else if bl.ContentType == "ico" {
if frame := FingerEngine.HashContentMatch(bl.Body); frame != nil {
bl.Frameworks.Add(frame)
if frame := FingerEngine.Favicon().Match(bl.Body); frame != nil {
bl.Frameworks.Merge(frame)
}
}
}

27
pkg/load.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"github.com/chainreactors/fingers"
"github.com/chainreactors/parsers"
"github.com/chainreactors/utils"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words/mask"
"os"
@ -19,9 +20,19 @@ var (
ActivePath []string
)
func LoadTemplates() error {
func LoadPorts() error {
var err error
var ports []*utils.PortConfig
err = json.Unmarshal(LoadConfig("port"), &ports)
if err != nil {
return err
}
utils.PrePort = utils.NewPortPreset(ports)
return nil
}
func LoadFingers() error {
var err error
// load fingers
FingerEngine, err = fingers.NewEngine()
if err != nil {
return err
@ -38,7 +49,11 @@ func LoadTemplates() error {
ActivePath = append(ActivePath, f.Path)
}
}
return nil
}
func LoadTemplates() error {
var err error
// load rule
var data map[string]interface{}
err = json.Unmarshal(LoadConfig("spray_rule"), &data)
@ -105,11 +120,15 @@ func LoadExtractorConfig(filename string) ([]*parsers.Extractor, error) {
}
func Load() error {
// load fingers
err := LoadTemplates()
err := LoadPorts()
if err != nil {
return err
}
err = LoadTemplates()
if err != nil {
return err
}
return nil
}

2
templates

@ -1 +1 @@
Subproject commit f1150d00253e0c888976dbfe55cf2669c51f0b58
Subproject commit 3e85234341b95f7e6e45b31468311f01093ac970