Merge pull request #52 from chainreactors/dev

merge v1.0.0

cmd/cmd.go

@@ -7,9 +7,7 @@ import (
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/internal"
 	"github.com/chainreactors/spray/internal/ihttp"
-	"github.com/chainreactors/spray/internal/pool"
 	"github.com/chainreactors/spray/pkg"
-	"github.com/chainreactors/utils/iutils"
 	"github.com/jessevdk/go-flags"
 	"os"
 	"os/signal"
@@ -17,7 +15,7 @@ import (
 	"time"
 )
 
-var ver = "v0.9.6"
+var ver = "v1.0.0"
 var DefaultConfig = "config.yaml"
 
 func init() {
@@ -113,27 +111,13 @@ func Spray() {
 		return
 	}
 
-	err = pkg.Load()
+	err = option.Prepare()
 	if err != nil {
-		iutils.Fatal(err.Error())
+		logs.Log.Errorf(err.Error())
+		return
 	}
 
-	// 初始化全局变量
+	runner, err := option.NewRunner()
-	pkg.Distance = uint8(option.SimhashDistance)
-	if option.MaxBodyLength == -1 {
-		ihttp.DefaultMaxBodySize = -1
-	} else {
-		ihttp.DefaultMaxBodySize = option.MaxBodyLength * 1024
-	}
-
-	pool.MaxCrawl = option.CrawlDepth
-
-	var runner *internal.Runner
-	if option.ResumeFrom != "" {
-		runner, err = option.PrepareRunner()
-	} else {
-		runner, err = option.PrepareRunner()
-	}
 	if err != nil {
 		logs.Log.Errorf(err.Error())
 		return
@@ -151,18 +135,29 @@ func Spray() {
 	}
 
 	go func() {
-		c := make(chan os.Signal, 2)
+		exitChan := make(chan os.Signal, 2)
-		signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+		signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)
+
 		go func() {
-			<-c
+			sigCount := 0
-			logs.Log.Important("exit signal, save stat and exit")
+			for {
-			canceler()
+				<-exitChan
+				sigCount++
+				if sigCount == 1 {
+					logs.Log.Infof("Exit signal received, saving task and exiting...")
+					canceler()
+				} else if sigCount == 2 {
+					logs.Log.Infof("forcing exit...")
+					os.Exit(1)
+				}
+			}
 		}()
 	}()
 
-	if runner.CheckOnly {
+	if runner.IsCheck {
 		runner.RunWithCheck(ctx)
 	} else {
 		runner.Run(ctx)
 	}
+	time.Sleep(1 * time.Second)
 }

config.yaml

@@ -101,7 +101,7 @@ mode:
   # Bool, skip error break
   force: false
   # Bool, check only
-  check-only: false
+  default: false
   # Bool, no scope
   no-scope: false
   # String, custom scope, e.g.: --scope *.example.com
@@ -132,7 +132,7 @@ mode:
   unique: false
   # Int, retry count
   retry: 0
-  distance: 5
+  sim-distance: 5
 misc:
   # String, path/host spray
   mod: path

go.mod

@@ -5,11 +5,11 @@ go 1.22
 toolchain go1.22.2
 
 require (
-	github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8
+	github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0
-	github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b
+	github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a
 	github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f
-	github.com/chainreactors/parsers v0.0.0-20240704071623-9d0ee90230a6
+	github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2
-	github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4
+	github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16
 	github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508
 	github.com/expr-lang/expr v1.16.9
 	github.com/gookit/config/v2 v2.2.5

go.sum

@@ -81,53 +81,23 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chainreactors/files v0.0.0-20230731174853-acee21c8c45a/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
 github.com/chainreactors/files v0.0.0-20231102192550-a652458cee26/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
-github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8 h1:8Plpi6haQbU8NzH+JtU6bkGDWF/OeC+GFj8DIDuY5yk=
 github.com/chainreactors/files v0.0.0-20231123083421-cea5b4ad18a8/go.mod h1:/Xa9YXhjBlaC33JTD6ZTJFig6pcplak2IDcovf42/6A=
-github.com/chainreactors/fingers v0.0.0-20240304115656-fa8ca9fc375f/go.mod h1:cO2a79lRNSaM6hu17xIyws5eWCWxjcRxY9IFPlss2lE=
+github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 h1:cU3sGEODXZsUZGBXfnz0nyxF6+37vA+ZGDx6L/FKN4o=
-github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541 h1:aDY5A+G53En6t3Pr4tbl+vxJle2p1VJsqHVlyhnWU8s=
+github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0/go.mod h1:NSxGNMRWryAyrDzZpVwmujI22wbGw6c52bQOd5zEvyU=
-github.com/chainreactors/fingers v0.0.0-20240603064620-e83951a40541/go.mod h1:s3lvNYcSW7NfM1inpgyn/wY3UEqQIvp6gE6BthFaOVo=
-github.com/chainreactors/fingers v0.0.0-20240628163007-c08576f96117 h1:EJqE2nST/CigTcfexGLdoovYP46LFsJrcYknT8DD5qA=
-github.com/chainreactors/fingers v0.0.0-20240628163007-c08576f96117/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
-github.com/chainreactors/fingers v0.0.0-20240628164555-606f64752b5c h1:DWcMYWDzHWJ8kgQoAV6Z6/gN047o3BeKttFDPMM1xok=
-github.com/chainreactors/fingers v0.0.0-20240628164555-606f64752b5c/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
-github.com/chainreactors/fingers v0.0.0-20240628165118-3f8dcb553f8b h1:1KeC7d+Pc5GPerODPPtT0C2mnGI/y+SC+7wYQGLGNsI=
-github.com/chainreactors/fingers v0.0.0-20240628165118-3f8dcb553f8b/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
-github.com/chainreactors/fingers v0.0.0-20240628165313-8c7e41bf9ab5 h1:tmr3wbiwZZ+d8pgmD7LdjRY5Qp8bDm1STU0u3ADazRI=
-github.com/chainreactors/fingers v0.0.0-20240628165313-8c7e41bf9ab5/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
-github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da h1:5kX/KxHFFjKzpZ7H8Ofdu7ukyrj7IXURhToWpeL3ID0=
-github.com/chainreactors/fingers v0.0.0-20240628190949-7257c400d3da/go.mod h1:P9RWZA2j7AALdNbpsZmxBtDa96y8CRpC4mRmwt9PwE8=
-github.com/chainreactors/fingers v0.0.0-20240701103336-582e82977506 h1:hH7PXBn+nMU0uBFn7PYVqJagZM7EsP3X3RwAcqaTQ8U=
-github.com/chainreactors/fingers v0.0.0-20240701103336-582e82977506/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4=
-github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673 h1:ezIijqDJmoKFVmXxnUufUzKo/HBxQxp6QnQZvHxaBaY=
-github.com/chainreactors/fingers v0.0.0-20240701112353-ecc7954b4673/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4=
-github.com/chainreactors/fingers v0.0.0-20240702104653-a66e34aa41df h1:1SrOHwdlU+X3/hxViU1ZCcwO+KRvSmJGtTstSph5xL0=
 github.com/chainreactors/fingers v0.0.0-20240702104653-a66e34aa41df/go.mod h1:l8AO6ZbIL8WQ8PkihCK/MD6Iww/O+LY/osAhRJjThs4=
-github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b h1:vqkkXIyeD0PGvWdJuKxinjEerfh8Rov/PqoPlnLVt0E=
+github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a h1:5l4i8TdHRlz088J5xZM30yvTUMLVcWJ6iXiO/VyD3ro=
-github.com/chainreactors/fingers v0.0.0-20240704063230-de8fec05ff8b/go.mod h1:vQ/LJzHnMdxbK6n1PwqZmvgPudfNpoQsyFAPdt3IlBo=
+github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a/go.mod h1:R03soobTE/AnZWtFgfQVYNM5QLH52NZ946wZTJVBXh4=
 github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460/go.mod h1:VZFqkFDGmp7/JOMeraW+YI7kTGcgz9fgc/HArVFnrGQ=
 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f h1:tcfp+CEdgiMvjyUzWab5edJtxUwRMSMEIkLybupIx0k=
 github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f/go.mod h1:6Mv6W70JrtL6VClulZhmMRZnoYpcTahcDTKLMNEjK0o=
-github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc h1:lGgglOE1FGWD7gVZuF0cufxd7i9HJ2gltUewxXCfvs4=
-github.com/chainreactors/parsers v0.0.0-20240422094636-b88693700dfc/go.mod h1:BuI21VlpmYHFr1jva/IN5I5jFvvCtYRyeldGK80wYCg=
-github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412 h1:GobKh1fztHQvslp8Ya6AgduYQshc1+CaomLYAf37yK8=
-github.com/chainreactors/parsers v0.0.0-20240628194456-a176ea53b412/go.mod h1:0zPZn7glHadCyL2lvThINGU57UrLS7Mf+gf/kFW+k28=
-github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8 h1:e0RDG17t6kcwOg4CLmwygn9Fh0alYa1GjxtRAcgm+so=
-github.com/chainreactors/parsers v0.0.0-20240628195954-201bdc68bdf8/go.mod h1:0zPZn7glHadCyL2lvThINGU57UrLS7Mf+gf/kFW+k28=
-github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29 h1:0RJnI9nDGao3LUSnqWhcg0ELex4RUlVvGY+SfAMuDSQ=
-github.com/chainreactors/parsers v0.0.0-20240701110332-2f9057256d29/go.mod h1:91mj0+pHkKsXd1SGD6+jVW1tl6W25TuXgoESwK5dJh8=
-github.com/chainreactors/parsers v0.0.0-20240702104902-1ce563b7ef76 h1:i4sHuonM50X/Tbgl6kNd7CYJqfRxgDUORIphcSuj4Bs=
 github.com/chainreactors/parsers v0.0.0-20240702104902-1ce563b7ef76/go.mod h1:G/XLE5RAaUdqADkbhQ59mPrUAbsJLiQ2DN6CwtwNpBQ=
-github.com/chainreactors/parsers v0.0.0-20240704062910-decf861def9e h1:42ILX5kS76M1D9IQvXgfelpgUJDi/K+4/egE0tLzuSE=
+github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2 h1:sE3SChgHLtPsEaqHo5tDSy8niDys1SO174C4eHlShSw=
-github.com/chainreactors/parsers v0.0.0-20240704062910-decf861def9e/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
+github.com/chainreactors/parsers v0.0.0-20240708072709-07deeece7ce2/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
-github.com/chainreactors/parsers v0.0.0-20240704071443-8a8558f34cf9 h1:XxPUVhP29vnbLuhxFt8VT3eyBR8d/GfHR7YK44zyDVo=
-github.com/chainreactors/parsers v0.0.0-20240704071443-8a8558f34cf9/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
-github.com/chainreactors/parsers v0.0.0-20240704071623-9d0ee90230a6 h1:jUxPo0RJ/f+/4x3ydeXqCeMq5VbvYBjtmpBePWFfNc8=
-github.com/chainreactors/parsers v0.0.0-20240704071623-9d0ee90230a6/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA=
-github.com/chainreactors/utils v0.0.0-20240302165634-2b8494c9cfc3/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
-github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1 h1:+awuysRKLmdLQbVK+HPSOGvO3dFGdNSbM2jyLh+VYOA=
 github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
-github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4 h1:pW7yzxGD19ykkWnKkuJ8oq+hLt1WuTq9HP+cJZibip8=
 github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs=
+github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
+github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16 h1:TCOshCp7PrWqhP/HSAM5kT3VxoOe7EoJbRseyoSX3RM=
+github.com/chainreactors/utils v0.0.0-20240716182459-e85f2b01ee16/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU=
 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508 h1:iT4HWkoZzUAfQYcQMRH8XyrMau9tCVE0zSuFQnkhrqw=
 github.com/chainreactors/words v0.4.1-0.20240510105042-5ba5c2edc508/go.mod h1:DUDx7PdsMEm5PvVhzkFyppzpiUhQb8dOJaWjVc1SMVk=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

internal/finger.go

@@ -0,0 +1,181 @@
+package internal
+
+import (
+	"fmt"
+	"github.com/chainreactors/files"
+	"github.com/chainreactors/fingers"
+	"github.com/chainreactors/fingers/resources"
+	"github.com/chainreactors/logs"
+	"github.com/chainreactors/utils/encode"
+	"github.com/chainreactors/utils/iutils"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	DefaultFingerPath     = "fingers"
+	DefaultFingerTemplate = "fingers/templates"
+
+	FingerConfigs = map[string]string{
+		fingers.FingersEngine:     "fingers_http.json.gz",
+		fingers.FingerPrintEngine: "fingerprinthub_v3.json.gz",
+		fingers.WappalyzerEngine:  "wappalyzer.json.gz",
+		fingers.EHoleEngine:       "ehole.json.gz",
+		fingers.GobyEngine:        "goby.json.gz",
+	}
+	baseURL = "https://raw.githubusercontent.com/chainreactors/fingers/master/resources/"
+)
+
+type FingerOptions struct {
+	Finger       bool   `long:"finger" description:"Bool, enable active finger detect" config:"finger"`
+	FingerUpdate bool   `long:"update" description:"Bool, update finger database" config:"update"`
+	FingerPath   string `long:"finger-path" default:"fingers" description:"String, 3rd finger config path" config:"finger-path"`
+	//FingersTemplatesPath string `long:"finger-template" default:"fingers/templates" description:"Bool, use finger templates path" config:"finger-template"`
+	FingerEngines string `long:"finger-engine" default:"all" description:"String, custom finger engine, e.g. --finger-engine ehole,goby" config:"finger-engine"`
+}
+
+func (opt *FingerOptions) Validate() error {
+	var err error
+	if opt.FingerUpdate {
+		if opt.FingerPath != DefaultFingerPath && !files.IsExist(opt.FingerPath) {
+			err = os.MkdirAll(opt.FingerPath, 0755)
+			if err != nil {
+				return err
+			}
+		} else if !files.IsExist(DefaultFingerPath) {
+			opt.FingerPath = DefaultFingerPath
+			err = os.MkdirAll(DefaultFingerPath, 0755)
+			if err != nil {
+				return err
+			}
+		}
+		//if opt.FingersTemplatesPath != DefaultFingerTemplate && !files.IsExist(opt.FingersTemplatesPath) {
+		//	err = os.MkdirAll(opt.FingersTemplatesPath, 0755)
+		//	if err != nil {
+		//		return err
+		//	}
+		//} else if !files.IsExist(DefaultFingerTemplate) {
+		//	err = os.MkdirAll(DefaultFingerTemplate, 0755)
+		//	if err != nil {
+		//		return err
+		//	}
+		//}
+	}
+
+	if opt.FingerEngines != "all" {
+		for _, name := range strings.Split(opt.FingerEngines, ",") {
+			if !iutils.StringsContains(fingers.AllEngines, name) {
+				return fmt.Errorf("invalid finger engine: %s, please input one of %v", name, fingers.FingersEngine)
+			}
+		}
+	}
+	return nil
+}
+
+func (opt *FingerOptions) LoadLocalFingerConfig() error {
+	for name, fingerPath := range FingerConfigs {
+		if content, err := os.ReadFile(fingerPath); err == nil {
+			if encode.Md5Hash(content) != resources.CheckSum[name] {
+				logs.Log.Importantf("found %s difference, use %s replace embed", name, fingerPath)
+				switch name {
+				case fingers.FingersEngine:
+					resources.FingersHTTPData = content
+				case fingers.FingerPrintEngine:
+					resources.Fingerprinthubdata = content
+				case fingers.EHoleEngine:
+					resources.EholeData = content
+				case fingers.GobyEngine:
+					resources.GobyData = content
+				case fingers.WappalyzerEngine:
+					resources.WappalyzerData = content
+				default:
+					return fmt.Errorf("unknown engine name")
+				}
+			} else {
+				logs.Log.Infof("%s config is up to date", name)
+			}
+		}
+	}
+	return nil
+}
+
+func (opt *FingerOptions) UpdateFinger() error {
+	modified := false
+	for name, _ := range FingerConfigs {
+		if ok, err := opt.downloadConfig(name); err != nil {
+			return err
+		} else {
+			if ok {
+				modified = ok
+			}
+		}
+	}
+	if !modified {
+		logs.Log.Importantf("everything is up to date")
+	}
+	return nil
+}
+
+func (opt *FingerOptions) downloadConfig(name string) (bool, error) {
+	fingerFile, ok := FingerConfigs[name]
+	if !ok {
+		return false, fmt.Errorf("unknown engine name")
+	}
+	url := baseURL + fingerFile
+	resp, err := http.Get(url)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return false, fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	content, err := io.ReadAll(resp.Body)
+	filePath := filepath.Join(files.GetExcPath(), opt.FingerPath, fingerFile)
+	if files.IsExist(filePath) {
+		origin, err := os.ReadFile(filePath)
+		if err != nil {
+			return false, err
+		}
+		if resources.CheckSum[name] != encode.Md5Hash(origin) {
+			logs.Log.Importantf("update %s config from %s save to %s", name, url, fingerFile)
+			err = os.WriteFile(filePath, content, 0644)
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		}
+	} else {
+		out, err := os.Create(filePath)
+		if err != nil {
+			return false, err
+		}
+		defer out.Close()
+		logs.Log.Importantf("download %s config from %s save to %s", name, url, fingerFile)
+		err = os.WriteFile(filePath, content, 0644)
+		if err != nil {
+			return false, err
+		}
+	}
+	if err != nil {
+		return false, err
+	}
+
+	if origin, err := os.ReadFile(filePath); err == nil {
+		if encode.Md5Hash(content) != encode.Md5Hash(origin) {
+			logs.Log.Infof("download %s config from %s save to %s", name, url, fingerFile)
+			err = os.WriteFile(filePath, content, 0644)
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

internal/format.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
-	"io/ioutil"
+	"io"
 	"os"
 )
 
@@ -13,9 +13,9 @@ func Format(filename string, color bool) {
 	var content []byte
 	var err error
 	if filename == "stdin" {
-		content, err = ioutil.ReadAll(os.Stdin)
+		content, err = io.ReadAll(os.Stdin)
 	} else {
-		content, err = ioutil.ReadFile(filename)
+		content, err = os.ReadFile(filename)
 	}
 
 	if err != nil {

internal/pool/checkpool.go

@@ -154,10 +154,10 @@ func (pool *CheckPool) Handler() {
 		if bl.IsValid {
 			if bl.RedirectURL != "" {
 				pool.doRedirect(bl, bl.ReqDepth)
-				pool.putToFuzzy(bl)
+				pool.putToOutput(bl)
 			} else if bl.Status == 400 {
 				pool.doUpgrade(bl)
-				pool.putToFuzzy(bl)
+				pool.putToOutput(bl)
 			} else {
 				params := map[string]interface{}{
 					"current": bl,

internal/pool/config.go

@@ -30,7 +30,7 @@ type Config struct {
 	ProcessCh       chan *pkg.Baseline
 	OutputCh        chan *pkg.Baseline
 	FuzzyCh         chan *pkg.Baseline
-	OutLocker       *sync.WaitGroup
+	Outwg           *sync.WaitGroup
 	RateLimit       int
 	CheckPeriod     int
 	ErrPeriod       int32

internal/pool/pool.go

@@ -153,12 +153,12 @@ func (pool *BasePool) putToOutput(bl *pkg.Baseline) {
 	if bl.IsValid || bl.IsFuzzy {
 		bl.Collect()
 	}
-	pool.OutLocker.Add(1)
+	pool.Outwg.Add(1)
 	pool.OutputCh <- bl
 }
 
 func (pool *BasePool) putToFuzzy(bl *pkg.Baseline) {
-	pool.OutLocker.Add(1)
+	pool.Outwg.Add(1)
 	bl.IsFuzzy = true
 	pool.FuzzyCh <- bl
 }

internal/runner.go

@@ -14,7 +14,6 @@ import (
 	"github.com/vbauerster/mpb/v8"
 	"github.com/vbauerster/mpb/v8/decor"
 	"sync"
-	"time"
 )
 
 var (
@@ -36,10 +35,10 @@ type Runner struct {
 	outputCh      chan *pkg.Baseline
 	fuzzyCh       chan *pkg.Baseline
 	bar           *mpb.Bar
-	finished      int
+	IsCheck       bool
 	Pools         *ants.PoolWithFunc
 	PoolName      map[string]bool
-	Tasks         chan *Task
+	Tasks         *TaskGenerator
 	Rules         *rule.Program
 	AppendRules   *rule.Program
 	Headers       map[string]string
@@ -73,7 +72,7 @@ func (r *Runner) PrepareConfig() *pool.Config {
 		Mod:            pool.ModMap[r.Mod],
 		OutputCh:       r.outputCh,
 		FuzzyCh:        r.fuzzyCh,
-		OutLocker:      r.outwg,
+		Outwg:          r.outwg,
 		Fuzzy:          r.Fuzzy,
 		CheckPeriod:    r.CheckPeriod,
 		ErrPeriod:      int32(r.ErrPeriod),
@@ -113,106 +112,106 @@ func (r *Runner) AppendFunction(fn func(string) []string) {
 
 func (r *Runner) Prepare(ctx context.Context) error {
 	var err error
-	if r.CheckOnly {
+	if r.IsCheck {
 		// 仅check, 类似httpx
 		r.Pools, err = ants.NewPoolWithFunc(1, func(i interface{}) {
 			config := r.PrepareConfig()
 
-			pool, err := pool.NewCheckPool(ctx, config)
+			checkPool, err := pool.NewCheckPool(ctx, config)
 			if err != nil {
 				logs.Log.Error(err.Error())
-				pool.Cancel()
+				checkPool.Cancel()
 				r.poolwg.Done()
 				return
 			}
 
 			ch := make(chan string)
 			go func() {
-				for t := range r.Tasks {
+				for t := range r.Tasks.tasks {
 					ch <- t.baseUrl
 				}
 				close(ch)
 			}()
-			pool.Worder = words.NewWorderWithChan(ch)
+			checkPool.Worder = words.NewWorderWithChan(ch)
-			pool.Worder.Fns = r.Fns
+			checkPool.Worder.Fns = r.Fns
-			pool.Bar = pkg.NewBar("check", r.Count-r.Offset, pool.Statistor, r.Progress)
+			checkPool.Bar = pkg.NewBar("check", r.Count-r.Offset, checkPool.Statistor, r.Progress)
-			pool.Run(ctx, r.Offset, r.Count)
+			checkPool.Run(ctx, r.Offset, r.Count)
 			r.poolwg.Done()
 		})
 	} else {
 		// 完整探测模式
 		go func() {
-			for t := range r.Tasks {
+			for t := range r.Tasks.tasks {
 				r.taskCh <- t
 			}
 			close(r.taskCh)
 		}()
 
 		if r.Count > 0 {
-			r.addBar(r.Count)
+			r.newBar(r.Count)
 		}
 
 		r.Pools, err = ants.NewPoolWithFunc(r.PoolSize, func(i interface{}) {
 			t := i.(*Task)
 			if t.origin != nil && t.origin.End == t.origin.Total {
-				r.StatFile.SafeWrite(t.origin.Json())
+				r.saveStat(t.origin.Json())
 				r.Done()
 				return
 			}
 			config := r.PrepareConfig()
 			config.BaseURL = t.baseUrl
 
-			pool, err := pool.NewBrutePool(ctx, config)
+			brutePool, err := pool.NewBrutePool(ctx, config)
 			if err != nil {
 				logs.Log.Error(err.Error())
-				pool.Cancel()
+				brutePool.Cancel()
 				r.Done()
 				return
 			}
 			if t.origin != nil && len(r.Wordlist) == 0 {
 				// 如果是从断点续传中恢复的任务, 则自动设置word,dict与rule, 不过优先级低于命令行参数
-				pool.Statistor = pkg.NewStatistorFromStat(t.origin.Statistor)
+				brutePool.Statistor = pkg.NewStatistorFromStat(t.origin.Statistor)
-				pool.Worder, err = t.origin.InitWorder(r.Fns)
+				brutePool.Worder, err = t.origin.InitWorder(r.Fns)
 				if err != nil {
 					logs.Log.Error(err.Error())
 					r.Done()
 					return
 				}
-				pool.Statistor.Total = t.origin.sum
+				brutePool.Statistor.Total = t.origin.sum
 			} else {
-				pool.Statistor = pkg.NewStatistor(t.baseUrl)
+				brutePool.Statistor = pkg.NewStatistor(t.baseUrl)
-				pool.Worder = words.NewWorder(r.Wordlist)
+				brutePool.Worder = words.NewWorder(r.Wordlist)
-				pool.Worder.Fns = r.Fns
+				brutePool.Worder.Fns = r.Fns
-				pool.Worder.Rules = r.Rules.Expressions
+				brutePool.Worder.Rules = r.Rules.Expressions
 			}
 
 			var limit int
-			if pool.Statistor.Total > r.Limit && r.Limit != 0 {
+			if brutePool.Statistor.Total > r.Limit && r.Limit != 0 {
 				limit = r.Limit
 			} else {
-				limit = pool.Statistor.Total
+				limit = brutePool.Statistor.Total
 			}
-			pool.Bar = pkg.NewBar(config.BaseURL, limit-pool.Statistor.Offset, pool.Statistor, r.Progress)
+			brutePool.Bar = pkg.NewBar(config.BaseURL, limit-brutePool.Statistor.Offset, brutePool.Statistor, r.Progress)
-			logs.Log.Importantf("[pool] task: %s, total %d words, %d threads, proxy: %s", pool.BaseURL, limit-pool.Statistor.Offset, pool.Thread, pool.ProxyAddr)
+			logs.Log.Importantf("[pool] task: %s, total %d words, %d threads, proxy: %s", brutePool.BaseURL, limit-brutePool.Statistor.Offset, brutePool.Thread, brutePool.ProxyAddr)
-			err = pool.Init()
+			err = brutePool.Init()
 			if err != nil {
-				pool.Statistor.Error = err.Error()
+				brutePool.Statistor.Error = err.Error()
 				if !r.Force {
 					// 如果没开启force, init失败将会关闭pool
-					pool.Close()
+					brutePool.Close()
-					r.PrintStat(pool)
+					r.PrintStat(brutePool)
 					r.Done()
 					return
 				}
 			}
 
-			pool.Run(pool.Statistor.Offset, limit)
+			brutePool.Run(brutePool.Statistor.Offset, limit)
 
-			if pool.IsFailed && len(pool.FailedBaselines) > 0 {
+			if brutePool.IsFailed && len(brutePool.FailedBaselines) > 0 {
 				// 如果因为错误积累退出, end将指向第一个错误发生时, 防止resume时跳过大量目标
-				pool.Statistor.End = pool.FailedBaselines[0].Number
+				brutePool.Statistor.End = brutePool.FailedBaselines[0].Number
 			}
-			r.PrintStat(pool)
+			r.PrintStat(brutePool)
 			r.Done()
 		})
 	}
@@ -224,28 +223,6 @@ func (r *Runner) Prepare(ctx context.Context) error {
 	return nil
 }
 
-func (r *Runner) AddRecursive(bl *pkg.Baseline) {
-	// 递归新任务
-	task := &Task{
-		baseUrl: bl.UrlString,
-		depth:   bl.RecuDepth + 1,
-		origin:  NewOrigin(pkg.NewStatistor(bl.UrlString)),
-	}
-
-	r.AddPool(task)
-}
-
-func (r *Runner) AddPool(task *Task) {
-	// 递归新任务
-	if _, ok := r.PoolName[task.baseUrl]; ok {
-		logs.Log.Importantf("already added pool, skip %s", task.baseUrl)
-		return
-	}
-	task.depth++
-	r.poolwg.Add(1)
-	r.Pools.Invoke(task)
-}
-
 func (r *Runner) Run(ctx context.Context) {
 Loop:
 	for {
@@ -254,10 +231,12 @@ Loop:
 			if len(r.taskCh) > 0 {
 				for t := range r.taskCh {
 					stat := pkg.NewStatistor(t.baseUrl)
-					r.StatFile.SafeWrite(stat.Json())
+					r.saveStat(stat.Json())
 				}
 			}
-			logs.Log.Importantf("already save all stat to %s", r.StatFile.Filename)
+			if r.StatFile != nil {
+				logs.Log.Importantf("already save all stat to %s", r.StatFile.Filename)
+			}
 			break Loop
 		case t, ok := <-r.taskCh:
 			if !ok {
@@ -294,16 +273,32 @@ Loop:
 		}
 	}
 
-	for {
+	r.outwg.Wait()
-		if len(r.outputCh) == 0 {
-			break
-		}
-	}
-
-	time.Sleep(100 * time.Millisecond) // 延迟100ms, 等所有数据处理完毕
 }
 
-func (r *Runner) addBar(total int) {
+func (r *Runner) AddRecursive(bl *pkg.Baseline) {
+	// 递归新任务
+	task := &Task{
+		baseUrl: bl.UrlString,
+		depth:   bl.RecuDepth + 1,
+		origin:  NewOrigin(pkg.NewStatistor(bl.UrlString)),
+	}
+
+	r.AddPool(task)
+}
+
+func (r *Runner) AddPool(task *Task) {
+	// 递归新任务
+	if _, ok := r.PoolName[task.baseUrl]; ok {
+		logs.Log.Importantf("already added pool, skip %s", task.baseUrl)
+		return
+	}
+	task.depth++
+	r.poolwg.Add(1)
+	r.Pools.Invoke(task)
+}
+
+func (r *Runner) newBar(total int) {
 	if r.Progress == nil {
 		return
 	}
@@ -329,7 +324,6 @@ func (r *Runner) Done() {
 	if r.bar != nil {
 		r.bar.Increment()
 	}
-	r.finished++
 	r.poolwg.Done()
 }
 
@@ -348,8 +342,12 @@ func (r *Runner) PrintStat(pool *pool.BrutePool) {
 		}
 	}
 
+	r.saveStat(pool.Statistor.Json())
+}
+
+func (r *Runner) saveStat(content string) {
 	if r.StatFile != nil {
-		r.StatFile.SafeWrite(pool.Statistor.Json())
+		r.StatFile.SafeWrite(content)
 		r.StatFile.SafeSync()
 	}
 }

internal/task.go

@@ -0,0 +1,73 @@
+package internal
+
+import (
+	"fmt"
+	"github.com/chainreactors/logs"
+	"github.com/chainreactors/utils"
+	"github.com/chainreactors/words/rule"
+	"net/url"
+)
+
+type Task struct {
+	baseUrl string
+	depth   int
+	rule    []rule.Expression
+	origin  *Origin
+}
+
+func NewTaskGenerator(port string) *TaskGenerator {
+	gen := &TaskGenerator{
+		ports: utils.ParsePortsString(port),
+		tasks: make(chan *Task),
+		In:    make(chan *Task),
+	}
+
+	go func() {
+		for task := range gen.In {
+			gen.tasks <- task
+		}
+		close(gen.tasks)
+	}()
+	return gen
+
+}
+
+type TaskGenerator struct {
+	Name  string
+	ports []string
+	tasks chan *Task
+	In    chan *Task
+}
+
+func (gen *TaskGenerator) Run(baseurl string) {
+	parsed, err := url.Parse(baseurl)
+	if err != nil {
+		logs.Log.Warnf("parse %s, %s ", baseurl, err.Error())
+		return
+	}
+
+	if parsed.Scheme == "" {
+		if parsed.Port() == "443" {
+			parsed.Scheme = "https"
+		} else {
+			parsed.Scheme = "http"
+		}
+	}
+
+	if len(gen.ports) == 0 {
+		gen.In <- &Task{baseUrl: parsed.String()}
+		return
+	}
+
+	for _, p := range gen.ports {
+		if parsed.Host == "" {
+			gen.In <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s", parsed.Scheme, parsed.Path, p)}
+		} else {
+			gen.In <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s/%s", parsed.Scheme, parsed.Host, p, parsed.Path)}
+		}
+	}
+}
+
+func (gen *TaskGenerator) Close() {
+	close(gen.tasks)
+}

internal/types.go

@@ -3,16 +3,8 @@ package internal
 import (
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/words"
-	"github.com/chainreactors/words/rule"
 )
 
-type Task struct {
-	baseUrl string
-	depth   int
-	rule    []rule.Expression
-	origin  *Origin
-}
-
 func NewOrigin(stat *pkg.Statistor) *Origin {
 	return &Origin{Statistor: stat}
 }

internal/utils.go

@@ -164,3 +164,11 @@ func wrapWordsFunc(f func(string) string) func(string) []string {
 		return []string{f(s)}
 	}
 }
+
+func safeFilename(filename string) string {
+	filename = strings.ReplaceAll(filename, "http://", "")
+	filename = strings.ReplaceAll(filename, "https://", "")
+	filename = strings.ReplaceAll(filename, ":", "_")
+	filename = strings.ReplaceAll(filename, "/", "_")
+	return filename
+}

pkg/baseline.go

@@ -155,8 +155,8 @@ func (bl *Baseline) Collect() {
 		if bl.ContentType == "html" {
 			bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
 		} else if bl.ContentType == "ico" {
-			if frame := FingerEngine.HashContentMatch(bl.Body); frame != nil {
+			if frame := FingerEngine.Favicon().Match(bl.Body); frame != nil {
-				bl.Frameworks.Add(frame)
+				bl.Frameworks.Merge(frame)
 			}
 		}
 	}

pkg/load.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"github.com/chainreactors/fingers"
 	"github.com/chainreactors/parsers"
+	"github.com/chainreactors/utils"
 	"github.com/chainreactors/utils/iutils"
 	"github.com/chainreactors/words/mask"
 	"os"
@@ -19,9 +20,19 @@ var (
 	ActivePath   []string
 )
 
-func LoadTemplates() error {
+func LoadPorts() error {
+	var err error
+	var ports []*utils.PortConfig
+	err = json.Unmarshal(LoadConfig("port"), &ports)
+	if err != nil {
+		return err
+	}
+	utils.PrePort = utils.NewPortPreset(ports)
+	return nil
+}
+
+func LoadFingers() error {
 	var err error
-	// load fingers
 	FingerEngine, err = fingers.NewEngine()
 	if err != nil {
 		return err
@@ -38,7 +49,11 @@ func LoadTemplates() error {
 			ActivePath = append(ActivePath, f.Path)
 		}
 	}
+	return nil
+}
 
+func LoadTemplates() error {
+	var err error
 	// load rule
 	var data map[string]interface{}
 	err = json.Unmarshal(LoadConfig("spray_rule"), &data)
@@ -105,11 +120,15 @@ func LoadExtractorConfig(filename string) ([]*parsers.Extractor, error) {
 }
 
 func Load() error {
-	// load fingers
+	err := LoadPorts()
-	err := LoadTemplates()
 	if err != nil {
 		return err
 	}
+	err = LoadTemplates()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 

templates

@@ -1 +1 @@
-Subproject commit f1150d00253e0c888976dbfe55cf2669c51f0b58
+Subproject commit 3e85234341b95f7e6e45b31468311f01093ac970