新增-c cidr与 -p port-range的支持

go.mod

@@ -6,10 +6,9 @@ require (
 	github.com/chainreactors/files v0.2.5-0.20221212083256-16ee4c1ae47e
 	github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8
 	github.com/chainreactors/gogo/v2 v2.10.4
-	github.com/chainreactors/ipcs v0.0.13
 	github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580
 	github.com/chainreactors/parsers v0.3.1-0.20230208070438-6903b0d366c9
-	github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361
+	github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2
 )
 
 require (
@@ -24,6 +23,8 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/chainreactors/ipcs v0.0.13 // indirect
+	github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56 // indirect
 	github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect
 	github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect
 	github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect

go.sum

@@ -30,12 +30,18 @@ github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599 h1:9PwMZzN
 github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA=
 github.com/chainreactors/parsers v0.3.1-0.20230208070438-6903b0d366c9 h1:JCm8SmLb1jMFp5T6bBXKn3GmqPTjLxqWiz5yQKlo5Bs=
 github.com/chainreactors/parsers v0.3.1-0.20230208070438-6903b0d366c9/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA=
+github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56 h1:1uhvEh7Of4fQJXRMsfGEZGy5NcETsM2yataQ0oYSw0k=
+github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56/go.mod h1:NKSu1V6EC4wa8QHtPfiJHlH9VjGfUQOx5HADK0xry3Y=
 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U=
 github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
 github.com/chainreactors/words v0.4.1-0.20230203114605-f305deb098a2 h1:51GoU85MLp/s8IvXcKLeedSxypkvZBFJWIBUlGV+MiI=
 github.com/chainreactors/words v0.4.1-0.20230203114605-f305deb098a2/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
 github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361 h1:5KTlGw3NL3whK61GkrKY+WccI9+fVm/ctOQH+XBC5NM=
 github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
+github.com/chainreactors/words v0.4.1-0.20230327062919-c1467e05e43e h1:hAMj3xno5XFuwuj/sk7bqSZ1xUhKwb85vN6djOiBJ/E=
+github.com/chainreactors/words v0.4.1-0.20230327062919-c1467e05e43e/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
+github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2 h1:/v8gTORQIRJl2lgNt82OOeP/04QZyNTGKcmjfstVN5E=
+github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w=
 github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

internal/option.go

@@ -8,6 +8,7 @@ import (
 	"github.com/chainreactors/parsers/iutils"
 	"github.com/chainreactors/spray/pkg"
 	"github.com/chainreactors/spray/pkg/ihttp"
+	"github.com/chainreactors/utils"
 	"github.com/chainreactors/words/mask"
 	"github.com/chainreactors/words/rule"
 	"github.com/gosuri/uiprogress"
@@ -18,6 +19,13 @@ import (
 	"strings"
 )
 
+var (
+	DefaultThreads = 20
+	//DefaultTimeout = 5
+	//DefaultPoolSize = 5
+	//DefaultRateLimit = 0
+)
+
 type Option struct {
 	InputOptions    `group:"Input Options"`
 	FunctionOptions `group:"Function Options"`
@@ -29,10 +37,12 @@ type Option struct {
 }
 
 type InputOptions struct {
-	ResumeFrom string   `long:"resume"`
+	ResumeFrom   string   `long:"resume"`
-	URL        []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"`
+	URL          []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"`
-	URLFile    string   `short:"l" long:"list" description:"File, input filename"`
+	URLFile      string   `short:"l" long:"list" description:"File, input filename"`
-	//Raw          string   `long:"raw" description:"File, input raw request filename"`
+	PortRange    string   `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"`
+	CIDRs        string   `short:"c" long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "`
+	Raw          string   `long:"raw" description:"File, input raw request filename"`
 	Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt"`
 	Offset       int      `long:"offset" description:"Int, wordlist offset"`
 	Limit        int      `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"`
@@ -46,7 +56,7 @@ type FunctionOptions struct {
 	Extensions        string            `short:"e" long:"extension" description:"String, add extensions (separated by commas), e.g.: -e jsp,jspx"`
 	ExcludeExtensions string            `long:"exclude-extension" description:"String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx"`
 	RemoveExtensions  string            `long:"remove-extension" description:"String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx"`
-	Uppercase         bool              `short:"U" long:"uppercase" desvcription:"Bool, upper wordlist, e.g.: --uppercase"`
+	Uppercase         bool              `short:"U" long:"uppercase" description:"Bool, upper wordlist, e.g.: --uppercase"`
 	Lowercase         bool              `short:"L" long:"lowercase" description:"Bool, lower wordlist, e.g.: --lowercase"`
 	Prefixes          []string          `long:"prefix" description:"Strings, add prefix, e.g.: --prefix aaa --prefix bbb"`
 	Suffixes          []string          `long:"suffix" description:"Strings, add suffix, e.g.: --suffix aaa --suffix bbb"`
@@ -108,15 +118,15 @@ type ModeOptions struct {
 
 type MiscOptions struct {
 	Deadline int    `long:"deadline" default:"999999" description:"Int, deadline (seconds)"` // todo 总的超时时间,适配云函数的deadline
-	Timeout  int    `long:"timeout" default:"2" description:"Int, timeout with request (seconds)"`
+	Timeout  int    `long:"timeout" default:"5" description:"Int, timeout with request (seconds)"`
-	PoolSize int    `short:"p" long:"pool" default:"5" description:"Int, Pool size"`
+	PoolSize int    `short:"P" long:"pool" default:"5" description:"Int, Pool size"`
 	Threads  int    `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool"`
 	Debug    bool   `long:"debug" description:"Bool, output debug info"`
 	Quiet    bool   `short:"q" long:"quiet" description:"Bool, Quiet"`
 	NoColor  bool   `long:"no-color" description:"Bool, no color"`
 	NoBar    bool   `long:"no-bar" description:"Bool, No progress bar"`
 	Mod      string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray"`
-	Client   string `short:"c" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type"`
+	Client   string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type"`
 }
 
 func (opt *Option) PrepareRunner() (*Runner, error) {
@@ -183,6 +193,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		r.ClientType = ihttp.STANDARD
 	}
 
+	if opt.Threads == DefaultThreads && opt.CheckOnly {
+		r.Threads = 1000
+	}
 	if opt.Recon {
 		pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"]
 	}
@@ -324,40 +337,75 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		}
 		r.AppendRules = rule.Compile(string(content), "")
 	}
+
+	ports := utils.ParsePort(opt.PortRange)
+
 	// prepare task
-	var tasks []*Task
+	tasks := make(chan *Task, opt.PoolSize)
 	var taskfrom string
 	if opt.ResumeFrom != "" {
 		stats, err := pkg.ReadStatistors(opt.ResumeFrom)
 		if err != nil {
-			return nil, err
+			logs.Log.Error(err.Error())
 		}
+		r.Count = len(stats)
 		taskfrom = "resume " + opt.ResumeFrom
-		for _, stat := range stats {
+		go func() {
-			task := &Task{baseUrl: stat.BaseUrl, origin: stat}
+			for _, stat := range stats {
-			tasks = append(tasks, task)
+				tasks <- &Task{baseUrl: stat.BaseUrl, origin: stat}
-		}
+			}
+			close(tasks)
+		}()
 	} else {
 		var file *os.File
-		var urls []string
+
+		// 根据不同的输入类型生成任务
 		if len(opt.URL) == 1 {
 			u, err := url.Parse(opt.URL[0])
 			if err != nil {
 				u, _ = url.Parse("http://" + opt.URL[0])
 			}
-			urls = append(urls, u.String())
+			go opt.GenerateTasks(tasks, u.Hostname(), ports)
-			tasks = append(tasks, &Task{baseUrl: opt.URL[0]})
 			taskfrom = u.Host
+			r.Count = 1
 		} else if len(opt.URL) > 1 {
-			for _, u := range opt.URL {
+			go func() {
-				urls = append(urls, u)
+				for _, u := range opt.URL {
-				tasks = append(tasks, &Task{baseUrl: u})
+					opt.GenerateTasks(tasks, u, ports)
-			}
+				}
+				close(tasks)
+			}()
+
 			taskfrom = "cmd"
+			r.Count = len(opt.URL)
+		} else if opt.CIDRs != "" {
+			if len(ports) == 0 {
+				ports = []string{"80", "443"}
+			}
+
+			for _, cidr := range strings.Split(opt.CIDRs, ",") {
+				ips := utils.ParseCIDR(cidr)
+				if ips != nil {
+					r.Count += ips.Count()
+				}
+			}
+			go func() {
+				for _, cidr := range strings.Split(opt.CIDRs, ",") {
+					ips := utils.ParseCIDR(cidr)
+					if ips == nil {
+						logs.Log.Error("cidr format error: " + cidr)
+					}
+					for ip := range ips.Range() {
+						opt.GenerateTasks(tasks, ip.String(), ports)
+					}
+				}
+				close(tasks)
+			}()
+			taskfrom = "cidr"
 		} else if opt.URLFile != "" {
 			file, err = os.Open(opt.URLFile)
 			if err != nil {
-				return nil, err
+				logs.Log.Error(err.Error())
 			}
 			taskfrom = opt.URLFile
 		} else if pkg.HasStdin() {
@@ -368,21 +416,37 @@ func (opt *Option) PrepareRunner() (*Runner, error) {
 		if file != nil {
 			content, err := ioutil.ReadAll(file)
 			if err != nil {
-				return nil, err
+				logs.Log.Error(err.Error())
 			}
-			urls = strings.Split(strings.TrimSpace(string(content)), "\n")
+			urls := strings.Split(strings.TrimSpace(string(content)), "\n")
-			for i, u := range urls {
+			for _, u := range urls {
-				urls[i] = strings.TrimSpace(u)
+				if _, err := url.Parse(u); err != nil {
-				tasks = append(tasks, &Task{baseUrl: urls[i]})
+					r.Count++
+				} else if ip := utils.ParseIP(u); ip != nil {
+					r.Count++
+				} else if cidr := utils.ParseCIDR(u); cidr != nil {
+					r.Count += cidr.Count()
+				}
 			}
-		}
 
-		if opt.CheckOnly {
+			go func() {
-			r.URLList = urls
+				for _, u := range urls {
-			r.Total = len(r.URLList)
+					if _, err := url.Parse(u); err != nil {
+						opt.GenerateTasks(tasks, u, ports)
+					} else if ip := utils.ParseIP(u); ip != nil {
+						opt.GenerateTasks(tasks, u, ports)
+					} else if cidr := utils.ParseCIDR(u); cidr != nil {
+						for ip := range cidr.Range() {
+							opt.GenerateTasks(tasks, ip.String(), ports)
+						}
+					}
+				}
+				close(tasks)
+			}()
 		}
 	}
 
+	r.Count = r.Count * len(ports)
 	r.Tasks = tasks
 	logs.Log.Importantf("Loaded %d urls from %s", len(tasks), taskfrom)
 
@@ -542,3 +606,33 @@ func (opt *Option) Validate() bool {
 	}
 	return true
 }
+
+// Generate Tasks
+func (opt *Option) GenerateTasks(ch chan *Task, u string, ports []string) {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		logs.Log.Warn(err.Error())
+		return
+	}
+
+	if parsed.Scheme == "" {
+		if parsed.Port() == "443" {
+			parsed.Scheme = "https"
+		} else {
+			parsed.Scheme = "http"
+		}
+	}
+
+	if len(ports) == 0 {
+		ch <- &Task{baseUrl: u}
+		return
+	}
+
+	for _, p := range ports {
+		if parsed.Host == "" {
+			ch <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s", parsed.Scheme, parsed.Path, p)}
+		} else {
+			ch <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s/%s", parsed.Scheme, parsed.Host, p, parsed.Path)}
+		}
+	}
+}

internal/runner.go

@@ -36,8 +36,8 @@ type Runner struct {
 	bar      *uiprogress.Bar
 	finished int
 
-	Tasks           []*Task
+	Tasks           chan *Task
-	URLList         []string
+	Count           int // tasks total number
 	Wordlist        []string
 	Rules           *rule.Program
 	AppendRules     *rule.Program
@@ -66,7 +66,7 @@ type Runner struct {
 	Offset          int
 	Limit           int
 	RateLimit       int
-	Total           int
+	Total           int // wordlist total number
 	Deadline        int
 	CheckPeriod     int
 	ErrPeriod       int
@@ -132,16 +132,24 @@ func (r *Runner) Prepare(ctx context.Context) error {
 				r.poolwg.Done()
 				return
 			}
-			pool.worder = words.NewWorder(r.URLList)
+
+			ch := make(chan string)
+			go func() {
+				for t := range r.Tasks {
+					ch <- t.baseUrl
+				}
+				close(ch)
+			}()
+			pool.worder = words.NewWorderWithChan(ch)
 			pool.worder.Fns = r.Fns
-			pool.bar = pkg.NewBar("check", r.Total-r.Offset, r.Progress)
+			pool.bar = pkg.NewBar("check", r.Count-r.Offset, r.Progress)
-			pool.Run(ctx, r.Offset, r.Total)
+			pool.Run(ctx, r.Offset, r.Count)
 			r.poolwg.Done()
 		})
 	} else {
 		// spray 完整探测模式
 		go func() {
-			for _, t := range r.Tasks {
+			for t := range r.Tasks {
 				r.taskCh <- t
 			}
 			close(r.taskCh)
@@ -151,7 +159,7 @@ func (r *Runner) Prepare(ctx context.Context) error {
 			r.bar = r.Progress.AddBar(len(r.Tasks))
 			r.bar.PrependCompleted()
 			r.bar.PrependFunc(func(b *uiprogress.Bar) string {
-				return fmt.Sprintf("total progressive: %d/%d ", r.finished, len(r.Tasks))
+				return fmt.Sprintf("total progressive: %d/%d ", r.finished, r.Count)
 			})
 			r.bar.AppendElapsed()
 		}