diff --git a/go.mod b/go.mod index 01c6a39..878f80d 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,9 @@ require ( github.com/chainreactors/files v0.2.5-0.20221212083256-16ee4c1ae47e github.com/chainreactors/go-metrics v0.0.0-20220926021830-24787b7a10f8 github.com/chainreactors/gogo/v2 v2.10.4 - github.com/chainreactors/ipcs v0.0.13 github.com/chainreactors/logs v0.7.1-0.20221214153111-85f123ff6580 github.com/chainreactors/parsers v0.3.1-0.20230208070438-6903b0d366c9 - github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361 + github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2 ) require ( @@ -24,6 +23,8 @@ require ( require ( github.com/andybalholm/brotli v1.0.4 // indirect + github.com/chainreactors/ipcs v0.0.13 // indirect + github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect diff --git a/go.sum b/go.sum index ae1cbda..e8bb00b 100644 --- a/go.sum +++ b/go.sum @@ -30,12 +30,18 @@ github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599 h1:9PwMZzN github.com/chainreactors/parsers v0.3.1-0.20230204104401-6e150669e599/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= github.com/chainreactors/parsers v0.3.1-0.20230208070438-6903b0d366c9 h1:JCm8SmLb1jMFp5T6bBXKn3GmqPTjLxqWiz5yQKlo5Bs= github.com/chainreactors/parsers v0.3.1-0.20230208070438-6903b0d366c9/go.mod h1:tA33N6UbYFnIT3k5tufOMfETxmEP20RZFyTSEnVXNUA= +github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56 h1:1uhvEh7Of4fQJXRMsfGEZGy5NcETsM2yataQ0oYSw0k= +github.com/chainreactors/utils v0.0.14-0.20230314084720-a4d745cabc56/go.mod h1:NKSu1V6EC4wa8QHtPfiJHlH9VjGfUQOx5HADK0xry3Y= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a h1:vRAMDJ6UQV73uyiRBQnuE/+S7Q7JTpfubSpyRlooZ2U= github.com/chainreactors/words v0.3.2-0.20230105161651-7c1fc4c9605a/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/chainreactors/words v0.4.1-0.20230203114605-f305deb098a2 h1:51GoU85MLp/s8IvXcKLeedSxypkvZBFJWIBUlGV+MiI= github.com/chainreactors/words v0.4.1-0.20230203114605-f305deb098a2/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361 h1:5KTlGw3NL3whK61GkrKY+WccI9+fVm/ctOQH+XBC5NM= github.com/chainreactors/words v0.4.1-0.20230203115443-ca934844e361/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/words v0.4.1-0.20230327062919-c1467e05e43e h1:hAMj3xno5XFuwuj/sk7bqSZ1xUhKwb85vN6djOiBJ/E= +github.com/chainreactors/words v0.4.1-0.20230327062919-c1467e05e43e/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= +github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2 h1:/v8gTORQIRJl2lgNt82OOeP/04QZyNTGKcmjfstVN5E= +github.com/chainreactors/words v0.4.1-0.20230327065326-448a905ac8c2/go.mod h1:QIWX1vMT5j/Mp9zx3/wgZh3FqskhjCbo/3Ffy/Hxj9w= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/option.go b/internal/option.go index 0eb8013..de601b4 100644 --- a/internal/option.go +++ b/internal/option.go @@ -8,6 +8,7 @@ import ( "github.com/chainreactors/parsers/iutils" "github.com/chainreactors/spray/pkg" "github.com/chainreactors/spray/pkg/ihttp" + "github.com/chainreactors/utils" "github.com/chainreactors/words/mask" "github.com/chainreactors/words/rule" "github.com/gosuri/uiprogress" @@ -18,6 +19,13 @@ import ( "strings" ) +var ( + DefaultThreads = 20 + //DefaultTimeout = 5 + //DefaultPoolSize = 5 + //DefaultRateLimit = 0 +) + type Option struct { InputOptions `group:"Input Options"` FunctionOptions `group:"Function Options"` @@ -29,10 +37,12 @@ type Option struct { } type InputOptions struct { - ResumeFrom string `long:"resume"` - URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` - URLFile string `short:"l" long:"list" description:"File, input filename"` - //Raw string `long:"raw" description:"File, input raw request filename"` + ResumeFrom string `long:"resume"` + URL []string `short:"u" long:"url" description:"Strings, input baseurl, e.g.: http://google.com"` + URLFile string `short:"l" long:"list" description:"File, input filename"` + PortRange string `short:"p" long:"port" description:"String, input port range, e.g.: 80,8080-8090,db"` + CIDRs string `short:"c" long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "` + Raw string `long:"raw" description:"File, input raw request filename"` Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt"` Offset int `long:"offset" description:"Int, wordlist offset"` Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"` @@ -46,7 +56,7 @@ type FunctionOptions struct { Extensions string `short:"e" long:"extension" description:"String, add extensions (separated by commas), e.g.: -e jsp,jspx"` ExcludeExtensions string `long:"exclude-extension" description:"String, exclude extensions (separated by commas), e.g.: --exclude-extension jsp,jspx"` RemoveExtensions string `long:"remove-extension" description:"String, remove extensions (separated by commas), e.g.: --remove-extension jsp,jspx"` - Uppercase bool `short:"U" long:"uppercase" desvcription:"Bool, upper wordlist, e.g.: --uppercase"` + Uppercase bool `short:"U" long:"uppercase" description:"Bool, upper wordlist, e.g.: --uppercase"` Lowercase bool `short:"L" long:"lowercase" description:"Bool, lower wordlist, e.g.: --lowercase"` Prefixes []string `long:"prefix" description:"Strings, add prefix, e.g.: --prefix aaa --prefix bbb"` Suffixes []string `long:"suffix" description:"Strings, add suffix, e.g.: --suffix aaa --suffix bbb"` @@ -108,15 +118,15 @@ type ModeOptions struct { type MiscOptions struct { Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)"` // todo 总的超时时间,适配云函数的deadline - Timeout int `long:"timeout" default:"2" description:"Int, timeout with request (seconds)"` - PoolSize int `short:"p" long:"pool" default:"5" description:"Int, Pool size"` + Timeout int `long:"timeout" default:"5" description:"Int, timeout with request (seconds)"` + PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size"` Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool"` Debug bool `long:"debug" description:"Bool, output debug info"` Quiet bool `short:"q" long:"quiet" description:"Bool, Quiet"` NoColor bool `long:"no-color" description:"Bool, no color"` NoBar bool `long:"no-bar" description:"Bool, No progress bar"` Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray"` - Client string `short:"c" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type"` + Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type"` } func (opt *Option) PrepareRunner() (*Runner, error) { @@ -183,6 +193,9 @@ func (opt *Option) PrepareRunner() (*Runner, error) { r.ClientType = ihttp.STANDARD } + if opt.Threads == DefaultThreads && opt.CheckOnly { + r.Threads = 1000 + } if opt.Recon { pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"] } @@ -324,40 +337,75 @@ func (opt *Option) PrepareRunner() (*Runner, error) { } r.AppendRules = rule.Compile(string(content), "") } + + ports := utils.ParsePort(opt.PortRange) + // prepare task - var tasks []*Task + tasks := make(chan *Task, opt.PoolSize) var taskfrom string if opt.ResumeFrom != "" { stats, err := pkg.ReadStatistors(opt.ResumeFrom) if err != nil { - return nil, err + logs.Log.Error(err.Error()) } + r.Count = len(stats) taskfrom = "resume " + opt.ResumeFrom - for _, stat := range stats { - task := &Task{baseUrl: stat.BaseUrl, origin: stat} - tasks = append(tasks, task) - } + go func() { + for _, stat := range stats { + tasks <- &Task{baseUrl: stat.BaseUrl, origin: stat} + } + close(tasks) + }() } else { var file *os.File - var urls []string + + // 根据不同的输入类型生成任务 if len(opt.URL) == 1 { u, err := url.Parse(opt.URL[0]) if err != nil { u, _ = url.Parse("http://" + opt.URL[0]) } - urls = append(urls, u.String()) - tasks = append(tasks, &Task{baseUrl: opt.URL[0]}) + go opt.GenerateTasks(tasks, u.Hostname(), ports) taskfrom = u.Host + r.Count = 1 } else if len(opt.URL) > 1 { - for _, u := range opt.URL { - urls = append(urls, u) - tasks = append(tasks, &Task{baseUrl: u}) - } + go func() { + for _, u := range opt.URL { + opt.GenerateTasks(tasks, u, ports) + } + close(tasks) + }() + taskfrom = "cmd" + r.Count = len(opt.URL) + } else if opt.CIDRs != "" { + if len(ports) == 0 { + ports = []string{"80", "443"} + } + + for _, cidr := range strings.Split(opt.CIDRs, ",") { + ips := utils.ParseCIDR(cidr) + if ips != nil { + r.Count += ips.Count() + } + } + go func() { + for _, cidr := range strings.Split(opt.CIDRs, ",") { + ips := utils.ParseCIDR(cidr) + if ips == nil { + logs.Log.Error("cidr format error: " + cidr) + } + for ip := range ips.Range() { + opt.GenerateTasks(tasks, ip.String(), ports) + } + } + close(tasks) + }() + taskfrom = "cidr" } else if opt.URLFile != "" { file, err = os.Open(opt.URLFile) if err != nil { - return nil, err + logs.Log.Error(err.Error()) } taskfrom = opt.URLFile } else if pkg.HasStdin() { @@ -368,21 +416,37 @@ func (opt *Option) PrepareRunner() (*Runner, error) { if file != nil { content, err := ioutil.ReadAll(file) if err != nil { - return nil, err + logs.Log.Error(err.Error()) } - urls = strings.Split(strings.TrimSpace(string(content)), "\n") - for i, u := range urls { - urls[i] = strings.TrimSpace(u) - tasks = append(tasks, &Task{baseUrl: urls[i]}) + urls := strings.Split(strings.TrimSpace(string(content)), "\n") + for _, u := range urls { + if _, err := url.Parse(u); err != nil { + r.Count++ + } else if ip := utils.ParseIP(u); ip != nil { + r.Count++ + } else if cidr := utils.ParseCIDR(u); cidr != nil { + r.Count += cidr.Count() + } } - } - if opt.CheckOnly { - r.URLList = urls - r.Total = len(r.URLList) + go func() { + for _, u := range urls { + if _, err := url.Parse(u); err != nil { + opt.GenerateTasks(tasks, u, ports) + } else if ip := utils.ParseIP(u); ip != nil { + opt.GenerateTasks(tasks, u, ports) + } else if cidr := utils.ParseCIDR(u); cidr != nil { + for ip := range cidr.Range() { + opt.GenerateTasks(tasks, ip.String(), ports) + } + } + } + close(tasks) + }() } } + r.Count = r.Count * len(ports) r.Tasks = tasks logs.Log.Importantf("Loaded %d urls from %s", len(tasks), taskfrom) @@ -542,3 +606,33 @@ func (opt *Option) Validate() bool { } return true } + +// Generate Tasks +func (opt *Option) GenerateTasks(ch chan *Task, u string, ports []string) { + parsed, err := url.Parse(u) + if err != nil { + logs.Log.Warn(err.Error()) + return + } + + if parsed.Scheme == "" { + if parsed.Port() == "443" { + parsed.Scheme = "https" + } else { + parsed.Scheme = "http" + } + } + + if len(ports) == 0 { + ch <- &Task{baseUrl: u} + return + } + + for _, p := range ports { + if parsed.Host == "" { + ch <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s", parsed.Scheme, parsed.Path, p)} + } else { + ch <- &Task{baseUrl: fmt.Sprintf("%s://%s:%s/%s", parsed.Scheme, parsed.Host, p, parsed.Path)} + } + } +} diff --git a/internal/runner.go b/internal/runner.go index 03f847b..8dcf928 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -36,8 +36,8 @@ type Runner struct { bar *uiprogress.Bar finished int - Tasks []*Task - URLList []string + Tasks chan *Task + Count int // tasks total number Wordlist []string Rules *rule.Program AppendRules *rule.Program @@ -66,7 +66,7 @@ type Runner struct { Offset int Limit int RateLimit int - Total int + Total int // wordlist total number Deadline int CheckPeriod int ErrPeriod int @@ -132,16 +132,24 @@ func (r *Runner) Prepare(ctx context.Context) error { r.poolwg.Done() return } - pool.worder = words.NewWorder(r.URLList) + + ch := make(chan string) + go func() { + for t := range r.Tasks { + ch <- t.baseUrl + } + close(ch) + }() + pool.worder = words.NewWorderWithChan(ch) pool.worder.Fns = r.Fns - pool.bar = pkg.NewBar("check", r.Total-r.Offset, r.Progress) - pool.Run(ctx, r.Offset, r.Total) + pool.bar = pkg.NewBar("check", r.Count-r.Offset, r.Progress) + pool.Run(ctx, r.Offset, r.Count) r.poolwg.Done() }) } else { // spray 完整探测模式 go func() { - for _, t := range r.Tasks { + for t := range r.Tasks { r.taskCh <- t } close(r.taskCh) @@ -151,7 +159,7 @@ func (r *Runner) Prepare(ctx context.Context) error { r.bar = r.Progress.AddBar(len(r.Tasks)) r.bar.PrependCompleted() r.bar.PrependFunc(func(b *uiprogress.Bar) string { - return fmt.Sprintf("total progressive: %d/%d ", r.finished, len(r.Tasks)) + return fmt.Sprintf("total progressive: %d/%d ", r.finished, r.Count) }) r.bar.AppendElapsed() }