admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-09-15 11:40:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

union load appendwords and dict

Browse Source
This commit is contained in:
M09Ic 2024-08-26 02:22:35 +08:00
parent 77a5e58a2a
commit 106f007693
7 changed files with 223 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
internal/option.go
View File

@ -2,7 +2,6 @@ package internal
import (
"bufio"
"bytes"
"errors"
"fmt"
"github.com/chainreactors/files"
@ -54,7 +53,6 @@ type InputOptions struct {
CIDRs []string `short:"i" long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "`
RawFile string `long:"raw" description:"File, input raw request filename"`
Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt" config:"dictionaries"`
PrintPreset bool `long:"print" description:"Bool, print preset all preset config "`
DefaultDict bool `short:"D" long:"default" description:"Bool, use default dictionary" config:"default"`
Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"`
Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt" config:"rules"`
@ -84,7 +82,6 @@ type OutputOptions struct {
Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter 'current.Body contains \"hello\"'" config:"filter"`
Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output" config:"fuzzy"`
OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty" config:"output-file"`
//FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty" config:"fuzzy-file"`
DumpFile string `long:"dump-file" description:"String, dump all request, and write to filename" config:"dump-file"`
Dump bool `long:"dump" description:"Bool, dump all request" config:"dump"`
AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" config:"auto-file"`
@ -122,7 +119,6 @@ type PluginOptions struct {
type ModeOptions struct {
RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100" config:"rate-limit"`
Force bool `long:"force" description:"Bool, skip error break" config:"force"`
//CheckOnly bool `long:"check-only" description:"Bool, check only" config:"check-only"`
NoScope bool `long:"no-scope" description:"Bool, no scope" config:"no-scope"`
Scope []string `long:"scope" description:"String, custom scope, e.g.: --scope *.example.com" config:"scope"`
Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()" config:"recursive"`
@ -153,6 +149,7 @@ type MiscOptions struct {
Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"`
Proxy string `long:"proxy" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"`
InitConfig bool `long:"init" description:"Bool, init config file"`
PrintPreset bool `long:"print" description:"Bool, print preset all preset config "`
}
func (opt *Option) Validate() error {
@ -240,18 +237,18 @@ func (opt *Option) Prepare() error {
ihttp.DefaultMaxBodySize = opt.MaxBodyLength * 1024
}
pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus)
pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus)
pkg.BlackStatus = pkg.ParseStatus(pkg.BlackStatus, opt.BlackStatus)
pkg.WhiteStatus = pkg.ParseStatus(pkg.WhiteStatus, opt.WhiteStatus)
if opt.FuzzyStatus == "all" {
pool.EnableAllFuzzy = true
} else {
pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus)
pkg.FuzzyStatus = pkg.ParseStatus(pkg.FuzzyStatus, opt.FuzzyStatus)
}
if opt.Unique {
pool.EnableAllUnique = true
} else {
pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus)
pkg.UniqueStatus = pkg.ParseStatus(pkg.UniqueStatus, opt.UniqueStatus)
}
pool.MaxCrawl = opt.CrawlDepth
@ -432,7 +429,7 @@ func (opt *Option) NewRunner() (*Runner, error) {
if opt.ResumeFrom != "" {
r.StatFile, err = files.NewFile(opt.ResumeFrom, false, true, true)
} else {
r.StatFile, err = files.NewFile(safeFilename(r.Tasks.Name)+".stat", false, true, true)
r.StatFile, err = files.NewFile(pkg.SafeFilename(r.Tasks.Name)+".stat", false, true, true)
}
if err != nil {
return nil, err
@ -500,7 +497,8 @@ func (opt *Option) BuildPlugin(r *Runner) error {
if opt.CommonPlugin {
r.bruteMod = true
r.AppendWords = append(r.AppendWords, pkg.GetPresetWordList([]string{"common_file", "log_file"})...)
r.AppendWords = append(r.AppendWords, pkg.Dicts["common"]...)
r.AppendWords = append(r.AppendWords, pkg.Dicts["log"]...)
}
if opt.ActivePlugin {
@ -527,20 +525,20 @@ func (opt *Option) BuildWords(r *Runner) error {
var dicts [][]string
var err error
if opt.DefaultDict {
//dicts = append(dicts, pkg.LoadDefaultDict())
//logs.Log.Info("use default dictionary: https://github.com/maurosoria/dirsearch/blob/master/db/dicc.txt")
dicts = append(dicts, pkg.Dicts["default"])
logs.Log.Info("use default dictionary: https://github.com/maurosoria/dirsearch/blob/master/db/dicc.txt")
}
for i, f := range opt.Dictionaries {
dict, err := loadFileToSlice(f)
dict, err := pkg.LoadFileToSlice(f)
if err != nil {
return err
}
dicts = append(dicts, dict)
if opt.ResumeFrom != "" {
dictCache[f] = dicts[i]
pkg.Dicts[f] = dicts[i]
}
logs.Log.Logf(pkg.LogVerbose, "Loaded %d word from %s", len(dicts[i]), f)
logs.Log.Logf(pkg.LogVerbose, "Loaded %d word from %s", len(dict), f)
}
if len(dicts) == 0 && opt.Word == "" && len(opt.Rules) == 0 && len(opt.AppendRule) == 0 {
@ -584,7 +582,7 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if len(opt.Rules) != 0 {
rules, err := loadRuleAndCombine(opt.Rules)
rules, err := pkg.LoadRuleAndCombine(opt.Rules)
if err != nil {
return err
}
@ -603,7 +601,7 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if len(opt.AppendRule) != 0 {
content, err := loadRuleAndCombine(opt.AppendRule)
content, err := pkg.LoadRuleAndCombine(opt.AppendRule)
if err != nil {
return err
}
@ -611,18 +609,13 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if len(opt.AppendFile) != 0 {
var bs bytes.Buffer
var lines []string
for _, f := range opt.AppendFile {
content, err := ioutil.ReadFile(f)
dict, err := pkg.LoadFileToSlice(f)
if err != nil {
return err
}
bs.Write(bytes.TrimSpace(content))
bs.WriteString("\n")
}
lines := strings.Split(bs.String(), "\n")
for i, line := range lines {
lines[i] = strings.TrimSpace(line)
lines = append(lines, dict...)
}
r.AppendWords = append(r.AppendWords, lines...)
}
@ -649,16 +642,16 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if opt.Uppercase {
r.AppendFunction(wrapWordsFunc(strings.ToUpper))
r.AppendFunction(pkg.WrapWordsFunc(strings.ToUpper))
}
if opt.Lowercase {
r.AppendFunction(wrapWordsFunc(strings.ToLower))
r.AppendFunction(pkg.WrapWordsFunc(strings.ToLower))
}
if opt.RemoveExtensions != "" {
rexts := strings.Split(opt.ExcludeExtensions, ",")
r.AppendFunction(func(s string) []string {
if ext := parseExtension(s); iutils.StringsContains(rexts, ext) {
if ext := pkg.ParseExtension(s); iutils.StringsContains(rexts, ext) {
return []string{strings.TrimSuffix(s, "."+ext)}
}
return []string{s}
@ -668,7 +661,7 @@ func (opt *Option) BuildWords(r *Runner) error {
if opt.ExcludeExtensions != "" {
exexts := strings.Split(opt.ExcludeExtensions, ",")
r.AppendFunction(func(s string) []string {
if ext := parseExtension(s); iutils.StringsContains(exexts, ext) {
if ext := pkg.ParseExtension(s); iutils.StringsContains(exexts, ext) {
return nil
}
return []string{s}
@ -702,7 +695,7 @@ func (opt *Option) BuildWords(r *Runner) error {
})
}
logs.Log.Logf(pkg.LogVerbose, "Loaded %d dictionaries and %d decorators", len(opt.Dictionaries), len(r.Fns))
logs.Log.Importantf("Loaded %d dictionaries, %d rules and %d decorators", len(opt.Dictionaries), len(opt.Rules), len(r.Fns))
return nil
}

8
internal/pool/brutepool.go
View File

@ -596,7 +596,13 @@ func (pool *BrutePool) doActive() {
func (pool *BrutePool) doCommonFile() {
defer pool.wg.Done()
for _, u := range pkg.GetPresetWordList([]string{"common_file", "log_file"}) {
for _, u := range pkg.Dicts["common"] {
pool.addAddition(&Unit{
path: pool.dir + u,
source: parsers.CommonFileSource,
})
}
for _, u := range pkg.Dicts["log"] {
pool.addAddition(&Unit{
path: pool.dir + u,
source: parsers.CommonFileSource,

6
internal/runner.go
View File

@ -20,12 +20,6 @@ var (
MAX = 2147483647
)
var (
dictCache = make(map[string][]string)
wordlistCache = make(map[string][]string)
ruleCache = make(map[string][]rule.Expression)
)
type Runner struct {
*Option

4
internal/types.go
View File

@ -16,13 +16,13 @@ type Origin struct {
func (o *Origin) InitWorder(fns []func(string) []string) (*words.Worder, error) {
var worder *words.Worder
wl, err := loadWordlist(o.Word, o.Dictionaries)
wl, err := pkg.LoadWordlist(o.Word, o.Dictionaries)
if err != nil {
return nil, err
}
worder = words.NewWorder(wl)
worder.Fns = fns
rules, err := loadRuleWithFiles(o.RuleFiles, o.RuleFilter)
rules, err := pkg.LoadRuleWithFiles(o.RuleFiles, o.RuleFilter)
if err != nil {
return nil, err
}

158
internal/utils.go
View File

@ -1,149 +1,5 @@
package internal
import (
"bytes"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/words/mask"
"github.com/chainreactors/words/rule"
"io/ioutil"
"strconv"
"strings"
)
func parseExtension(s string) string {
if i := strings.Index(s, "."); i != -1 {
return s[i+1:]
}
return ""
}
func parseStatus(preset []int, changed string) []int {
if changed == "" {
return preset
}
if strings.HasPrefix(changed, "+") {
for _, s := range strings.Split(changed[1:], ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
} else if strings.HasPrefix(changed, "!") {
for _, s := range strings.Split(changed[1:], ",") {
for i, status := range preset {
if t, err := strconv.Atoi(s); err != nil {
break
} else if t == status {
preset = append(preset[:i], preset[i+1:]...)
break
}
}
}
} else {
preset = []int{}
for _, s := range strings.Split(changed, ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
}
return preset
}
func loadFileToSlice(filename string) ([]string, error) {
var ss []string
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
ss = strings.Split(strings.TrimSpace(string(content)), "\n")
// 统一windows与linux的回车换行差异
for i, word := range ss {
ss[i] = strings.TrimSpace(word)
}
return ss, nil
}
func loadRuleAndCombine(filename []string) (string, error) {
var bs bytes.Buffer
for _, f := range filename {
if data, ok := pkg.Rules[f]; ok {
bs.WriteString(strings.TrimSpace(data))
bs.WriteString("\n")
} else {
content, err := ioutil.ReadFile(f)
if err != nil {
return "", err
}
bs.Write(bytes.TrimSpace(content))
bs.WriteString("\n")
}
}
return bs.String(), nil
}
func loadFileWithCache(filename string) ([]string, error) {
if dict, ok := dictCache[filename]; ok {
return dict, nil
}
dict, err := loadFileToSlice(filename)
if err != nil {
return nil, err
}
dictCache[filename] = dict
return dict, nil
}
func loadDictionaries(filenames []string) ([][]string, error) {
dicts := make([][]string, len(filenames))
for i, name := range filenames {
dict, err := loadFileWithCache(name)
if err != nil {
return nil, err
}
dicts[i] = dict
}
return dicts, nil
}
func loadWordlist(word string, dictNames []string) ([]string, error) {
if wl, ok := wordlistCache[word+strings.Join(dictNames, ",")]; ok {
return wl, nil
}
dicts, err := loadDictionaries(dictNames)
if err != nil {
return nil, err
}
wl, err := mask.Run(word, dicts, nil)
if err != nil {
return nil, err
}
wordlistCache[word] = wl
return wl, nil
}
func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, error) {
if rules, ok := ruleCache[strings.Join(ruleFiles, ",")]; ok {
return rules, nil
}
var rules bytes.Buffer
for _, filename := range ruleFiles {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
rules.Write(content)
rules.WriteString("\n")
}
return rule.Compile(rules.String(), filter).Expressions, nil
}
//type bytesPatcher struct{}
//
//func (p *bytesPatcher) Visit(node *ast.Node) {
@ -158,17 +14,3 @@ func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, er
// })
// }
//}
func wrapWordsFunc(f func(string) string) func(string) []string {
return func(s string) []string {
return []string{f(s)}
}
}
func safeFilename(filename string) string {
filename = strings.ReplaceAll(filename, "http://", "")
filename = strings.ReplaceAll(filename, "https://", "")
filename = strings.ReplaceAll(filename, ":", "_")
filename = strings.ReplaceAll(filename, "/", "_")
return filename
}

14
pkg/load.go
View File

@ -12,14 +12,6 @@ import (
"strings"
)
var (
ExtractRegexps = make(parsers.Extractors)
Extractors = make(parsers.Extractors)
FingerEngine *fingers.Engine
ActivePath []string
)
func LoadPorts() error {
var err error
var ports []*utils.PortConfig
@ -68,7 +60,11 @@ func LoadTemplates() error {
return err
}
for name, wordlist := range dicts {
Dicts[strings.TrimSuffix(name, ".txt")] = strings.Split(strings.TrimSpace(wordlist), "\n")
dict := strings.Split(strings.TrimSpace(wordlist), "\n")
for i, d := range dict {
dict[i] = strings.TrimSpace(d)
}
Dicts[strings.TrimSuffix(name, ".txt")] = dict
}
// load mask

165
pkg/utils.go
View File

@ -3,11 +3,16 @@ package pkg
import (
"bufio"
"bytes"
"github.com/chainreactors/files"
"github.com/chainreactors/fingers"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words/mask"
"github.com/chainreactors/words/rule"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
"io/ioutil"
"math/rand"
"net/http"
"net/url"
@ -34,9 +39,15 @@ var (
var (
Rules map[string]string = make(map[string]string)
Dicts map[string][]string = make(map[string][]string)
wordlistCache = make(map[string][]string)
ruleCache = make(map[string][]rule.Expression)
BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4", ".zip", ".rar"}
BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"}
ExtractRegexps = make(parsers.Extractors)
Extractors = make(parsers.Extractors)
FingerEngine *fingers.Engine
ActivePath []string
ContentTypeMap = map[string]string{
"application/javascript": "js",
"application/json": "json",
@ -403,3 +414,157 @@ func GetPresetWordList(key []string) []string {
}
return wordlist
}
func ParseExtension(s string) string {
if i := strings.Index(s, "."); i != -1 {
return s[i+1:]
}
return ""
}
func ParseStatus(preset []int, changed string) []int {
if changed == "" {
return preset
}
if strings.HasPrefix(changed, "+") {
for _, s := range strings.Split(changed[1:], ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
} else if strings.HasPrefix(changed, "!") {
for _, s := range strings.Split(changed[1:], ",") {
for i, status := range preset {
if t, err := strconv.Atoi(s); err != nil {
break
} else if t == status {
preset = append(preset[:i], preset[i+1:]...)
break
}
}
}
} else {
preset = []int{}
for _, s := range strings.Split(changed, ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
}
return preset
}
func LoadFileToSlice(filename string) ([]string, error) {
var ss []string
if dicts, ok := Dicts[filename]; ok {
if files.IsExist(filename) {
logs.Log.Warnf("load and overwrite %s from preset", filename)
}
return dicts, nil
}
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
ss = strings.Split(strings.TrimSpace(string(content)), "\n")
// 统一windows与linux的回车换行差异
for i, word := range ss {
ss[i] = strings.TrimSpace(word)
}
return ss, nil
}
func LoadRuleAndCombine(filename []string) (string, error) {
var bs bytes.Buffer
for _, f := range filename {
if data, ok := Rules[f]; ok {
bs.WriteString(strings.TrimSpace(data))
bs.WriteString("\n")
} else {
content, err := ioutil.ReadFile(f)
if err != nil {
return "", err
}
bs.Write(bytes.TrimSpace(content))
bs.WriteString("\n")
}
}
return bs.String(), nil
}
func loadFileWithCache(filename string) ([]string, error) {
if dict, ok := Dicts[filename]; ok {
return dict, nil
}
dict, err := LoadFileToSlice(filename)
if err != nil {
return nil, err
}
Dicts[filename] = dict
return dict, nil
}
func loadDictionaries(filenames []string) ([][]string, error) {
dicts := make([][]string, len(filenames))
for i, name := range filenames {
dict, err := loadFileWithCache(name)
if err != nil {
return nil, err
}
dicts[i] = dict
}
return dicts, nil
}
func LoadWordlist(word string, dictNames []string) ([]string, error) {
if wl, ok := wordlistCache[word+strings.Join(dictNames, ",")]; ok {
return wl, nil
}
dicts, err := loadDictionaries(dictNames)
if err != nil {
return nil, err
}
wl, err := mask.Run(word, dicts, nil)
if err != nil {
return nil, err
}
wordlistCache[word] = wl
return wl, nil
}
func LoadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, error) {
if rules, ok := ruleCache[strings.Join(ruleFiles, ",")]; ok {
return rules, nil
}
var rules bytes.Buffer
for _, filename := range ruleFiles {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
rules.Write(content)
rules.WriteString("\n")
}
return rule.Compile(rules.String(), filter).Expressions, nil
}
func WrapWordsFunc(f func(string) string) func(string) []string {
return func(s string) []string {
return []string{f(s)}
}
}
func SafeFilename(filename string) string {
filename = strings.ReplaceAll(filename, "http://", "")
filename = strings.ReplaceAll(filename, "https://", "")
filename = strings.ReplaceAll(filename, ":", "_")
filename = strings.ReplaceAll(filename, "/", "_")
return filename
}