admin/spray
1
0
Fork 0
mirror of https://github.com/chainreactors/spray.git synced 2025-05-05 10:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #65 from chainreactors/dev

Browse Source 
merge v1.1.0
This commit is contained in:
M09Ic 2024-08-27 14:30:40 +08:00 committed by GitHub
commit f6037d7a1e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 564 additions and 530 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/cmd.go
View File

@ -8,6 +8,7 @@ import (
"github.com/chainreactors/spray/internal"
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/utils/iutils"
"github.com/jessevdk/go-flags"
"os"
"os/signal"
@ -112,8 +113,23 @@ func Spray() {
return
}
if option.PrintPreset {
err = pkg.Load()
if err != nil {
iutils.Fatal(err.Error())
}
err = pkg.LoadFingers()
if err != nil {
iutils.Fatal(err.Error())
}
internal.PrintPreset()
return
}
if option.Format != "" {
internal.Format(option.Format, !option.NoColor)
internal.Format(option)
return
}
@ -128,7 +144,7 @@ func Spray() {
logs.Log.Errorf(err.Error())
return
}
if option.ReadAll || runner.Crawl {
if option.ReadAll || runner.CrawlPlugin {
ihttp.DefaultMaxBodySize = -1
}

64
internal/format.go
View File

@ -5,23 +5,26 @@ import (
"encoding/json"
"github.com/chainreactors/logs"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/words/mask"
"io"
"net/url"
"os"
"strings"
)
func Format(filename string, color bool) {
func Format(opts Option) {
var content []byte
var err error
if filename == "stdin" {
if opts.Format == "stdin" {
content, err = io.ReadAll(os.Stdin)
} else {
content, err = os.ReadFile(filename)
content, err = os.ReadFile(opts.Format)
}
if err != nil {
return
}
var results []*pkg.Baseline
group := make(map[string][]*pkg.Baseline)
for _, line := range bytes.Split(bytes.TrimSpace(content), []byte("\n")) {
var result pkg.Baseline
err := json.Unmarshal(line, &result)
@ -29,13 +32,54 @@ func Format(filename string, color bool) {
logs.Log.Error(err.Error())
return
}
results = append(results, &result)
result.Url, err = url.Parse(result.UrlString)
if err != nil {
continue
}
group[result.Url.Host] = append(group[result.Url.Host], &result)
}
for _, result := range results {
if color {
logs.Log.Info(result.ColorString())
} else {
logs.Log.Info(result.String())
// 分组
for _, results := range group {
for _, result := range results {
if !opts.Fuzzy && result.IsFuzzy {
continue
}
if !opts.NoColor {
logs.Log.Console(result.ColorString() + "\n")
} else {
logs.Log.Console(result.String() + "\n")
}
}
}
}
func PrintPreset() {
logs.Log.Console("internal rules:\n")
for name, rule := range pkg.Rules {
logs.Log.Consolef("\t%s\t%d rules\n", name, len(strings.Split(rule, "\n")))
}
logs.Log.Console("\ninternal dicts:\n")
for name, dict := range pkg.Dicts {
logs.Log.Consolef("\t%s\t%d items\n", name, len(dict))
}
logs.Log.Console("\ninternal words keyword:\n")
for name, words := range mask.SpecialWords {
logs.Log.Consolef("\t%s\t%d words\n", name, len(words))
}
logs.Log.Console("\ninternal extractor:\n")
for name, _ := range pkg.ExtractRegexps {
logs.Log.Consolef("\t%s\n", name)
}
logs.Log.Console("\ninternal fingers:\n")
for name, engine := range pkg.FingerEngine.EnginesImpl {
logs.Log.Consolef("\t%s\t%d fingerprints \n", name, engine.Len())
}
logs.Log.Consolef("\nload %d active path\n", len(pkg.ActivePath))
}

4
internal/ihttp/client.go
View File

@ -62,10 +62,8 @@ func NewClient(config *ClientConfig) *Client {
client = &Client{
standardClient: &http.Client{
Transport: &http.Transport{
//Proxy: Proxy,
//TLSHandshakeTimeout : delay * time.Second,
TLSClientConfig: &tls.Config{
Renegotiation: tls.RenegotiateOnceAsClient,
Renegotiation: tls.RenegotiateNever,
InsecureSkipVerify: true,
},
TLSHandshakeTimeout: config.Timeout,

240
internal/option.go
View File

@ -2,7 +2,6 @@ package internal
import (
"bufio"
"bytes"
"errors"
"fmt"
"github.com/chainreactors/files"
@ -54,15 +53,14 @@ type InputOptions struct {
CIDRs []string `short:"i" long:"cidr" description:"String, input cidr, e.g.: 1.1.1.1/24 "`
RawFile string `long:"raw" description:"File, input raw request filename"`
Dictionaries []string `short:"d" long:"dict" description:"Files, Multi,dict files, e.g.: -d 1.txt -d 2.txt" config:"dictionaries"`
//NoDict bool `long:"no-dict" description:"Bool, no dictionary" config:"no-dict"`
DefaultDict bool `short:"D" long:"default" description:"Bool, use default dictionary" config:"default"`
Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"`
Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt" config:"rules"`
AppendRule []string `long:"append-rule" description:"Files, when found valid path , use append rule generator new word with current path" config:"append-rules"`
FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'" config:"filter-rule"`
AppendFile []string `long:"append-file" description:"Files, when found valid path , use append file new word with current path" config:"append-files"`
Offset int `long:"offset" description:"Int, wordlist offset"`
Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"`
DefaultDict bool `short:"D" long:"default" description:"Bool, use default dictionary" config:"default"`
Word string `short:"w" long:"word" description:"String, word generate dsl, e.g.: -w test{?ld#4}" config:"word"`
Rules []string `short:"r" long:"rules" description:"Files, rule files, e.g.: -r rule1.txt -r rule2.txt" config:"rules"`
AppendRule []string `long:"append-rule" description:"Files, when found valid path , use append rule generator new word with current path" config:"append-rules"`
FilterRule string `long:"filter-rule" description:"String, filter rule, e.g.: --rule-filter '>8 <4'" config:"filter-rule"`
AppendFile []string `long:"append" description:"Files, when found valid path , use append file new word with current path" config:"append-files"`
Offset int `long:"offset" description:"Int, wordlist offset"`
Limit int `long:"limit" description:"Int, wordlist limit, start with offset. e.g.: --offset 1000 --limit 100"`
}
type FunctionOptions struct {
@ -84,7 +82,6 @@ type OutputOptions struct {
Filter string `long:"filter" description:"String, custom filter function, e.g.: --filter 'current.Body contains \"hello\"'" config:"filter"`
Fuzzy bool `long:"fuzzy" description:"String, open fuzzy output" config:"fuzzy"`
OutputFile string `short:"f" long:"file" description:"String, output filename" json:"output_file,omitempty" config:"output-file"`
FuzzyFile string `long:"fuzzy-file" description:"String, fuzzy output filename" json:"fuzzy_file,omitempty" config:"fuzzy-file"`
DumpFile string `long:"dump-file" description:"String, dump all request, and write to filename" config:"dump-file"`
Dump bool `long:"dump" description:"Bool, dump all request" config:"dump"`
AutoFile bool `long:"auto-file" description:"Bool, auto generator output and fuzzy filename" config:"auto-file"`
@ -111,19 +108,17 @@ type PluginOptions struct {
Advance bool `short:"a" long:"advance" description:"Bool, enable all plugin" config:"all" `
Extracts []string `long:"extract" description:"Strings, extract response, e.g.: --extract js --extract ip --extract version:(.*?)" config:"extract"`
ExtractConfig string `long:"extract-config" description:"String, extract config filename" config:"extract-config"`
Active bool `long:"active" description:"Bool, enable active finger path"`
Recon bool `long:"recon" description:"Bool, enable recon" config:"recon"`
Bak bool `long:"bak" description:"Bool, enable bak found" config:"bak"`
FileBak bool `long:"file-bak" description:"Bool, enable valid result bak found, equal --append-rule rule/filebak.txt" config:"file-bak"`
Common bool `long:"common" description:"Bool, enable common file found" config:"common"`
Crawl bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"`
ActivePlugin bool `long:"active" description:"Bool, enable active finger path"`
ReconPlugin bool `long:"recon" description:"Bool, enable recon" config:"recon"`
BakPlugin bool `long:"bak" description:"Bool, enable bak found" config:"bak"`
CommonPlugin bool `long:"common" description:"Bool, enable common file found" config:"common"`
CrawlPlugin bool `long:"crawl" description:"Bool, enable crawl" config:"crawl"`
CrawlDepth int `long:"crawl-depth" default:"3" description:"Int, crawl depth" config:"crawl-depth"`
}
type ModeOptions struct {
RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100" config:"rate-limit"`
Force bool `long:"force" description:"Bool, skip error break" config:"force"`
//CheckOnly bool `long:"check-only" description:"Bool, check only" config:"check-only"`
RateLimit int `long:"rate-limit" default:"0" description:"Int, request rate limit (rate/s), e.g.: --rate-limit 100" config:"rate-limit"`
Force bool `long:"force" description:"Bool, skip error break" config:"force"`
NoScope bool `long:"no-scope" description:"Bool, no scope" config:"no-scope"`
Scope []string `long:"scope" description:"String, custom scope, e.g.: --scope *.example.com" config:"scope"`
Recursive string `long:"recursive" default:"current.IsDir()" description:"String,custom recursive rule, e.g.: --recursive current.IsDir()" config:"recursive"`
@ -135,7 +130,7 @@ type ModeOptions struct {
BreakThreshold int `long:"error-threshold" default:"20" description:"Int, break when the error exceeds the threshold" config:"error-threshold"`
BlackStatus string `long:"black-status" default:"400,410" description:"Strings (comma split),custom black status" config:"black-status"`
WhiteStatus string `long:"white-status" default:"200" description:"Strings (comma split), custom white status" config:"white-status"`
FuzzyStatus string `long:"fuzzy-status" default:"500,501,502,503" description:"Strings (comma split), custom fuzzy status" config:"fuzzy-status"`
FuzzyStatus string `long:"fuzzy-status" default:"500,501,502,503,301,302" description:"Strings (comma split), custom fuzzy status" config:"fuzzy-status"`
UniqueStatus string `long:"unique-status" default:"403,200,404" description:"Strings (comma split), custom unique status" config:"unique-status"`
Unique bool `long:"unique" description:"Bool, unique response" config:"unique"`
RetryCount int `long:"retry" default:"0" description:"Int, retry count" config:"retry"`
@ -143,17 +138,18 @@ type ModeOptions struct {
}
type MiscOptions struct {
Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray" config:"mod"`
Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type" config:"client"`
Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)" config:"deadline"` // todo 总的超时时间,适配云函数的deadline
Timeout int `short:"T" long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"`
PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size" config:"pool"`
Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool" config:"thread"`
Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"`
Version bool `long:"version" description:"Bool, show version"`
Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"`
Proxy string `long:"proxy" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"`
InitConfig bool `long:"init" description:"Bool, init config file"`
Mod string `short:"m" long:"mod" default:"path" choice:"path" choice:"host" description:"String, path/host spray" config:"mod"`
Client string `short:"C" long:"client" default:"auto" choice:"fast" choice:"standard" choice:"auto" description:"String, Client type" config:"client"`
Deadline int `long:"deadline" default:"999999" description:"Int, deadline (seconds)" config:"deadline"` // todo 总的超时时间,适配云函数的deadline
Timeout int `short:"T" long:"timeout" default:"5" description:"Int, timeout with request (seconds)" config:"timeout"`
PoolSize int `short:"P" long:"pool" default:"5" description:"Int, Pool size" config:"pool"`
Threads int `short:"t" long:"thread" default:"20" description:"Int, number of threads per pool" config:"thread"`
Debug bool `long:"debug" description:"Bool, output debug info" config:"debug"`
Version bool `long:"version" description:"Bool, show version"`
Verbose []bool `short:"v" description:"Bool, log verbose level ,default 0, level1: -v level2 -vv " config:"verbose"`
Proxy string `long:"proxy" description:"String, proxy address, e.g.: --proxy socks5://127.0.0.1:1080" config:"proxy"`
InitConfig bool `long:"init" description:"Bool, init config file"`
PrintPreset bool `long:"print" description:"Bool, print preset all preset config "`
}
func (opt *Option) Validate() error {
@ -241,18 +237,18 @@ func (opt *Option) Prepare() error {
ihttp.DefaultMaxBodySize = opt.MaxBodyLength * 1024
}
pkg.BlackStatus = parseStatus(pkg.BlackStatus, opt.BlackStatus)
pkg.WhiteStatus = parseStatus(pkg.WhiteStatus, opt.WhiteStatus)
pkg.BlackStatus = pkg.ParseStatus(pkg.BlackStatus, opt.BlackStatus)
pkg.WhiteStatus = pkg.ParseStatus(pkg.WhiteStatus, opt.WhiteStatus)
if opt.FuzzyStatus == "all" {
pool.EnableAllFuzzy = true
} else {
pkg.FuzzyStatus = parseStatus(pkg.FuzzyStatus, opt.FuzzyStatus)
pkg.FuzzyStatus = pkg.ParseStatus(pkg.FuzzyStatus, opt.FuzzyStatus)
}
if opt.Unique {
pool.EnableAllUnique = true
} else {
pkg.UniqueStatus = parseStatus(pkg.UniqueStatus, opt.UniqueStatus)
pkg.UniqueStatus = pkg.ParseStatus(pkg.UniqueStatus, opt.UniqueStatus)
}
pool.MaxCrawl = opt.CrawlDepth
@ -265,6 +261,7 @@ func (opt *Option) NewRunner() (*Runner, error) {
Option: opt,
taskCh: make(chan *Task),
outputCh: make(chan *pkg.Baseline, 256),
poolwg: &sync.WaitGroup{},
outwg: &sync.WaitGroup{},
fuzzyCh: make(chan *pkg.Baseline, 256),
Headers: make(map[string]string),
@ -309,52 +306,9 @@ func (opt *Option) NewRunner() (*Runner, error) {
r.Threads = 1000
}
if opt.Recon {
pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"]
}
if opt.Finger {
pkg.EnableAllFingerEngine = true
}
// brute only
if opt.Advance {
r.Crawl = true
r.Finger = true
r.Bak = true
r.Common = true
r.Active = true
pkg.EnableAllFingerEngine = true
pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"]
r.bruteMod = true
opt.AppendRule = append(opt.AppendRule, "filebak")
}
if opt.FileBak {
r.bruteMod = true
opt.AppendRule = append(opt.AppendRule, "filebak")
}
if opt.Common {
r.bruteMod = true
r.AppendWords = append(r.AppendWords, mask.SpecialWords["common_file"]...)
}
if opt.Active {
r.bruteMod = true
r.AppendWords = append(r.AppendWords, pkg.ActivePath...)
}
if opt.Crawl {
r.bruteMod = true
}
opt.PrintPlugin()
if r.bruteMod {
logs.Log.Important("enabling brute mod, because of enabled brute plugin")
}
if opt.NoScope {
r.Scope = []string{"*"}
err = opt.BuildPlugin(r)
if err != nil {
return nil, err
}
err = opt.BuildWords(r)
@ -449,17 +403,17 @@ func (opt *Option) NewRunner() (*Runner, error) {
}
}
if opt.FuzzyFile != "" {
r.FuzzyFile, err = files.NewFile(opt.FuzzyFile, false, false, true)
if err != nil {
return nil, err
}
} else if opt.AutoFile {
r.FuzzyFile, err = files.NewFile("fuzzy.json", false, false, true)
if err != nil {
return nil, err
}
}
//if opt.FuzzyFile != "" {
// r.FuzzyFile, err = files.NewFile(opt.FuzzyFile, false, false, true)
// if err != nil {
// return nil, err
// }
//} else if opt.AutoFile {
// r.FuzzyFile, err = files.NewFile("fuzzy.json", false, false, true)
// if err != nil {
// return nil, err
// }
//}
if opt.DumpFile != "" {
r.DumpFile, err = files.NewFile(opt.DumpFile, false, false, true)
@ -475,7 +429,7 @@ func (opt *Option) NewRunner() (*Runner, error) {
if opt.ResumeFrom != "" {
r.StatFile, err = files.NewFile(opt.ResumeFrom, false, true, true)
} else {
r.StatFile, err = files.NewFile(safeFilename(r.Tasks.Name)+".stat", false, true, true)
r.StatFile, err = files.NewFile(pkg.SafeFilename(r.Tasks.Name)+".stat", false, true, true)
}
if err != nil {
return nil, err
@ -493,25 +447,21 @@ func (opt *Option) NewRunner() (*Runner, error) {
func (opt *Option) PrintPlugin() {
var s strings.Builder
if opt.Crawl {
if opt.CrawlPlugin {
s.WriteString("crawl enable; ")
}
if opt.Finger {
s.WriteString("active fingerprint enable; ")
}
if opt.Bak {
if opt.BakPlugin {
s.WriteString("bak file enable; ")
}
if opt.Common {
if opt.CommonPlugin {
s.WriteString("common file enable; ")
}
if opt.Recon {
if opt.ReconPlugin {
s.WriteString("recon enable; ")
}
if opt.FileBak {
s.WriteString("file bak enable; ")
}
if opt.RetryCount > 0 {
s.WriteString("Retry Count: " + strconv.Itoa(opt.RetryCount))
}
@ -521,27 +471,78 @@ func (opt *Option) PrintPlugin() {
}
}
func (opt *Option) BuildPlugin(r *Runner) error {
// brute only
if opt.Advance {
opt.CrawlPlugin = true
opt.Finger = true
opt.BakPlugin = true
opt.CommonPlugin = true
opt.ActivePlugin = true
opt.ReconPlugin = true
}
if opt.ReconPlugin {
pkg.Extractors["recon"] = pkg.ExtractRegexps["pentest"]
}
if opt.Finger {
pkg.EnableAllFingerEngine = true
}
if opt.BakPlugin {
r.bruteMod = true
opt.AppendRule = append(opt.AppendRule, "filebak")
r.AppendWords = append(r.AppendWords, pkg.GetPresetWordList([]string{"bak_file"})...)
}
if opt.CommonPlugin {
r.bruteMod = true
r.AppendWords = append(r.AppendWords, pkg.Dicts["common"]...)
r.AppendWords = append(r.AppendWords, pkg.Dicts["log"]...)
}
if opt.ActivePlugin {
r.bruteMod = true
r.AppendWords = append(r.AppendWords, pkg.ActivePath...)
}
if opt.CrawlPlugin {
r.bruteMod = true
}
opt.PrintPlugin()
if r.bruteMod {
logs.Log.Important("enabling brute mod, because of enabled brute plugin")
}
if opt.NoScope {
r.Scope = []string{"*"}
}
return nil
}
func (opt *Option) BuildWords(r *Runner) error {
var dicts [][]string
var err error
if opt.DefaultDict {
dicts = append(dicts, pkg.LoadDefaultDict())
dicts = append(dicts, pkg.Dicts["default"])
logs.Log.Info("use default dictionary: https://github.com/maurosoria/dirsearch/blob/master/db/dicc.txt")
}
for i, f := range opt.Dictionaries {
dict, err := loadFileToSlice(f)
dict, err := pkg.LoadFileToSlice(f)
if err != nil {
return err
}
dicts = append(dicts, dict)
if opt.ResumeFrom != "" {
dictCache[f] = dicts[i]
pkg.Dicts[f] = dicts[i]
}
logs.Log.Logf(pkg.LogVerbose, "Loaded %d word from %s", len(dicts[i]), f)
logs.Log.Logf(pkg.LogVerbose, "Loaded %d word from %s", len(dict), f)
}
if len(dicts) == 0 && opt.Word == "" {
if len(dicts) == 0 && opt.Word == "" && len(opt.Rules) == 0 && len(opt.AppendRule) == 0 {
r.IsCheck = true
}
@ -582,7 +583,7 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if len(opt.Rules) != 0 {
rules, err := loadRuleAndCombine(opt.Rules)
rules, err := pkg.LoadRuleAndCombine(opt.Rules)
if err != nil {
return err
}
@ -601,7 +602,7 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if len(opt.AppendRule) != 0 {
content, err := loadRuleAndCombine(opt.AppendRule)
content, err := pkg.LoadRuleAndCombine(opt.AppendRule)
if err != nil {
return err
}
@ -609,18 +610,13 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if len(opt.AppendFile) != 0 {
var bs bytes.Buffer
var lines []string
for _, f := range opt.AppendFile {
content, err := ioutil.ReadFile(f)
dict, err := pkg.LoadFileToSlice(f)
if err != nil {
return err
}
bs.Write(bytes.TrimSpace(content))
bs.WriteString("\n")
}
lines := strings.Split(bs.String(), "\n")
for i, line := range lines {
lines[i] = strings.TrimSpace(line)
lines = append(lines, dict...)
}
r.AppendWords = append(r.AppendWords, lines...)
}
@ -647,16 +643,16 @@ func (opt *Option) BuildWords(r *Runner) error {
}
if opt.Uppercase {
r.AppendFunction(wrapWordsFunc(strings.ToUpper))
r.AppendFunction(pkg.WrapWordsFunc(strings.ToUpper))
}
if opt.Lowercase {
r.AppendFunction(wrapWordsFunc(strings.ToLower))
r.AppendFunction(pkg.WrapWordsFunc(strings.ToLower))
}
if opt.RemoveExtensions != "" {
rexts := strings.Split(opt.ExcludeExtensions, ",")
r.AppendFunction(func(s string) []string {
if ext := parseExtension(s); iutils.StringsContains(rexts, ext) {
if ext := pkg.ParseExtension(s); iutils.StringsContains(rexts, ext) {
return []string{strings.TrimSuffix(s, "."+ext)}
}
return []string{s}
@ -666,7 +662,7 @@ func (opt *Option) BuildWords(r *Runner) error {
if opt.ExcludeExtensions != "" {
exexts := strings.Split(opt.ExcludeExtensions, ",")
r.AppendFunction(func(s string) []string {
if ext := parseExtension(s); iutils.StringsContains(exexts, ext) {
if ext := pkg.ParseExtension(s); iutils.StringsContains(exexts, ext) {
return nil
}
return []string{s}
@ -700,7 +696,7 @@ func (opt *Option) BuildWords(r *Runner) error {
})
}
logs.Log.Logf(pkg.LogVerbose, "Loaded %d dictionaries and %d decorators", len(opt.Dictionaries), len(r.Fns))
logs.Log.Importantf("Loaded %d dictionaries, %d rules and %d decorators", len(opt.Dictionaries), len(opt.Rules), len(r.Fns))
return nil
}

186
internal/pool/brutepool.go
View File

@ -10,11 +10,13 @@ import (
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words"
"github.com/chainreactors/words/rule"
"github.com/panjf2000/ants/v2"
"github.com/valyala/fasthttp"
"golang.org/x/time/rate"
"math/rand"
"net/url"
"path"
"strings"
"sync"
"sync/atomic"
@ -378,12 +380,9 @@ func (pool *BrutePool) Invoke(v interface{}) {
pool.locker.Lock()
pool.index = bl
pool.locker.Unlock()
if bl.Status == 200 || (bl.Status/100) == 3 {
// 保留index输出结果
pool.wg.Add(1)
pool.doCrawl(bl)
pool.putToOutput(bl)
}
pool.doCrawl(bl)
pool.doAppend(bl)
pool.putToOutput(bl)
pool.initwg.Done()
case parsers.CheckSource:
if bl.ErrString != "" {
@ -520,14 +519,9 @@ func (pool *BrutePool) Handler() {
bl.IsValid = false
}
if bl.IsValid || bl.IsFuzzy {
pool.wg.Add(2)
if bl.IsValid || (bl.IsFuzzy && pool.Fuzzy) {
pool.doCrawl(bl)
pool.doRule(bl)
if iutils.IntsContains(pkg.WhiteStatus, bl.Status) || iutils.IntsContains(pkg.UniqueStatus, bl.Status) {
pool.wg.Add(1)
pool.doAppendWords(bl)
}
pool.doAppend(bl)
}
// 如果要进行递归判断, 要满足 bl有效, mod为path-spray, 当前深度小于最大递归深度
@ -549,6 +543,73 @@ func (pool *BrutePool) Handler() {
pool.analyzeDone = true
}
func (pool *BrutePool) doAppendRule(bl *pkg.Baseline) {
if pool.AppendRule == nil || bl.Source == parsers.RuleSource {
pool.wg.Done()
return
}
go func() {
defer pool.wg.Done()
for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) {
pool.addAddition(&Unit{
path: pkg.Dir(bl.Url.Path) + u,
source: parsers.RuleSource,
})
}
}()
}
func (pool *BrutePool) doAppendWords(bl *pkg.Baseline) {
if pool.AppendWords == nil || bl.Source == parsers.AppendSource || bl.Source == parsers.RuleSource {
// 防止自身递归
pool.wg.Done()
return
}
go func() {
defer pool.wg.Done()
for _, u := range pool.AppendWords {
pool.addAddition(&Unit{
path: pkg.SafePath(bl.Path, u),
source: parsers.AppendSource,
})
}
}()
}
func (pool *BrutePool) doAppend(bl *pkg.Baseline) {
pool.wg.Add(2)
pool.doAppendWords(bl)
pool.doAppendRule(bl)
}
func (pool *BrutePool) doActive() {
defer pool.wg.Done()
for _, u := range pkg.ActivePath {
pool.addAddition(&Unit{
path: pool.dir + u[1:],
source: parsers.FingerSource,
})
}
}
func (pool *BrutePool) doCommonFile() {
defer pool.wg.Done()
for _, u := range pkg.Dicts["common"] {
pool.addAddition(&Unit{
path: pool.dir + u,
source: parsers.CommonFileSource,
})
}
for _, u := range pkg.Dicts["log"] {
pool.addAddition(&Unit{
path: pool.dir + u,
source: parsers.CommonFileSource,
})
}
}
func (pool *BrutePool) PreCompare(resp *ihttp.Response) error {
status := resp.StatusCode()
if iutils.IntsContains(pkg.WhiteStatus, status) {
@ -629,6 +690,50 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool {
return true
}
func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) {
if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) {
bl.Collect()
pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取
pool.baselines[bl.Status] = bl
logs.Log.Logf(pkg.LogVerbose, "[baseline.%dinit] %s", bl.Status, bl.Format([]string{"status", "length", "spend", "title", "frame", "redirect"}))
}
}
func (pool *BrutePool) recover() {
logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.wordOffset)
for i, bl := range pool.FailedBaselines {
if i > int(pool.BreakThreshold) {
break
}
logs.Log.Errorf("[failed.%d] %s", i, bl.String())
}
}
func (pool *BrutePool) Close() {
for pool.analyzeDone {
// 等待缓存的待处理任务完成
time.Sleep(time.Duration(100) * time.Millisecond)
}
close(pool.additionCh) // 关闭addition管道
close(pool.checkCh) // 关闭check管道
pool.Statistor.EndTime = time.Now().Unix()
pool.Bar.Close()
}
func (pool *BrutePool) safePath(u string) string {
// 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common
if pool.isDir {
return pkg.SafePath(pool.dir, u)
} else {
return pkg.SafePath(pool.url.Path+"/", u)
}
}
func (pool *BrutePool) resetFailed() {
pool.failedCount = 1
pool.FailedBaselines = nil
}
func (pool *BrutePool) doCheck() {
if pool.failedCount > pool.BreakThreshold {
// 当报错次数超过上限是, 结束任务
@ -647,16 +752,15 @@ func (pool *BrutePool) doCheck() {
func (pool *BrutePool) doCrawl(bl *pkg.Baseline) {
if !pool.Crawl || bl.ReqDepth >= MaxCrawl {
pool.wg.Done()
return
}
bl.CollectURL()
if bl.URLs == nil {
pool.wg.Done()
return
}
pool.wg.Add(1)
bl.CollectURL()
if bl.URLs == nil {
return
}
pool.wg.Add(2)
pool.doScopeCrawl(bl)
go func() {
@ -700,16 +804,6 @@ func (pool *BrutePool) doScopeCrawl(bl *pkg.Baseline) {
}()
}
func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) {
if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) {
bl.Collect()
pool.wg.Add(1)
pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取
pool.baselines[bl.Status] = bl
logs.Log.Logf(pkg.LogVerbose, "[baseline.%dinit] %s", bl.Status, bl.Format([]string{"status", "length", "spend", "title", "frame", "redirect"}))
}
}
func (pool *BrutePool) doBak() {
defer pool.wg.Done()
worder, err := words.NewWorderWithDsl("{?0}.{?@bak_ext}", [][]string{pkg.BakGenerator(pool.url.Host)}, nil)
@ -736,35 +830,3 @@ func (pool *BrutePool) doBak() {
})
}
}
func (pool *BrutePool) recover() {
logs.Log.Errorf("%s ,failed request exceeds the threshold , task will exit. Breakpoint %d", pool.BaseURL, pool.wordOffset)
for i, bl := range pool.FailedBaselines {
logs.Log.Errorf("[failed.%d] %s", i, bl.String())
}
}
func (pool *BrutePool) Close() {
for pool.analyzeDone {
// 等待缓存的待处理任务完成
time.Sleep(time.Duration(100) * time.Millisecond)
}
close(pool.additionCh) // 关闭addition管道
close(pool.checkCh) // 关闭check管道
pool.Statistor.EndTime = time.Now().Unix()
pool.Bar.Close()
}
func (pool *BrutePool) safePath(u string) string {
// 自动生成的目录将采用safepath的方式拼接到相对目录中, 避免出现//的情况. 例如init, check, common
if pool.isDir {
return pkg.SafePath(pool.dir, u)
} else {
return pkg.SafePath(pool.url.Path+"/", u)
}
}
func (pool *BrutePool) resetFailed() {
pool.failedCount = 1
pool.FailedBaselines = nil
}

2
internal/pool/config.go
View File

@ -51,7 +51,7 @@ type Config struct {
Active bool
Bak bool
Common bool
Retry int
RetryLimit int
RandomUserAgent bool
Random string
Index string

67
internal/pool/pool.go
View File

@ -7,10 +7,7 @@ import (
"github.com/chainreactors/spray/internal/ihttp"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/words"
"github.com/chainreactors/words/mask"
"github.com/chainreactors/words/rule"
"github.com/panjf2000/ants/v2"
"path"
"sync"
)
@ -49,50 +46,8 @@ func (pool *BasePool) doRedirect(bl *pkg.Baseline, depth int) {
}()
}
func (pool *BasePool) doRule(bl *pkg.Baseline) {
if pool.AppendRule == nil {
pool.wg.Done()
return
}
if bl.Source == parsers.RuleSource {
pool.wg.Done()
return
}
go func() {
defer pool.wg.Done()
for u := range rule.RunAsStream(pool.AppendRule.Expressions, path.Base(bl.Path)) {
pool.addAddition(&Unit{
path: pkg.Dir(bl.Url.Path) + u,
source: parsers.RuleSource,
})
}
}()
}
func (pool *BasePool) doAppendWords(bl *pkg.Baseline) {
if pool.AppendWords == nil {
pool.wg.Done()
return
}
if bl.Source == parsers.AppendSource {
pool.wg.Done()
return
}
go func() {
defer pool.wg.Done()
for _, u := range pool.AppendWords {
pool.addAddition(&Unit{
path: pkg.SafePath(bl.Path, u),
source: parsers.AppendSource,
})
}
}()
}
func (pool *BasePool) doRetry(bl *pkg.Baseline) {
if bl.Retry >= pool.Retry {
if bl.Retry >= pool.RetryLimit {
return
}
pool.wg.Add(1)
@ -106,26 +61,6 @@ func (pool *BasePool) doRetry(bl *pkg.Baseline) {
}()
}
func (pool *BasePool) doActive() {
defer pool.wg.Done()
for _, u := range pkg.ActivePath {
pool.addAddition(&Unit{
path: pool.dir + u[1:],
source: parsers.FingerSource,
})
}
}
func (pool *BasePool) doCommonFile() {
defer pool.wg.Done()
for _, u := range mask.SpecialWords["common_file"] {
pool.addAddition(&Unit{
path: pool.dir + u,
source: parsers.CommonFileSource,
})
}
}
func (pool *BasePool) addAddition(u *Unit) {
// 强行屏蔽报错, 防止goroutine泄露
pool.wg.Add(1)

131
internal/runner.go
View File

@ -20,17 +20,11 @@ var (
MAX = 2147483647
)
var (
dictCache = make(map[string][]string)
wordlistCache = make(map[string][]string)
ruleCache = make(map[string][]rule.Expression)
)
type Runner struct {
*Option
taskCh chan *Task
poolwg sync.WaitGroup
poolwg *sync.WaitGroup
outwg *sync.WaitGroup
outputCh chan *pkg.Baseline
fuzzyCh chan *pkg.Baseline
@ -47,20 +41,20 @@ type Runner struct {
MatchExpr *vm.Program
RecursiveExpr *vm.Program
OutputFile *files.File
FuzzyFile *files.File
DumpFile *files.File
StatFile *files.File
Progress *mpb.Progress
Fns []func(string) []string
Count int // tasks total number
Wordlist []string
AppendWords []string
RecuDepth int
ClientType int
Probes []string
Total int // wordlist total number
Color bool
Jsonify bool
//FuzzyFile *files.File
DumpFile *files.File
StatFile *files.File
Progress *mpb.Progress
Fns []func(string) []string
Count int // tasks total number
Wordlist []string
AppendWords []string
RecuDepth int
ClientType int
Probes []string
Total int // wordlist total number
Color bool
Jsonify bool
}
func (r *Runner) PrepareConfig() *pool.Config {
@ -81,15 +75,15 @@ func (r *Runner) PrepareConfig() *pool.Config {
MatchExpr: r.MatchExpr,
FilterExpr: r.FilterExpr,
RecuExpr: r.RecursiveExpr,
AppendRule: r.AppendRules,
AppendWords: r.AppendWords,
AppendRule: r.AppendRules, // 对有效目录追加规则, 根据rule生成
AppendWords: r.AppendWords, // 对有效目录追加字典
//IgnoreWaf: r.IgnoreWaf,
Crawl: r.Crawl,
Crawl: r.CrawlPlugin,
Scope: r.Scope,
Active: r.Finger,
Bak: r.Bak,
Common: r.Common,
Retry: r.RetryCount,
Bak: r.BakPlugin,
Common: r.CommonPlugin,
RetryLimit: r.RetryCount,
ClientType: r.ClientType,
RandomUserAgent: r.RandomUserAgent,
Random: r.Random,
@ -358,62 +352,31 @@ func (r *Runner) saveStat(content string) {
}
}
func (r *Runner) OutputHandler() {
debugPrint := func(bl *pkg.Baseline) {
if r.Color {
logs.Log.Debug(bl.ColorString())
} else {
logs.Log.Debug(bl.String())
}
func (r *Runner) Output(bl *pkg.Baseline) {
var out string
if r.Option.Json {
out = bl.Jsonify()
} else if len(r.Probes) > 0 {
out = bl.Format(r.Probes)
} else if r.Color {
out = bl.ColorString()
} else {
out = bl.String()
}
if bl.IsFuzzy {
logs.Log.Console("[fuzzy] " + out + "\n")
} else {
logs.Log.Console(out + "\n")
}
var saveFunc func(string)
if r.OutputFile != nil {
saveFunc = func(line string) {
r.OutputFile.SafeWrite(line + "\n")
r.OutputFile.SafeSync()
}
} else {
saveFunc = func(line string) {
logs.Log.Console(line + "\n")
}
}
var fuzzySaveFunc func(string)
if r.FuzzyFile != nil {
fuzzySaveFunc = func(line string) {
r.FuzzyFile.SafeWrite(line + "\n")
r.FuzzyFile.SafeSync()
}
} else {
fuzzySaveFunc = func(line string) {
logs.Log.Console("[fuzzy] " + line + "\n")
}
}
outputPrint := func(bl *pkg.Baseline) {
var outFunc func(string)
if bl.IsFuzzy {
outFunc = fuzzySaveFunc
} else {
outFunc = saveFunc
}
if r.Option.Json {
outFunc(bl.Jsonify())
} else if r.Color {
if len(r.Probes) > 0 {
outFunc(logs.GreenBold(bl.Format(r.Probes)))
} else {
outFunc(logs.GreenBold(bl.ColorString()))
}
} else {
if len(r.Probes) > 0 {
outFunc(bl.Format(r.Probes))
} else {
outFunc(bl.String())
}
}
r.OutputFile.SafeWrite(bl.Jsonify() + "\n")
r.OutputFile.SafeSync()
}
}
func (r *Runner) OutputHandler() {
go func() {
for {
select {
@ -426,12 +389,16 @@ func (r *Runner) OutputHandler() {
r.DumpFile.SafeSync()
}
if bl.IsValid {
outputPrint(bl)
r.Output(bl)
if bl.Recu {
r.AddRecursive(bl)
}
} else {
debugPrint(bl)
if r.Color {
logs.Log.Debug(bl.ColorString())
} else {
logs.Log.Debug(bl.String())
}
}
r.outwg.Done()
}
@ -445,9 +412,7 @@ func (r *Runner) OutputHandler() {
if !ok {
return
}
if r.Fuzzy {
outputPrint(bl)
}
r.Output(bl)
r.outwg.Done()
}
}

4
internal/types.go
View File

@ -16,13 +16,13 @@ type Origin struct {
func (o *Origin) InitWorder(fns []func(string) []string) (*words.Worder, error) {
var worder *words.Worder
wl, err := loadWordlist(o.Word, o.Dictionaries)
wl, err := pkg.LoadWordlist(o.Word, o.Dictionaries)
if err != nil {
return nil, err
}
worder = words.NewWorder(wl)
worder.Fns = fns
rules, err := loadRuleWithFiles(o.RuleFiles, o.RuleFilter)
rules, err := pkg.LoadRuleWithFiles(o.RuleFiles, o.RuleFilter)
if err != nil {
return nil, err
}

158
internal/utils.go
View File

@ -1,149 +1,5 @@
package internal
import (
"bytes"
"github.com/chainreactors/spray/pkg"
"github.com/chainreactors/words/mask"
"github.com/chainreactors/words/rule"
"io/ioutil"
"strconv"
"strings"
)
func parseExtension(s string) string {
if i := strings.Index(s, "."); i != -1 {
return s[i+1:]
}
return ""
}
func parseStatus(preset []int, changed string) []int {
if changed == "" {
return preset
}
if strings.HasPrefix(changed, "+") {
for _, s := range strings.Split(changed[1:], ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
} else if strings.HasPrefix(changed, "!") {
for _, s := range strings.Split(changed[1:], ",") {
for i, status := range preset {
if t, err := strconv.Atoi(s); err != nil {
break
} else if t == status {
preset = append(preset[:i], preset[i+1:]...)
break
}
}
}
} else {
preset = []int{}
for _, s := range strings.Split(changed, ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
}
return preset
}
func loadFileToSlice(filename string) ([]string, error) {
var ss []string
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
ss = strings.Split(strings.TrimSpace(string(content)), "\n")
// 统一windows与linux的回车换行差异
for i, word := range ss {
ss[i] = strings.TrimSpace(word)
}
return ss, nil
}
func loadRuleAndCombine(filename []string) (string, error) {
var bs bytes.Buffer
for _, f := range filename {
if data, ok := pkg.Rules[f]; ok {
bs.WriteString(strings.TrimSpace(data))
bs.WriteString("\n")
} else {
content, err := ioutil.ReadFile(f)
if err != nil {
return "", err
}
bs.Write(bytes.TrimSpace(content))
bs.WriteString("\n")
}
}
return bs.String(), nil
}
func loadFileWithCache(filename string) ([]string, error) {
if dict, ok := dictCache[filename]; ok {
return dict, nil
}
dict, err := loadFileToSlice(filename)
if err != nil {
return nil, err
}
dictCache[filename] = dict
return dict, nil
}
func loadDictionaries(filenames []string) ([][]string, error) {
dicts := make([][]string, len(filenames))
for i, name := range filenames {
dict, err := loadFileWithCache(name)
if err != nil {
return nil, err
}
dicts[i] = dict
}
return dicts, nil
}
func loadWordlist(word string, dictNames []string) ([]string, error) {
if wl, ok := wordlistCache[word+strings.Join(dictNames, ",")]; ok {
return wl, nil
}
dicts, err := loadDictionaries(dictNames)
if err != nil {
return nil, err
}
wl, err := mask.Run(word, dicts, nil)
if err != nil {
return nil, err
}
wordlistCache[word] = wl
return wl, nil
}
func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, error) {
if rules, ok := ruleCache[strings.Join(ruleFiles, ",")]; ok {
return rules, nil
}
var rules bytes.Buffer
for _, filename := range ruleFiles {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
rules.Write(content)
rules.WriteString("\n")
}
return rule.Compile(rules.String(), filter).Expressions, nil
}
//type bytesPatcher struct{}
//
//func (p *bytesPatcher) Visit(node *ast.Node) {
@ -158,17 +14,3 @@ func loadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, er
// })
// }
//}
func wrapWordsFunc(f func(string) string) func(string) []string {
return func(s string) []string {
return []string{f(s)}
}
}
func safeFilename(filename string) string {
filename = strings.ReplaceAll(filename, "http://", "")
filename = strings.ReplaceAll(filename, "https://", "")
filename = strings.ReplaceAll(filename, ":", "_")
filename = strings.ReplaceAll(filename, "/", "_")
return filename
}

31
pkg/load.go
View File

@ -12,14 +12,6 @@ import (
"strings"
)
var (
ExtractRegexps = make(parsers.Extractors)
Extractors = make(parsers.Extractors)
FingerEngine *fingers.Engine
ActivePath []string
)
func LoadPorts() error {
var err error
var ports []*utils.PortConfig
@ -55,13 +47,24 @@ func LoadFingers() error {
func LoadTemplates() error {
var err error
// load rule
var data map[string]interface{}
err = json.Unmarshal(LoadConfig("spray_rule"), &data)
err = json.Unmarshal(LoadConfig("spray_rule"), &Rules)
if err != nil {
return err
}
for k, v := range data {
Rules[k] = v.(string)
// load default words
var dicts map[string]string
err = json.Unmarshal(LoadConfig("spray_dict"), &dicts)
if err != nil {
return err
}
for name, wordlist := range dicts {
dict := strings.Split(strings.TrimSpace(wordlist), "\n")
for i, d := range dict {
dict[i] = strings.TrimSpace(d)
}
Dicts[strings.TrimSuffix(name, ".txt")] = dict
}
// load mask
@ -131,7 +134,3 @@ func Load() error {
return nil
}
func LoadDefaultDict() []string {
return strings.Split(strings.TrimSpace(string(LoadConfig("spray_default"))), "\n")
}

185
pkg/utils.go
View File

@ -3,10 +3,16 @@ package pkg
import (
"bufio"
"bytes"
"github.com/chainreactors/files"
"github.com/chainreactors/fingers"
"github.com/chainreactors/logs"
"github.com/chainreactors/parsers"
"github.com/chainreactors/utils/iutils"
"github.com/chainreactors/words/mask"
"github.com/chainreactors/words/rule"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
"io/ioutil"
"math/rand"
"net/http"
"net/url"
@ -31,11 +37,17 @@ var (
EnableAllFingerEngine = false
)
var (
Rules map[string]string = make(map[string]string)
BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4", ".zip", ".rar"}
BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"}
Rules map[string]string = make(map[string]string)
Dicts map[string][]string = make(map[string][]string)
wordlistCache = make(map[string][]string)
ruleCache = make(map[string][]rule.Expression)
BadExt = []string{".js", ".css", ".scss", ".,", ".jpeg", ".jpg", ".png", ".gif", ".svg", ".vue", ".ts", ".swf", ".pdf", ".mp4", ".zip", ".rar"}
BadURL = []string{";", "}", "\\n", "webpack://", "{", "www.w3.org", ".src", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path"}
ExtractRegexps = make(parsers.Extractors)
Extractors = make(parsers.Extractors)
FingerEngine *fingers.Engine
ActivePath []string
ContentTypeMap = map[string]string{
"application/javascript": "js",
"application/json": "json",
@ -391,3 +403,168 @@ func ParseRawResponse(raw []byte) (*http.Response, error) {
defer resp.Body.Close()
return resp, nil
}
func GetPresetWordList(key []string) []string {
var wordlist []string
for _, k := range key {
if v, ok := mask.SpecialWords[k]; ok {
wordlist = append(wordlist, v...)
}
}
return wordlist
}
func ParseExtension(s string) string {
if i := strings.Index(s, "."); i != -1 {
return s[i+1:]
}
return ""
}
func ParseStatus(preset []int, changed string) []int {
if changed == "" {
return preset
}
if strings.HasPrefix(changed, "+") {
for _, s := range strings.Split(changed[1:], ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
} else if strings.HasPrefix(changed, "!") {
for _, s := range strings.Split(changed[1:], ",") {
for i, status := range preset {
if t, err := strconv.Atoi(s); err != nil {
break
} else if t == status {
preset = append(preset[:i], preset[i+1:]...)
break
}
}
}
} else {
preset = []int{}
for _, s := range strings.Split(changed, ",") {
if t, err := strconv.Atoi(s); err != nil {
continue
} else {
preset = append(preset, t)
}
}
}
return preset
}
func LoadFileToSlice(filename string) ([]string, error) {
var ss []string
if dicts, ok := Dicts[filename]; ok {
if files.IsExist(filename) {
logs.Log.Warnf("load and overwrite %s from preset", filename)
}
return dicts, nil
}
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
ss = strings.Split(strings.TrimSpace(string(content)), "\n")
// 统一windows与linux的回车换行差异
for i, word := range ss {
ss[i] = strings.TrimSpace(word)
}
return ss, nil
}
func LoadRuleAndCombine(filename []string) (string, error) {
var bs bytes.Buffer
for _, f := range filename {
if data, ok := Rules[f]; ok {
bs.WriteString(strings.TrimSpace(data))
bs.WriteString("\n")
} else {
content, err := ioutil.ReadFile(f)
if err != nil {
return "", err
}
bs.Write(bytes.TrimSpace(content))
bs.WriteString("\n")
}
}
return bs.String(), nil
}
func loadFileWithCache(filename string) ([]string, error) {
if dict, ok := Dicts[filename]; ok {
return dict, nil
}
dict, err := LoadFileToSlice(filename)
if err != nil {
return nil, err
}
Dicts[filename] = dict
return dict, nil
}
func loadDictionaries(filenames []string) ([][]string, error) {
dicts := make([][]string, len(filenames))
for i, name := range filenames {
dict, err := loadFileWithCache(name)
if err != nil {
return nil, err
}
dicts[i] = dict
}
return dicts, nil
}
func LoadWordlist(word string, dictNames []string) ([]string, error) {
if wl, ok := wordlistCache[word+strings.Join(dictNames, ",")]; ok {
return wl, nil
}
dicts, err := loadDictionaries(dictNames)
if err != nil {
return nil, err
}
wl, err := mask.Run(word, dicts, nil)
if err != nil {
return nil, err
}
wordlistCache[word] = wl
return wl, nil
}
func LoadRuleWithFiles(ruleFiles []string, filter string) ([]rule.Expression, error) {
if rules, ok := ruleCache[strings.Join(ruleFiles, ",")]; ok {
return rules, nil
}
var rules bytes.Buffer
for _, filename := range ruleFiles {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
rules.Write(content)
rules.WriteString("\n")
}
return rule.Compile(rules.String(), filter).Expressions, nil
}
func WrapWordsFunc(f func(string) string) func(string) []string {
return func(s string) []string {
return []string{f(s)}
}
}
func SafeFilename(filename string) string {
filename = strings.ReplaceAll(filename, "http://", "")
filename = strings.ReplaceAll(filename, "https://", "")
filename = strings.ReplaceAll(filename, ":", "_")
filename = strings.ReplaceAll(filename, "/", "_")
return filename
}

2
templates

@ -1 +1 @@
Subproject commit f2980b8d312c8088f3947d914499e96cfc40d975
Subproject commit eb3f934b623c0e9bcc895cc8828ec88913ea8311