mirror of
https://github.com/chainreactors/spray.git
synced 2026-10-24 08:54:30 +00:00
fix: regenerate templates and fix logs API for updated submodule
go generate with the updated templates submodule (7d5675c) produces different embedded data and code. Key changes: - cmd/cmd.go: logs.Info/Important → logs.InfoLevel/ImportantLevel - pkg/templates.go: remove proton_rules.bin/neutron.bin embeds (no longer needed) - core/baseline: use ProtonExtract instead of Extractors.Extract - core/format: list proton extractor names - Update embedded binary data files Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
M09IcandClaude Opus 4.6 committed
1 parent
f4b8adf642
commit
d1b63ad67b
12 files changed
+393
-404
No files matched your search
+3
-3
@@ -19,9 +19,9 @@ var DefaultConfig = "config.yaml"
|
||||
|
||||
func init() {
|
||||
logs.Log.SetColorMap(map[logs.Level]func(string) string{
|
||||
logs.Info: logs.PurpleBold,
|
||||
logs.Important: logs.GreenBold,
|
||||
pkg.LogVerbose: logs.Green,
|
||||
logs.InfoLevel: logs.PurpleBold,
|
||||
logs.ImportantLevel: logs.GreenBold,
|
||||
pkg.LogVerbose: logs.Green,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -129,7 +129,7 @@ mode:
|
||||
# Strings (comma split), custom fuzzy status
|
||||
fuzzy-status: 500,501,502,503
|
||||
# Strings (comma split), custom unique status
|
||||
unique-status: 403,200,404
|
||||
unique-status: 403,200,404
|
||||
# Bool, unique response
|
||||
unique: false
|
||||
# Int, retry count
|
||||
|
||||
+263
-264
@@ -1,264 +1,263 @@
|
||||
package baseline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/chainreactors/fingers/common"
|
||||
"github.com/chainreactors/logs"
|
||||
"github.com/chainreactors/parsers"
|
||||
"github.com/chainreactors/spray/core/ihttp"
|
||||
"github.com/chainreactors/spray/pkg"
|
||||
"github.com/chainreactors/utils/encode"
|
||||
"github.com/chainreactors/utils/iutils"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
|
||||
var err error
|
||||
bl := &Baseline{
|
||||
SprayResult: &parsers.SprayResult{
|
||||
UrlString: u,
|
||||
Status: resp.StatusCode(),
|
||||
IsValid: true,
|
||||
Frameworks: make(common.Frameworks),
|
||||
},
|
||||
}
|
||||
|
||||
if t, ok := pkg.MatchContentType(resp.ContentType()); ok {
|
||||
bl.ContentType = t
|
||||
bl.Title = t + " data"
|
||||
} else {
|
||||
bl.ContentType = "other"
|
||||
}
|
||||
|
||||
header := resp.Header()
|
||||
bl.Header = make([]byte, len(header))
|
||||
copy(bl.Header, header)
|
||||
bl.HeaderLength = len(bl.Header)
|
||||
if i := resp.ContentLength(); ihttp.CheckBodySize(i) {
|
||||
if body := resp.Body(); body != nil {
|
||||
bl.Body = make([]byte, len(body))
|
||||
copy(bl.Body, body)
|
||||
}
|
||||
|
||||
if i == -1 {
|
||||
bl.Chunked = true
|
||||
bl.BodyLength = len(bl.Body)
|
||||
} else {
|
||||
bl.BodyLength = int(i)
|
||||
}
|
||||
}
|
||||
|
||||
bl.Raw = append(bl.Header, bl.Body...)
|
||||
bl.Response, err = pkg.ParseRawResponse(bl.Raw)
|
||||
if err != nil {
|
||||
// raw 重解析失败不影响 baseline 有效性，live response 已提供所有需要的数据
|
||||
logs.Log.Debugf("ParseRawResponse failed for %s: %s", u, err.Error())
|
||||
}
|
||||
// 始终从 live response 读取 Location
|
||||
if r := resp.GetHeader("Location"); r != "" {
|
||||
bl.RedirectURL = r
|
||||
} else {
|
||||
bl.RedirectURL = resp.GetHeader("location")
|
||||
}
|
||||
|
||||
bl.Dir = bl.IsDir()
|
||||
uu, err := url.Parse(u)
|
||||
if err == nil {
|
||||
bl.Path = uu.Path
|
||||
bl.Url = uu
|
||||
if uu.Host != host {
|
||||
bl.Host = host
|
||||
}
|
||||
} else {
|
||||
bl.IsValid = false
|
||||
bl.Reason = pkg.ErrUrlError.Error()
|
||||
bl.ErrString = err.Error()
|
||||
}
|
||||
bl.Unique = UniqueHash(bl)
|
||||
return bl
|
||||
}
|
||||
|
||||
func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline {
|
||||
bl := &Baseline{
|
||||
SprayResult: &parsers.SprayResult{
|
||||
UrlString: u,
|
||||
Status: resp.StatusCode(),
|
||||
IsValid: false,
|
||||
Reason: reason,
|
||||
},
|
||||
}
|
||||
|
||||
// 无效数据也要读取body, 否则keep-alive不生效
|
||||
resp.Body()
|
||||
bl.BodyLength = int(resp.ContentLength())
|
||||
bl.RedirectURL = string(resp.GetHeader("Location"))
|
||||
|
||||
bl.Dir = bl.IsDir()
|
||||
uu, err := url.Parse(u)
|
||||
if err == nil {
|
||||
bl.Path = uu.Path
|
||||
bl.Url = uu
|
||||
} else {
|
||||
return bl
|
||||
}
|
||||
|
||||
if bl.Url.Host != host {
|
||||
bl.Host = host
|
||||
}
|
||||
|
||||
return bl
|
||||
}
|
||||
|
||||
type Baseline struct {
|
||||
*parsers.SprayResult
|
||||
Url *url.URL `json:"-"`
|
||||
Dir bool `json:"-"`
|
||||
Chunked bool `json:"-"`
|
||||
Body pkg.BS `json:"-"`
|
||||
Header pkg.BS `json:"-"`
|
||||
Raw pkg.BS `json:"-"`
|
||||
Response *http.Response `json:"-"`
|
||||
Recu bool `json:"-"`
|
||||
RecuDepth int `json:"-"`
|
||||
URLs []string `json:"-"`
|
||||
Collected bool `json:"-"`
|
||||
Retry int `json:"-"`
|
||||
SameRedirectDomain bool `json:"-"`
|
||||
IsBaseline bool `json:"-"`
|
||||
}
|
||||
|
||||
func (bl *Baseline) IsDir() bool {
|
||||
if strings.HasSuffix(bl.Path, "/") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Collect 深度收集信息
|
||||
func (bl *Baseline) Collect() {
|
||||
if bl.Collected { // 防止重复收集
|
||||
return
|
||||
} else {
|
||||
bl.Collected = true
|
||||
}
|
||||
|
||||
if bl.ContentType == "html" || bl.ContentType == "json" || bl.ContentType == "txt" {
|
||||
// 指纹库设计的时候没考虑js,css文件的指纹, 跳过非必要的指纹收集减少误报提高性能
|
||||
//fmt.Println(bl.Source, bl.Url.String()+bl.Path, bl.RedirectURL, "call fingersengine")
|
||||
if pkg.EnableAllFingerEngine {
|
||||
bl.Frameworks = pkg.EngineDetect(bl.Raw)
|
||||
} else {
|
||||
bl.Frameworks = pkg.FingersDetect(bl.Raw)
|
||||
}
|
||||
}
|
||||
|
||||
if len(bl.Body) > 0 {
|
||||
if bl.ContentType == "html" {
|
||||
bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
|
||||
} else if bl.ContentType == "ico" && pkg.FingerEngine != nil && pkg.FingerEngine.Favicon() != nil {
|
||||
if frame := pkg.FingerEngine.Favicon().WebMatch(bl.Body); frame != nil {
|
||||
bl.Frameworks.Merge(frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bl.Hashes = parsers.NewHashes(bl.Raw)
|
||||
bl.Extracteds.Merge(pkg.Extractors.Extract(string(bl.Raw), true))
|
||||
bl.Unique = UniqueHash(bl)
|
||||
}
|
||||
|
||||
func (bl *Baseline) CollectURL() {
|
||||
if len(bl.Body) == 0 {
|
||||
return
|
||||
}
|
||||
for _, reg := range pkg.ExtractRegexps["js"][0].CompiledRegexps {
|
||||
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
|
||||
for _, u := range urls {
|
||||
u[1] = pkg.CleanURL(u[1])
|
||||
if u[1] != "" && !pkg.FilterJs(u[1]) {
|
||||
bl.URLs = append(bl.URLs, u[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, reg := range pkg.ExtractRegexps["url"][0].CompiledRegexps {
|
||||
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
|
||||
for _, u := range urls {
|
||||
u[1] = pkg.CleanURL(u[1])
|
||||
if u[1] != "" && !pkg.FilterUrl(u[1]) {
|
||||
bl.URLs = append(bl.URLs, u[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bl.URLs = iutils.StringsUnique(bl.URLs)
|
||||
if len(bl.URLs) != 0 {
|
||||
bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{
|
||||
Name: "crawl",
|
||||
ExtractResult: bl.URLs,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Compare
|
||||
// if totally equal return 1
|
||||
// if maybe equal return 0
|
||||
// not equal return -1
|
||||
func (bl *Baseline) Compare(other *Baseline) int {
|
||||
if other.RedirectURL != "" && bl.RedirectURL == other.RedirectURL {
|
||||
// 如果重定向url不为空, 且与base不相同, 则说明不是同一个页面
|
||||
return 1
|
||||
}
|
||||
|
||||
if bl.BodyLength == other.BodyLength {
|
||||
// 如果body length相等且md5相等, 则说明是同一个页面
|
||||
if bytes.Equal(bl.Body, other.Body) {
|
||||
// 如果length相等, md5也相等, 则判断为全同
|
||||
return 1
|
||||
} else {
|
||||
// 如果长度相等, 但是md5不相等, 可能是存在csrftoken之类的随机值
|
||||
return 0
|
||||
}
|
||||
} else if i := bl.BodyLength - other.BodyLength; (i < 16 && i > 0) || (i > -16 && i < 0) {
|
||||
// 如果body length绝对值小于16, 则可能是存在csrftoken之类的随机值, 需要模糊判断
|
||||
return 0
|
||||
} else {
|
||||
// 如果body length绝对值大于16, 则认为大概率存在较大差异
|
||||
if strings.Contains(string(other.Body), other.Path) {
|
||||
// 如果包含路径本身, 可能是路径自身的随机值影响结果
|
||||
return 0
|
||||
} else {
|
||||
// 如果不包含路径本身, 则认为是不同页面
|
||||
return -1
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (bl *Baseline) ProbeOutput(format []string) string {
|
||||
var s strings.Builder
|
||||
for _, f := range format {
|
||||
s.WriteString("\t")
|
||||
s.WriteString(bl.Get(f))
|
||||
}
|
||||
return strings.TrimSpace(s.String())
|
||||
}
|
||||
|
||||
var Distance uint8 = 5 // 数字越小越相似, 数字为0则为完全一致.
|
||||
|
||||
func (bl *Baseline) FuzzyCompare(other *Baseline) bool {
|
||||
// 这里使用rawsimhash, 是为了保证一定数量的字符串, 否则超短的body会导致simhash偏差指较大
|
||||
if other.Distance = encode.SimhashCompare(other.RawSimhash, bl.RawSimhash); other.Distance < Distance {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func UniqueHash(bl *Baseline) uint16 {
|
||||
// 由host+状态码+重定向url+content-type+title+length舍去个位组成的hash
|
||||
// body length可能会导致一些误报, 目前没有更好的解决办法
|
||||
return pkg.CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/10*10)))
|
||||
}
|
||||
package baseline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/chainreactors/fingers/common"
|
||||
"github.com/chainreactors/logs"
|
||||
"github.com/chainreactors/parsers"
|
||||
"github.com/chainreactors/spray/core/ihttp"
|
||||
"github.com/chainreactors/spray/pkg"
|
||||
"github.com/chainreactors/utils/encode"
|
||||
"github.com/chainreactors/utils/iutils"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func NewBaseline(u, host string, resp *ihttp.Response) *Baseline {
|
||||
var err error
|
||||
bl := &Baseline{
|
||||
SprayResult: &parsers.SprayResult{
|
||||
UrlString: u,
|
||||
Status: resp.StatusCode(),
|
||||
IsValid: true,
|
||||
Frameworks: make(common.Frameworks),
|
||||
},
|
||||
}
|
||||
|
||||
if t, ok := pkg.MatchContentType(resp.ContentType()); ok {
|
||||
bl.ContentType = t
|
||||
bl.Title = t + " data"
|
||||
} else {
|
||||
bl.ContentType = "other"
|
||||
}
|
||||
|
||||
header := resp.Header()
|
||||
bl.Header = make([]byte, len(header))
|
||||
copy(bl.Header, header)
|
||||
bl.HeaderLength = len(bl.Header)
|
||||
if i := resp.ContentLength(); ihttp.CheckBodySize(i) {
|
||||
if body := resp.Body(); body != nil {
|
||||
bl.Body = make([]byte, len(body))
|
||||
copy(bl.Body, body)
|
||||
}
|
||||
|
||||
if i == -1 {
|
||||
bl.Chunked = true
|
||||
bl.BodyLength = len(bl.Body)
|
||||
} else {
|
||||
bl.BodyLength = int(i)
|
||||
}
|
||||
}
|
||||
|
||||
bl.Raw = append(bl.Header, bl.Body...)
|
||||
bl.Response, err = pkg.ParseRawResponse(bl.Raw)
|
||||
if err != nil {
|
||||
// raw 重解析失败不影响 baseline 有效性，live response 已提供所有需要的数据
|
||||
logs.Log.Debugf("ParseRawResponse failed for %s: %s", u, err.Error())
|
||||
}
|
||||
// 始终从 live response 读取 Location
|
||||
if r := resp.GetHeader("Location"); r != "" {
|
||||
bl.RedirectURL = r
|
||||
} else {
|
||||
bl.RedirectURL = resp.GetHeader("location")
|
||||
}
|
||||
|
||||
bl.Dir = bl.IsDir()
|
||||
uu, err := url.Parse(u)
|
||||
if err == nil {
|
||||
bl.Path = uu.Path
|
||||
bl.Url = uu
|
||||
if uu.Host != host {
|
||||
bl.Host = host
|
||||
}
|
||||
} else {
|
||||
bl.IsValid = false
|
||||
bl.Reason = pkg.ErrUrlError.Error()
|
||||
bl.ErrString = err.Error()
|
||||
}
|
||||
bl.Unique = UniqueHash(bl)
|
||||
return bl
|
||||
}
|
||||
|
||||
func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Baseline {
|
||||
bl := &Baseline{
|
||||
SprayResult: &parsers.SprayResult{
|
||||
UrlString: u,
|
||||
Status: resp.StatusCode(),
|
||||
IsValid: false,
|
||||
Reason: reason,
|
||||
},
|
||||
}
|
||||
|
||||
// 无效数据也要读取body, 否则keep-alive不生效
|
||||
resp.Body()
|
||||
bl.BodyLength = int(resp.ContentLength())
|
||||
bl.RedirectURL = string(resp.GetHeader("Location"))
|
||||
|
||||
bl.Dir = bl.IsDir()
|
||||
uu, err := url.Parse(u)
|
||||
if err == nil {
|
||||
bl.Path = uu.Path
|
||||
bl.Url = uu
|
||||
} else {
|
||||
return bl
|
||||
}
|
||||
|
||||
if bl.Url.Host != host {
|
||||
bl.Host = host
|
||||
}
|
||||
|
||||
return bl
|
||||
}
|
||||
|
||||
type Baseline struct {
|
||||
*parsers.SprayResult
|
||||
Url *url.URL `json:"-"`
|
||||
Dir bool `json:"-"`
|
||||
Chunked bool `json:"-"`
|
||||
Body pkg.BS `json:"-"`
|
||||
Header pkg.BS `json:"-"`
|
||||
Raw pkg.BS `json:"-"`
|
||||
Response *http.Response `json:"-"`
|
||||
Recu bool `json:"-"`
|
||||
RecuDepth int `json:"-"`
|
||||
URLs []string `json:"-"`
|
||||
Collected bool `json:"-"`
|
||||
Retry int `json:"-"`
|
||||
SameRedirectDomain bool `json:"-"`
|
||||
IsBaseline bool `json:"-"`
|
||||
}
|
||||
|
||||
func (bl *Baseline) IsDir() bool {
|
||||
if strings.HasSuffix(bl.Path, "/") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Collect 深度收集信息
|
||||
func (bl *Baseline) Collect() {
|
||||
if bl.Collected { // 防止重复收集
|
||||
return
|
||||
} else {
|
||||
bl.Collected = true
|
||||
}
|
||||
|
||||
if bl.ContentType == "html" || bl.ContentType == "json" || bl.ContentType == "txt" {
|
||||
// 指纹库设计的时候没考虑js,css文件的指纹, 跳过非必要的指纹收集减少误报提高性能
|
||||
//fmt.Println(bl.Source, bl.Url.String()+bl.Path, bl.RedirectURL, "call fingersengine")
|
||||
if pkg.EnableAllFingerEngine {
|
||||
bl.Frameworks = pkg.EngineDetect(bl.Raw)
|
||||
} else {
|
||||
bl.Frameworks = pkg.FingersDetect(bl.Raw)
|
||||
}
|
||||
}
|
||||
|
||||
if len(bl.Body) > 0 {
|
||||
if bl.ContentType == "html" {
|
||||
bl.Title = iutils.AsciiEncode(parsers.MatchTitle(bl.Body))
|
||||
} else if bl.ContentType == "ico" && pkg.FingerEngine != nil && pkg.FingerEngine.Favicon() != nil {
|
||||
if frame := pkg.FingerEngine.Favicon().WebMatch(bl.Body); frame != nil {
|
||||
bl.Frameworks.Merge(frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bl.Hashes = parsers.NewHashes(bl.Raw)
|
||||
bl.Extracteds.Merge(pkg.ProtonExtract(bl.Raw))
|
||||
bl.Unique = UniqueHash(bl)
|
||||
}
|
||||
|
||||
func (bl *Baseline) CollectURL() {
|
||||
if len(bl.Body) == 0 {
|
||||
return
|
||||
}
|
||||
for _, reg := range pkg.ExtractRegexps["js"][0].CompiledRegexps {
|
||||
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
|
||||
for _, u := range urls {
|
||||
u[1] = pkg.CleanURL(u[1])
|
||||
if u[1] != "" && !pkg.FilterJs(u[1]) {
|
||||
bl.URLs = append(bl.URLs, u[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, reg := range pkg.ExtractRegexps["url"][0].CompiledRegexps {
|
||||
urls := reg.FindAllStringSubmatch(string(bl.Body), -1)
|
||||
for _, u := range urls {
|
||||
u[1] = pkg.CleanURL(u[1])
|
||||
if u[1] != "" && !pkg.FilterUrl(u[1]) {
|
||||
bl.URLs = append(bl.URLs, u[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bl.URLs = iutils.StringsUnique(bl.URLs)
|
||||
if len(bl.URLs) != 0 {
|
||||
bl.Extracteds = append(bl.Extracteds, &parsers.Extracted{
|
||||
Name: "crawl",
|
||||
ExtractResult: bl.URLs,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Compare
|
||||
// if totally equal return 1
|
||||
// if maybe equal return 0
|
||||
// not equal return -1
|
||||
func (bl *Baseline) Compare(other *Baseline) int {
|
||||
if other.RedirectURL != "" && bl.RedirectURL == other.RedirectURL {
|
||||
// 如果重定向url不为空, 且与base不相同, 则说明不是同一个页面
|
||||
return 1
|
||||
}
|
||||
|
||||
if bl.BodyLength == other.BodyLength {
|
||||
// 如果body length相等且md5相等, 则说明是同一个页面
|
||||
if bytes.Equal(bl.Body, other.Body) {
|
||||
// 如果length相等, md5也相等, 则判断为全同
|
||||
return 1
|
||||
} else {
|
||||
// 如果长度相等, 但是md5不相等, 可能是存在csrftoken之类的随机值
|
||||
return 0
|
||||
}
|
||||
} else if i := bl.BodyLength - other.BodyLength; (i < 16 && i > 0) || (i > -16 && i < 0) {
|
||||
// 如果body length绝对值小于16, 则可能是存在csrftoken之类的随机值, 需要模糊判断
|
||||
return 0
|
||||
} else {
|
||||
// 如果body length绝对值大于16, 则认为大概率存在较大差异
|
||||
if strings.Contains(string(other.Body), other.Path) {
|
||||
// 如果包含路径本身, 可能是路径自身的随机值影响结果
|
||||
return 0
|
||||
} else {
|
||||
// 如果不包含路径本身, 则认为是不同页面
|
||||
return -1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (bl *Baseline) ProbeOutput(format []string) string {
|
||||
var s strings.Builder
|
||||
for _, f := range format {
|
||||
s.WriteString("\t")
|
||||
s.WriteString(bl.Get(f))
|
||||
}
|
||||
return strings.TrimSpace(s.String())
|
||||
}
|
||||
|
||||
var Distance uint8 = 5 // 数字越小越相似, 数字为0则为完全一致.
|
||||
|
||||
func (bl *Baseline) FuzzyCompare(other *Baseline) bool {
|
||||
// 这里使用rawsimhash, 是为了保证一定数量的字符串, 否则超短的body会导致simhash偏差指较大
|
||||
if other.Distance = encode.SimhashCompare(other.RawSimhash, bl.RawSimhash); other.Distance < Distance {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func UniqueHash(bl *Baseline) uint16 {
|
||||
// 由host+状态码+重定向url+content-type+title+length舍去个位组成的hash
|
||||
// body length可能会导致一些误报, 目前没有更好的解决办法
|
||||
return pkg.CRC16Hash([]byte(bl.Host + strconv.Itoa(bl.Status) + bl.RedirectURL + bl.ContentType + bl.Title + strconv.Itoa(bl.BodyLength/10*10)))
|
||||
}
|
||||
+2
-2
@@ -264,8 +264,8 @@ func PrintPreset() {
|
||||
logs.Log.Consolef("\t%s\t%d words\n", name, len(words))
|
||||
}
|
||||
|
||||
logs.Log.Console("\ninternal extractor:\n")
|
||||
for name, _ := range pkg.ExtractRegexps {
|
||||
logs.Log.Console("\ninternal extractor (proton):\n")
|
||||
for _, name := range pkg.ProtonExtractorNames() {
|
||||
logs.Log.Consolef("\t%s\n", name)
|
||||
}
|
||||
|
||||
|
||||
@@ -130,7 +130,8 @@ func customDialFunc(dialer proxyclient.Dial, timeout time.Duration) fasthttp.Dia
|
||||
}
|
||||
}
|
||||
return func(addr string) (net.Conn, error) {
|
||||
ctx, _ := context.WithTimeout(context.Background(), timeout)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
return dialer.DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -24,13 +24,6 @@ var extractData []byte
|
||||
//go:embed data/port.bin
|
||||
var portData []byte
|
||||
|
||||
//go:embed data/proton_rules.bin
|
||||
var protonRulesData []byte
|
||||
|
||||
//go:embed data/neutron.bin
|
||||
var neutronData []byte
|
||||
|
||||
var RandomDir = "/g8kZMwp4oeKsL2in"
|
||||
|
||||
func loadEmbeddedConfig(typ string) []byte {
|
||||
if typ == "spray_rule" {
|
||||
@@ -43,10 +36,6 @@ func loadEmbeddedConfig(typ string) []byte {
|
||||
return encode.MustDeflateDeCompress(extractData)
|
||||
}else if typ == "port" {
|
||||
return encode.MustDeflateDeCompress(portData)
|
||||
}else if typ == "proton_rules" {
|
||||
return encode.MustDeflateDeCompress(protonRulesData)
|
||||
}else if typ == "neutron" {
|
||||
return encode.MustDeflateDeCompress(neutronData)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+122
-122
@@ -1,122 +1,122 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseRawResponse(t *testing.T) {
|
||||
t.Run("valid complete response", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 5\r\n\r\nhello"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("expected status 200, got %d", resp.StatusCode)
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "text/html" {
|
||||
t.Fatalf("expected Content-Type text/html, got %s", ct)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil input", func(t *testing.T) {
|
||||
_, err := ParseRawResponse(nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for nil input")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty input", func(t *testing.T) {
|
||||
_, err := ParseRawResponse([]byte{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty input")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("status line only no headers", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("expected status 200, got %d", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("truncated status line", func(t *testing.T) {
|
||||
raw := "HTTP/1."
|
||||
_, err := ParseRawResponse([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for truncated status line")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("redirect response no body", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 302 Found\r\nLocation: https://example.com/new\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 302 {
|
||||
t.Fatalf("expected status 302, got %d", resp.StatusCode)
|
||||
}
|
||||
if loc := resp.Header.Get("Location"); loc != "https://example.com/new" {
|
||||
t.Fatalf("expected Location https://example.com/new, got %s", loc)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("chunked transfer encoding header", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if te := resp.Header.Get("Transfer-Encoding"); te == "" {
|
||||
// Transfer-Encoding may be consumed by http.ReadResponse, just verify no panic
|
||||
}
|
||||
_ = resp
|
||||
})
|
||||
|
||||
t.Run("large header value", func(t *testing.T) {
|
||||
largeValue := strings.Repeat("A", 8192)
|
||||
raw := "HTTP/1.1 200 OK\r\nX-Large: " + largeValue + "\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := resp.Header.Get("X-Large"); got != largeValue {
|
||||
t.Fatalf("large header value mismatch, got length %d", len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid status code", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 xyz OK\r\n\r\n"
|
||||
_, err := ParseRawResponse([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid status code")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("incomplete header no terminator", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html"
|
||||
_, err := ParseRawResponse([]byte(raw))
|
||||
// Should return error or at least not panic
|
||||
// http.ReadResponse may or may not error on missing \r\n\r\n,
|
||||
// the key requirement is no panic
|
||||
_ = err
|
||||
})
|
||||
|
||||
t.Run("binary body", func(t *testing.T) {
|
||||
body := []byte{0x00, 0x01, 0x02, 0xff, 0xfe, 0xfd}
|
||||
raw := append([]byte("HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n"), body...)
|
||||
resp, err := ParseRawResponse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("expected status 200, got %d", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseRawResponse(t *testing.T) {
|
||||
t.Run("valid complete response", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 5\r\n\r\nhello"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("expected status 200, got %d", resp.StatusCode)
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "text/html" {
|
||||
t.Fatalf("expected Content-Type text/html, got %s", ct)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil input", func(t *testing.T) {
|
||||
_, err := ParseRawResponse(nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for nil input")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty input", func(t *testing.T) {
|
||||
_, err := ParseRawResponse([]byte{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty input")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("status line only no headers", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("expected status 200, got %d", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("truncated status line", func(t *testing.T) {
|
||||
raw := "HTTP/1."
|
||||
_, err := ParseRawResponse([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for truncated status line")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("redirect response no body", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 302 Found\r\nLocation: https://example.com/new\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 302 {
|
||||
t.Fatalf("expected status 302, got %d", resp.StatusCode)
|
||||
}
|
||||
if loc := resp.Header.Get("Location"); loc != "https://example.com/new" {
|
||||
t.Fatalf("expected Location https://example.com/new, got %s", loc)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("chunked transfer encoding header", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if te := resp.Header.Get("Transfer-Encoding"); te == "" {
|
||||
// Transfer-Encoding may be consumed by http.ReadResponse, just verify no panic
|
||||
}
|
||||
_ = resp
|
||||
})
|
||||
|
||||
t.Run("large header value", func(t *testing.T) {
|
||||
largeValue := strings.Repeat("A", 8192)
|
||||
raw := "HTTP/1.1 200 OK\r\nX-Large: " + largeValue + "\r\n\r\n"
|
||||
resp, err := ParseRawResponse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := resp.Header.Get("X-Large"); got != largeValue {
|
||||
t.Fatalf("large header value mismatch, got length %d", len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid status code", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 xyz OK\r\n\r\n"
|
||||
_, err := ParseRawResponse([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid status code")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("incomplete header no terminator", func(t *testing.T) {
|
||||
raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html"
|
||||
_, err := ParseRawResponse([]byte(raw))
|
||||
// Should return error or at least not panic
|
||||
// http.ReadResponse may or may not error on missing \r\n\r\n,
|
||||
// the key requirement is no panic
|
||||
_ = err
|
||||
})
|
||||
|
||||
t.Run("binary body", func(t *testing.T) {
|
||||
body := []byte{0x00, 0x01, 0x02, 0xff, 0xfe, 0xfd}
|
||||
raw := append([]byte("HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n"), body...)
|
||||
resp, err := ParseRawResponse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("expected status 200, got %d", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user