修改webtitle模块,加入gbk解码,减少乱码

Plugins/webtitle.go

@@ -1,16 +1,24 @@
 package Plugins
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/shadow1ng/fscan/WebScan"
 	"github.com/shadow1ng/fscan/WebScan/lib"
 	"github.com/shadow1ng/fscan/common"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/transform"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strings"
 )
 
+var (
+	Charsets = []string{"utf-8", "gbk", "gb2312"}
+)
+
 func WebTitle(info *common.HostInfo) error {
 	err := GOWebTitle(info)
 	if err != nil {
@@ -20,6 +28,11 @@ func WebTitle(info *common.HostInfo) error {
 	return err
 }
 
+//flag 1 first try
+//flag 2 /favicon.ico
+//flag 3 302
+//flag 4 400 -> https
+
 func GOWebTitle(info *common.HostInfo) error {
 	var CheckData []WebScan.CheckDatas
 	if info.Url == "" {
@@ -36,19 +49,42 @@ func GOWebTitle(info *common.HostInfo) error {
 		}
 	}
 
-	err, result, CheckData := geturl(info, true, CheckData)
+	err, result, CheckData := geturl(info, 1, CheckData)
 	if err != nil {
 		return err
 	}
-
+	if strings.Contains(result, "://") {
-	if result == "https" {
+		//有跳转
-		err, _, CheckData = geturl(info, true, CheckData)
+		redirecturl, err := url.Parse(result)
-		if err != nil {
+		if err == nil {
-			return err
+			info.Url = redirecturl.String()
+			err, result, CheckData = geturl(info, 3, CheckData)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
-	err, _, CheckData = geturl(info, false, CheckData)
+	if result == "https" {
+		err, result, CheckData = geturl(info, 1, CheckData)
+		if strings.Contains(result, "://") {
+			//有跳转
+			redirecturl, err := url.Parse(result)
+			if err == nil {
+				info.Url = redirecturl.String()
+				err, result, CheckData = geturl(info, 3, CheckData)
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	err, _, CheckData = geturl(info, 2, CheckData)
 	if err != nil {
 		return err
 	}
@@ -61,11 +97,17 @@ func GOWebTitle(info *common.HostInfo) error {
 	return err
 }
 
-func geturl(info *common.HostInfo, flag bool, CheckData []WebScan.CheckDatas) (error, string, []WebScan.CheckDatas) {
+func geturl(info *common.HostInfo, flag int, CheckData []WebScan.CheckDatas) (error, string, []WebScan.CheckDatas) {
 	Url := info.Url
-	if flag == false {
+	if flag == 2 {
-		Url += "/favicon.ico"
+		URL, err := url.Parse(Url)
+		if err == nil {
+			Url = fmt.Sprintf("%s://%s/favicon.ico", URL.Scheme, URL.Host)
+		} else {
+			Url += "/favicon.ico"
+		}
 	}
+
 	res, err := http.NewRequest("GET", Url, nil)
 	if err == nil {
 		res.Header.Set("User-agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36")
@@ -73,34 +115,84 @@ func geturl(info *common.HostInfo, flag bool, CheckData []WebScan.CheckDatas) (e
 		res.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
 		res.Header.Set("Accept-Encoding", "gzip, deflate")
 		if common.Pocinfo.Cookie != "" {
-			res.Header.Set("Cookie", common.Pocinfo.Cookie)
-		}
-		if flag == true {
 			res.Header.Set("Cookie", "rememberMe=1;"+common.Pocinfo.Cookie)
+		} else {
+			res.Header.Set("Cookie", "rememberMe=1")
 		}
 		res.Header.Set("Connection", "close")
-		resp, err := lib.Client.Do(res)
+
+		var client *http.Client
+		if flag == 1 {
+			client = lib.ClientNoRedirect
+		} else {
+			client = lib.Client
+		}
+
+		resp, err := client.Do(res)
 		if err == nil {
 			defer resp.Body.Close()
 			var title string
 			body, _ := ioutil.ReadAll(resp.Body)
-			re := regexp.MustCompile("<title>(.*)</title>")
+			if flag != 2 {
-			find := re.FindAllStringSubmatch(string(body), -1)
+				re := regexp.MustCompile("(?im)<title>(.*)</title>")
-			if len(find) > 0 {
+				find := re.FindSubmatch(body)
-				title = find[0][1]
+				if len(find) > 1 {
+					text := find[1]
+					GetEncoding := func() string { // 判断Content-Type
+						r1, err := regexp.Compile(`(?im)charset=\s*?([\w-]+)`)
+						if err != nil {
+							return ""
+						}
+						headerCharset := r1.FindString(resp.Header.Get("Content-Type"))
+						if headerCharset != "" {
+							for _, v := range Charsets { // headers 编码优先，所以放在前面
+								if strings.Contains(strings.ToLower(headerCharset), v) == true {
+									return v
+								}
+							}
+						}
+
+						r2, err := regexp.Compile(`(?im)<meta.*?charset=['"]?([\w-]+)["']?.*?>`)
+						if err != nil {
+							return ""
+						}
+						htmlCharset := r2.FindString(string(body))
+						if htmlCharset != "" {
+							for _, v := range Charsets {
+								if strings.Contains(strings.ToLower(htmlCharset), v) == true {
+									return v
+								}
+							}
+						}
+						return ""
+					}
+					encoding := GetEncoding()
+					if encoding == "gbk" || encoding == "gb2312" {
+						titleGBK, err := Decodegbk(text)
+						if err == nil {
+							title = string(titleGBK)
+						}
+					} else {
+						title = string(text)
+					}
+				} else {
+					title = "None"
+				}
+				title = strings.Trim(title, "\r\n \t")
+				title = strings.Replace(title, "\n", "", -1)
+				title = strings.Replace(title, "\r", "", -1)
+				title = strings.Replace(title, "&nbsp;", " ", -1)
 				if len(title) > 100 {
 					title = title[:100]
 				}
-			} else {
-				title = "None"
-			}
-			if flag == true {
 				result := fmt.Sprintf("[*] WebTitle:%-25v %-3v %v", Url, resp.StatusCode, title)
 				common.LogSuccess(result)
 			}
-
 			CheckData = append(CheckData, WebScan.CheckDatas{body, fmt.Sprintf("%s", resp.Header)})
-
+			redirURL, err1 := resp.Location()
+			if err1 == nil {
+				return nil, redirURL.String(), CheckData
+			}
 			if resp.StatusCode == 400 && info.Url[:5] != "https" {
 				info.Url = strings.Replace(info.Url, "http://", "https://", 1)
 				return err, "https", CheckData
@@ -111,3 +203,13 @@ func geturl(info *common.HostInfo, flag bool, CheckData []WebScan.CheckDatas) (e
 	}
 	return err, "", CheckData
 }
+
+func Decodegbk(s []byte) ([]byte, error) { // GBK解码
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}

README.md

@@ -152,6 +152,7 @@ fscan 是 404Team [星链计划2.0](https://github.com/knownsec/404StarLink2.0-G
 
 
 ## 最近更新
+[+] 2021/4/22 修改webtitle模块,加入gbk解码  
 [+] 2021/4/21 加入netbios探测、域控识别  
 [+] 2021/3/4 支持-u url或者-uf url.txt,对url进行批量扫描  
 [+] 2021/2/25 修改yaml解析模块,支持密码爆破,如tomcat弱口令。yaml中新增sets参数,类型为数组,用于存放密码,具体看tomcat-manager-week.yaml  

WebScan/lib/http.go

@@ -15,7 +15,7 @@ import (
 
 var (
 	Client           *http.Client
-	clientNoRedirect *http.Client
+	ClientNoRedirect *http.Client
 	dialTimout       = 5 * time.Second
 	keepAlive        = 15 * time.Second
 )
@@ -51,11 +51,11 @@ func InitHttpClient(ThreadsNum int, DownProxy string, Timeout time.Duration) err
 		Transport: tr,
 		Timeout:   Timeout,
 	}
-	clientNoRedirect = &http.Client{
+	ClientNoRedirect = &http.Client{
 		Transport: tr,
 		Timeout:   Timeout,
 	}
-	clientNoRedirect.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+	ClientNoRedirect.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 		return http.ErrUseLastResponse
 	}
 	return nil
@@ -75,7 +75,7 @@ func DoRequest(req *http.Request, redirect bool) (*Response, error) {
 	if redirect {
 		oResp, err = Client.Do(req)
 	} else {
-		oResp, err = clientNoRedirect.Do(req)
+		oResp, err = ClientNoRedirect.Do(req)
 	}
 	if err != nil {
 		return nil, err

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/stacktitan/smb v0.0.0-20190531122847-da9a425dceb8
 	golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc
 	golang.org/x/net v0.0.0-20210420072503-d25e30425868
+	golang.org/x/text v0.3.6
 	google.golang.org/genproto v0.0.0-20210416161957-9910b6c460de
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )