添加proxy代理选项，这样可以使用代理进行fofa搜索，用于防止fofa账号被锁 (akkuman#7)

* 添加proxy代理选项，这样可以使用代理进行fofa搜索，用于防止fofa账号被锁

* remove vendor

Author: izj007

cmd/rotateproxy/main.go

@@ -14,6 +14,7 @@ var (
 	token       string
 	rule        string
 	pageCount   int
+	proxy       string
 	portPattern = regexp.MustCompile(`^\d+$`)
 )
 
@@ -23,7 +24,8 @@ func init() {
 	flag.StringVar(&baseCfg.Password, "pass", "", "authentication password")
 	flag.StringVar(&email, "email", "", "email address")
 	flag.StringVar(&token, "token", "", "token")
-	flag.StringVar(&rule, "rule", `protocol=="socks5" && "Version:5 Method:No Authentication(0x00)" && after="2021-08-01" && country="CN"`, "search rule")
+	flag.StringVar(&proxy, "proxy", "", "proxy")
+	flag.StringVar(&rule, "rule", `protocol=="socks5" && "Version:5 Method:No Authentication(0x00)" && after="2022-02-01" && country="CN"`, "search rule")
 	flag.IntVar(&baseCfg.IPRegionFlag, "region", 0, "0: all 1: cannot bypass gfw 2: bypass gfw")
 	flag.IntVar(&baseCfg.SelectStrategy, "strategy", 3, "0: random, 1: Select the one with the shortest timeout, 2: Select the two with the shortest timeout, ...")
 	flag.IntVar(&pageCount, "page", 5, "the page count you want to crawl")
@@ -52,7 +54,7 @@ func main() {
 		baseCfg.ListenAddr = ":" + baseCfg.ListenAddr
 	}
 
-	rotateproxy.StartRunCrawler(token, email, rule, pageCount)
+	rotateproxy.StartRunCrawler(token, email, rule, pageCount, proxy)
 	rotateproxy.StartCheckProxyAlive()
 	c := rotateproxy.NewRedirectClient(rotateproxy.WithConfig(&baseCfg))
 	c.Serve()

crawler.go

@@ -1,10 +1,12 @@
 package rotateproxy
 
 import (
+	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -23,11 +25,20 @@ func addProxyURL(url string) {
 	CreateProxyURL(url)
 }
 
-func RunCrawler(fofaApiKey, fofaEmail, rule string, pageNum int) (err error) {
+func RunCrawler(fofaApiKey, fofaEmail, rule string, pageNum int, proxy string) (err error) {
 	req, err := http.NewRequest("GET", "https://fofa.info/api/v1/search/all", nil)
 	if err != nil {
 		return err
 	}
+	tr := &http.Transport{TLSClientConfig: &tls.Config{
+		InsecureSkipVerify: true,
+	}}
+	if proxy != "" {
+		proxyUrl, err := url.Parse(proxy)
+		if err == nil { // 使用传入代理
+			tr.Proxy = http.ProxyURL(proxyUrl)
+		}
+	}
 	rule = base64.StdEncoding.EncodeToString([]byte(rule))
 	q := req.URL.Query()
 	q.Add("email", fofaEmail)
@@ -37,7 +48,8 @@ func RunCrawler(fofaApiKey, fofaEmail, rule string, pageNum int) (err error) {
 	q.Add("page", fmt.Sprintf("%d", pageNum))
 	q.Add("fields", "host,title,ip,domain,port,country,city,server,protocol")
 	req.URL.RawQuery = q.Encode()
-	resp, err := http.DefaultClient.Do(req)
+	// resp, err := http.DefaultClient.Do(req)
+	resp, err := (&http.Client{Transport: tr}).Do(req)
 	if err != nil {
 		return err
 	}
@@ -57,10 +69,10 @@ func RunCrawler(fofaApiKey, fofaEmail, rule string, pageNum int) (err error) {
 	return
 }
 
-func StartRunCrawler(fofaApiKey, fofaEmail, rule string, pageCount int) {
+func StartRunCrawler(fofaApiKey, fofaEmail, rule string, pageCount int, proxy string) {
 	runCrawlerFunc := func() {
 		for i := 1; i <= 3; i++ {
-			err := RunCrawler(fofaApiKey, fofaEmail, rule, i)
+			err := RunCrawler(fofaApiKey, fofaEmail, rule, i, proxy)
 			if err != nil {
 				fmt.Printf("[!] error: %v\n", err)
 			}