feat: verify URL content,use baidu for alive test

Author: mark0smith

README.md

@@ -10,8 +10,12 @@
 ## 帮助
 
 ```shell
-> .\rotateproxy.exe -h
+.\rotateproxy.exe -h
 Usage of rotateproxy.exe:
+  -check string
+        check url (default "https://www.google.com")
+  -checkWords string
+        words in check url (default "Copyright The Closure Library Authors")
   -email string
         email address
   -l string
@@ -20,10 +24,14 @@ Usage of rotateproxy.exe:
         the page count you want to crawl (default 5)
   -pass string
         authentication password
+  -proxy string
+        proxy
   -region int
         0: all 1: cannot bypass gfw 2: bypass gfw
   -rule string
-        search rule (default "protocol==\"socks5\" && \"Version:5 Method:No Authentication(0x00)\" && after=\"2021-08-01\" && country=\"CN\"")
+        search rule (default "protocol==\"socks5\" && \"Version:5 Method:No Authentication(0x00)\" && after=\"2022-02-01\" && country=\"CN\"")
+  -strategy int
+        0: random, 1: Select the one with the shortest timeout, 2: Select the two with the shortest timeout, ... (default 3)
   -token string
         token
   -user string

check.go

@@ -3,7 +3,7 @@ package rotateproxy
 import (
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -38,23 +38,26 @@ func CheckProxyAlive(proxyURL string) (respBody string, timeout int64, avail boo
 		Timeout: 20 * time.Second,
 	}
 	startTime := time.Now()
-	resp, err := httpclient.Get("http://cip.cc/")
+
+	// http://cip.cc isn't stable enough for proxies alive test.
+	resp, err := httpclient.Get("https://www.baidu.com/robots.txt")
+
 	if err != nil {
 		return "", 0, false
 	}
 	defer resp.Body.Close()
 	timeout = int64(time.Since(startTime))
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", 0, false
 	}
-	if !strings.Contains(string(body), "地址") {
+	if !strings.Contains(string(body), "Baiduspider-image") {
 		return "", 0, false
 	}
 	return string(body), timeout, true
 }
 
-func CheckProxyWithCheckURL(proxyURL string, checkURL string) (timeout int64, avail bool) {
+func CheckProxyWithCheckURL(proxyURL string, checkURL string, checkURLwords string) (timeout int64, avail bool) {
 	fmt.Printf("check %s： %s\n", proxyURL, checkURL)
 	proxy, _ := url.Parse(proxyURL)
 	httpclient := &http.Client{
@@ -72,32 +75,41 @@ func CheckProxyWithCheckURL(proxyURL string, checkURL string) (timeout int64, av
 	}
 	defer resp.Body.Close()
 	timeout = int64(time.Since(startTime))
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		return 0, false
+	}
 
 	// TODO: support regex
 	if resp.StatusCode != 200 {
 		return 0, false
 	}
 
+	if !strings.Contains(string(body), checkURLwords) {
+		return 0, false
+	}
+
 	return timeout, true
 }
 
-func StartCheckProxyAlive(checkURL string) {
+func StartCheckProxyAlive(checkURL string, checkURLwords string) {
 	go func() {
 		ticker := time.NewTicker(120 * time.Second)
 		for {
 			select {
 			case <-crawlDone:
 				fmt.Println("Checking")
-				checkAlive(checkURL)
+				checkAlive(checkURL, checkURLwords)
 				fmt.Println("Check done")
 			case <-ticker.C:
-				checkAlive(checkURL)
+				checkAlive(checkURL, checkURLwords)
 			}
 		}
 	}()
 }
 
-func checkAlive(checkURL string) {
+func checkAlive(checkURL string, checkURLwords string) {
 	proxies, err := QueryProxyURL()
 	if err != nil {
 		fmt.Printf("[!] query db error: %v\n", err)
@@ -108,7 +120,7 @@ func checkAlive(checkURL string) {
 			respBody, timeout, avail := CheckProxyAlive(proxy.URL)
 			if avail {
 				if checkURL != "" {
-					timeout, avail = CheckProxyWithCheckURL(proxy.URL, checkURL)
+					timeout, avail = CheckProxyWithCheckURL(proxy.URL, checkURL, checkURLwords)
 				}
 				if avail {
 					fmt.Printf("%v 可用\n", proxy.URL)

cmd/rotateproxy/main.go

@@ -2,21 +2,24 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"os"
 	"regexp"
 	"strings"
 
 	"github.com/akkuman/rotateproxy"
 )
 
 var (
-	baseCfg     rotateproxy.BaseConfig
-	email       string
-	token       string
-	rule        string
-	pageCount   int
-	proxy       string
-	checkURL    string
-	portPattern = regexp.MustCompile(`^\d+$`)
+	baseCfg       rotateproxy.BaseConfig
+	email         string
+	token         string
+	rule          string
+	pageCount     int
+	proxy         string
+	checkURL      string
+	checkURLwords string
+	portPattern   = regexp.MustCompile(`^\d+$`)
 )
 
 func init() {
@@ -28,10 +31,17 @@ func init() {
 	flag.StringVar(&proxy, "proxy", "", "proxy")
 	flag.StringVar(&rule, "rule", `protocol=="socks5" && "Version:5 Method:No Authentication(0x00)" && after="2022-02-01" && country="CN"`, "search rule")
 	flag.StringVar(&checkURL, "check", `https://www.google.com`, "check url")
+	flag.StringVar(&checkURLwords, "checkWords", `Copyright The Closure Library Authors`, "words in check url")
 	flag.IntVar(&baseCfg.IPRegionFlag, "region", 0, "0: all 1: cannot bypass gfw 2: bypass gfw")
 	flag.IntVar(&baseCfg.SelectStrategy, "strategy", 3, "0: random, 1: Select the one with the shortest timeout, 2: Select the two with the shortest timeout, ...")
 	flag.IntVar(&pageCount, "page", 5, "the page count you want to crawl")
 	flag.Parse()
+
+	if checkURL != "https://www.google.com" && checkURLwords == "Copyright The Closure Library Authors" {
+		fmt.Println("You set check url but forget to set `-checkWords`!")
+		os.Exit(1)
+	}
+
 }
 
 func isFlagPassed(name string) bool {
@@ -57,7 +67,7 @@ func main() {
 	}
 
 	rotateproxy.StartRunCrawler(token, email, rule, pageCount, proxy)
-	rotateproxy.StartCheckProxyAlive(checkURL)
+	rotateproxy.StartCheckProxyAlive(checkURL, checkURLwords)
 	c := rotateproxy.NewRedirectClient(rotateproxy.WithConfig(&baseCfg))
 	c.Serve()
 	select {}