add debug

Author: taro

check.go

@@ -41,8 +41,9 @@ func CheckProxyAlive(proxyURL string) (respBody string, timeout int64, avail boo
 
 	// http://cip.cc isn't stable enough for proxies alive test.
 	resp, err := httpclient.Get("https://www.baidu.com/robots.txt")
-	
+
 	if err != nil {
+		DebugLog(Warn("[!] %v: %v", proxyURL, err))
 		return "", 0, false
 	}
 	defer resp.Body.Close()
@@ -52,7 +53,8 @@ func CheckProxyAlive(proxyURL string) (respBody string, timeout int64, avail boo
 		return "", 0, false
 	}
 	if !strings.Contains(string(body), "Baiduspider-image") {
-			return "", 0, false
+		DebugLog(Warn("[!] err  live  %s resp: %v", proxyURL, err))
+		return "", 0, false
 	}
 	return string(body), timeout, true
 }
@@ -71,13 +73,15 @@ func CheckProxyWithCheckURL(proxyURL string, checkURL string, checkURLwords stri
 	startTime := time.Now()
 	resp, err := httpclient.Get(checkURL)
 	if err != nil {
+		DebugLog(Warn("[!] httpclient.Get: %v", err))
 		return 0, false
 	}
 	defer resp.Body.Close()
 	timeout = int64(time.Since(startTime))
 	body, err := io.ReadAll(resp.Body)
 
 	if err != nil {
+		DebugLog(Warn("[!] io.ReadAll: %v", err))
 		return 0, false
 	}
 

cmd/rotateproxy/main.go

@@ -41,6 +41,7 @@ func init() {
 	flag.IntVar(&baseCfg.IPRegionFlag, "region", 0, "0: all 1: cannot bypass gfw 2: bypass gfw")
 	flag.IntVar(&baseCfg.SelectStrategy, "strategy", 3, "0: random, 1: Select the one with the shortest timeout, 2: Select the two with the shortest timeout, ...")
 	flag.IntVar(&pageCount, "page", 5, "the page count you want to crawl")
+	flag.BoolVar(&rotateproxy.IsDebug, "debug", false, "debug mode to see log")
 	flag.Parse()
 
 	if checkURL != "https://www.google.com" && checkURLwords == "Copyright The Closure Library Authors" {

util.go

@@ -2,6 +2,7 @@ package rotateproxy
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"math/rand"
@@ -102,9 +103,16 @@ var (
 	Noticeln = color.New(color.FgBlue).SprintFunc()
 	Info     = color.New(color.FgGreen).SprintfFunc()
 	Warn     = color.New(color.FgRed).SprintfFunc()
+	IsDebug  = false
 )
 
 var (
 	InfoLog  = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile).Println
 	ErrorLog = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile).Println
 )
+
+func DebugLog(content string) {
+	if IsDebug {
+		fmt.Print(content)
+	}
+}