modified URLChecker

Author: troyliu0105

.gitignore

@@ -1,2 +1,4 @@
 Utils/URLChecker/out
 Utils/URLChecker/*.class
+Utils/.idea
+Utils/Utils.iml

Utils/URLChecker/BadUrlsSaver.java

@@ -18,20 +18,22 @@ public class BadUrlsSaver {
 	 */
 	public static void save(Map<String, String> map, boolean isSaveAlLMatterUrls) throws IOException {
 		File file = new File("..\\..\\badUrls.txt");
-		if (!file.exists()) {
-			file.createNewFile();
+		if (file.exists()) {
+			System.out.println("删除先前文件: " + (file.delete() ? "成功" : "失败"));
 		}
 		FileWriter writer = new FileWriter(file);
 		BufferedWriter bufferedWriter = new BufferedWriter(writer);
+		int index = 0;
 		for (Map.Entry<String, String> entry : map.entrySet()) {
 			if (isSaveAlLMatterUrls) {
-				bufferedWriter.write(entry.getValue() + "\t\t" + entry.getKey() + "\r\n");
+				bufferedWriter.write((index++) + ". " + "[" + entry.getValue() + "]+ \"(\"+ entry.getKey() + +\")\"+ \"\r\n");
 			} else {
 				if (entry.getValue().equals("TIMEOUT")) {
-					bufferedWriter.write(entry.getValue() + "\t\t" + entry.getKey() + "\r\n");
+					bufferedWriter.write((index++) + ". " + "[" + entry.getValue() + "]+ \"(\"+ entry.getKey() + +\")\"+ \"\r\n");
 				}
 			}
 		}
 		bufferedWriter.close();
 	}
+
 }

Utils/URLChecker/NetUtils.java

@@ -4,6 +4,7 @@
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
+import java.net.ProtocolException;
 import java.net.URL;
 import java.util.HashMap;
 import java.util.List;
@@ -27,11 +28,16 @@ public static String getBundle(String urlMain) {
 		try {
 			URL url = new URL(urlMain);
 			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+			connection.setRequestProperty("Content-Type", "text/html");
+			connection.setRequestProperty("Accept-Charset", "utf-8");
+			connection.setRequestProperty("Content-Type", "utf-8");
+			connection.setRequestProperty("Charset", "utf-8");
+			connection.setRequestMethod("GET");
 			InputStream inputStream;
 			BufferedReader bufferedReader = null;
 			if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
 				inputStream = connection.getInputStream();
-				bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
+				bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
 				String line = null;
 				while ((line = bufferedReader.readLine()) != null) {
 					stringBuilder.append(line);
@@ -58,7 +64,7 @@ public static Map<String, String> getNotAvailableUrl(String urlMain) {
 			threadPool.execute(new ItemThread(url, new ICallback() {
 				@Override
 				public void finish(int code) {
-					matterUrls.put(url, "BAD");
+					matterUrls.put(url, "BAD" + "[Code: " + code + "]");
 				}
 
 				@Override
@@ -75,6 +81,7 @@ public void timeout() {
 		}
 	}
 
+
 }
 
 class ItemThread implements Runnable {
@@ -89,25 +96,32 @@ public ItemThread(String url, ICallback callback) {
 
 	@Override
 	public void run() {
+		HttpURLConnection connection = null;
 		try {
 			URL tmp = new URL(url);
-			HttpURLConnection connection = (HttpURLConnection) tmp.openConnection();
+			connection = (HttpURLConnection) tmp.openConnection();
 			connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
 			connection.setRequestMethod("GET");
-			connection.setConnectTimeout(1000);
+			connection.setReadTimeout(10000);
+			connection.setConnectTimeout(10000);
 			int responseCode = connection.getResponseCode();
 			if (responseCode == HttpURLConnection.HTTP_OK) {
 				System.out.println("checking [" + url + "]" + "----->[OK]");
 			} else {
 				System.out.println("checking [" + url + "]" + "----->[BAD]" + "; Code--->" + responseCode);
 				callback.finish(responseCode);
 			}
-			connection.disconnect();
 		} catch (MalformedURLException e) {
 			e.printStackTrace();
+		} catch (ProtocolException e) {
+			e.printStackTrace();
 		} catch (IOException e) {
-			System.out.println("checking [" + url + "]" + "[Time Out]");
+			System.out.println("checking [" + url + "]" + e.getMessage());
 			callback.timeout();
+		} finally {
+			if (connection != null) {
+				connection.disconnect();
+			}
 		}
 	}
 }

Utils/URLChecker/UrlMatcher.java

@@ -8,15 +8,19 @@
  */
 public class UrlMatcher {
 	public static List<String> getUrlStr(String response) {
-		Pattern pattern = Pattern.compile("(http|ftp|https):\\/\\/[\\w\\-_]+(\\.[\\w\\-_]+)+([\\w\\-\\.,@?^=%&amp;:/~\\+#]*[\\w\\-\\@?^=%&amp;/~\\+#])?");
+		Pattern pattern = Pattern.compile("[a-zA-z]+://[^\\s]*", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
 		Matcher matcher = pattern.matcher(response);
-		List<String> urlList = new ArrayList<String>();
+		List<String> urls = new ArrayList<String>();
 		while (matcher.find()) {
-			urlList.add(matcher.group());
+			String tmp = matcher.group();
+			int index = tmp.lastIndexOf(")");
+			tmp = tmp.substring(0, index);
+			urls.add(tmp);
 		}
-		return urlList;
+		return urls;
 	}
 
+
 	public static String getUrlTitle(String urlResponse) {
 		Pattern pattern = Pattern.compile("<title>.*?</title>");
 		Matcher matcher = pattern.matcher(urlResponse);