update proxy check

src/proxy.ts

@@ -4,17 +4,49 @@ const proxySourceList = [
 	"http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.66ip.cn/mo.php?tqsl=9999"
 ]
 
-for (let i = 1; i <= 2000; ++i) {
-	proxySourceList.push(`http://www.xiladaili.com/http/${i}/`);
+//for (let i = 1; i <= 2000; ++i) {
+//	proxySourceList.push(`http://www.xiladaili.com/http/${i}`);
+//}
+
+const agentList = [
+	'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
+	'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
+	'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
+	'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
+	'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+	'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
+	'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
+	'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
+	'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
+	'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',
+	'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+]
+
+async function testProxy(proxy: AxiosProxyConfig) {
+	await axios.get("http://mirrors.aliyun.com/debian/pool", {
+		proxy,
+		headers: {
+			"User-Agent": agentList[4]
+		},
+		timeout: 1000,
+	});
+	return proxy;
 }
 
 async function checkProxy(proxy: AxiosProxyConfig) {
 	let isProxyUsable = false;
 	try {
-		await axios.get("http://mirrors.aliyun.com/debian/pool", { proxy });
+		await testProxy(proxy);
+		//console.log(`Proxy ${proxy.host} is ok.`);
 		isProxyUsable = true;
 	} catch (e) {
-		console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
+		//console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
 	}
 	return isProxyUsable;
 }
@@ -26,6 +58,10 @@ async function filterProxies(proxies: AxiosProxyConfig[]) {
 	});
 }
 
+//async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) {
+//	return [await Promise.any(proxies.map(testProxy))];
+//}
+
 export class ProxyFetcher {
 	proxies: AxiosProxyConfig[];
 	counter: number;
@@ -38,30 +74,30 @@ export class ProxyFetcher {
 			return;
 		}
 		console.log(`Fetching proxies from ${url}.`)
-		//while (true) {
-		try {
-			const proxyPage: string = (await axios.get(url, {
-				responseType: "document"
-			})).data;
-			const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => {
-				const [host, _port] = proxyString.split(":");
-				const port = parseInt(_port);
-				const proxy = { host, port };
-				return proxy;
-			});
-			const usableProxies = await filterProxies(proxies);
-			for (let proxy of usableProxies) {
-				this.proxies.push(proxy);
+		while (true) {
+			try {
+				const proxyPage: string = (await axios.get(url, {
+					responseType: "document",
+				})).data;
+				const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => {
+					const [host, _port] = proxyString.split(":");
+					const port = parseInt(_port);
+					const proxy = { host, port };
+					return proxy;
+				});
+				//const usableProxies = await filterProxies(proxies);
+				for (let proxy of proxies) {
+					this.proxies.push(proxy);
+				}
+				console.error(`Got ${proxies.length} proxies from ${url}.`);
+				return;
+			} catch (e) {
+				console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`)
 			}
-			console.error(`Got ${usableProxies.length} proxies from ${url}.`);
-			return;
-		} catch (e) {
-			console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`)
 		}
-		//}
 	}
 	async initProxies() {
-		await Promise.race(proxySourceList.map((m) => {
+		await Promise.all(proxySourceList.map((m) => {
 			return this.initProxiesFrom(m);
 		}));
 	}
@@ -71,14 +107,15 @@ export class ProxyFetcher {
 				await this.initProxies();
 			}
 			const proxyIndex = process.env.NO_PROXY ? null : (++this.counter) % this.proxies.length;
+			//const proxyIndex = 0;
 			const proxy = process.env.NO_PROXY ? null : this.proxies[proxyIndex];
 			try {
 				const data = (await axios.get(url, {
 					proxy,
 					headers: {
-						"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0"
+						"User-Agent": agentList[this.counter % agentList.length]
 					},
-					timeout: 10000,
+					timeout: 5000,
 					...options
 				})).data;
 				return data;