fetch list only

src/fetcher.ts

@@ -76,6 +76,32 @@ export class Tx3Fetcher {
 		return _.flatten(res);
 	}
 	async fetchUsers(school: number, server: string, page: number): Promise<User[]> {
+		const playerRows = await this.fetchList(school, server, page);
+		return await Promise.all(playerRows.map(row => this.fetchUser(row)));
+	}
+	async fetchListFromServer(server: string): Promise<PlayerRow[]> {
+		console.log(`Fetching user list from server ${server}.`);
+		const resPromises: Promise<PlayerRow[]>[] = [];
+		for (let school = 1; school < 12; ++school) {
+			resPromises.push(this.fetchListFromSchoolAndServer(school, server));
+		}
+		const result = _.flatten(await Promise.all(resPromises));
+		console.log(`Fetched user list with ${result.length} users from server ${server}.`);
+		return result;
+	}
+	async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
+		console.log(`Fetching users from server ${server} with school ${school}.`);
+		const res: PlayerRow[][] = [];
+		for (let page = 1; page <= 25; ++page) {
+			const list = await this.fetchList(school, server, page);
+			if (!list.length) {
+				break;
+			}
+			res.push(list);
+		}
+		return _.flatten(res);
+	}
+	async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> {
 		console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
 		try { 
 			const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, {
@@ -92,7 +118,7 @@ export class Tx3Fetcher {
 				}
 			});
 			const playerRows = parsePlayerRows(content);
-			return await Promise.all(playerRows.map(row => this.fetchUser(row)));
+			return playerRows;
 		} catch(e) {
 			console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`);
 			return [];

src/proxy.ts

 import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
 
+const NO_PROXY = true;
+
 const proxySourceList = [
 	"http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.66ip.cn/mo.php?tqsl=9999"
 ]
@@ -70,7 +72,7 @@ export class ProxyFetcher {
 		this.counter = 0;
 	}
 	async initProxiesFrom(url: string) {
-		if (process.env.NO_PROXY) {
+		if (NO_PROXY) {
 			return;
 		}
 		console.log(`Fetching proxies from ${url}.`)
@@ -103,24 +105,24 @@ export class ProxyFetcher {
 	}
 	async getWithProxy(url: string, options: AxiosRequestConfig) {
 		while (true) {
-			if (!process.env.NO_PROXY && !this.proxies.length) {
+			if (!NO_PROXY && !this.proxies.length) {
 				await this.initProxies();
 			}
-			const proxyIndex = process.env.NO_PROXY ? null : (++this.counter) % this.proxies.length;
+			const proxyIndex = NO_PROXY ? null : (++this.counter) % this.proxies.length;
 			//const proxyIndex = 0;
-			const proxy = process.env.NO_PROXY ? null : this.proxies[proxyIndex];
+			const proxy = NO_PROXY ? null : this.proxies[proxyIndex];
 			try {
 				const data = (await axios.get(url, {
 					proxy,
 					headers: {
 						"User-Agent": agentList[this.counter % agentList.length]
 					},
-					timeout: 5000,
+					timeout: 30000,
 					...options
 				})).data;
 				return data;
 			} catch (e) {
-				if (!process.env.NO_PROXY) {
+				if (!NO_PROXY) {
 					this.proxies.splice(proxyIndex, 1);
 				}
 				console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`)

src/run.ts

@@ -5,7 +5,7 @@ import _ from "underscore";
 const fetcher = new Tx3Fetcher();
 
 async function runServer(server: string) {
-	const users = await fetcher.fetchUsersFromServer(server);
+	const users = await fetcher.fetchListFromServer(server);
 	await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify(users, null, 2));
 	return users;
 }