finish

package.json

@@ -5,7 +5,7 @@
   "main": "build/run.js",
   "scripts": {
     "build": "./node_modules/.bin/tsc",
-    "test": "node build/tests/test.js"
+    "fetch": "node build/src/run.js"
   },
   "repository": {
     "type": "git",

src/fetcher.ts

 import axios from "axios";
 import _ from "underscore";
-import parseHTML from "posthtml-parser";
 import { User } from "./user";
+import { ProxyFetcher } from "./proxy";
+import { PlayerRow, parsePlayerRows } from "./playerlist";
+import qs from "querystring";
 
-class Tx3Fetcher {
-	async fetchRole(id: string): Promise<User> {
-		//try { 
-			const content = (await axios.get(`http://bang.tx3.163.com/bang/role/${id}`, {
+export const servers = [
+	"东方明珠",
+	"君临天下",
+	"幻龙诀",
+	"气壮山河",
+	"紫禁之巅",
+	"绝代风华",
+	"问鼎天下",
+	"鼎立山河",
+	"天下无双",
+	"情动大荒",
+	"昆仑变",
+	"琉璃月",
+	"齐鲁天下",
+	"剑舞香江",
+	"白云山",
+	"瘦西湖",
+	"逐鹿中原",
+	"天府之国",
+	"黄鹤楼",
+	"墨倾天下",
+	"武夷九曲",
+	"上善若水",
+	"飞龙在天",
+	"烽火关东",
+	"盛世长安",
+	"荒火山神",
+	"一诺山海",
+	"梦回山海",
+	"锦绣花朝"
+]
+
+export class Tx3Fetcher {
+	proxyFetcher: ProxyFetcher;
+	constructor() {
+		this.proxyFetcher = new ProxyFetcher();
+	}
+	async initProxies() {
+		await this.proxyFetcher.initProxies();
+	}
+	async fetch(): Promise<Map<string, User[]>> {
+		const res = new Map<string, User[]>();
+		const userLists = await Promise.all(servers.map(server => this.fetchUsersFromServer(server)));
+		for (let i = 1; i < userLists.length; ++i){
+			res.set(servers[i], userLists[i])
+		}
+		return res;
+	}
+	async fetchUsersFromServer(server: string): Promise<User[]> {
+		const userList = await this.fetchListFromServer(server);
+		return await Promise.all(userList.map(r => this.fetchUser(r)));
+	}
+	async fetchListFromServer(server: string): Promise<PlayerRow[]> {
+		console.log(`Fetching user list from server ${server}.`);
+		const resPromises: Promise<PlayerRow[]>[] = [];
+		for (let school = 1; school < 12; ++school) {
+			resPromises.push(this.fetchListFromSchoolAndServer(school, server));
+		}
+		const result = _.flatten(await Promise.all(resPromises));
+		console.log(`Fetched user list with ${result.length} users from server ${server}.`);
+		return result;
+	}
+	async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
+		console.log(`Fetching user list from server ${server} with school ${school}.`);
+		const res: PlayerRow[][] = [];
+		for (let page = 1; page <= 25; ++page) {
+			const list = await this.fetchUserList(school, server, page);
+			if (!list.length) {
+				console.log(`User list from server ${server} with school ${school} page ${page} is blank.`);
+				break;
+			}
+			res.push(list);
+		}
+		return _.flatten(res);
+	}
+	async fetchUserList(school: number, server: string, page: number): Promise<PlayerRow[]> {
+		console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
+		try { 
+			const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, {
+				responseType: "document",
+				params: {
+					order_key: "equ_xiuwei",
+					count: 20,
+					school,
+					server,
+					page
+				},
+				paramsSerializer: (params) => {
+					return qs.stringify(params);
+				}
+			});
+			return parsePlayerRows(content);
+		} catch(e) {
+			console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`);
+			return [];
+		}
+	}
+	async fetchUser(playerRow: PlayerRow): Promise<User> {
+		const id = playerRow.url.split("/").pop();
+		try { 
+			console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`);
+			const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com${playerRow.url}`, {
 				responseType: "document"
-			})).data;
-			const parsedContent = parseHTML(content);
-			return new User(id, content, null);
-		/*} catch(e) {
+			});
+			const user = new User(id, content, playerRow.region);
+			console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`);
+			return user;
+		} catch(e) {
 			console.error(`Errored fetching role data from ${id}: ${e.toString()}`);
 			return null;
-		}*/
+		}
 	}
 }
-
-async function main() {
-	const fetcher = new Tx3Fetcher();
-	console.log(JSON.stringify(await fetcher.fetchRole("28_20588"), null, 2));
-}
-main();

src/proxy.ts

@@ -8,6 +8,9 @@ export class ProxyFetcher {
 		this.counter = 0;
 	}
 	async initProxiesFrom(url: string) {
+		if (process.env.NO_PROXY) {
+			return;
+		}
 		while (true) {
 			try {
 				const proxyPage: string = (await axios.get(url, {
@@ -33,19 +36,25 @@ export class ProxyFetcher {
 	}
 	async getWithProxy(url: string, options: AxiosRequestConfig) {
 		while (true) {
-			if (!this.proxies.length) {
+			if (!process.env.NO_PROXY && !this.proxies.length) {
 				await this.initProxies();
 			}
-			const proxyIndex = (++this.counter) % this.proxies.length;
-			const proxy = this.proxies[proxyIndex];
+			const proxyIndex = process.env.NO_PROXY ? null : (++this.counter) % this.proxies.length;
+			const proxy = process.env.NO_PROXY ? null : this.proxies[proxyIndex];
 			try {
 				const data = (await axios.get(url, {
 					proxy,
+					headers: {
+						"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0"
+					},
+					timeout: 5000,
 					...options
 				})).data;
 				return data;
 			} catch (e) {
-				this.proxies.splice(proxyIndex, 1);
+				if (!process.env.NO_PROXY) {
+					this.proxies.splice(proxyIndex, 1);
+				}
 				console.error(`Failed fetching data from ${url}: ${e.toString()}`)
 			}
 		}

src/run.ts

+import { Tx3Fetcher, servers } from "./fetcher";
+import fs from "fs";
+import _ from "underscore";
+async function main() {
+	try {
+		await fs.promises.access("./output/servers");
+	} catch (e) {
+		await fs.promises.mkdir("./output/servers", {
+			recursive: true
+		});
+	}
+	const fetcher = new Tx3Fetcher();
+	await fetcher.initProxies();
+	const fetchResult = await fetcher.fetch();
+	const allUsers = _.flatten(Array.from(fetchResult.values));
+	await fs.promises.writeFile("./output/all.json", JSON.stringify(allUsers, null, 2));
+	await Promise.all(servers.map(server => fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify(fetchResult.get(server), null, 2))));
+}
+main();