Commit cfb001c8 authored by nanahira's avatar nanahira

fetch list only

parent 8baa9789
Pipeline #215 passed with stages
in 1 minute and 12 seconds
......@@ -76,6 +76,32 @@ export class Tx3Fetcher {
return _.flatten(res);
}
async fetchUsers(school: number, server: string, page: number): Promise<User[]> {
const playerRows = await this.fetchList(school, server, page);
return await Promise.all(playerRows.map(row => this.fetchUser(row)));
}
async fetchListFromServer(server: string): Promise<PlayerRow[]> {
console.log(`Fetching user list from server ${server}.`);
const resPromises: Promise<PlayerRow[]>[] = [];
for (let school = 1; school < 12; ++school) {
resPromises.push(this.fetchListFromSchoolAndServer(school, server));
}
const result = _.flatten(await Promise.all(resPromises));
console.log(`Fetched user list with ${result.length} users from server ${server}.`);
return result;
}
async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
console.log(`Fetching users from server ${server} with school ${school}.`);
const res: PlayerRow[][] = [];
for (let page = 1; page <= 25; ++page) {
const list = await this.fetchList(school, server, page);
if (!list.length) {
break;
}
res.push(list);
}
return _.flatten(res);
}
async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> {
console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
try {
const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, {
......@@ -92,7 +118,7 @@ export class Tx3Fetcher {
}
});
const playerRows = parsePlayerRows(content);
return await Promise.all(playerRows.map(row => this.fetchUser(row)));
return playerRows;
} catch(e) {
console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`);
return [];
......
import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
const NO_PROXY = true;
const proxySourceList = [
"http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.66ip.cn/mo.php?tqsl=9999"
]
......@@ -70,7 +72,7 @@ export class ProxyFetcher {
this.counter = 0;
}
async initProxiesFrom(url: string) {
if (process.env.NO_PROXY) {
if (NO_PROXY) {
return;
}
console.log(`Fetching proxies from ${url}.`)
......@@ -103,24 +105,24 @@ export class ProxyFetcher {
}
async getWithProxy(url: string, options: AxiosRequestConfig) {
while (true) {
if (!process.env.NO_PROXY && !this.proxies.length) {
if (!NO_PROXY && !this.proxies.length) {
await this.initProxies();
}
const proxyIndex = process.env.NO_PROXY ? null : (++this.counter) % this.proxies.length;
const proxyIndex = NO_PROXY ? null : (++this.counter) % this.proxies.length;
//const proxyIndex = 0;
const proxy = process.env.NO_PROXY ? null : this.proxies[proxyIndex];
const proxy = NO_PROXY ? null : this.proxies[proxyIndex];
try {
const data = (await axios.get(url, {
proxy,
headers: {
"User-Agent": agentList[this.counter % agentList.length]
},
timeout: 5000,
timeout: 30000,
...options
})).data;
return data;
} catch (e) {
if (!process.env.NO_PROXY) {
if (!NO_PROXY) {
this.proxies.splice(proxyIndex, 1);
}
console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`)
......
......@@ -5,7 +5,7 @@ import _ from "underscore";
const fetcher = new Tx3Fetcher();
async function runServer(server: string) {
const users = await fetcher.fetchUsersFromServer(server);
const users = await fetcher.fetchListFromServer(server);
await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify(users, null, 2));
return users;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment