Skip to content

T
tx3-bang-reader
Commit 593d03fb authored by nanahira's avatar nanahira
Browse files
Options

rework

parent 85d2aa62
Pipeline #9850 failed with stages
        in 1 minute and 33 seconds
        Expand all Hide whitespace changes
        Inline Side-by-side
        Showing with 12316 additions and 1266 deletions
        .eslintignore 0 → 100644
        View file @ 593d03fb
        webpack.config.js
        dist/*
        build/*
        *.js
        .eslintrc.js 0 → 100644
        View file @ 593d03fb
        module.exports = {
        parser: '@typescript-eslint/parser',
        parserOptions: {
        project: 'tsconfig.json',
        sourceType: 'module',
        },
        plugins: ['@typescript-eslint/eslint-plugin'],
        extends: [
        'plugin:@typescript-eslint/recommended',
        'plugin:prettier/recommended',
        ],
        root: true,
        env: {
        node: true,
        jest: true,
        },
        ignorePatterns: ['.eslintrc.js'],
        rules: {
        '@typescript-eslint/interface-name-prefix': 'off',
        '@typescript-eslint/explicit-function-return-type': 'off',
        '@typescript-eslint/explicit-module-boundary-types': 'off',
        '@typescript-eslint/no-explicit-any': 'off',
        },
        };
        .gitlab-ci.yml
        View file @ 593d03fb
        stages:
        - install
        - build
        - deploy
        variables:
        GIT_DEPTH: "1"
        CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
        CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
        docker:
        npm_ci:
        stage: install
        tags:
        - linux
        script:
        - npm ci
        artifacts:
        paths:
        - node_modules
        .build_base:
        stage: build
        tags:
        - docker
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        tags:
        - linux
        dependencies:
        - npm_ci
        build:
        extends:
        - .build_base
        script:
        - docker build --pull -t $CONTAINER_TEST_IMAGE .
        - docker push $CONTAINER_TEST_IMAGE
        - npm run build
        artifacts:
        paths:
        - dist/
        deploy_latest:
        stage: deploy
        tags:
        - docker
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        unit-test:
        extends:
        - .build_base
        script:
        - docker pull $CONTAINER_TEST_IMAGE
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
        - docker push $CONTAINER_RELEASE_IMAGE
        only:
        - master
        - npm run test
        deploy_tag:
        deploy_npm:
        stage: deploy
        tags:
        - docker
        variables:
        CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        dependencies:
        - build
        tags:
        - linux
        script:
        - docker pull $CONTAINER_TEST_IMAGE
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE
        - docker push $CONTAINER_TAG_IMAGE
        - apt update;apt -y install coreutils
        - echo $NPMRC | base64 --decode > ~/.npmrc
        - npm publish . || true
        only:
        - tags
        - master
        .idea/inspectionProfiles/Project_Default.xml 0 → 100644
        View file @ 593d03fb
        <component name="InspectionProjectProfileManager">
        <profile version="1.0">
        <option name="myName" value="Project Default" />
        <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
        </profile>
        </component>
        \ No newline at end of file
        .idea/jsLinters/eslint.xml 0 → 100644
        View file @ 593d03fb
        <?xml version="1.0" encoding="UTF-8"?>
        <project version="4">
        <component name="EslintConfiguration">
        <option name="fix-on-save" value="true" />
        </component>
        </project>
        \ No newline at end of file
        .npmignore 0 → 100644
        View file @ 593d03fb
        /install-npm.sh
        .git*
        /data
        /output
        /config.yaml
        .idea
        .dockerignore
        Dockerfile
        /src
        /coverage
        /tests
        /dist/tests
        Dockerfile
        View file @ 593d03fb
        FROM node:bullseye-slim
        RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/*
        FROM node:bullseye-slim as base
        LABEL Author="Nanahira <nanahira@momobako.com>"
        RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
        WORKDIR /usr/src/app
        COPY ./package*.json ./
        RUN npm ci
        FROM base as builder
        RUN npm ci && npm cache clean --force
        COPY . ./
        RUN npm run build
        CMD ["npm", "run", "start"]
        FROM base
        ENV NODE_ENV production
        RUN npm ci && npm cache clean --force
        COPY --from=builder /usr/src/app/dist ./dist
        CMD [ "npm", "start" ]
        package-lock.json
        View file @ 593d03fb
        This diff is collapsed.
        package.json
        View file @ 593d03fb
        ......@@ -2,12 +2,11 @@
        "name": "tx3-bang-reader",
        "version": "1.0.0",
        "description": "Read TX3 bang and parse it",
        "main": "build/src/run.js",
        "main": "dist/run.js",
        "scripts": {
        "build": "./node_modules/.bin/tsc",
        "pack": "mkdir dist ; ./node_modules/.bin/pkg --out-path dist .",
        "fetch": "node build/src/run.js",
        "start": "node build/src/run.js cron"
        "build": "tsc",
        "fetch": "node dist/src/run.js",
        "start": "node dist/src/run.js cron"
        },
        "repository": {
        "type": "git",
        ......@@ -22,17 +21,10 @@
        "url": "https://github.com/purerosefallen/tx3-bang-reader/issues"
        },
        "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
        "bin": "build/src/run.js",
        "pkg": {
        "scripts": [
        "build/src/*.js"
        ],
        "assets": []
        },
        "bin": "dist/src/run.js",
        "dependencies": {
        "@types/cron": "^1.7.2",
        "@types/csv-parse": "^1.2.2",
        "@types/node": "^14.0.14",
        "@types/underscore": "^1.10.2",
        "@types/yaml": "^1.9.7",
        "axios": "^0.19.2",
        ......@@ -44,8 +36,21 @@
        "posthtml-parser": "^0.4.2",
        "promise-mysql": "^4.1.3",
        "querystring": "^0.2.0",
        "typescript": "^3.9.5",
        "underscore": "^1.10.2",
        "yaml": "^1.10.0"
        },
        "devDependencies": {
        "@types/jest": "^27.4.0",
        "@types/node": "^17.0.18",
        "@typescript-eslint/eslint-plugin": "^4.33.0",
        "@typescript-eslint/parser": "^4.33.0",
        "eslint": "^7.32.0",
        "eslint-config-prettier": "^8.3.0",
        "eslint-plugin-prettier": "^3.4.1",
        "jest": "^27.5.1",
        "prettier": "^2.5.1",
        "rimraf": "^3.0.2",
        "ts-jest": "^27.1.3",
        "typescript": "^4.5.5"
        }
        }
        src/fetcher.ts
        View file @ 593d03fb
        This diff is collapsed.
        src/migrate.ts
        View file @ 593d03fb
        ......@@ -2,100 +2,119 @@ import mysql from "promise-mysql";
        import moment from "moment";
        import fs from "fs";
        import _csv_parse from "csv-parse";
        import util from 'util';
        import util from "util";
        import { Config } from "./fetcher";
        import { PlayerRow, PlayerRowDated } from "./playerlist";
        import _ from "underscore";
        import yaml from "yaml";
        const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse);
        const parse_csv: (
        input: Buffer | string,
        options?: _csv_parse.Options
        ) => Promise<any[]> = util.promisify(_csv_parse);
        let config: Config;
        let db: mysql.Pool;
        const serverAreaCache = new Map<string, string>();
        async function getServerAreaFromServer(server: string) {
        if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server);
        } else {
        const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server);
        if (!res) {
        return "none";
        }
        const serverArea = res.serverArea;
        serverAreaCache.set(server, serverArea);
        return serverArea;
        }
        if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server);
        } else {
        const [res] = await db.query(
        "select serverArea from userdata where server = ? limit 1",
        server
        );
        if (!res) {
        return "none";
        }
        const serverArea = res.serverArea;
        serverAreaCache.set(server, serverArea);
        return serverArea;
        }
        }
        async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> {
        let pointer = offset;
        const newRecord: PlayerRowDated = {
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        ...(_.clone(base))
        };
        newRecord.name = col[pointer++];
        newRecord.server = col[pointer++];
        newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
        const _region = col[pointer++];
        newRecord.region = _region.length ? _region : "none";
        newRecord.category = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]);
        newRecord.totalScore = newRecord.equip + newRecord.score;
        return newRecord;
        async function readSingleRecord(
        col: string[],
        offset: number,
        base: PlayerRow
        ): Promise<PlayerRowDated> {
        let pointer = offset;
        const newRecord: PlayerRowDated = {
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        ..._.clone(base),
        };
        newRecord.name = col[pointer++];
        newRecord.server = col[pointer++];
        newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
        const _region = col[pointer++];
        newRecord.region = _region.length ? _region : "none";
        newRecord.category = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]);
        newRecord.totalScore = newRecord.equip + newRecord.score;
        return newRecord;
        }
        let leftCount: number;
        async function readColumn(col: string[]): Promise<void> {
        const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`);
        let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[];
        if (!base) {
        console.error(`Base record of ${url} not found. Using default values.`);
        base = {
        url,
        rank: 500,
        name: null,
        server: null,
        serverArea: null,
        category: null,
        level: 80,
        region: "none",
        score: 0,
        equip: 0,
        totalScore: 0
        }
        }
        for (let i = 0; i < recordCount; ++i) {
        const offset = 4 + (i * 6);
        if (!col[offset].length) {
        continue;
        }
        const record = await readSingleRecord(col, offset, base);
        const sql = "insert into userdata set ?";
        console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record)));
        }
        console.error(`Read column ${url}. ${--leftCount} columns left.`);
        const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`);
        let [base] = (await db.query(
        "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id asc limit 1",
        url
        )) as PlayerRow[];
        if (!base) {
        console.error(`Base record of ${url} not found. Using default values.`);
        base = {
        url,
        rank: 500,
        name: null,
        server: null,
        serverArea: null,
        category: null,
        level: 80,
        region: "none",
        score: 0,
        equip: 0,
        totalScore: 0,
        };
        }
        for (let i = 0; i < recordCount; ++i) {
        const offset = 4 + i * 6;
        if (!col[offset].length) {
        continue;
        }
        const record = await readSingleRecord(col, offset, base);
        const sql = "insert into userdata set ?";
        console.log(
        sql,
        JSON.stringify(record),
        JSON.stringify(await db.query(sql, record))
        );
        }
        console.error(`Read column ${url}. ${--leftCount} columns left.`);
        }
        async function loadCsv(path: string): Promise<string[][]> {
        const data = await fs.promises.readFile(path);
        return await parse_csv(data, {
        trim: true
        });
        const data = await fs.promises.readFile(path);
        return await parse_csv(data, {
        trim: true,
        });
        }
        async function main() {
        console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        db = await mysql.createPool(config.MySQLConfig);
        const data = await loadCsv(process.argv[2]);
        leftCount = data.length;
        //await Promise.all(data.map(col => readColumn(col)));
        for (let col of data) {
        await readColumn(col);
        }
        console.error("Finished.");
        process.exit();
        console.error("Started.");
        const config: Config = yaml.parse(
        await fs.promises.readFile("./config.yaml", "utf8")
        );
        db = await mysql.createPool(config.MySQLConfig);
        const data = await loadCsv(process.argv[2]);
        leftCount = data.length;
        //await Promise.all(data.map(col => readColumn(col)));
        for (const col of data) {
        await readColumn(col);
        }
        console.error("Finished.");
        process.exit();
        }
        main();
        src/playerlist.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser";
        import _, { first } from "underscore";
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, findAllNodeIndex, getContinuousData, getContinuousNodes, getString} from "./utility";
        import {
        findAllNodeIndex,
        findNodeIndexByTag,
        getContinuousNodes,
        getDepthOfTree,
        getNumber,
        getString,
        } from "./utility";
        export interface PlayerRow {
        url: string;
        rank: number;
        name: string;
        category: string;
        serverArea: string;
        server: string;
        level: number;
        region: string;
        score: number;
        equip: number;
        totalScore: number;
        url: string;
        rank: number;
        name: string;
        category: string;
        serverArea: string;
        server: string;
        level: number;
        region: string;
        score: number;
        equip: number;
        totalScore: number;
        }
        export interface PlayerRowDated extends PlayerRow {
        date: string;
        date: string;
        }
        export interface PlayerRowFull extends PlayerRowDated {
        id: number;
        id: number;
        }
        function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
        const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
        return {
        url: (nodes[1] as HTML.NodeTag).attrs.href as string,
        rank: getNumber(nodes[0]),
        name: getString(nodes[1], 7),
        serverArea: getString(nodes[2]),
        server: getString(nodes[3]),
        level: getNumber(nodes[4]),
        category: getString(nodes[5]),
        region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9])
        }
        const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
        return {
        url: (nodes[1] as HTML.NodeTag).attrs.href as string,
        rank: getNumber(nodes[0]),
        name: getString(nodes[1], 7),
        serverArea: getString(nodes[2]),
        server: getString(nodes[3]),
        level: getNumber(nodes[4]),
        category: getString(nodes[5]),
        region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9]),
        };
        }
        export function parsePlayerRows(content: string) {
        const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(tableTree, (node) => {
        return typeof (node) !== "string" && node.tag === "tr" && node.attrs.class !== "trTop2";
        }, []);
        return playerPoses.map(pos => {
        const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree);
        });
        const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(
        tableTree,
        (node) => {
        return (
        typeof node !== "string" &&
        node.tag === "tr" &&
        node.attrs.class !== "trTop2"
        );
        },
        []
        );
        return playerPoses.map((pos) => {
        const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree);
        });
        }
        src/proxy.ts
        View file @ 593d03fb
        ......@@ -9,59 +9,59 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
        //}
        export interface ProxyConfig {
        useProxy: boolean,
        proxySource: string[],
        timeout: number
        useProxy: boolean;
        proxySource: string[];
        timeout: number;
        }
        const agentList = [
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
        ]
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
        "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
        "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        ];
        async function testProxy(proxy: AxiosProxyConfig) {
        await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy,
        headers: {
        "User-Agent": agentList[4]
        },
        timeout: this.config.timeout,
        });
        return proxy;
        await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy,
        headers: {
        "User-Agent": agentList[4],
        },
        timeout: this.config.timeout,
        });
        return proxy;
        }
        async function checkProxy(proxy: AxiosProxyConfig) {
        let isProxyUsable = false;
        try {
        await testProxy(proxy);
        //console.log(`Proxy ${proxy.host} is ok.`);
        isProxyUsable = true;
        } catch (e) {
        //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
        }
        return isProxyUsable;
        let isProxyUsable = false;
        try {
        await testProxy(proxy);
        //console.log(`Proxy ${proxy.host} is ok.`);
        isProxyUsable = true;
        } catch (e) {
        //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
        }
        return isProxyUsable;
        }
        async function filterProxies(proxies: AxiosProxyConfig[]) {
        const proxiesUsableList = await Promise.all(proxies.map(checkProxy));
        return proxies.filter((proxy, index) => {
        return proxiesUsableList[index];
        });
        const proxiesUsableList = await Promise.all(proxies.map(checkProxy));
        return proxies.filter((proxy, index) => {
        return proxiesUsableList[index];
        });
        }
        //async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) {
        ......@@ -69,70 +69,86 @@ async function filterProxies(proxies: AxiosProxyConfig[]) {
        //}
        export class ProxyFetcher {
        proxies: AxiosProxyConfig[];
        counter: number;
        config: ProxyConfig;
        constructor(config: ProxyConfig) {
        this.config = config;
        this.proxies = [];
        this.counter = 0;
        }
        async initProxiesFrom(url: string) {
        if (!this.config.useProxy) {
        return;
        }
        console.log(`Fetching proxies from ${url}.`)
        while (true) {
        try {
        const proxyPage: string = (await axios.get(url, {
        responseType: "document",
        })).data;
        const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => {
        const [host, _port] = proxyString.split(":");
        const port = parseInt(_port);
        const proxy = { host, port };
        return proxy;
        });
        //const usableProxies = await filterProxies(proxies);
        for (let proxy of proxies) {
        this.proxies.push(proxy);
        }
        console.error(`Got ${proxies.length} proxies from ${url}.`);
        return;
        } catch (e) {
        console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`)
        }
        }
        }
        async initProxies() {
        await Promise.all(this.config.proxySource.map((m) => {
        return this.initProxiesFrom(m);
        }));
        }
        async getWithProxy(url: string, options: AxiosRequestConfig) {
        while (true) {
        if (this.config.useProxy && !this.proxies.length) {
        await this.initProxies();
        }
        const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length;
        //const proxyIndex = 0;
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        try {
        const data = (await axios.get(url, {
        proxy,
        headers: {
        "User-Agent": agentList[this.counter % agentList.length]
        },
        timeout: this.config.timeout,
        ...options
        })).data;
        return data;
        } catch (e) {
        if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1);
        }
        console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`)
        }
        }
        }
        proxies: AxiosProxyConfig[];
        counter: number;
        config: ProxyConfig;
        constructor(config: ProxyConfig) {
        this.config = config;
        this.proxies = [];
        this.counter = 0;
        }
        async initProxiesFrom(url: string) {
        if (!this.config.useProxy) {
        return;
        }
        console.log(`Fetching proxies from ${url}.`);
        while (true) {
        try {
        const proxyPage: string = (
        await axios.get(url, {
        responseType: "document",
        })
        ).data;
        const proxies: AxiosProxyConfig[] = proxyPage
        .match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)
        .map((proxyString) => {
        const [host, _port] = proxyString.split(":");
        const port = parseInt(_port);
        const proxy = { host, port };
        return proxy;
        });
        //const usableProxies = await filterProxies(proxies);
        for (const proxy of proxies) {
        this.proxies.push(proxy);
        }
        console.error(`Got ${proxies.length} proxies from ${url}.`);
        return;
        } catch (e) {
        console.error(
        `Failed fetching proxy list from ${url}: ${e.toString()}`
        );
        }
        }
        }
        async initProxies() {
        await Promise.all(
        this.config.proxySource.map((m) => {
        return this.initProxiesFrom(m);
        })
        );
        }
        async getWithProxy(url: string, options: AxiosRequestConfig) {
        while (true) {
        if (this.config.useProxy && !this.proxies.length) {
        await this.initProxies();
        }
        const proxyIndex = !this.config.useProxy
        ? null
        : ++this.counter % this.proxies.length;
        //const proxyIndex = 0;
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        try {
        const data = (
        await axios.get(url, {
        proxy,
        headers: {
        "User-Agent": agentList[this.counter % agentList.length],
        },
        timeout: this.config.timeout,
        ...options,
        })
        ).data;
        return data;
        } catch (e) {
        if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1);
        }
        console.error(
        `Failed fetching data from ${url}: ${e.toString()} ${
        this.proxies.length
        } proxies left.`
        );
        }
        }
        }
        }
        src/remove-dup.ts
        View file @ 593d03fb
        ......@@ -6,31 +6,40 @@ import _ from "underscore";
        import yaml from "yaml";
        function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) {
        return _.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field]);
        return _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        );
        }
        async function main() {
        console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        const db = await mysql.createPool(config.MySQLConfig);
        const urlDataCache = new Map<string, PlayerRowFull>();
        const deleteList: number[] = [];
        const datas: PlayerRowFull[] = await db.query(`select * from userdata order by date asc`);
        for (let row of datas) {
        if (urlDataCache.has(row.url)) {
        const oldRow = urlDataCache.get(row.url);
        if (checkSameRow(row, oldRow)) {
        deleteList.push(row.id);
        }
        }
        urlDataCache.set(row.url, row);
        }
        console.error(`Deletes: ${deleteList.length}`);
        for (let id of deleteList) {
        const sql = `delete from userdata where id = ?`;
        console.error(`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`);
        }
        console.error("Finished.");
        process.exit();
        console.error("Started.");
        const config: Config = yaml.parse(
        await fs.promises.readFile("./config.yaml", "utf8")
        );
        const db = await mysql.createPool(config.MySQLConfig);
        const urlDataCache = new Map<string, PlayerRowFull>();
        const deleteList: number[] = [];
        const datas: PlayerRowFull[] = await db.query(
        `select * from userdata order by id asc`
        );
        for (const row of datas) {
        if (urlDataCache.has(row.url)) {
        const oldRow = urlDataCache.get(row.url);
        if (checkSameRow(row, oldRow)) {
        deleteList.push(row.id);
        }
        }
        urlDataCache.set(row.url, row);
        }
        console.error(`Deletes: ${deleteList.length}`);
        for (const id of deleteList) {
        const sql = `delete from userdata where id = ?`;
        console.error(
        `Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`
        );
        }
        console.error("Finished.");
        process.exit();
        }
        main();
        src/run.ts
        View file @ 593d03fb
        import {Tx3Fetcher, servers, Config} from "./fetcher";
        import { Config, servers, Tx3Fetcher } from "./fetcher";
        import fs from "fs";
        import _ from "underscore";
        import yaml from "yaml";
        import { CronJob } from "cron";
        let config: Config;
        async function loadConfig() {
        config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        }
        async function runServer(fetcher: Tx3Fetcher, server: string) {
        const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({
        date: fetcher.curDate,
        data: users
        }, null, 2));
        return users;
        const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(
        `./output/servers/${server}.json`,
        JSON.stringify(
        {
        date: fetcher.curDate,
        data: users,
        },
        null,
        2
        )
        );
        return users;
        }
        async function run() {
        console.log(`Fetch started.`);
        try {
        await fs.promises.access("./output/servers");
        } catch (e) {
        await fs.promises.mkdir("./output/servers", {
        recursive: true
        });
        }
        const fetcher = new Tx3Fetcher(config);
        await fetcher.init();
        if (config.server) {
        await Promise.all(config.server.map(server => {
        return runServer(fetcher, server)
        }));
        } else {
        const userListWithServer = await Promise.all(servers.map(server => {
        return runServer(fetcher, server)
        }));
        const allServersList: any = {};
        for (let i = 0; i < servers.length;++i) {
        allServersList[servers[i]] = userListWithServer[i];
        }
        await fs.promises.writeFile(`./output/all.json`, JSON.stringify({
        date: fetcher.curDate,
        data: allServersList
        }, null, 2));
        }
        console.log("Finished.");
        console.log(`Fetch started.`);
        try {
        await fs.promises.access("./output/servers");
        } catch (e) {
        await fs.promises.mkdir("./output/servers", {
        recursive: true,
        });
        }
        const fetcher = new Tx3Fetcher(config);
        await fetcher.init();
        if (config.server) {
        await Promise.all(
        config.server.map((server) => {
        return runServer(fetcher, server);
        })
        );
        } else {
        const userListWithServer = await Promise.all(
        servers.map((server) => {
        return runServer(fetcher, server);
        })
        );
        const allServersList: any = {};
        for (let i = 0; i < servers.length; ++i) {
        allServersList[servers[i]] = userListWithServer[i];
        }
        await fs.promises.writeFile(
        `./output/all.json`,
        JSON.stringify(
        {
        date: fetcher.curDate,
        data: allServersList,
        },
        null,
        2
        )
        );
        }
        console.log("Finished.");
        }
        async function main() {
        await loadConfig();
        if (process.argv[2] === "cron") {
        const job = new CronJob(config.cronString, run, null, true, "Asia/Shanghai", null, true);
        job.start();
        } else {
        await run();
        process.exit();
        }
        await loadConfig();
        if (process.argv[2] === "cron") {
        const job = new CronJob(
        config.cronString,
        run,
        null,
        true,
        "Asia/Shanghai",
        null,
        true
        );
        job.start();
        } else {
        await run();
        process.exit();
        }
        }
        main();
        src/user.ts
        View file @ 593d03fb
        This diff is collapsed.
        src/utility.ts
        View file @ 593d03fb
        This diff is collapsed.
        tests/test-list.ts
        View file @ 593d03fb
        ......@@ -3,9 +3,9 @@ import fs from "fs";
        import { parsePlayerRows } from "../src/playerlist";
        async function main() {
        let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
        console.log(parsePlayerRows(html));
        html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
        console.log(parsePlayerRows(html));
        let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
        console.log(parsePlayerRows(html));
        html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
        console.log(parsePlayerRows(html));
        }
        main();
        tests/test-proxy.ts
        View file @ 593d03fb
        import { ProxyFetcher } from "../src/proxy";
        async function main() {
        const fetcher = new ProxyFetcher({
        useProxy: true,
        proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999"
        ],
        timeout: 10000
        });
        await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        async function main() {
        const fetcher = new ProxyFetcher({
        useProxy: true,
        proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999",
        ],
        timeout: 10000,
        });
        await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        }
        main();
        tests/test-user.ts
        View file @ 593d03fb
        ......@@ -3,13 +3,19 @@ import fs from "fs";
        import HTML from "posthtml-parser";
        async function main() {
        let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2));
        console.log(new User(id, html, null));
        id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2));
        console.log(new User(id, html, null));
        let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null));
        id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null));
        }
        main();
        tsconfig.json
        View file @ 593d03fb
        {
        "compilerOptions": {
        "outDir": "build",
        "outDir": "dist",
        "module": "commonjs",
        "target": "esnext",
        "target": "es2021",
        "esModuleInterop": true,
        "emitDecoratorMetadata": true,
        "experimentalDecorators": true,
        "declaration": true,
        "sourceMap": true
        },
        "compileOnSave": true,
        "allowJs": true,
        "include": [
        "src/*.ts",
        "tests/*.ts"
        "*.ts",
        "src/**/*.ts",
        "test/**/*.ts",
        "tests/**/*.ts"
        ]
        }
        Markdown is supported
        0% or
        You are about to add 0 people to the discussion. Proceed with caution.
        Finish editing this message first!
        Please register or to comment