Commit 3e9c29ed authored by nanahira's avatar nanahira

add cron

parent 60bc0108
FROM node:buster-slim
RUN apt update && apt -y install cron python3 && rm -rf /var/lib/apt/lists/*
RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/*
WORKDIR /usr/src/app
COPY ./package*.json ./
RUN npm ci
COPY . ./
RUN npm run build
RUN echo '0 0 * * * bash -c "cd /usr/src/app && npm run fetch"' > /etc/cron.d/tx3 && \
crontab /etc/cron.d/tx3
CMD ["bash", "-c", "npm run fetch && cron -f"]
CMD ["npm", "run", "start"]
......@@ -11,4 +11,4 @@ proxy:
proxySource:
- "http://www.89ip.cn/tqdl.html?api=1&num=9999"
- "http://www.66ip.cn/mo.php?tqsl=9999"
timeout: 10000
\ No newline at end of file
timeout: 10000
......@@ -9,6 +9,15 @@
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
"integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g=="
},
"@types/cron": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-1.7.2.tgz",
"integrity": "sha512-AEpNLRcsVSc5AdseJKNHpz0d4e8+ow+abTaC0fKDbAU86rF1evoFF0oC2fV9FdqtfVXkG2LKshpLTJCFOpyvTg==",
"requires": {
"@types/node": "*",
"moment": ">=2.14.0"
}
},
"@types/mysql": {
"version": "2.15.14",
"resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.14.tgz",
......@@ -58,6 +67,14 @@
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cron": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/cron/-/cron-1.8.2.tgz",
"integrity": "sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==",
"requires": {
"moment-timezone": "^0.5.x"
}
},
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz",
......@@ -108,6 +125,14 @@
"resolved": "https://registry.npmjs.org/moment/-/moment-2.27.0.tgz",
"integrity": "sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ=="
},
"moment-timezone": {
"version": "0.5.31",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.31.tgz",
"integrity": "sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==",
"requires": {
"moment": ">= 2.9.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npm.taobao.org/ms/download/ms-2.0.0.tgz",
......
......@@ -5,7 +5,8 @@
"main": "build/fetcher.js",
"scripts": {
"build": "./node_modules/.bin/tsc",
"fetch": "node build/src/run.js"
"fetch": "node build/src/run.js",
"start": "node build/src/run.js cron"
},
"repository": {
"type": "git",
......@@ -21,10 +22,12 @@
},
"homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
"dependencies": {
"@types/cron": "^1.7.2",
"@types/node": "^14.0.14",
"@types/underscore": "^1.10.2",
"@types/yaml": "^1.9.7",
"axios": "^0.19.2",
"cron": "^1.8.2",
"moment": "^2.27.0",
"mysql-promise": "^5.0.0",
"posthtml-parser": "^0.4.2",
......
......@@ -121,7 +121,7 @@ export class Tx3Fetcher {
resPromises.push(this.fetchListFromSchoolAndServer(school, server));
}
const result = _.flatten(await Promise.all(resPromises));
console.log(`Fetched user list with ${result.length} users from server ${server}.`);
console.log(`Fetched user list from server ${server}. ${result.length} users found.`);
return result;
}
async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
......@@ -134,7 +134,9 @@ export class Tx3Fetcher {
}
res.push(list);
}
return _.flatten(res);
const ret = _.flatten(res);
console.log(`Fetched users from server ${server} with school ${school}.`);
return ret;
}
async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> {
console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
......@@ -153,7 +155,9 @@ export class Tx3Fetcher {
}
});
const playerRows = parsePlayerRows(content);
if(this.db) {
console.log(`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`);
if (this.db) {
//await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
await Promise.all(playerRows.map(m => this.db.query("insert into userdata set ?", {
date: this.curDate,
...m
......
......@@ -2,19 +2,25 @@ import {Tx3Fetcher, servers, Config} from "./fetcher";
import fs from "fs";
import _ from "underscore";
import yaml from "yaml";
import { CronJob } from "cron";
let fetcher;
let config: Config;
async function runServer(server: string) {
async function loadConfig() {
config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
}
async function runServer(fetcher: Tx3Fetcher, server: string) {
const users = await fetcher.fetchListFromServer(server);
await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({
date: fetcher.date,
date: fetcher.curDate,
data: users
}, null, 2));
return users;
}
async function main() {
async function run() {
console.log(`Fetch started.`);
try {
await fs.promises.access("./output/servers");
} catch (e) {
......@@ -22,23 +28,37 @@ async function main() {
recursive: true
});
}
const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
fetcher = new Tx3Fetcher(config);
const fetcher = new Tx3Fetcher(config);
await fetcher.init();
if (config.server) {
await Promise.all(config.server.map(runServer));
await Promise.all(config.server.map(server => {
return runServer(fetcher, server)
}));
} else {
const userListWithServer = await Promise.all(servers.map(runServer));
const userListWithServer = await Promise.all(servers.map(server => {
return runServer(fetcher, server)
}));
const allServersList: any = {};
for (let i = 0; i < servers.length;++i) {
allServersList[servers[i]] = userListWithServer[i];
}
await fs.promises.writeFile(`./output/all.json`, JSON.stringify({
date: fetcher.date,
date: fetcher.curDate,
data: allServersList
}, null, 2));
}
console.log("Finished.");
process.exit();
}
async function main() {
await loadConfig();
if (process.argv[2] === "cron") {
const job = new CronJob("0 0 1 * * *", run, null, true, "Asia/Shanghai");
job.start();
} else {
await run();
process.exit();
}
}
main();
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment