Commit 17477995 authored by nanahira's avatar nanahira

workaround

parent 6ba4653d
FROM node:latest
FROM node:12-buster-slim
RUN apt update && apt -y install libmagick++-dev python3 build-essential mariadb-client-10.3 && rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*
WORKDIR /usr/src/app
COPY package*.json ./
RUN npm install
RUN npm ci && npm cache clean --force
COPY . .
......@@ -14,4 +16,4 @@ EXPOSE 3000
ENTRYPOINT [ "npm", "run" ]
CMD [ "start" ]
ONBUILD RUN mv /usr/src/app /usr/src/server
\ No newline at end of file
ONBUILD RUN mv /usr/src/app /usr/src/server
# Minimum Viable Express Typescript Application
# Avatar Handler
## Quick start
......@@ -24,4 +24,4 @@ Generate test coverage
```
npm run test:coverage
```
\ No newline at end of file
```
#!/usr/bin/env node
var cluster = require('cluster');
if (cluster.isWorker) {
var worker = require('../dist/worker').default;
return worker();
}
/**
* Module dependencies.
*/
......
......@@ -1258,6 +1258,11 @@
"@babel/types": "^7.3.0"
}
},
"@types/bluebird": {
"version": "3.5.32",
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
"integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g=="
},
"@types/body-parser": {
"version": "1.19.0",
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz",
......@@ -1321,6 +1326,14 @@
"@types/range-parser": "*"
}
},
"@types/gm": {
"version": "1.18.9",
"resolved": "https://registry.npmjs.org/@types/gm/-/gm-1.18.9.tgz",
"integrity": "sha512-b7ybB60YvNQzYauJs1ufviskjWvWRO3MizlU/4RAts77z4HzDbN3OMGAneQnrot0S28EMdNRo98epRUJ/orofQ==",
"requires": {
"@types/node": "*"
}
},
"@types/graceful-fs": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.3.tgz",
......@@ -1380,11 +1393,18 @@
"@types/express": "*"
}
},
"@types/mysql": {
"version": "2.15.15",
"resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.15.tgz",
"integrity": "sha512-1GJnq7RwuFPRicMHdT53vza5v39nep9OKIbozxNUpFXP04CydcdWrqpZQ+MlVdlLFCisWnnt09xughajjWpFsw==",
"requires": {
"@types/node": "*"
}
},
"@types/node": {
"version": "14.0.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.9.tgz",
"integrity": "sha512-0sCTiXKXELOBxvZLN4krQ0FPOAA7ij+6WwvD0k/PHd9/KAkr4dXel5J9fh6F4x1FwAQILqAWkmpeuS6mjf1iKA==",
"dev": true
"integrity": "sha512-0sCTiXKXELOBxvZLN4krQ0FPOAA7ij+6WwvD0k/PHd9/KAkr4dXel5J9fh6F4x1FwAQILqAWkmpeuS6mjf1iKA=="
},
"@types/normalize-package-data": {
"version": "2.4.0",
......@@ -1649,6 +1669,14 @@
"integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==",
"dev": true
},
"axios": {
"version": "0.20.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.20.0.tgz",
"integrity": "sha512-ANA4rr2BDcmmAQLOKft2fufrtuvlqR+cXNNinUmvfeSNCOF98PZL+7M/v1zIdGo7OLjEA9J2gXJL+j4zGsl0bA==",
"requires": {
"follow-redirects": "^1.10.0"
}
},
"babel-jest": {
"version": "26.0.1",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-26.0.1.tgz",
......@@ -1859,12 +1887,22 @@
"tweetnacl": "^0.14.3"
}
},
"bignumber.js": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
"integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="
},
"binary-extensions": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
"integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
"dev": true
},
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"body-parser": {
"version": "1.18.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz",
......@@ -2381,8 +2419,7 @@
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cross-spawn": {
"version": "6.0.5",
......@@ -3095,6 +3132,11 @@
"locate-path": "^3.0.0"
}
},
"follow-redirects": {
"version": "1.13.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz",
"integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA=="
},
"for-in": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
......@@ -3425,6 +3467,14 @@
"integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=",
"dev": true
},
"imagemagick-native-12": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/imagemagick-native-12/-/imagemagick-native-12-1.9.3.tgz",
"integrity": "sha512-TyG+guvlluP+BrGGIfH1CdesV7585sGMbWqP35dATCPUdHjtdVELmNbfN452ZozSQnr25J5qUcp5NTvoItNfLg==",
"requires": {
"nan": "2.x"
}
},
"import-lazy": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz",
......@@ -3460,8 +3510,7 @@
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"dev": true
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ini": {
"version": "1.3.5",
......@@ -3695,8 +3744,7 @@
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isexe": {
"version": "2.0.0",
......@@ -6477,6 +6525,22 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"dev": true
},
"mysql": {
"version": "2.18.1",
"resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
"integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
"requires": {
"bignumber.js": "9.0.0",
"readable-stream": "2.3.7",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
}
},
"nan": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
"integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw=="
},
"nanomatch": {
"version": "1.2.13",
"resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
......@@ -6991,8 +7055,18 @@
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"promise-mysql": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/promise-mysql/-/promise-mysql-4.1.3.tgz",
"integrity": "sha512-TSqIBjeflVLK8LDv6dszWfhn+NJggwf5e8o3k8OdUe4XHiJRiX/caDoX8ohNeIk4xsT0b60PBHlGk8J5wUj/0Q==",
"requires": {
"@types/bluebird": "^3.5.26",
"@types/mysql": "^2.15.2",
"bluebird": "^3.5.1",
"mysql": "^2.18.1"
}
},
"prompts": {
"version": "2.3.2",
......@@ -7193,7 +7267,6 @@
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dev": true,
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
......@@ -7938,6 +8011,11 @@
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true
},
"sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
},
"sshpk": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
......@@ -8046,7 +8124,6 @@
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"requires": {
"safe-buffer": "~5.1.0"
}
......@@ -8634,8 +8711,7 @@
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"utils-merge": {
"version": "1.0.1",
......
......@@ -20,10 +20,14 @@
"test:coverage": "npm run test -- --coverage"
},
"dependencies": {
"@types/gm": "^1.18.9",
"axios": "^0.20.0",
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"express": "~4.16.1",
"morgan": "^1.9.1"
"imagemagick-native-12": "^1.9.3",
"morgan": "^1.9.1",
"promise-mysql": "^4.1.3"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.2",
......
......@@ -2,6 +2,7 @@ import express from 'express';
import cookieParser from 'cookie-parser';
import logger from 'morgan';
import config from './config';
import { Avatar } from './avatar';
const app = express();
......@@ -14,4 +15,23 @@ app.get('/healthcheck', (req, res, next) => {
res.status(200).send('OK');
});
const avatar = new Avatar();
avatar.init();
app.get('/user_avatar/ygobbs.com/:username/:size/:filename.png', async(req, res, next) => {
const { username, size, filename } = req.params;
const parsedSize = parseInt(size);
if (!parsedSize || parsedSize < 8 || parsedSize > 1000) {
res.status(400).send("Invalid size.");
}
const buffer = await avatar.getImage(username, parsedSize, `${filename}.png`);
if (buffer) {
res.status(200).header(
{ 'Content-Type': 'image/png' }
).send(buffer);
} else {
res.status(400).send(`Failed to fetch avatar.`);
}
});
export default app;
import config from './config';
import mysql, { ConnectionConfig } from "promise-mysql";
import axios from "axios";
import { Processor } from "./processor";
import { CutData } from "./utility";
async function getRealUsername(originalUsername: string) {
try {
const { user: { name, username } } = (await axios.get(`https://ygobbs.com/users/${encodeURIComponent(originalUsername)}.json`, {
responseType: "json"
})).data;
return [name, username];
} catch (e) {
console.error(`User ${originalUsername} not found: ${e.toString()}`);
return [originalUsername];
}
}
async function getAvatarURL(usernames: string[]) {
for (let username of usernames) {
try {
let url = (await axios.get(`https://api.moecube.com/accounts/users/${encodeURIComponent(username)}.avatar`, {
responseType: "text"
})).data as string;
if (url.match(/^http(s)?:\/\//)) {
return url;
}
} catch (e) {
console.error(`Avatar of ${username} not found: ${e.toString()}`);
}
}
return "https://accounts.moecube.com/default_avatar.jpg";
}
async function getURLFromUsername(username: string) {
const possibleNames = await getRealUsername(username);
return await getAvatarURL(possibleNames);
}
interface QueryResult{
content: string;
}
export class Avatar {
db: mysql.Pool;
inited: boolean;
processor: Processor;
async init() {
console.error(`Initing...`);
console.error(`Connectiing to database...`);
this.db = await mysql.createPool({
host: config.dbHost,
port: config.dbPort as number,
user: config.dbUser,
password: config.dbPassword,
database: config.dbName
});
console.error(`Creating table...`);
await this.db.query("CREATE TABLE IF NOT EXISTS `avatar_cache` (\n" +
" `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
" `url` text COLLATE utf8_unicode_ci NOT NULL,\n" +
" `size` smallint UNSIGNED NOT NULL,\n" +
" `content` MEDIUMTEXT COLLATE utf8_unicode_ci NOT NULL,\n" +
" PRIMARY KEY (`id`),\n" +
" INDEX(`url`(42)),\n" +
" INDEX(`url`(42), `size`)\n" +
") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;");
console.error(`Staring workers...`);
this.processor = new Processor(config.workers);
await this.processor.startWorkers();
console.error(`Ready.`);
this.inited = true;
}
async getImage(username: string, size: number, filename: string) {
if (!this.inited) {
return null;
}
if (size < 8 || size > 1000) {
return null;
}
const url = await getURLFromUsername(username);
let buffer: Buffer;
let content: string;
let queryResult: QueryResult[]
queryResult = await this.db.query("select content from `avatar_cache` where url = ? and size = ? limit 1", [url, size]);
console.log(queryResult);
if (queryResult.length) {
content = queryResult[0].content;
buffer = Buffer.from(content, "base64");
return buffer;
}
console.error(`Resizing image of ${username} ${url} with size ${size}.`);
queryResult = await this.db.query("select content from `avatar_cache` where url = ? and size = 0 limit 1", [url, size]);
if (queryResult.length) {
content = queryResult[0].content;
buffer = Buffer.from(content, "base64");
} else {
try {
buffer = (await axios.get(url, {
responseType: "arraybuffer"
})).data as Buffer;
} catch (e) {
console.error(`Error fetching original image of ${username} ${url}: ${e.toString()}`);
return null;
}
content = buffer.toString("base64");
await this.db.query("insert into `avatar_cache` set ?", {
url,
size: 0,
content
});
}
const resizedContent: string = await this.processor.addTask("cut", {
image: content,
filename,
size
} as CutData);
if(!resizedContent) {
console.error(`Error resizing image of ${username} ${url} with size ${size}.`);
return null;
}
await this.db.query("insert into `avatar_cache` set ?", {
url,
size,
content: resizedContent
});
return Buffer.from(resizedContent, "base64");
}
}
import os from "os";
export default {
port: process.env.PORT || 3000,
loggerLevel: process.env.LOGGER_LEVEL || 'dev'
loggerLevel: process.env.LOGGER_LEVEL || 'dev',
workers: process.env.WORKERS ? parseInt(process.env.WORKERS) : os.cpus().length,
dbHost: process.env.DB_HOST,
dbPort: process.env.DB_PORT ? parseInt(process.env.DB_PORT) : 3306,
dbUser: process.env.DB_USER,
dbPassword: process.env.DB_PASS,
dbName: process.env.DB_NAME
}
declare module 'imagemagick-native-12';
import cluster from "cluster";
export interface SendData {
id: number,
proto: string,
param: any
}
class Task {
id: number;
solved: boolean;
callback: (ret: any) => void;
constructor(id: number, proto: string, param: any, callback: (ret: any) => void, worker?: cluster.Worker) {
this.id = id;
this.callback = callback;
this.solved = false;
const sendData: SendData = {
id,
proto,
param
}
if (cluster.isMaster && worker) {
worker.send(sendData);
} else {
process.send(sendData);
}
}
solve(ret: any) {
if (this.solved) {
return;
}
this.solved = true;
this.callback(ret);
}
}
async function sleep(ms: number) {
return new Promise(callback => {
setTimeout(callback, ms);
})
}
export class Processor {
curID: number = 0;
queue: Map<number, Task>;
handlers: Map<string, (param: any, id: number, workerID?: number) => Promise<any>>;
nproc: number;
workerReadyCallbacks: Map<number, () => void>
constructor(nproc?: number) {
if (cluster.isMaster && nproc === undefined) {
throw "A value of nproc is needed.";
}
this.queue = new Map();
this.handlers = new Map();
this.workerReadyCallbacks = new Map();
this.nproc = nproc || 0;
if (cluster.isMaster) {
this.addHandler("ready", async (param: any, dataID: number, workerID: number) => {
const callback = this.workerReadyCallbacks.get(workerID);
if (callback) {
callback();
}
});
cluster.on("message", Processor.masterHandler(this));
} else {
process.on("message", Processor.workerHandler(this));
}
}
async startWorkers(env?: any) {
if (cluster.isMaster) {
let readyPromises = [];
for (let i = 0; i < this.nproc; ++i) {
readyPromises.push(new Promise(callback => {
const worker = cluster.fork(env);
const ID = worker.id;
this.workerReadyCallbacks.set(ID, callback);
}));
}
await Promise.all(readyPromises);
}
}
async ready() {
if (cluster.isWorker) {
await this.addTask("ready", null);
}
}
addHandler(proto: string, handler: (param: any, id: number, workerID?: number) => Promise<any>) {
this.handlers.set(proto, handler);
}
solveTask(data: SendData) {
const task = this.queue.get(data.id);
this.queue.delete(data.id);
if (task && !task.solved) {
task.solve(data.param);
}
}
static masterHandler(_this: Processor) {
return async (worker: cluster.Worker, data: SendData) => {
if (data.proto === "solve") {
_this.solveTask(data);
} else if (_this.handlers.has(data.proto)) {
const handler = _this.handlers.get(data.proto);
const ret = await handler(data.param, data.id, worker.id);
const sendData: SendData = {
id: data.id,
proto: "solve",
param: ret
};
worker.send(sendData);
} else {
console.error(`Unknown task: ${data.proto}`);
const sendData: SendData = {
id: data.id,
proto: "solve",
param: null
};
worker.send(sendData);
}
};
}
static workerHandler(_this: Processor) {
return async (data: SendData) => {
if (data.proto === "solve") {
_this.solveTask(data);
} else if (_this.handlers.has(data.proto)) {
const handler = _this.handlers.get(data.proto);
const ret = await handler(data.param, data.id);
const sendData: SendData = {
id: data.id,
proto: "solve",
param: ret
};
process.send(sendData);
} else {
console.error(`Unknown task: ${data.proto}`);
const sendData: SendData = {
id: data.id,
proto: "solve",
param: null
};
process.send(sendData);
}
};
}
addTask(proto: string, param: any, targetWorker?: string): any {
return new Promise((callback: (ret: any) => void) => {
let worker: cluster.Worker;
if (cluster.isMaster) {
if (!targetWorker) {
targetWorker = Object.keys(cluster.workers)[this.curID % this.nproc];
}
worker = cluster.workers[targetWorker]
}
const task = new Task(++this.curID, proto, param, callback, worker);
this.queue.set(task.id, task);
});
}
}
export interface CutData {
image: string, // base64
filename: string,
size: number
}
import { Processor } from "./processor";
import magick from "imagemagick-native-12";
import { CutData } from "./utility";
import cluster from "cluster";
import util from "util";
export default async function worker() {
const processor = new Processor();
processor.addHandler("cut", async (cutData: CutData) => {
try {
console.log(cutData.image);
const buf = Buffer.from(cutData.image);
const targetBuffer: Buffer = await util.promisify(magick.convert)({
srcFormat: "PNG",
format: "PNG",
srcData: buf,
width: cutData.size,
height: cutData.size
});
return targetBuffer.toString("base64");
} catch (e) {
console.error(`Worker ${cluster.worker.id}: Error resizing image to ${cutData.size}: ${e.toString()}`);
return null;
}
});
await processor.ready();
console.error(`Worker ${cluster.worker.id} is ready.`);
}
......@@ -6,7 +6,7 @@
],
"module": "commonjs",
"esModuleInterop": true,
"target": "es6",
"target": "esnext",
"noImplicitAny": true,
"moduleResolution": "node",
"sourceMap": true,
......@@ -29,4 +29,4 @@
"exclude": [
"dist"
]
}
\ No newline at end of file
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment