Commit 3f7750af authored by nanahira's avatar nanahira

use redlock autoquit

parent eb9d33d1
Pipeline #24199 passed with stages
in 4 minutes and 35 seconds
...@@ -23,7 +23,6 @@ ...@@ -23,7 +23,6 @@
"@nestjs/swagger": "^5.0.9", "@nestjs/swagger": "^5.0.9",
"@nestjs/typeorm": "^8.0.2", "@nestjs/typeorm": "^8.0.2",
"axios": "^0.21.1", "axios": "^0.21.1",
"better-lock": "^2.0.3",
"busboy": "^0.2.14", "busboy": "^0.2.14",
"class-transformer": "^0.4.0", "class-transformer": "^0.4.0",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
...@@ -4203,11 +4202,6 @@ ...@@ -4203,11 +4202,6 @@
} }
] ]
}, },
"node_modules/better-lock": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/better-lock/-/better-lock-2.0.3.tgz",
"integrity": "sha512-3bCaToLrmEXZcIOOVWgi1STvp3/6EpoZAmlWBeuX2MvDB0Ql2ctl/vQ0CbhQIJYQiptdGypllP3ez+TeEmdnKQ=="
},
"node_modules/binary-extensions": { "node_modules/binary-extensions": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
...@@ -15213,11 +15207,6 @@ ...@@ -15213,11 +15207,6 @@
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
}, },
"better-lock": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/better-lock/-/better-lock-2.0.3.tgz",
"integrity": "sha512-3bCaToLrmEXZcIOOVWgi1STvp3/6EpoZAmlWBeuX2MvDB0Ql2ctl/vQ0CbhQIJYQiptdGypllP3ez+TeEmdnKQ=="
},
"binary-extensions": { "binary-extensions": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
......
...@@ -35,7 +35,6 @@ ...@@ -35,7 +35,6 @@
"@nestjs/swagger": "^5.0.9", "@nestjs/swagger": "^5.0.9",
"@nestjs/typeorm": "^8.0.2", "@nestjs/typeorm": "^8.0.2",
"axios": "^0.21.1", "axios": "^0.21.1",
"better-lock": "^2.0.3",
"busboy": "^0.2.14", "busboy": "^0.2.14",
"class-transformer": "^0.4.0", "class-transformer": "^0.4.0",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
......
...@@ -5,11 +5,39 @@ import Redis from 'ioredis'; ...@@ -5,11 +5,39 @@ import Redis from 'ioredis';
@Injectable() @Injectable()
export class LockService { export class LockService {
constructor(private config: ConfigService) { } constructor(private config: ConfigService) {}
getLockInstance() { createRedisClient() {
const url = this.config.get<string>('REDIS_URL'); const url = this.config.get<string>('REDIS_URL');
const redis = new Redis(url); return new Redis(url);
}
getLockInstance() {
const redis = this.createRedisClient();
return new Redlock([redis]); return new Redlock([redis]);
} }
async useTmpRedisClient<T>(cb: (redis: Redis.Redis) => Promise<T>) {
const redis = this.createRedisClient();
try {
return await cb(redis);
} finally {
await redis.quit();
}
}
async useTmpLockInstance<T>(cb: (lock: Redlock) => Promise<T>) {
const lock = this.getLockInstance();
try {
return await cb(lock);
} finally {
await lock.quit();
}
}
async using<T>(key: string[], timeout: number, cb: () => Promise<T>) {
return this.useTmpLockInstance(async (lock) => {
return await lock.using(key, timeout, cb);
});
}
} }
...@@ -185,7 +185,7 @@ export class MirrorService extends ConsoleLogger implements OnApplicationBootstr ...@@ -185,7 +185,7 @@ export class MirrorService extends ConsoleLogger implements OnApplicationBootstr
if (!middleware.singleton) { if (!middleware.singleton) {
return this.uploadWithMiddlewareProcess(uploadInfo, middleware); return this.uploadWithMiddlewareProcess(uploadInfo, middleware);
} }
return this.redlock.getLockInstance().using([`mirror:${middleware.identifier}`], 5000, () => this.uploadWithMiddlewareProcess(uploadInfo, middleware)); return this.redlock.using([`mirror:${middleware.identifier}`], 5000, () => this.uploadWithMiddlewareProcess(uploadInfo, middleware));
} }
async uploadWithRandomMiddleware(uploadInfo: UploadInfo, middlewares = Array.from(this.middlewares.values())) { async uploadWithRandomMiddleware(uploadInfo: UploadInfo, middlewares = Array.from(this.middlewares.values())) {
......
...@@ -17,7 +17,6 @@ import { createHash } from 'crypto'; ...@@ -17,7 +17,6 @@ import { createHash } from 'crypto';
import PQueue from 'p-queue'; import PQueue from 'p-queue';
import { LockService } from 'src/lock/lock.service'; import { LockService } from 'src/lock/lock.service';
import { InjectRedis, Redis } from '@nestjs-modules/ioredis'; import { InjectRedis, Redis } from '@nestjs-modules/ioredis';
import BetterLock from 'better-lock';
interface FileWithHash { interface FileWithHash {
file: readdirp.EntryInfo; file: readdirp.EntryInfo;
...@@ -209,10 +208,8 @@ export class PackagerService extends ConsoleLogger { ...@@ -209,10 +208,8 @@ export class PackagerService extends ConsoleLogger {
]); ]);
} }
private localLock = new BetterLock();
async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> { async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> {
return this.localLock.acquire([`archive:${archiveTask.path}`], async () => this.archiveProcess(root, archiveTask)); return this.redlock.using([`archive:${archiveTask.path}`], 30000, async () => this.archiveProcess(root, archiveTask));
} }
private archiveQueue = new PQueue({ concurrency: parseInt(process.env.PACKAGE_COCURRENCY) || os.cpus().length }); private archiveQueue = new PQueue({ concurrency: parseInt(process.env.PACKAGE_COCURRENCY) || os.cpus().length });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment