Commit efe8a3ad authored by nanahira's avatar nanahira

limit mirror concurrent

parent 7e1abb5c
Pipeline #17471 passed with stages
in 1 minute and 55 seconds
......@@ -38,6 +38,7 @@ export interface UploadResult {
export class MirrorService extends ConsoleLogger {
private middlewares = new Map<string, Middleware>();
private maximumMirroredSize = 0x7fffffff;
private mirrorConcurrent = this.config.get('MIRROR_CONCURRENT') || 100;
constructor(
@InjectConnection('app')
private db: Connection,
......@@ -133,10 +134,11 @@ export class MirrorService extends ConsoleLogger {
.distinctOn(['archive.path'])
.from(Archive, 'archive')
.where('archive.size <= :maximumMirroredSize', { maximumMirroredSize: this.maximumMirroredSize });
query.andWhere(
query
.andWhere(
`not exists ${query.subQuery().select('mirror.path').from(ArchiveMirror, 'mirror').where('archive.path = mirror.path').getQuery()}`
);
//.take(100);
)
.take(this.mirrorConcurrent);
this.log(`Searching for archives to mirror`);
const archives = await query.getMany();
if (!archives.length) {
......@@ -149,13 +151,7 @@ export class MirrorService extends ConsoleLogger {
return false;
}
this.log(`Saving ${uploadResults.length} mirror records.`);
await this.db.transaction(async (edb) => {
const entTrunks = _.chunk(uploadResults, 20000);
for (const chunk of entTrunks) {
await edb.getRepository(ArchiveMirror).save(chunk);
}
});
return true;
await this.db.getRepository(ArchiveMirror).save(uploadResults);
}
private async uploadWithMiddlewareProcess(uploadInfo: UploadInfo, middleware: Middleware): Promise<UploadResult> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment