limit mirror concurrent

console-api/src/mirror/mirror.service.ts

@@ -38,6 +38,7 @@ export interface UploadResult {
 export class MirrorService extends ConsoleLogger {
   private middlewares = new Map<string, Middleware>();
   private maximumMirroredSize = 0x7fffffff;
+  private mirrorConcurrent = this.config.get('MIRROR_CONCURRENT') || 100;
   constructor(
     @InjectConnection('app')
     private db: Connection,
@@ -133,10 +134,11 @@ export class MirrorService extends ConsoleLogger {
       .distinctOn(['archive.path'])
       .from(Archive, 'archive')
       .where('archive.size <= :maximumMirroredSize', { maximumMirroredSize: this.maximumMirroredSize });
-    query.andWhere(
-      `not exists ${query.subQuery().select('mirror.path').from(ArchiveMirror, 'mirror').where('archive.path = mirror.path').getQuery()}`
-    );
-    //.take(100);
+    query
+      .andWhere(
+        `not exists ${query.subQuery().select('mirror.path').from(ArchiveMirror, 'mirror').where('archive.path = mirror.path').getQuery()}`
+      )
+      .take(this.mirrorConcurrent);
     this.log(`Searching for archives to mirror`);
     const archives = await query.getMany();
     if (!archives.length) {
@@ -149,13 +151,7 @@ export class MirrorService extends ConsoleLogger {
       return false;
     }
     this.log(`Saving ${uploadResults.length} mirror records.`);
-    await this.db.transaction(async (edb) => {
-      const entTrunks = _.chunk(uploadResults, 20000);
-      for (const chunk of entTrunks) {
-        await edb.getRepository(ArchiveMirror).save(chunk);
-      }
-    });
-    return true;
+    await this.db.getRepository(ArchiveMirror).save(uploadResults);
   }
 
   private async uploadWithMiddlewareProcess(uploadInfo: UploadInfo, middleware: Middleware): Promise<UploadResult> {