check file exists before upload

console-api/src/mirror/mirror.service.ts

@@ -76,8 +76,11 @@ export class MirrorService extends ConsoleLogger {
   }
 
   private async saveMirrorFromPath(archive: Archive) {
+    if (!(await this.packageS3.fileExists(archive.archiveFullPath))) {
+      return [];
+    }
     const uploadInfo: UploadInfo = {
-      url: this.packageS3.getCdnUrl(`${archive.path}.tar.zst`),
+      url: this.packageS3.getCdnUrl(archive.archiveFullPath),
       size: archive.size,
     };
     return _.compact(
@@ -128,11 +131,10 @@ export class MirrorService extends ConsoleLogger {
       .distinctOn(['archive.path'])
       .from(Archive, 'archive')
       .where('archive.size <= :maximumMirroredSize', { maximumMirroredSize: this.maximumMirroredSize });
-    query
-      .andWhere(
-        `not exists ${query.subQuery().select('mirror.path').from(ArchiveMirror, 'mirror').where('archive.path = mirror.path').getQuery()}`
-      )
-      .take(100);
+    query.andWhere(
+      `not exists ${query.subQuery().select('mirror.path').from(ArchiveMirror, 'mirror').where('archive.path = mirror.path').getQuery()}`
+    );
+    //.take(100);
     this.log(`Searching for archives to mirror`);
     const archives = await query.getMany();
     if (!archives.length) {
@@ -141,7 +143,7 @@ export class MirrorService extends ConsoleLogger {
     this.log(`Uploading ${archives.length} archives.`);
     const uploadResults = _.flatten(await Promise.all(archives.map((a) => this.saveMirrorFromPath(a))));
     if (!uploadResults) {
-      this.error(`All uploads failed, exiting.`);
+      this.error(`Nothing uploaded, exiting.`);
       return false;
     }
     this.log(`Saving ${uploadResults.length} mirror records.`);