async middleware download support

console-api/src/entities/Archive.entity.ts

@@ -38,7 +38,7 @@ export class Archive extends TimeBase {
   role: ArchiveType;
 
   // should not be a relation
-  mirrors: ArchiveMirror[];
+  mirrors: string[];
 
   get archiveFullPath() {
     return `${this.path}.tar.zst`;

console-api/src/mirror/mirror.service.ts

@@ -11,14 +11,11 @@ import { PackageS3Service } from '../package-s3/package-s3.service';
 import _ from 'lodash';
 import delay from 'delay';
 
-export interface MiddlewareInfo {
-  identifier: string;
-  maxSize?: number;
-}
-
 export interface Middleware {
   url: string;
-  info: MiddlewareInfo;
+  identifier: string;
+  maxSize?: number;
+  callback?: string;
 }
 
 export interface UploadInfo {
@@ -35,7 +32,7 @@ export interface UploadResult {
 
 @Injectable()
 export class MirrorService extends ConsoleLogger {
-  private middlewares: Middleware[] = [];
+  private middlewares = new Map<string, Middleware>();
   private maximumMirroredSize = 0xffffffff;
   constructor(
     @InjectConnection('app')
@@ -57,9 +54,9 @@ export class MirrorService extends ConsoleLogger {
     for (const url of middlewares) {
       try {
         this.log(`Loading middleware ${url}`);
-        const { data } = await lastValueFrom(this.http.get<ReturnMessage<MiddlewareInfo>>(url, { responseType: 'json' }));
-        const middleware: Middleware = { url, info: data.data };
-        this.middlewares.push(middleware);
+        const { data } = await lastValueFrom(this.http.get<ReturnMessage<Middleware>>(url, { responseType: 'json' }));
+        const middleware: Middleware = { ...data.data, url };
+        this.middlewares.set(data.data.identifier, middleware);
         if (data.data.maxSize) {
           this.maximumMirroredSize = Math.min(this.maximumMirroredSize, data.data.maxSize);
         }
@@ -68,7 +65,7 @@ export class MirrorService extends ConsoleLogger {
         this.log(`Loading middleware ${url} failed: ${e.toString()}`);
       }
     }
-    if (this.middlewares.length) {
+    if (this.middlewares.size) {
       this.mainLoop().then();
     }
   }
@@ -83,14 +80,14 @@ export class MirrorService extends ConsoleLogger {
     };
     return _.compact(
       await Promise.all(
-        this.middlewares.map(async (middleware) => {
+        Array.from(this.middlewares.values()).map(async (middleware) => {
           const result = await this.uploadWithMiddleware(uploadInfo, middleware);
           if (!result) {
             return null;
           }
           const mirrorEnt = new ArchiveMirror();
           mirrorEnt.path = archive.path;
-          mirrorEnt.middleware = middleware.info.identifier;
+          mirrorEnt.middleware = middleware.identifier;
           mirrorEnt.url = result.url;
           return mirrorEnt;
         })
@@ -156,7 +153,7 @@ export class MirrorService extends ConsoleLogger {
 
   private async uploadWithMiddleware(uploadInfo: UploadInfo, middleware: Middleware): Promise<UploadResult> {
     try {
-      this.log(`Uploading ${uploadInfo.url} with middleware ${middleware.info.identifier}.`);
+      this.log(`Uploading ${uploadInfo.url} with middleware ${middleware.identifier}.`);
       const { data } = await lastValueFrom(
         this.http.post<ReturnMessage<string>>(middleware.url, uploadInfo, { responseType: 'json', timeout: 60 * 60 * 1000 })
       );
@@ -166,12 +163,12 @@ export class MirrorService extends ConsoleLogger {
         middleware,
       };
     } catch (e) {
-      this.error(`Failed uploading ${uploadInfo.url} with middleware ${middleware.info.identifier}: ${e.toString()} ${e.data}`);
+      this.error(`Failed uploading ${uploadInfo.url} with middleware ${middleware.identifier}: ${e.toString()} ${e.data}`);
       return null;
     }
   }
 
-  async uploadWithRandomMiddleware(uploadInfo: UploadInfo, middlewares = this.middlewares) {
+  async uploadWithRandomMiddleware(uploadInfo: UploadInfo, middlewares = Array.from(this.middlewares.values())) {
     if (!middlewares.length) {
       return null;
     }
@@ -182,17 +179,49 @@ export class MirrorService extends ConsoleLogger {
     }
     return this.uploadWithRandomMiddleware(
       uploadInfo,
-      middlewares.filter((m) => m.info.identifier !== middleware.info.identifier)
+      middlewares.filter((m) => m.identifier !== middleware.identifier)
     );
   }
 
+  async getMirrorUrl(mirror: ArchiveMirror): Promise<string> {
+    const middleware = this.middlewares.get(mirror.middleware);
+    if (!middleware) {
+      return;
+    }
+    if (!middleware.callback) {
+      return mirror.url;
+    }
+    try {
+      const { data } = await lastValueFrom(
+        this.http.patch<{ url: string }>(middleware.url, mirror, { responseType: 'json', timeout: 30000 })
+      );
+      return data.url;
+    } catch (e) {
+      this.error(`Failed getting mirror url for ${mirror.path} with middleware ${middleware.identifier}: ${e.toString()}`);
+      return;
+    }
+  }
+
   async lookForArchiveMirror(archive: Archive) {
     if (archive.size > this.maximumMirroredSize) {
       return;
     }
-    const mirrors = await this.db.getRepository(ArchiveMirror).find({ where: { path: archive.path, disabled: false }, select: ['url'] });
+    const mirrors = await this.db
+      .getRepository(ArchiveMirror)
+      .find({ where: { path: archive.path, disabled: false }, select: ['url', 'middleware', 'path'] });
     if (mirrors.length) {
-      archive.mirrors = mirrors;
+      const urls: string[] = [];
+      await Promise.all(
+        mirrors.map(async (m) => {
+          const url = await this.getMirrorUrl(m);
+          if (url) {
+            urls.push(url);
+          }
+        })
+      );
+      if (urls.length) {
+        archive.mirrors = urls;
+      }
     }
     return archive;
   }

console-api/views/metalinks.mustache

@@ -7,7 +7,7 @@
       <hash type="sha-256">{{.}}</hash>
       {{/hash}}
       {{#mirrors}}
-      <url priority="1">{{&url}}</url>
+      <url priority="1">{{&.}}</url>
       {{/mirrors}}
       <url priority="2">{{&cdnUrl}}/{{path}}.tar.zst</url>
     </file>