mirror

console-api/src/mirror/mirror.service.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { MirrorService } from './mirror.service';
+
+describe('MirrorService', () => {
+  let service: MirrorService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [MirrorService],
+    }).compile();
+
+    service = module.get<MirrorService>(MirrorService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

console-api/src/mirror/mirror.service.ts

+import { ConsoleLogger, Injectable } from '@nestjs/common';
+import { InjectConnection } from '@nestjs/typeorm';
+import { Connection } from 'typeorm';
+import { ConfigService } from '@nestjs/config';
+import { HttpService } from '@nestjs/axios';
+import { lastValueFrom } from 'rxjs';
+import { ReturnMessage } from '../dto/ReturnMessage.dto';
+import { Archive } from '../entities/Archive.entity';
+import { ArchiveMirror } from '../entities/ArchiveMirror.dto';
+
+export interface MiddlewareInfo {
+  identifier: string;
+  maxSize?: number;
+}
+
+export interface Middleware {
+  url: string;
+  info: MiddlewareInfo;
+}
+
+export interface UploadInfo {
+  url: string;
+  size?: number;
+  customSuffix?: string;
+  customMime?: string;
+}
+
+export interface UploadResult {
+  url: string;
+  middleware: Middleware;
+}
+
+@Injectable()
+export class MirrorService extends ConsoleLogger {
+  private middlewares: Middleware[] = [];
+  private maximumMirroredSize = 0xffffffff;
+  constructor(
+    @InjectConnection('app')
+    private db: Connection,
+    private config: ConfigService,
+    private http: HttpService
+  ) {
+    super('mirror');
+    this.init().then();
+  }
+
+  private async init() {
+    const middlewareUrlStrings = this.config.get<string>('MIRROR_MIDDLEWARES');
+    if (!middlewareUrlStrings) {
+      return;
+    }
+    const middlewares = middlewareUrlStrings.split(',');
+    for (const url of middlewares) {
+      try {
+        this.log(`Loading middleware ${url}`);
+        const { data } = await lastValueFrom(
+          this.http.get<ReturnMessage<MiddlewareInfo>>(url, { responseType: 'json' })
+        );
+        const middleware: Middleware = { url, info: data.data };
+        this.middlewares.push(middleware);
+        if (data.data.maxSize) {
+          this.maximumMirroredSize = Math.min(this.maximumMirroredSize, data.data.maxSize);
+        }
+        this.log(`Loaded middleware ${JSON.stringify(middleware)}`);
+      } catch (e) {
+        this.log(`Loading middleware ${url} failed: ${e.toString()}`);
+      }
+    }
+  }
+
+  private async uploadWithMiddleware(uploadInfo: UploadInfo, middleware: Middleware): Promise<UploadResult> {
+    try {
+      this.log(`Uploading ${uploadInfo.url} with middleware ${middleware.info.identifier}.`);
+      const { data } = await lastValueFrom(
+        this.http.post<ReturnMessage<string>>(middleware.url, uploadInfo, { responseType: 'json', timeout: 60 * 60 * 1000 })
+      );
+      this.log(`Uploaded ${uploadInfo.url} to ${data.data}.`);
+      return {
+        url: data.data,
+        middleware,
+      };
+    } catch (e) {
+      this.error(`Failed uploading ${uploadInfo.url} with middleware ${middleware.info.identifier}: ${e.toString()} ${e.data}`);
+    }
+  }
+
+  async uploadWithRandomMiddleware(uploadInfo: UploadInfo, middlewares = this.middlewares) {
+    if (!middlewares.length) {
+      return null;
+    }
+    const middleware = middlewares[Math.floor(Math.random() * middlewares.length)];
+    const result = await this.uploadWithMiddleware(uploadInfo, middleware);
+    if (result) {
+      return result;
+    }
+    return this.uploadWithRandomMiddleware(
+      uploadInfo,
+      middlewares.filter((m) => m.info.identifier !== middleware.info.identifier)
+    );
+  }
+
+  async lookForArchiveMirror(archive: Archive) {
+    if (archive.size > this.maximumMirroredSize) {
+      return;
+    }
+    const mirrors = await this.db.getRepository(ArchiveMirror).find({ where: { path: archive.path, disabled: false }, select: ['url'] });
+    if (mirrors.length) {
+      archive.mirrors = mirrors;
+    }
+    return archive;
+  }
+
+  async lookForArchivesMirror(archives: Archive[]) {
+    //const suitableArchives = archives.filter((a) => a.size <= this.maximumMirroredSize);
+    //const paths = _.uniq(suitableArchives.map((a) => a.path));
+    //const mirrors = await this.db.getRepository(ArchiveMirror).find({ where: { path: archive.path, disabled: false }, select: ['url'] });
+    await Promise.all(archives.map((a) => this.lookForArchiveMirror(a)));
+    return archives;
+  }
+}