mirror

console-api/package-lock.json

@@ -11,6 +11,7 @@
         "@aws-sdk/client-s3": "^3.26.0",
         "@aws-sdk/lib-storage": "^3.26.0",
         "@cityssm/map-expire": "^1.1.1",
+        "@nestjs/axios": "^0.0.1",
         "@nestjs/cli": "^8.0.0",
         "@nestjs/common": "^8.0.0",
         "@nestjs/config": "^1.0.1",
@@ -2388,6 +2389,19 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
+    "node_modules/@nestjs/axios": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz",
+      "integrity": "sha512-6H06VmDp5mmjjjLgglEdtzwtmli5pz5UdcY7E9v5SPoz5KeM4uLQvOF5ZJ1S/R4wub0WAqXUQL8mwuXxdjKIQQ==",
+      "dependencies": {
+        "axios": "0.21.1"
+      },
+      "peerDependencies": {
+        "@nestjs/common": "^7.0.0 || ^8.0.0",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^6.0.0 || ^7.0.0"
+      }
+    },
     "node_modules/@nestjs/cli": {
       "version": "8.1.1",
       "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-8.1.1.tgz",
@@ -13315,6 +13329,14 @@
         }
       }
     },
+    "@nestjs/axios": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz",
+      "integrity": "sha512-6H06VmDp5mmjjjLgglEdtzwtmli5pz5UdcY7E9v5SPoz5KeM4uLQvOF5ZJ1S/R4wub0WAqXUQL8mwuXxdjKIQQ==",
+      "requires": {
+        "axios": "0.21.1"
+      }
+    },
     "@nestjs/cli": {
       "version": "8.1.1",
       "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-8.1.1.tgz",

console-api/package.json

@@ -24,6 +24,7 @@
     "@aws-sdk/client-s3": "^3.26.0",
     "@aws-sdk/lib-storage": "^3.26.0",
     "@cityssm/map-expire": "^1.1.1",
+    "@nestjs/axios": "^0.0.1",
     "@nestjs/cli": "^8.0.0",
     "@nestjs/common": "^8.0.0",
     "@nestjs/config": "^1.0.1",

console-api/src/app.controller.ts

@@ -20,7 +20,6 @@ import { AppsJson } from './utility/apps-json-type';
 import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { FileUploadDto } from './dto/FileUpload.dto';
-import { AssetsS3Service } from './assets-s3/assets-s3.service';
 import Busboy from 'busboy';
 import { Request } from 'express';
 import { PackageResult } from './dto/PackageResult.dto';
@@ -29,7 +28,7 @@ import AppClass = AppsJson.AppClass;
 
 @Controller('api')
 export class AppController {
-  constructor(private readonly appService: AppService, private readonly s3: AssetsS3Service) {}
+  constructor(private readonly appService: AppService) {}
 
   @Get('app')
   @ApiOperation({
@@ -69,7 +68,7 @@ export class AppController {
     if (!file) {
       throw new BlankReturnMessageDto(400, 'no file').toException();
     }
-    const res = await this.s3.uploadAssets(file);
+    const res = await this.appService.uploadAssets(file);
     if (res) {
       return new ReturnMessageDto(201, 'success', res);
     } else {
@@ -117,7 +116,12 @@ export class AppController {
   ) {
     let busboy: busboy.Busboy;
     try {
-      busboy = new Busboy({ headers: req.headers });
+      busboy = new Busboy({
+        headers: req.headers,
+        limits: {
+          fileSize: 0xffffffff,
+        },
+      });
     } catch (e) {
       throw new BlankReturnMessageDto(400, `Creation failed: ${e.toString()}`).toException();
     }

console-api/src/app.module.ts

@@ -16,6 +16,9 @@ import { UpdateController } from './update/update.controller';
 import { UpdateService } from './update/update.service';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import path from 'path';
+import { ArchiveMirror } from './entities/ArchiveMirror.dto';
+import { MirrorService } from './mirror/mirror.service';
+import { HttpModule } from '@nestjs/axios';
 
 const configModule = ConfigModule.forRoot();
 
@@ -25,6 +28,7 @@ const configModule = ConfigModule.forRoot();
       rootPath: path.join(__dirname, '..', 'public'),
     }),
     configModule,
+    HttpModule,
     TypeOrmModule.forRootAsync({
       name: 'app',
       imports: [configModule],
@@ -32,7 +36,7 @@ const configModule = ConfigModule.forRoot();
       useFactory: async (config: ConfigService) => {
         return {
           type: 'postgres',
-          entities: [App, AppHistory, Archive, Build, Depot], // entities here
+          entities: [App, AppHistory, Archive, Build, Depot, ArchiveMirror], // entities here
           synchronize: !config.get('DB_NO_INIT'),
           host: config.get('DB_HOST'),
           port: parseInt(config.get('DB_PORT')) || 5432,
@@ -45,6 +49,6 @@ const configModule = ConfigModule.forRoot();
     }),
   ],
   controllers: [AppController, AdminController, UpdateController],
-  providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService],
+  providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService, MirrorService],
 })
 export class AppModule {}

console-api/src/app.service.ts

@@ -13,8 +13,11 @@ import { Build } from './entities/Build.entity';
 import { ConfigService } from '@nestjs/config';
 import { Archive, ArchiveType } from './entities/Archive.entity';
 import { PackageS3Service } from './package-s3/package-s3.service';
-import axios from 'axios';
 import { createHash } from 'crypto';
+import { HttpService } from '@nestjs/axios';
+import { lastValueFrom } from 'rxjs';
+import { MirrorService } from './mirror/mirror.service';
+import { AssetsS3Service } from './assets-s3/assets-s3.service';
 
 @Injectable()
 export class AppService extends ConsoleLogger {
@@ -27,7 +30,10 @@ export class AppService extends ConsoleLogger {
     private db: Connection,
     private packager: PackagerService,
     private packageS3: PackageS3Service,
-    config: ConfigService
+    private assetsS3: AssetsS3Service,
+    config: ConfigService,
+    private http: HttpService,
+    private mirror: MirrorService
   ) {
     super('app');
     // 打包追溯几个版本的更新包
@@ -346,7 +352,9 @@ export class AppService extends ConsoleLogger {
 
   async checkHashFromUrl(url: string) {
     this.log(`Downloading ${url} for checking hash.`);
-    const { data } = await axios.get<internal.Readable>(url, { responseType: 'stream' });
+    const { data } = await lastValueFrom(
+      this.http.get<internal.Readable>(url, { responseType: 'stream' })
+    );
     return new Promise<string>((resolve, reject) => {
       const hashObject = createHash('sha256');
       data.on('data', (data) => hashObject.update(data));
@@ -356,9 +364,8 @@ export class AppService extends ConsoleLogger {
   }
 
   async getHashForMigrate(fullPath: string): Promise<string> {
-    const url = `${this.packageS3.cdnUrl}/${fullPath}`;
+    const url = this.packageS3.getCdnUrl(fullPath);
     this.log(`Migrating hash: ${url} `);
-
     const hash = await this.checkHashFromUrl(url);
     this.log(`Migrated hash: ${url} => ${hash}`);
     return hash;
@@ -378,6 +385,20 @@ export class AppService extends ConsoleLogger {
     return new BlankReturnMessageDto(200, 'success');
   }
 
+  async uploadAssets(file: Express.Multer.File) {
+    const s3Url = await this.assetsS3.uploadAssets(file);
+    if (!s3Url) {
+      return null;
+    }
+    const resultFromMirror = await this.mirror.uploadWithRandomMiddleware({
+      url: s3Url,
+      size: file.size,
+      customMime: file.mimetype,
+      // customSuffix: path.extname(file.originalname) || undefined;
+    });
+    return resultFromMirror ? resultFromMirror.url : s3Url;
+  }
+
   /*
   async migrateFilesField() {
     const archives = await this.db.getRepository(Archive).find({ select: ['id', 'files'], where: { files: IsNull() } });

console-api/src/dto/ReturnMessage.dto.ts

@@ -3,7 +3,17 @@ import { HttpException } from '@nestjs/common';
 import { AppsJson } from '../utility/apps-json-type';
 import { Build } from '../entities/Build.entity';
 
-export class BlankReturnMessageDto {
+export interface BlankReturnMessage {
+  statusCode: number;
+  message: string;
+  success: boolean;
+}
+
+export interface ReturnMessage<T> extends BlankReturnMessage {
+  data?: T;
+}
+
+export class BlankReturnMessageDto implements BlankReturnMessage {
   @ApiProperty({ description: '返回状态' })
   statusCode: number;
   @ApiProperty({ description: '返回信息' })
@@ -21,7 +31,7 @@ export class BlankReturnMessageDto {
   }
 }
 
-export class ReturnMessageDto<T> extends BlankReturnMessageDto {
+export class ReturnMessageDto<T> extends BlankReturnMessageDto implements ReturnMessage<T> {
   @ApiProperty({ description: '返回内容' })
   data?: T;
   constructor(statusCode: number, message?: string, data?: T) {

console-api/src/entities/Archive.entity.ts

 import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
 import { Build } from './Build.entity';
 import { TimeBase } from './TimeBase.entity';
+import { ArchiveMirror } from './ArchiveMirror.dto';
 
 export enum ArchiveType {
   Full = 'full',
@@ -36,6 +37,9 @@ export class Archive extends TimeBase {
   @Column({ type: 'enum', enum: ArchiveType })
   role: ArchiveType;
 
+  // should not be a relation
+  mirrors: ArchiveMirror[];
+
   get archiveFullPath() {
     return `${this.path}.tar.zst`;
   }

console-api/src/entities/ArchiveMirror.dto.ts

+import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
+import { TimeBase } from './TimeBase.entity';
+
+@Entity()
+export class ArchiveMirror extends TimeBase {
+  @PrimaryColumn('varchar', { length: 128 })
+  @Index()
+  path: string;
+
+  @PrimaryColumn('varchar', { length: 64 })
+  middleware: string;
+
+  @Column('boolean', { default: false })
+  disabled: boolean;
+
+  @Column('varchar', { length: 512 })
+  url: string;
+}

console-api/src/packager/packager.service.ts

@@ -16,8 +16,6 @@ import { AppService } from '../app.service';
 import { createHash } from 'crypto';
 import delay from 'delay';
 import { Cache } from '@cityssm/map-expire';
-import axios from 'axios';
-import internal from 'stream';
 
 export interface FileWithHash {
   file: readdirp.EntryInfo;

console-api/src/s3/s3.service.ts

@@ -3,7 +3,6 @@ import { ConfigService } from '@nestjs/config';
 import { _Object, DeleteObjectsCommand, ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
 import { createHash } from 'crypto';
 import internal from 'stream';
-import Path from 'path';
 import { Upload } from '@aws-sdk/lib-storage';
 
 export interface S3StreamUploadResult {
@@ -21,6 +20,10 @@ export class S3Service extends ConsoleLogger {
     return this.config.get(`${this.servicePrefix}_${field}`) || this.config.get(field);
   }
 
+  getCdnUrl(path: string) {
+    return `${this.cdnUrl}/${path}`;
+  }
+
   constructor(private servicePrefix: string, private config: ConfigService) {
     super(`${servicePrefix} s3`);
     this.bucket = this.getConfig('S3_BUCKET');
@@ -67,11 +70,23 @@ export class S3Service extends ConsoleLogger {
     return this.prefix ? `${this.prefix}/${filename}` : filename;
   }
 
+  private getFileSuffix(filename: string) {
+    const lastPattern = filename.split('/').pop();
+    if (!lastPattern.length) {
+      return '';
+    }
+    const dotSplit = lastPattern.split('.');
+    if (dotSplit.length <= 1) {
+      return '';
+    }
+    return `.${dotSplit.slice(1).join('.')}`;
+  }
+
   async uploadAssets(file: Express.Multer.File) {
-    const fileSuffix = Path.extname(file.originalname);
+    const fileSuffix = this.getFileSuffix(file.originalname);
     const hash = createHash('sha512').update(file.buffer).digest('hex');
-    const path = fileSuffix ? `${hash}.${fileSuffix}` : hash;
-    const downloadUrl = `${this.cdnUrl}/${path}`;
+    const path = fileSuffix ? `${hash}${fileSuffix}` : hash;
+    const downloadUrl = this.getCdnUrl(path);
     try {
       if (await this.fileExists(path)) {
         // already uploaded
@@ -94,7 +109,7 @@ export class S3Service extends ConsoleLogger {
         ...extras,
       })
     );
-    return `${this.cdnUrl}/${path}`;
+    return this.getCdnUrl(path);
   }
 
   async uploadStream(path: string, stream: internal.Readable, extras: Partial<PutObjectCommandInput> = {}) {
@@ -112,6 +127,6 @@ export class S3Service extends ConsoleLogger {
     const {
       Contents: [object],
     } = await this.listObjects(path);
-    return { object, url: `${this.cdnUrl}/${path}` };
+    return { object, url: this.getCdnUrl(path) };
   }
 }

console-api/src/update/update.service.ts

@@ -9,8 +9,8 @@ import { Archive, ArchiveType } from '../entities/Archive.entity';
 import { PackageS3Service } from '../package-s3/package-s3.service';
 import _ from 'lodash';
 import { AppsJson } from '../utility/apps-json-type';
+import { MirrorService } from '../mirror/mirror.service';
 import Platform = AppsJson.Platform;
-import moment from 'moment';
 
 @Injectable()
 export class UpdateService extends ConsoleLogger {
@@ -18,7 +18,8 @@ export class UpdateService extends ConsoleLogger {
   constructor(
     @InjectConnection('app')
     private db: Connection,
-    packageS3: PackageS3Service
+    packageS3: PackageS3Service,
+    private mirror: MirrorService
   ) {
     super('update');
     this.cdnUrl = packageS3.cdnUrl;
@@ -142,6 +143,7 @@ export class UpdateService extends ConsoleLogger {
     const archives = await this.getArchives(id, depotDto, version, (qb) =>
       qb.select(['archive.hash', 'archive.path', 'archive.size']).andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full })
     );
+    await this.mirror.lookForArchivesMirror(archives);
     return {
       cdnUrl: this.cdnUrl,
       archives: archives,
@@ -308,6 +310,7 @@ export class UpdateService extends ConsoleLogger {
     if (!allPartArchives.length || (fullArchive && this.getCostOfArchives([fullArchive]) <= this.getCostOfArchives(allPartArchives))) {
       archives = [fullArchive];
     }
+    await this.mirror.lookForArchivesMirror(archives);
     //this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
     return {
       cdnUrl: this.cdnUrl,

console-api/views/metalinks.mustache

@@ -6,7 +6,10 @@
       {{#hash}}
       <hash type="sha-256">{{.}}</hash>
       {{/hash}}
-      <url priority="1">{{&cdnUrl}}/{{path}}.tar.zst</url>
+      {{#mirrors}}
+      <url priority="1">{{&url}}</url>
+      {{/mirrors}}
+      <url priority="2">{{&cdnUrl}}/{{path}}.tar.zst</url>
     </file>
   {{/archives}}
 </metalink>