so far

console-api/src/app.controller.ts

@@ -22,6 +22,7 @@ import { FileUploadDto } from './dto/FileUpload.dto';
 import AppClass = AppsJson.AppClass;
 import { AssetsS3Service } from './assets-s3/assets-s3.service';
 import { MulterDirectEngine } from './packager/MulterStreamEngine';
+import * as fs from 'fs';
 
 @Controller('api')
 export class AppController {
@@ -89,23 +90,23 @@ export class AppController {
   @ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform })
   @ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale })
   @ApiParam({ name: 'version', description: 'APP 的 版本号' })
-  @ApiBody({
+  /*@ApiBody({
     description: 'app 的 tar.gz 文件',
     type: FileUploadDto,
-  })
+  })*/
   @ApiCreatedResponse({ type: BlankReturnMessageDto })
   async makeBuild(
     @FetchMyCardUser() user: MyCardUser,
-    @UploadedFile() file: Express.Multer.File,
+    //@UploadedFile() file: Express.Multer.File,
     @Param('id') id: string,
     @Param('platform') platform: AppsJson.Platform,
     @Param('locale') locale: AppsJson.Locale,
     @Param('version') version: string
   ) {
-    console.log(file.stream);
+    /*console.log(file.stream);
     if (!file) {
       throw new BlankReturnMessageDto(400, 'no file').toException();
-    }
-    return this.appService.makeBuild(user, file, id, platform, locale, version);
+    }*/
+    return this.appService.makeBuild(user, fs.createReadStream('/tmp/test1.tar.gz'), id, platform, locale, version);
   }
 }

console-api/src/app.service.ts

@@ -6,6 +6,7 @@ import { App } from './entities/App.entity';
 import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto';
 import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
 import { PackagerService } from './packager/packager.service';
+import internal from 'stream';
 
 @Injectable()
 export class AppService extends ConsoleLogger {
@@ -133,7 +134,7 @@ export class AppService extends ConsoleLogger {
 
   async makeBuild(
     user: MyCardUser,
-    file: Express.Multer.File,
+    stream: internal.Readable,
     id: string,
     platform: AppsJson.Platform,
     locale: AppsJson.Locale,
@@ -142,7 +143,7 @@ export class AppService extends ConsoleLogger {
     if (!user) {
       throw new BlankReturnMessageDto(401, 'Needs login').toException();
     }
-    this.log('Build: Checking app.');
+    //this.log('Build: Checking app.');
     const app = await this.db.getRepository(App).findOne({
       where: { id },
       select: ['id', 'packagePrefix', 'author'],
@@ -153,7 +154,8 @@ export class AppService extends ConsoleLogger {
     if (!app.isUserCanEditApp(user)) {
       throw new BlankReturnMessageDto(403, 'Permission denied').toException();
     }
-    const result = await this.packager.build(file.stream, app.packagePrefix);
+    this.log(`Start packaging ${app.id}.`);
+    const result = await this.packager.build(stream, app.packagePrefix);
     return new ReturnMessageDto(201, 'success', result);
   }
 

console-api/src/packager/packager.service.ts

@@ -28,12 +28,13 @@ export class PackagerService extends ConsoleLogger {
   async build(stream: internal.Readable, pathPrefix?: string): Promise<PackageResult> {
     this.log(`Start packaging.`);
     const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'mycard-console-'));
+    const tarballRoot = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'mycard-console-tarball-'));
     let extractRoot = root;
     if (pathPrefix) {
       extractRoot = path.join(root, pathPrefix);
       await fs.promises.mkdir(extractRoot, { recursive: true });
     }
-    await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'inherit', 'inherit'] });
+    await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'ignore', 'ignore'] });
 
     this.log(`Package extracted to ${extractRoot}.`);
 
@@ -42,17 +43,17 @@ export class PackagerService extends ConsoleLogger {
     const [directories, files] = _.partition(entries, (item) => item.stats.isDirectory());
 
     // checksum
-    const c = this.checksum(
+    const checksum = await this.checksum(
       root,
       directories.map((d) => d.path),
       files.map((f) => f.path)
     );
-    const promises = [c];
+    const promises: Promise<[string, string]>[] = [];
 
     // 整包
     const archive = `${uuidv4()}.tar.gz`;
-    packages[archive] = [];
-    promises.push(this.archive(archive, root, await fs.promises.readdir(root)));
+    packages[archive] = ['_full'];
+    promises.push(this.archive(archive, root, tarballRoot, await fs.promises.readdir(root)));
 
     // 散包
     const buckets: Record<string, [string[], number]> = {};
@@ -64,7 +65,7 @@ export class PackagerService extends ConsoleLogger {
         if (bucket[1] + file.stats.size >= this.bucket_max) {
           const archive = `${uuidv4()}.tar.gz`;
           packages[archive] = bucket[0];
-          promises.push(this.archive(archive, root, bucket[0]));
+          promises.push(this.archive(archive, root, tarballRoot, bucket[0], checksum));
           bucket[0] = [];
           bucket[1] = 0;
         } else {
@@ -74,27 +75,35 @@ export class PackagerService extends ConsoleLogger {
       } else {
         const archive = `${uuidv4()}.tar.gz`;
         packages[archive] = [file.path];
-        promises.push(this.archive(archive, root, [file.path]));
+        promises.push(this.archive(archive, root, tarballRoot, [file.path], checksum));
       }
     }
     for (const bucket of Object.values(buckets)) {
       if (bucket[0].length) {
         const archive = `${uuidv4()}.tar.gz`;
         packages[archive] = bucket[0];
-        promises.push(this.archive(archive, root, bucket[0]));
+        promises.push(this.archive(archive, root, tarballRoot, bucket[0], checksum));
       }
     }
 
     // TODO: 更新包
 
-    const [checksum] = await Promise.all(promises); // 这个 await 过后，checksum 和 打包上传都已经跑完了
+    const gotPackages = await Promise.all(promises); // 这个 await 过后，checksum 和 打包上传都已经跑完了
+
+    for (const differentPackages of gotPackages.filter((p) => p[0] !== p[1])) {
+      const [originalPackage, gotPackage] = differentPackages;
+      packages[gotPackage] = packages[originalPackage];
+      delete packages[originalPackage];
+    }
+
     this.log({ checksum, packages });
-    await fs.promises.unlink(root);
+    await fs.promises.rm(root, { recursive: true });
+    await fs.promises.rm(tarballRoot, { recursive: true });
     return new PackageResult(checksum, packages);
   }
 
-  async checksum(root: string, directories: string[], files: string[]) {
-    const { stdout } = await util.promisify(child_process.execFile)('sha256sum', files, { maxBuffer: 1 * 1024 ** 2 });
+  async checksum(root: string, directories: string[], files: string[]): Promise<Record<string, string>> {
+    const { stdout } = await util.promisify(child_process.execFile)('sha256sum', files, { maxBuffer: 1 * 1024 ** 2, cwd: root });
     return Object.fromEntries([
       ['.', ''],
       ...directories.map((d) => [d, '']),
@@ -102,12 +111,20 @@ export class PackagerService extends ConsoleLogger {
     ]);
   }
 
-  archive(archive: string = `${uuidv4()}.tar.gz`, root: string, files: string[]) {
-    const child = child_process.spawn('tar', ['-zcvf', '-'].concat(files), {
+  async archive(
+    archive: string = `${uuidv4()}.tar.gz`,
+    root: string,
+    tarballRoot: string,
+    files: string[],
+    checksum: Record<string, string> = {}
+  ): Promise<[string, string]> {
+    const archivePath = path.join(tarballRoot, archive);
+    await this.spawnAsync('tar', ['-zcvf', archivePath].concat(files), {
       cwd: root,
-      stdio: ['ignore', 'pipe', 'inherit'],
     });
-    return this.s3.uploadFile(archive, child.stdout, 'application/tar+gzip');
+    const fileSize = (await fs.promises.stat(archivePath)).size;
+    await this.s3.uploadFile(archive, fs.createReadStream(archivePath), { ContentType: 'application/tar+gzip', ContentLength: fileSize });
+    return [archive, archive];
   }
 
   private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {

console-api/src/s3/s3.service.ts

 import { ConsoleLogger, Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
-import { ListObjectsCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
 import { createHash } from 'crypto';
 import internal from 'stream';
 
@@ -54,7 +54,7 @@ export class S3Service extends ConsoleLogger {
         // already uploaded
         return downloadUrl;
       } else {
-        return this.uploadFile(filename, file.buffer, file.mimetype);
+        return this.uploadFile(filename, file.buffer, { ContentType: file.mimetype });
       }
     } catch (e) {
       this.error(`Failed to upload assets ${path}: ${e.toString()}`);
@@ -62,13 +62,13 @@ export class S3Service extends ConsoleLogger {
     }
   }
 
-  async uploadFile(path: string, content: Buffer | internal.Readable, mime?: string) {
+  async uploadFile(path: string, content: Buffer | internal.Readable, extras: Partial<PutObjectCommandInput> = {}) {
     await this.s3.send(
       new PutObjectCommand({
         Bucket: this.bucket,
         Key: this.getPathWithPrefix(path),
         Body: content,
-        ContentType: mime,
+        ...extras,
       })
     );
     return `${this.cdnUrl}/${path}`;