use array back..

console-api/src/app.controller.ts

@@ -14,13 +14,7 @@ import {
 } from '@nestjs/common';
 import { AppService } from './app.service';
 import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
-import {
-  BlankReturnMessageDto,
-  BuildReturnMessageDto,
-  GetAppReturnMessageDto,
-  ReturnMessageDto,
-  StringReturnMessageDto,
-} from './dto/ReturnMessage.dto';
+import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
 import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
 import { AppsJson } from './utility/apps-json-type';
 import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
@@ -29,11 +23,9 @@ import { FileUploadDto } from './dto/FileUpload.dto';
 import { AssetsS3Service } from './assets-s3/assets-s3.service';
 import Busboy from 'busboy';
 import { Request } from 'express';
-import { Stream } from 'stream';
 import { PackageResult } from './dto/PackageResult.dto';
-import { platform } from 'os';
-import AppClass = AppsJson.AppClass;
 import { DepotDto } from './dto/Depot.dto';
+import AppClass = AppsJson.AppClass;
 
 @Controller('api')
 export class AppController {

console-api/src/app.module.ts

@@ -16,7 +16,6 @@ import { UpdateController } from './update/update.controller';
 import { UpdateService } from './update/update.service';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import path from 'path';
-import { ArchiveFile } from './entities/ArchiveFile.entity';
 
 const configModule = ConfigModule.forRoot();
 
@@ -33,7 +32,7 @@ const configModule = ConfigModule.forRoot();
       useFactory: async (config: ConfigService) => {
         return {
           type: 'postgres',
-          entities: [App, AppHistory, Archive, Build, Depot, ArchiveFile], // entities here
+          entities: [App, AppHistory, Archive, Build, Depot], // entities here
           synchronize: !config.get('DB_NO_INIT'),
           host: config.get('DB_HOST'),
           port: parseInt(config.get('DB_PORT')) || 5432,

console-api/src/app.service.ts

-import { Connection, In, IsNull, Not, SelectQueryBuilder } from 'typeorm';
+import { Connection, IsNull, Not, SelectQueryBuilder } from 'typeorm';
 import { InjectConnection } from '@nestjs/typeorm';
 import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
 import { AppsJson } from './utility/apps-json-type';
@@ -15,8 +15,6 @@ import { Archive, ArchiveType } from './entities/Archive.entity';
 import { PackageS3Service } from './package-s3/package-s3.service';
 import axios from 'axios';
 import { createHash } from 'crypto';
-import { ArchiveFile } from './entities/ArchiveFile.entity';
-import _ from 'lodash';
 
 @Injectable()
 export class AppService extends ConsoleLogger {
@@ -203,7 +201,9 @@ export class AppService extends ConsoleLogger {
       const result = await this.packager.build(stream, app.packagePrefix, previousTracingBuildChecksums);
       this.log(`Saving package info of ${app.id} ${version} for ${JSON.stringify(depot)}`);
       build.checksum = result.checksum;
-      // build.archives = result.archives;
+      build.archives = result.archives;
+      build = await this.db.getRepository(Build).save(build);
+      /*
       await this.db.transaction(async (edb) => {
         this.log(`Saving build info.`);
         build = await edb.getRepository(Build).save(build);
@@ -226,6 +226,7 @@ export class AppService extends ConsoleLogger {
           await edb.getRepository(Archive).save(archivePot);
         }
       });
+       */
       this.log(`Finished packaging ${app.id} ${version} for ${JSON.stringify(depot)}`);
       return new BlankReturnMessageDto(201, 'success');
     } catch (e) {
@@ -371,6 +372,7 @@ export class AppService extends ConsoleLogger {
     return new BlankReturnMessageDto(200, 'success');
   }
 
+  /*
   async migrateFilesField() {
     const archives = await this.db.getRepository(Archive).find({ select: ['id', 'files'], where: { files: IsNull() } });
     await this.db.transaction(async (edb) => {
@@ -384,5 +386,5 @@ export class AppService extends ConsoleLogger {
     });
     this.log(`Done.`);
     return new BlankReturnMessageDto(200, 'success');
-  }
+  }*/
 }

console-api/src/dto/Depot.dto.ts

 import { AppsJson } from '../utility/apps-json-type';
-import Platform = AppsJson.Platform;
-import Locale = AppsJson.Locale;
-import { ApiParam, ApiProperty } from '@nestjs/swagger';
-import { Brackets, SelectQueryBuilder } from 'typeorm';
+import { ApiProperty } from '@nestjs/swagger';
+import { Brackets } from 'typeorm';
 
 export interface DepotLike {
   platform?: string;

console-api/src/entities/Archive.entity.ts

-import { Column, Entity, ManyToOne, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
+import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
 import { Build } from './Build.entity';
-import { Index } from 'typeorm';
 import { TimeBase } from './TimeBase.entity';
-import { ArchiveFile } from './ArchiveFile.entity';
 
 export enum ArchiveType {
   Full = 'full',
@@ -15,11 +13,11 @@ export class Archive extends TimeBase {
   @PrimaryGeneratedColumn({ type: 'int8' })
   id: number;
 
-  @Column('varchar', { length: 256, array: true, nullable: true })
+  @Column('varchar', { length: 256, array: true, default: [] })
   files: string[];
 
-  @OneToMany(() => ArchiveFile, (file) => file.archive, { cascade: true })
-  containingFiles: ArchiveFile[];
+  //@OneToMany(() => ArchiveFile, (file) => file.archive, { cascade: true })
+  //containingFiles: ArchiveFile[];
 
   @Index()
   @Column('varchar', { length: 128 })
@@ -42,9 +40,9 @@ export class Archive extends TimeBase {
     return `${this.path}.tar.zst`;
   }
 
-  get paramSize() {
-    return 5 + (this.containingFiles ? this.containingFiles.length * 2 : 0);
-  }
+  //get paramSize() {
+  //  return 5 + (this.containingFiles ? this.containingFiles.length * 2 : 0);
+  //}
 
   toMetalinkView() {
     return {

console-api/src/entities/ArchiveFile.entity.ts

-import { Entity, Index, ManyToOne, PrimaryColumn } from 'typeorm';
+import { Index, PrimaryColumn } from 'typeorm';
 import { Archive } from './Archive.entity';
 
-@Entity()
+//@Entity()
 export class ArchiveFile {
-  @ManyToOne(() => Archive, (a) => a.containingFiles, { primary: true, onDelete: 'CASCADE' })
+  //@ManyToOne(() => Archive, (a) => a.containingFiles, { primary: true, onDelete: 'CASCADE' })
   archive: Archive;
 
   @Index()

console-api/src/entities/Build.entity.ts

-import { Column, Entity, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique } from 'typeorm';
+import { Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
 import { Depot } from './Depot.entity';
 import { Archive } from './Archive.entity';
-import { Index } from 'typeorm';
 import { TimeBase } from './TimeBase.entity';
 
 @Entity()

console-api/src/packager/packager.service.ts

@@ -14,7 +14,6 @@ import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
 import { Archive, ArchiveType } from '../entities/Archive.entity';
 import { AppService } from '../app.service';
 import { createHash } from 'crypto';
-import { ArchiveFile } from '../entities/ArchiveFile.entity';
 
 export interface FileWithHash {
   file: readdirp.EntryInfo;
@@ -45,8 +44,8 @@ export class ArchiveTask {
     const archive = new Archive();
     archive.path = this.path;
     archive.role = this.role;
-    //archive.files = this.exactFilePaths;
-    archive.containingFiles = this.exactFilePaths.map((filePath) => ArchiveFile.fromPath(filePath));
+    archive.files = this.exactFilePaths;
+    //archive.containingFiles = this.exactFilePaths.map((filePath) => ArchiveFile.fromPath(filePath));
     return archive;
   }
 

console-api/src/update/update.controller.ts

 import { Body, Controller, Get, Header, Param, ParseArrayPipe, Post, Query, Render, ValidationPipe } from '@nestjs/common';
 import { UpdateService } from './update.service';
-import { ApiBody, ApiOkResponse, ApiOperation, ApiParam, ApiProperty, ApiQuery, ApiTags } from '@nestjs/swagger';
+import { ApiBody, ApiOkResponse, ApiOperation, ApiParam, ApiQuery, ApiTags } from '@nestjs/swagger';
 import { DepotDto } from '../dto/Depot.dto';
 
 @Controller('update')

console-api/src/update/update.service.ts

@@ -8,11 +8,9 @@ import { BlankReturnMessageDto } from '../dto/ReturnMessage.dto';
 import { Archive, ArchiveType } from '../entities/Archive.entity';
 import { PackageS3Service } from '../package-s3/package-s3.service';
 import _ from 'lodash';
-import { ArchiveFile } from '../entities/ArchiveFile.entity';
-import * as os from 'os';
-import moment from 'moment';
 import { AppsJson } from '../utility/apps-json-type';
 import Platform = AppsJson.Platform;
+import moment from 'moment';
 
 @Injectable()
 export class UpdateService extends ConsoleLogger {
@@ -187,6 +185,21 @@ export class UpdateService extends ConsoleLogger {
     return trafficCost + requestCost;
   }
 
+  private pickArchives(archives: Archive[], requestedFiles: string[]) {
+    const aset = archives.map((a) => ({ archive: a, files: new Set(a.files) }));
+
+    const result = new Set<Archive>();
+
+    for (const file of requestedFiles) {
+      try {
+        result.add(aset.find((a) => a.files.has(file)).archive);
+      } catch (e) {
+        //throw new BlankReturnMessageDto(404, `File ${file} not found in archives.`).toException();
+      }
+    }
+    return Array.from(result);
+  }
+
   async getPartPackageMetalink(id: string, depotDto: DepotDto, version: string, requestedFiles: string[]) {
     const build = await this.getBuild(id, depotDto, version, (qb) =>
       qb
@@ -194,20 +207,21 @@ export class UpdateService extends ConsoleLogger {
         .leftJoin('build.archives', 'archive', 'archive.role = :partRole', { partRole: ArchiveType.Part })
     );
     // console.log(build.archives);
-    //let clock = moment();
-    //this.log('part 1');
+    let clock = moment();
+    this.log('part 1');
     const tryExactArchiveQuery = this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
       .select(['archive.hash', 'archive.path', 'archive.size'])
       .where('archive.buildId = :buildId', { buildId: build.id })
-      .andWhere('archive.role != :partRole', { partRole: ArchiveType.Part });
-    /*.addSelect(`array(${qb
+      .andWhere('archive.role != :partRole', { partRole: ArchiveType.Part })
+      .andWhere(':requestedFiles = archive.files', { requestedFiles })
+      /*.addSelect(`array(${qb
             .subQuery()
             .select('file.path')
             .from(ArchiveFile, 'file')
             .where('file.archiveId = archive.id')
-            .getQuery()})`, 'allFiles')*/
+            .getQuery()})`, 'allFiles')
     tryExactArchiveQuery
       .andWhere(
         `:requestedFiles = array(${tryExactArchiveQuery
@@ -217,7 +231,7 @@ export class UpdateService extends ConsoleLogger {
           .where('file.archiveId = archive.id')
           .getQuery()})`,
         { requestedFiles: requestedFiles }
-      )
+      )*/
       //  .orderBy('archive.size', 'ASC')
       .limit(1);
     const tryExactArchives = await tryExactArchiveQuery.getMany();
@@ -239,19 +253,21 @@ export class UpdateService extends ConsoleLogger {
         .innerJoin('file.archive', 'archive')
         .andWhere('archive.buildId = :buildId', { buildId: build.id })
         .getRawMany()
-    ).map((obj) => obj.archiveId);
-    this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);*/
-    //clock = moment();
-    //this.log('part 2');
-    const allPartArchivesQuery = this.db
+    ).map((obj) => obj.archiveId);*/
+    this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
+    clock = moment();
+    this.log('part 2');
+    const allPossiblePartArchives = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
-      .select(['archive.hash', 'archive.path', 'archive.size'])
+      .select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
       .where('archive.buildId = :buildId', { buildId: build.id })
-      .andWhere('archive.role = :partRole', { partRole: ArchiveType.Part });
+      .andWhere('archive.role = :partRole', { partRole: ArchiveType.Part })
+      .getMany();
     //.innerJoin('archive.containingFiles', 'file')
     //.andWhere('file.path = any(:requestedFiles)', { requestedFiles: requestedFiles });
 
+    /*
     allPartArchivesQuery.andWhere(
       `archive.id in (${allPartArchivesQuery
         .subQuery()
@@ -261,11 +277,16 @@ export class UpdateService extends ConsoleLogger {
         .andWhere('file.path = any(:requestedFiles)', { requestedFiles: requestedFiles })
         .getQuery()})`
     );
+     */
 
-    const allPartArchives = await allPartArchivesQuery.getMany();
-    //this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
-    //clock = moment();
-    //this.log('part 3');
+    //const allPartArchives = await allPartArchivesQuery.getMany();
+    this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
+    clock = moment();
+    this.log('part 3');
+    const allPartArchives = this.pickArchives(allPossiblePartArchives, requestedFiles);
+    this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
+    clock = moment();
+    this.log('part 4');
     const fullArchive = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
@@ -279,7 +300,7 @@ export class UpdateService extends ConsoleLogger {
     if (!allPartArchives.length || (fullArchive && this.getCostOfArchives([fullArchive]) <= this.getCostOfArchives(allPartArchives))) {
       archives = [fullArchive];
     }
-    //this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
+    this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
     return {
       cdnUrl: this.cdnUrl,
       archives,