better packager system

console-api/src/update/update.service.ts

@@ -209,26 +209,29 @@ export class UpdateService extends ConsoleLogger {
     if (!requestedFiles || !requestedFiles.length) {
       throw new BlankReturnMessageDto(400, 'empty files').toException();
     }
+    const requestedFilesSet = new Set(requestedFiles);
     const build = await this.getBuild(id, depotDto, version, (qb) => qb.select('build.id'));
-    console.log(1);
-    const bestUpdateArchive = await this.db
+    const updateArchives = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
       .select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
       .where('archive.buildId = :buildId', { buildId: build.id })
       .andWhere('archive.role = :updateRole', { updateRole: ArchiveType.Update })
-      .andWhere(':requestedFiles @> archive.files', { requestedFiles })
-      .orderBy('array_length(archive.files, 1)', 'DESC')
-      .take(1)
-      .getOne();
+      .getMany();
+    const suitableUpdateArchives = updateArchives.filter((a) => a.files.every((f) => requestedFilesSet.has(f)));
+    let bestUpdateArchive: Archive;
+    if (suitableUpdateArchives.length) {
+      bestUpdateArchive = _.maxBy(suitableUpdateArchives, (a) => a.files.length);
+    }
 
-    console.log(1.5);
     let remainingFiles: string[];
     if (bestUpdateArchive) {
+      // console.log(`Found best update archive: ${bestUpdateArchive}`);
       const exactFiles = new Set(bestUpdateArchive.files);
       remainingFiles = requestedFiles.filter((f) => !exactFiles.has(f));
       // If single update archive satisfies all requested files, return it.
       if (!remainingFiles.length) {
+        // console.log('exact', bestUpdateArchive);
         await this.mirror.lookForArchivesMirror([bestUpdateArchive]);
         return {
           cdnUrl: this.cdnUrl,
@@ -238,8 +241,7 @@ export class UpdateService extends ConsoleLogger {
     }
     const packagePlans: Archive[][] = [];
 
-    console.log(2);
-    const allPossiblePartArchives = await this.db
+    const partArchives = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
       .select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
@@ -248,40 +250,45 @@ export class UpdateService extends ConsoleLogger {
       .getMany();
 
     // Plan 1: use all part archives
-    console.log(3);
-    const allPartArchives = this.pickArchives(allPossiblePartArchives, requestedFiles);
-    if (allPartArchives.length) {
-      packagePlans.push(allPartArchives);
+    const suitablePartArchives = this.pickArchives(partArchives, requestedFiles);
+    if (suitablePartArchives.length) {
+      // console.log(`1 ${suitablePartArchives}`);
+      packagePlans.push(suitablePartArchives);
     }
 
     // Plan 2: use a smallest single as-large-possible archive
-    console.log(4);
-    const smallestSingleArchive = await this.db
-      .getRepository(Archive)
-      .createQueryBuilder('archive')
-      .select(['archive.hash', 'archive.path', 'archive.size'])
-      .where('archive.buildId = :buildId', { buildId: build.id })
-      .andWhere('archive.role != :partRole', { partRole: ArchiveType.Part })
-      .andWhere('archive.files @> :requestedFiles', { requestedFiles })
-      .orderBy('archive.size', 'ASC')
-      .limit(1)
-      .getOne();
-    if (smallestSingleArchive) {
-      packagePlans.push([smallestSingleArchive]);
+    const satisfyingUpdateArchives = updateArchives.filter((a) => {
+      const files = new Set(a.files);
+      return requestedFiles.every((f) => files.has(f));
+    });
+    if (satisfyingUpdateArchives.length) {
+      const useArchive = _.minBy(satisfyingUpdateArchives, (a) => a.size);
+      // console.log(`2 update ${useArchive}`);
+      packagePlans.push([useArchive]);
+    } else {
+      const fullArchive = await this.db
+        .getRepository(Archive)
+        .createQueryBuilder('archive')
+        .select(['archive.hash', 'archive.path', 'archive.size'])
+        .where('archive.buildId = :buildId', { buildId: build.id })
+        .andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full })
+        .limit(1)
+        .getOneOrFail();
+      // console.log(`2 full ${fullArchive}`);
+      packagePlans.push([fullArchive]);
     }
 
     if (bestUpdateArchive) {
-      console.log(5);
       // Plan 3: use single update archive + part archives for remaining files
-      const remainingPartArchives = this.pickArchives(allPossiblePartArchives, remainingFiles);
+      const remainingPartArchives = this.pickArchives(partArchives, remainingFiles);
       if (remainingPartArchives.length) {
         packagePlans.push([bestUpdateArchive, ...remainingPartArchives]);
+        // console.log(`3 ${bestUpdateArchive} ${remainingPartArchives}`);
       }
     }
 
-    console.log(6);
     const archives = _.minBy(packagePlans, (plan) => this.getCostOfArchives(plan));
-    console.log(archives);
+    // console.log('use', archives);
     await this.mirror.lookForArchivesMirror(archives);
     return {
       cdnUrl: this.cdnUrl,