too slow

console-api/src/update/update.service.ts

@@ -36,11 +36,7 @@ export class UpdateService extends ConsoleLogger {
       if (platform === 'generic') {
         continue;
       }
-      /*const depotDto = new DepotDto();
-      depotDto.locale = 'generic';
-      depotDto.arch = 'generic';
-      depotDto.platform = platform;*/
-      // this.log(platform);
+
       for (const app of data) {
         app.version ??= {};
         const tryCache = appVersionCacheMap.get(app.id);
@@ -199,10 +195,11 @@ export class UpdateService extends ConsoleLogger {
     const result = new Set<Archive>();
 
     for (const file of requestedFiles) {
-      try {
-        result.add(aset.find((a) => a.files.has(file)).archive);
-      } catch (e) {
-        //throw new BlankReturnMessageDto(404, `File ${file} not found in archives.`).toException();
+      const satisfyingArchive = aset.find((a) => a.files.has(file))?.archive;
+      if (satisfyingArchive) {
+        result.add(satisfyingArchive);
+      } else {
+        // Ignore missing file for now
       }
     }
     return Array.from(result);
@@ -213,22 +210,35 @@ export class UpdateService extends ConsoleLogger {
       throw new BlankReturnMessageDto(400, 'empty files').toException();
     }
     const build = await this.getBuild(id, depotDto, version, (qb) => qb.select('build.id'));
-    const tryExactArchiveQuery = this.db
+    console.log(1);
+    const bestUpdateArchive = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
-      .select(['archive.hash', 'archive.path', 'archive.size'])
+      .select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
       .where('archive.buildId = :buildId', { buildId: build.id })
-      .andWhere('archive.role != :partRole', { partRole: ArchiveType.Part })
-      .andWhere(':requestedFiles = archive.files', { requestedFiles })
-      .limit(1);
-    const tryExactArchives = await tryExactArchiveQuery.getMany();
-    if (tryExactArchives.length) {
-      return {
-        cdnUrl: this.cdnUrl,
-        archives: tryExactArchives,
-      };
+      .andWhere('archive.role = :updateRole', { updateRole: ArchiveType.Update })
+      .andWhere(':requestedFiles @> archive.files', { requestedFiles })
+      .orderBy('array_length(archive.files, 1)', 'DESC')
+      .take(1)
+      .getOne();
+
+    console.log(1.5);
+    let remainingFiles: string[];
+    if (bestUpdateArchive) {
+      const exactFiles = new Set(bestUpdateArchive.files);
+      remainingFiles = requestedFiles.filter((f) => !exactFiles.has(f));
+      // If single update archive satisfies all requested files, return it.
+      if (!remainingFiles.length) {
+        await this.mirror.lookForArchivesMirror([bestUpdateArchive]);
+        return {
+          cdnUrl: this.cdnUrl,
+          archives: [bestUpdateArchive],
+        };
+      }
     }
+    const packagePlans: Archive[][] = [];
 
+    console.log(2);
     const allPossiblePartArchives = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
@@ -237,20 +247,41 @@ export class UpdateService extends ConsoleLogger {
       .andWhere('archive.role = :partRole', { partRole: ArchiveType.Part })
       .getMany();
 
+    // Plan 1: use all part archives
+    console.log(3);
     const allPartArchives = this.pickArchives(allPossiblePartArchives, requestedFiles);
-    const fullArchive = await this.db
+    if (allPartArchives.length) {
+      packagePlans.push(allPartArchives);
+    }
+
+    // Plan 2: use a smallest single as-large-possible archive
+    console.log(4);
+    const smallestSingleArchive = await this.db
       .getRepository(Archive)
       .createQueryBuilder('archive')
       .select(['archive.hash', 'archive.path', 'archive.size'])
       .where('archive.buildId = :buildId', { buildId: build.id })
-      .andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full })
+      .andWhere('archive.role != :partRole', { partRole: ArchiveType.Part })
+      .andWhere('archive.files @> :requestedFiles', { requestedFiles })
+      .orderBy('archive.size', 'ASC')
       .limit(1)
       .getOne();
+    if (smallestSingleArchive) {
+      packagePlans.push([smallestSingleArchive]);
+    }
 
-    let archives = allPartArchives;
-    if (!allPartArchives.length || (fullArchive && this.getCostOfArchives([fullArchive]) <= this.getCostOfArchives(allPartArchives))) {
-      archives = [fullArchive];
+    if (bestUpdateArchive) {
+      console.log(5);
+      // Plan 3: use single update archive + part archives for remaining files
+      const remainingPartArchives = this.pickArchives(allPossiblePartArchives, remainingFiles);
+      if (remainingPartArchives.length) {
+        packagePlans.push([bestUpdateArchive, ...remainingPartArchives]);
+      }
     }
+
+    console.log(6);
+    const archives = _.minBy(packagePlans, (plan) => this.getCostOfArchives(plan));
+    console.log(archives);
     await this.mirror.lookForArchivesMirror(archives);
     return {
       cdnUrl: this.cdnUrl,