Commit 46c72c45 authored by nanahira's avatar nanahira

better packager system

parent dfb739a7
Pipeline #15363 passed with stages
in 5 minutes and 58 seconds
......@@ -209,26 +209,29 @@ export class UpdateService extends ConsoleLogger {
if (!requestedFiles || !requestedFiles.length) {
throw new BlankReturnMessageDto(400, 'empty files').toException();
}
const requestedFilesSet = new Set(requestedFiles);
const build = await this.getBuild(id, depotDto, version, (qb) => qb.select('build.id'));
console.log(1);
const bestUpdateArchive = await this.db
const updateArchives = await this.db
.getRepository(Archive)
.createQueryBuilder('archive')
.select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
.where('archive.buildId = :buildId', { buildId: build.id })
.andWhere('archive.role = :updateRole', { updateRole: ArchiveType.Update })
.andWhere(':requestedFiles @> archive.files', { requestedFiles })
.orderBy('array_length(archive.files, 1)', 'DESC')
.take(1)
.getOne();
.getMany();
const suitableUpdateArchives = updateArchives.filter((a) => a.files.every((f) => requestedFilesSet.has(f)));
let bestUpdateArchive: Archive;
if (suitableUpdateArchives.length) {
bestUpdateArchive = _.maxBy(suitableUpdateArchives, (a) => a.files.length);
}
console.log(1.5);
let remainingFiles: string[];
if (bestUpdateArchive) {
// console.log(`Found best update archive: ${bestUpdateArchive}`);
const exactFiles = new Set(bestUpdateArchive.files);
remainingFiles = requestedFiles.filter((f) => !exactFiles.has(f));
// If single update archive satisfies all requested files, return it.
if (!remainingFiles.length) {
// console.log('exact', bestUpdateArchive);
await this.mirror.lookForArchivesMirror([bestUpdateArchive]);
return {
cdnUrl: this.cdnUrl,
......@@ -238,8 +241,7 @@ export class UpdateService extends ConsoleLogger {
}
const packagePlans: Archive[][] = [];
console.log(2);
const allPossiblePartArchives = await this.db
const partArchives = await this.db
.getRepository(Archive)
.createQueryBuilder('archive')
.select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
......@@ -248,40 +250,45 @@ export class UpdateService extends ConsoleLogger {
.getMany();
// Plan 1: use all part archives
console.log(3);
const allPartArchives = this.pickArchives(allPossiblePartArchives, requestedFiles);
if (allPartArchives.length) {
packagePlans.push(allPartArchives);
const suitablePartArchives = this.pickArchives(partArchives, requestedFiles);
if (suitablePartArchives.length) {
// console.log(`1 ${suitablePartArchives}`);
packagePlans.push(suitablePartArchives);
}
// Plan 2: use a smallest single as-large-possible archive
console.log(4);
const smallestSingleArchive = await this.db
const satisfyingUpdateArchives = updateArchives.filter((a) => {
const files = new Set(a.files);
return requestedFiles.every((f) => files.has(f));
});
if (satisfyingUpdateArchives.length) {
const useArchive = _.minBy(satisfyingUpdateArchives, (a) => a.size);
// console.log(`2 update ${useArchive}`);
packagePlans.push([useArchive]);
} else {
const fullArchive = await this.db
.getRepository(Archive)
.createQueryBuilder('archive')
.select(['archive.hash', 'archive.path', 'archive.size'])
.where('archive.buildId = :buildId', { buildId: build.id })
.andWhere('archive.role != :partRole', { partRole: ArchiveType.Part })
.andWhere('archive.files @> :requestedFiles', { requestedFiles })
.orderBy('archive.size', 'ASC')
.andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full })
.limit(1)
.getOne();
if (smallestSingleArchive) {
packagePlans.push([smallestSingleArchive]);
.getOneOrFail();
// console.log(`2 full ${fullArchive}`);
packagePlans.push([fullArchive]);
}
if (bestUpdateArchive) {
console.log(5);
// Plan 3: use single update archive + part archives for remaining files
const remainingPartArchives = this.pickArchives(allPossiblePartArchives, remainingFiles);
const remainingPartArchives = this.pickArchives(partArchives, remainingFiles);
if (remainingPartArchives.length) {
packagePlans.push([bestUpdateArchive, ...remainingPartArchives]);
// console.log(`3 ${bestUpdateArchive} ${remainingPartArchives}`);
}
}
console.log(6);
const archives = _.minBy(packagePlans, (plan) => this.getCostOfArchives(plan));
console.log(archives);
// console.log('use', archives);
await this.mirror.lookForArchivesMirror(archives);
return {
cdnUrl: this.cdnUrl,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment