Skip to content

C
console
Commit 8862a77e authored by nanahira's avatar nanahira
Browse files
Options

Merge branch 'migrate-fileent-to-array'

parents 27c00edd 8110bc6c
Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 71 deletions
console-api/src/app.controller.ts
View file @ 8862a77e
......@@ -14,13 +14,7 @@ import {
} from '@nestjs/common';
import { AppService } from './app.service';
import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
import {
BlankReturnMessageDto,
BuildReturnMessageDto,
GetAppReturnMessageDto,
ReturnMessageDto,
StringReturnMessageDto,
} from './dto/ReturnMessage.dto';
import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
import { AppsJson } from './utility/apps-json-type';
import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
......@@ -29,11 +23,9 @@ import { FileUploadDto } from './dto/FileUpload.dto';
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import Busboy from 'busboy';
import { Request } from 'express';
import { Stream } from 'stream';
import { PackageResult } from './dto/PackageResult.dto';
import { platform } from 'os';
import AppClass = AppsJson.AppClass;
import { DepotDto } from './dto/Depot.dto';
import AppClass = AppsJson.AppClass;
@Controller('api')
export class AppController {
......
console-api/src/app.module.ts
View file @ 8862a77e
......@@ -16,7 +16,6 @@ import { UpdateController } from './update/update.controller';
import { UpdateService } from './update/update.service';
import { ServeStaticModule } from '@nestjs/serve-static';
import path from 'path';
import { ArchiveFile } from './entities/ArchiveFile.entity';
const configModule = ConfigModule.forRoot();
......@@ -33,7 +32,7 @@ const configModule = ConfigModule.forRoot();
useFactory: async (config: ConfigService) => {
return {
type: 'postgres',
entities: [App, AppHistory, Archive, Build, Depot, ArchiveFile], // entities here
entities: [App, AppHistory, Archive, Build, Depot], // entities here
synchronize: !config.get('DB_NO_INIT'),
host: config.get('DB_HOST'),
port: parseInt(config.get('DB_PORT')) || 5432,
......
console-api/src/app.service.ts
View file @ 8862a77e
import { Connection, In, IsNull, Not, SelectQueryBuilder } from 'typeorm';
import { Connection, IsNull, Not, SelectQueryBuilder } from 'typeorm';
import { InjectConnection } from '@nestjs/typeorm';
import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
import { AppsJson } from './utility/apps-json-type';
......@@ -15,8 +15,6 @@ import { Archive, ArchiveType } from './entities/Archive.entity';
import { PackageS3Service } from './package-s3/package-s3.service';
import axios from 'axios';
import { createHash } from 'crypto';
import { ArchiveFile } from './entities/ArchiveFile.entity';
import _ from 'lodash';
@Injectable()
export class AppService extends ConsoleLogger {
......@@ -203,7 +201,9 @@ export class AppService extends ConsoleLogger {
const result = await this.packager.build(stream, app.packagePrefix, previousTracingBuildChecksums);
this.log(`Saving package info of ${app.id} ${version} for ${JSON.stringify(depot)}`);
build.checksum = result.checksum;
// build.archives = result.archives;
build.archives = result.archives;
build = await this.db.getRepository(Build).save(build);
/*
await this.db.transaction(async (edb) => {
this.log(`Saving build info.`);
build = await edb.getRepository(Build).save(build);
......@@ -226,6 +226,7 @@ export class AppService extends ConsoleLogger {
await edb.getRepository(Archive).save(archivePot);
}
});
*/
this.log(`Finished packaging ${app.id} ${version} for ${JSON.stringify(depot)}`);
return new BlankReturnMessageDto(201, 'success');
} catch (e) {
......@@ -373,21 +374,17 @@ export class AppService extends ConsoleLogger {
/*
async migrateFilesField() {
const archives = await this.db.getRepository(Archive).find({ select: ['id', 'files'] });
const archives = await this.db.getRepository(Archive).find({ select: ['id', 'files'], where: { files: IsNull() } });
await this.db.transaction(async (edb) => {
for (const a of archives) {
this.log(`Processing archive ${a.id} with ${a.files.length} files.`);
const files = a.files.map((f) => {
const fileEnt = ArchiveFile.fromPath(f);
fileEnt.archive = a;
return fileEnt;
});
await edb.getRepository(ArchiveFile).save(files);
this.log(`Processing archive ${a.id}.`);
const fileEnts = await this.db.getRepository(ArchiveFile).find({ select: ['path'], where: { archive: a } });
this.log(`${fileEnts.length} files found for archive ${a.id}`);
a.files = fileEnts.map((f) => f.path);
await edb.getRepository(Archive).save(a);
}
});
this.log(`Done.`);
return new BlankReturnMessageDto(200, 'success');
}
*/
}*/
}
console-api/src/dto/Depot.dto.ts
View file @ 8862a77e
import { AppsJson } from '../utility/apps-json-type';
import Platform = AppsJson.Platform;
import Locale = AppsJson.Locale;
import { ApiParam, ApiProperty } from '@nestjs/swagger';
import { Brackets, SelectQueryBuilder } from 'typeorm';
import { ApiProperty } from '@nestjs/swagger';
import { Brackets } from 'typeorm';
export interface DepotLike {
platform?: string;
......
console-api/src/entities/Archive.entity.ts
View file @ 8862a77e
import { Column, Entity, ManyToOne, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
import { Build } from './Build.entity';
import { Index } from 'typeorm';
import { TimeBase } from './TimeBase.entity';
import { ArchiveFile } from './ArchiveFile.entity';
export enum ArchiveType {
Full = 'full',
......@@ -15,11 +13,11 @@ export class Archive extends TimeBase {
@PrimaryGeneratedColumn({ type: 'int8' })
id: number;
//@Column('varchar', { length: 256, array: true })
//files: string[];
@Column('varchar', { length: 256, array: true, default: [] })
files: string[];
@OneToMany(() => ArchiveFile, (file) => file.archive, { cascade: true })
containingFiles: ArchiveFile[];
//@OneToMany(() => ArchiveFile, (file) => file.archive, { cascade: true })
//containingFiles: ArchiveFile[];
@Index()
@Column('varchar', { length: 128 })
......@@ -42,9 +40,9 @@ export class Archive extends TimeBase {
return `${this.path}.tar.zst`;
}
get paramSize() {
return 5 + (this.containingFiles ? this.containingFiles.length * 2 : 0);
}
//get paramSize() {
// return 5 + (this.containingFiles ? this.containingFiles.length * 2 : 0);
//}
toMetalinkView() {
return {
......
console-api/src/entities/ArchiveFile.entity.ts
View file @ 8862a77e
import { Entity, Index, ManyToOne, PrimaryColumn } from 'typeorm';
import { Index, PrimaryColumn } from 'typeorm';
import { Archive } from './Archive.entity';
@Entity()
//@Entity()
export class ArchiveFile {
@ManyToOne(() => Archive, (a) => a.containingFiles, { primary: true, onDelete: 'CASCADE' })
//@ManyToOne(() => Archive, (a) => a.containingFiles, { primary: true, onDelete: 'CASCADE' })
archive: Archive;
@Index()
......
console-api/src/entities/Build.entity.ts
View file @ 8862a77e
import { Column, Entity, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique } from 'typeorm';
import { Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
import { Depot } from './Depot.entity';
import { Archive } from './Archive.entity';
import { Index } from 'typeorm';
import { TimeBase } from './TimeBase.entity';
@Entity()
......
console-api/src/packager/packager.service.ts
View file @ 8862a77e
......@@ -14,7 +14,6 @@ import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
import { Archive, ArchiveType } from '../entities/Archive.entity';
import { AppService } from '../app.service';
import { createHash } from 'crypto';
import { ArchiveFile } from '../entities/ArchiveFile.entity';
export interface FileWithHash {
file: readdirp.EntryInfo;
......@@ -45,8 +44,8 @@ export class ArchiveTask {
const archive = new Archive();
archive.path = this.path;
archive.role = this.role;
//archive.files = this.exactFilePaths;
archive.containingFiles = this.exactFilePaths.map((filePath) => ArchiveFile.fromPath(filePath));
archive.files = this.exactFilePaths;
//archive.containingFiles = this.exactFilePaths.map((filePath) => ArchiveFile.fromPath(filePath));
return archive;
}
......
console-api/src/update/update.controller.ts
View file @ 8862a77e
import { Body, Controller, Get, Header, Param, ParseArrayPipe, Post, Query, Render, ValidationPipe } from '@nestjs/common';
import { UpdateService } from './update.service';
import { ApiBody, ApiOkResponse, ApiOperation, ApiParam, ApiProperty, ApiQuery, ApiTags } from '@nestjs/swagger';
import { ApiBody, ApiOkResponse, ApiOperation, ApiParam, ApiQuery, ApiTags } from '@nestjs/swagger';
import { DepotDto } from '../dto/Depot.dto';
@Controller('update')
......
console-api/src/update/update.service.ts
View file @ 8862a77e
......@@ -8,11 +8,9 @@ import { BlankReturnMessageDto } from '../dto/ReturnMessage.dto';
import { Archive, ArchiveType } from '../entities/Archive.entity';
import { PackageS3Service } from '../package-s3/package-s3.service';
import _ from 'lodash';
import { ArchiveFile } from '../entities/ArchiveFile.entity';
import * as os from 'os';
import moment from 'moment';
import { AppsJson } from '../utility/apps-json-type';
import Platform = AppsJson.Platform;
import moment from 'moment';
@Injectable()
export class UpdateService extends ConsoleLogger {
......@@ -187,6 +185,21 @@ export class UpdateService extends ConsoleLogger {
return trafficCost + requestCost;
}
private pickArchives(archives: Archive[], requestedFiles: string[]) {
const aset = archives.map((a) => ({ archive: a, files: new Set(a.files) }));
const result = new Set<Archive>();
for (const file of requestedFiles) {
try {
result.add(aset.find((a) => a.files.has(file)).archive);
} catch (e) {
//throw new BlankReturnMessageDto(404, `File ${file} not found in archives.`).toException();
}
}
return Array.from(result);
}
async getPartPackageMetalink(id: string, depotDto: DepotDto, version: string, requestedFiles: string[]) {
const build = await this.getBuild(id, depotDto, version, (qb) =>
qb
......@@ -194,20 +207,21 @@ export class UpdateService extends ConsoleLogger {
.leftJoin('build.archives', 'archive', 'archive.role = :partRole', { partRole: ArchiveType.Part })
);
// console.log(build.archives);
//let clock = moment();
//this.log('part 1');
let clock = moment();
this.log('part 1');
const tryExactArchiveQuery = this.db
.getRepository(Archive)
.createQueryBuilder('archive')
.select(['archive.hash', 'archive.path', 'archive.size'])
.where('archive.buildId = :buildId', { buildId: build.id })
.andWhere('archive.role != :partRole', { partRole: ArchiveType.Part });
/*.addSelect(`array(${qb
.andWhere('archive.role != :partRole', { partRole: ArchiveType.Part })
.andWhere(':requestedFiles = archive.files', { requestedFiles })
/*.addSelect(`array(${qb
.subQuery()
.select('file.path')
.from(ArchiveFile, 'file')
.where('file.archiveId = archive.id')
.getQuery()})`, 'allFiles')*/
.getQuery()})`, 'allFiles')
tryExactArchiveQuery
.andWhere(
`:requestedFiles = array(${tryExactArchiveQuery
......@@ -217,7 +231,7 @@ export class UpdateService extends ConsoleLogger {
.where('file.archiveId = archive.id')
.getQuery()})`,
{ requestedFiles: requestedFiles }
)
)*/
// .orderBy('archive.size', 'ASC')
.limit(1);
const tryExactArchives = await tryExactArchiveQuery.getMany();
......@@ -239,19 +253,21 @@ export class UpdateService extends ConsoleLogger {
.innerJoin('file.archive', 'archive')
.andWhere('archive.buildId = :buildId', { buildId: build.id })
.getRawMany()
).map((obj) => obj.archiveId);
this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);*/
//clock = moment();
//this.log('part 2');
const allPartArchivesQuery = this.db
).map((obj) => obj.archiveId);*/
this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
clock = moment();
this.log('part 2');
const allPossiblePartArchives = await this.db
.getRepository(Archive)
.createQueryBuilder('archive')
.select(['archive.hash', 'archive.path', 'archive.size'])
.select(['archive.hash', 'archive.path', 'archive.size', 'archive.files'])
.where('archive.buildId = :buildId', { buildId: build.id })
.andWhere('archive.role = :partRole', { partRole: ArchiveType.Part });
.andWhere('archive.role = :partRole', { partRole: ArchiveType.Part })
.getMany();
//.innerJoin('archive.containingFiles', 'file')
//.andWhere('file.path = any(:requestedFiles)', { requestedFiles: requestedFiles });
/*
allPartArchivesQuery.andWhere(
`archive.id in (${allPartArchivesQuery
.subQuery()
......@@ -261,11 +277,16 @@ export class UpdateService extends ConsoleLogger {
.andWhere('file.path = any(:requestedFiles)', { requestedFiles: requestedFiles })
.getQuery()})`
);
*/
const allPartArchives = await allPartArchivesQuery.getMany();
//this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
//clock = moment();
//this.log('part 3');
//const allPartArchives = await allPartArchivesQuery.getMany();
this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
clock = moment();
this.log('part 3');
const allPartArchives = this.pickArchives(allPossiblePartArchives, requestedFiles);
this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
clock = moment();
this.log('part 4');
const fullArchive = await this.db
.getRepository(Archive)
.createQueryBuilder('archive')
......@@ -279,7 +300,7 @@ export class UpdateService extends ConsoleLogger {
if (!allPartArchives.length || (fullArchive && this.getCostOfArchives([fullArchive]) <= this.getCostOfArchives(allPartArchives))) {
archives = [fullArchive];
}
//this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
this.log(`Time used: ${moment().diff(clock, 'seconds')} s`);
return {
cdnUrl: this.cdnUrl,
archives,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment