add IP specific mirror url

console-api/package-lock.json

@@ -28,6 +28,7 @@
         "class-validator": "^0.13.1",
         "delay": "^5.0.0",
         "ioredis": "^4.28.5",
+        "ip6addr": "^0.2.5",
         "lodash": "^4.17.21",
         "moment": "^2.29.1",
         "mustache": "^4.2.0",
@@ -47,6 +48,7 @@
         "@types/busboy": "^0.2.4",
         "@types/express": "^4.17.13",
         "@types/ioredis": "^4.28.10",
+        "@types/ip6addr": "^0.2.3",
         "@types/jest": "^26.0.24",
         "@types/lodash": "^4.14.172",
         "@types/multer": "^1.4.7",
@@ -3337,6 +3339,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/ip6addr": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/@types/ip6addr/-/ip6addr-0.2.3.tgz",
+      "integrity": "sha512-oe7hzc+P9DU6+gql8+bLKuUf4WL4aakyCSXZMZq2cjhhGK75qYwH1zJ4s94XOlnb4cAhrGKwnbrmMBaqDK8+Ww==",
+      "dev": true
+    },
     "node_modules/@types/istanbul-lib-coverage": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
@@ -4018,6 +4026,14 @@
         "node": ">=8"
       }
     },
+    "node_modules/assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/astral-regex": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
@@ -5921,6 +5937,14 @@
         "node": ">=4"
       }
     },
+    "node_modules/extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==",
+      "engines": [
+        "node >=0.6.0"
+      ]
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -6719,6 +6743,15 @@
         "url": "https://opencollective.com/ioredis"
       }
     },
+    "node_modules/ip6addr": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/ip6addr/-/ip6addr-0.2.5.tgz",
+      "integrity": "sha512-9RGGSB6Zc9Ox5DpDGFnJdIeF0AsqXzdH+FspCfPPaU/L/4tI6P+5lIoFUFm9JXs9IrJv1boqAaNCQmoDADTSKQ==",
+      "dependencies": {
+        "assert-plus": "^1.0.0",
+        "jsprim": "^2.0.2"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -8218,6 +8251,11 @@
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
       "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="
     },
+    "node_modules/json-schema": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
+      "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="
+    },
     "node_modules/json-schema-traverse": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
@@ -8259,6 +8297,20 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/jsprim": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz",
+      "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==",
+      "engines": [
+        "node >=0.6.0"
+      ],
+      "dependencies": {
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.4.0",
+        "verror": "1.10.0"
+      }
+    },
     "node_modules/kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -11364,6 +11416,19 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/verror": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==",
+      "engines": [
+        "node >=0.6.0"
+      ],
+      "dependencies": {
+        "assert-plus": "^1.0.0",
+        "core-util-is": "1.0.2",
+        "extsprintf": "^1.2.0"
+      }
+    },
     "node_modules/w3c-hr-time": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",
@@ -14461,6 +14526,12 @@
         "@types/node": "*"
       }
     },
+    "@types/ip6addr": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/@types/ip6addr/-/ip6addr-0.2.3.tgz",
+      "integrity": "sha512-oe7hzc+P9DU6+gql8+bLKuUf4WL4aakyCSXZMZq2cjhhGK75qYwH1zJ4s94XOlnb4cAhrGKwnbrmMBaqDK8+Ww==",
+      "dev": true
+    },
     "@types/istanbul-lib-coverage": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
@@ -15008,6 +15079,11 @@
       "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
       "dev": true
     },
+    "assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw=="
+    },
     "astral-regex": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
@@ -16432,6 +16508,11 @@
         "tmp": "^0.0.33"
       }
     },
+    "extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g=="
+    },
     "fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -17025,6 +17106,15 @@
         "standard-as-callback": "^2.1.0"
       }
     },
+    "ip6addr": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/ip6addr/-/ip6addr-0.2.5.tgz",
+      "integrity": "sha512-9RGGSB6Zc9Ox5DpDGFnJdIeF0AsqXzdH+FspCfPPaU/L/4tI6P+5lIoFUFm9JXs9IrJv1boqAaNCQmoDADTSKQ==",
+      "requires": {
+        "assert-plus": "^1.0.0",
+        "jsprim": "^2.0.2"
+      }
+    },
     "ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -18161,6 +18251,11 @@
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
       "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="
     },
+    "json-schema": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
+      "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="
+    },
     "json-schema-traverse": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
@@ -18194,6 +18289,17 @@
         "universalify": "^2.0.0"
       }
     },
+    "jsprim": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz",
+      "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==",
+      "requires": {
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.4.0",
+        "verror": "1.10.0"
+      }
+    },
     "kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -20445,6 +20551,16 @@
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
       "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
     },
+    "verror": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==",
+      "requires": {
+        "assert-plus": "^1.0.0",
+        "core-util-is": "1.0.2",
+        "extsprintf": "^1.2.0"
+      }
+    },
     "w3c-hr-time": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",

console-api/package.json

@@ -40,6 +40,7 @@
     "class-validator": "^0.13.1",
     "delay": "^5.0.0",
     "ioredis": "^4.28.5",
+    "ip6addr": "^0.2.5",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
     "mustache": "^4.2.0",
@@ -62,6 +63,7 @@
     "@types/busboy": "^0.2.4",
     "@types/express": "^4.17.13",
     "@types/ioredis": "^4.28.10",
+    "@types/ip6addr": "^0.2.3",
     "@types/jest": "^26.0.24",
     "@types/lodash": "^4.14.172",
     "@types/multer": "^1.4.7",

console-api/src/mirror/mirror.service.ts

@@ -11,6 +11,7 @@ import { PackageS3Service } from '../package-s3/package-s3.service';
 import _ from 'lodash';
 import delay from 'delay';
 import { LockService } from 'src/lock/lock.service';
+import { createCIDR } from 'ip6addr';
 
 export interface Middleware {
   url: string;
@@ -193,7 +194,7 @@ export class MirrorService extends ConsoleLogger {
     );
   }
 
-  async getMirrorUrl(mirror: ArchiveMirror): Promise<string> {
+  private async getMirrorUrl(mirror: ArchiveMirror, ip: string): Promise<string> {
     const middleware = this.middlewares.get(mirror.middleware);
     if (!middleware) {
       return;
@@ -203,7 +204,7 @@ export class MirrorService extends ConsoleLogger {
     }
     try {
       const { data } = await lastValueFrom(
-        this.http.patch<ReturnMessage<string>>(middleware.url, mirror, { responseType: 'json', timeout: 30000 })
+        this.http.patch<ReturnMessage<string>>(middleware.url, { ...mirror, ip }, { responseType: 'json', timeout: 30000 })
       );
       return data.data;
     } catch (e) {
@@ -212,7 +213,7 @@ export class MirrorService extends ConsoleLogger {
     }
   }
 
-  async lookForArchiveMirror(archive: Archive) {
+  private async lookForArchiveMirror(archive: Archive, ip: string) {
     if (archive.size > this.maximumMirroredSize) {
       return;
     }
@@ -223,7 +224,7 @@ export class MirrorService extends ConsoleLogger {
       const urls: string[] = [];
       await Promise.all(
         mirrors.map(async (m) => {
-          const url = await this.getMirrorUrl(m);
+          const url = await this.getMirrorUrl(m, ip);
           if (url) {
             urls.push(url);
           }
@@ -236,11 +237,22 @@ export class MirrorService extends ConsoleLogger {
     return archive;
   }
 
-  async lookForArchivesMirror(archives: Archive[]) {
+  mcnetworkCidrs = [
+    '10.0.0.0/8',
+    '192.168.0.0/16',
+    '172.16.0.0/12',
+    '114.112.159.128/25', // cain
+  ].map((cidr) => createCIDR(cidr));
+
+  async lookForArchivesMirror(archives: Archive[], ip: string) {
+    if (this.mcnetworkCidrs.some((cidr) => cidr.contains(ip))) {
+      // local network no mirror
+      return archives;
+    }
     //const suitableArchives = archives.filter((a) => a.size <= this.maximumMirroredSize);
     //const paths = _.uniq(suitableArchives.map((a) => a.path));
     //const mirrors = await this.db.getRepository(ArchiveMirror).find({ where: { path: archive.path, disabled: false }, select: ['url'] });
-    await Promise.all(archives.map((a) => this.lookForArchiveMirror(a)));
+    await Promise.all(archives.map((a) => this.lookForArchiveMirror(a, ip)));
     return archives;
   }
 }

console-api/src/update/update.controller.ts

-import { Body, Controller, Get, Header, Param, ParseArrayPipe, Post, Query, Render, Res, ValidationPipe } from '@nestjs/common';
+import { Body, Controller, Get, Header, Ip, Param, ParseArrayPipe, Post, Query, Render, Res, ValidationPipe } from '@nestjs/common';
 import { UpdateService } from './update.service';
 import { ApiBody, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiTags } from '@nestjs/swagger';
 import { DepotDto } from '../dto/Depot.dto';
@@ -48,9 +48,10 @@ export class UpdateController {
     @Param('id') id: string,
     @Query(new ValidationPipe({ transform: true })) depot: DepotDto,
     @Param('version') version: string,
-    @Res({ passthrough: true }) res: Response
+    @Res({ passthrough: true }) res: Response,
+    @Ip() ip: string
   ) {
-    return this.cacheResult(res, this.updateService.getFullPackageMetalink(id, depot, version));
+    return this.cacheResult(res, this.updateService.getFullPackageMetalink(id, depot, version, ip));
   }
 
   @Post('update/:id/:version')
@@ -64,9 +65,10 @@ export class UpdateController {
     @Param('id') id: string,
     @Query(new ValidationPipe({ transform: true })) depot: DepotDto,
     @Param('version') version: string,
-    @Body(new ParseArrayPipe()) requestedFiles: string[]
+    @Body(new ParseArrayPipe()) requestedFiles: string[],
+    @Ip() ip: string
   ) {
     //return requestedFiles;
-    return this.updateService.getPartPackageMetalink(id, depot, version, requestedFiles);
+    return this.updateService.getPartPackageMetalink(id, depot, version, requestedFiles, ip);
   }
 }

console-api/src/update/update.service.ts

@@ -138,11 +138,11 @@ export class UpdateService extends ConsoleLogger {
     };
   }
 
-  async getFullPackageMetalink(id: string, depotDto: DepotDto, version: string) {
+  async getFullPackageMetalink(id: string, depotDto: DepotDto, version: string, ip: string) {
     const archives = await this.getArchives(id, depotDto, version, (qb) =>
       qb.select(['archive.hash', 'archive.path', 'archive.size']).andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full })
     );
-    await this.mirror.lookForArchivesMirror(archives);
+    await this.mirror.lookForArchivesMirror(archives, ip);
     return {
       cdnUrl: this.cdnUrl,
       archives: archives,
@@ -205,7 +205,7 @@ export class UpdateService extends ConsoleLogger {
     return Array.from(result);
   }
 
-  async getPartPackageMetalink(id: string, depotDto: DepotDto, version: string, requestedFiles: string[]) {
+  async getPartPackageMetalink(id: string, depotDto: DepotDto, version: string, requestedFiles: string[], ip: string) {
     if (!requestedFiles || !requestedFiles.length) {
       throw new BlankReturnMessageDto(400, 'empty files').toException();
     }
@@ -232,7 +232,7 @@ export class UpdateService extends ConsoleLogger {
       // If single update archive satisfies all requested files, return it.
       if (!remainingFiles.length) {
         // console.log('exact', bestUpdateArchive);
-        await this.mirror.lookForArchivesMirror([bestUpdateArchive]);
+        await this.mirror.lookForArchivesMirror([bestUpdateArchive], ip);
         return {
           cdnUrl: this.cdnUrl,
           archives: [bestUpdateArchive],
@@ -289,7 +289,7 @@ export class UpdateService extends ConsoleLogger {
 
     const archives = _.minBy(packagePlans, (plan) => this.getCostOfArchives(plan));
     // console.log('use', archives);
-    await this.mirror.lookForArchivesMirror(archives);
+    await this.mirror.lookForArchivesMirror(archives, ip);
     return {
       cdnUrl: this.cdnUrl,
       archives,