add queue to archive file

console-api/package-lock.json

@@ -5,6 +5,7 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "console-api",
       "version": "0.0.1",
       "license": "UNLICENSED",
       "dependencies": {
@@ -29,6 +30,7 @@
         "moment": "^2.29.1",
         "mustache": "^4.2.0",
         "mustache-express": "^1.3.1",
+        "p-queue": "6.6.2",
         "pg": "^8.7.1",
         "readdirp": "^3.6.0",
         "reflect-metadata": "^0.1.13",
@@ -5432,6 +5434,11 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/eventemitter3": {
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
+      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
+    },
     "node_modules/events": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -8552,6 +8559,14 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-finally": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+      "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/p-limit": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
@@ -8579,6 +8594,32 @@
         "node": ">=8"
       }
     },
+    "node_modules/p-queue": {
+      "version": "6.6.2",
+      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
+      "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
+      "dependencies": {
+        "eventemitter3": "^4.0.4",
+        "p-timeout": "^3.2.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-timeout": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
+      "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
+      "dependencies": {
+        "p-finally": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -15628,6 +15669,11 @@
       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
       "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
     },
+    "eventemitter3": {
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
+      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
+    },
     "events": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -18006,6 +18052,11 @@
       "integrity": "sha512-ycIL2+1V32th+8scbpTvyHNaHe02z0sjgh91XXjAk+ZeXoPN4Z46DVUnzdso0aX4KckKw0FNNFHdjZ2UsZvxiA==",
       "dev": true
     },
+    "p-finally": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+      "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow=="
+    },
     "p-limit": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
@@ -18024,6 +18075,23 @@
         "p-limit": "^2.2.0"
       }
     },
+    "p-queue": {
+      "version": "6.6.2",
+      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
+      "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
+      "requires": {
+        "eventemitter3": "^4.0.4",
+        "p-timeout": "^3.2.0"
+      }
+    },
+    "p-timeout": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
+      "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
+      "requires": {
+        "p-finally": "^1.0.0"
+      }
+    },
     "p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",

console-api/package.json

@@ -42,6 +42,7 @@
     "moment": "^2.29.1",
     "mustache": "^4.2.0",
     "mustache-express": "^1.3.1",
+    "p-queue": "6.6.2",
     "pg": "^8.7.1",
     "readdirp": "^3.6.0",
     "reflect-metadata": "^0.1.13",

console-api/src/packager/packager.service.ts

@@ -16,6 +16,7 @@ import { AppService } from '../app.service';
 import { createHash } from 'crypto';
 import delay from 'delay';
 import { Cache } from '@cityssm/map-expire';
+import PQueue from 'p-queue';
 
 export interface FileWithHash {
   file: readdirp.EntryInfo;
@@ -220,9 +221,15 @@ export class PackagerService extends ConsoleLogger {
   }
 
   async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> {
+    await this.waitForLock(archiveTask.path);
+    return this.archiveQueue.add(() => this.archiveProcess(root, archiveTask));
+  }
+
+  private archiveQueue = new PQueue({ concurrency: os.cpus().length });
+
+  private async archiveProcess(root: string, archiveTask: ArchiveTask): Promise<Archive> {
     const archive = archiveTask.archive;
     const archiveName = archiveTask.archiveFullPath;
-    await this.waitForLock(archiveTask.path);
     try {
       const existing = await this.s3.fileExists(archiveName);
       if (existing) {