Commit 075d0d39 authored by nanahira's avatar nanahira

move to redis

parent d8bb19f4
Pipeline #14454 failed with stages
in 2 minutes and 52 seconds
......@@ -12,6 +12,7 @@
"@aws-sdk/client-s3": "^3.26.0",
"@aws-sdk/lib-storage": "^3.26.0",
"@cityssm/map-expire": "^1.1.1",
"@nestjs-modules/ioredis": "^1.0.1",
"@nestjs/axios": "^0.0.1",
"@nestjs/cli": "^8.0.0",
"@nestjs/common": "^8.0.0",
......@@ -25,7 +26,7 @@
"busboy": "^0.2.14",
"class-transformer": "^0.4.0",
"class-validator": "^0.13.1",
"delay": "^5.0.0",
"ioredis": "^4.28.5",
"lodash": "^4.17.21",
"moment": "^2.29.1",
"mustache": "^4.2.0",
......@@ -33,6 +34,7 @@
"p-queue": "6.6.2",
"pg": "^8.7.1",
"readdirp": "^3.6.0",
"redlock": "^5.0.0-beta.2",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0",
......@@ -44,6 +46,7 @@
"@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4",
"@types/express": "^4.17.13",
"@types/ioredis": "^4.28.10",
"@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172",
"@types/multer": "^1.4.7",
......@@ -2391,6 +2394,16 @@
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/@nestjs-modules/ioredis": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@nestjs-modules/ioredis/-/ioredis-1.0.1.tgz",
"integrity": "sha512-UhIOqL5pXeJlCucHMkJONwqecvLFySW/95uw34Umh0ld7v+6tvYRK3JHQYcJqzlmNiq52bfNxf7CUpuHCSdA3A==",
"peerDependencies": {
"@nestjs/common": "^6.7.0 || ^7.0.0 || ^8.0.0",
"@nestjs/core": "^6.7.0 || ^7.0.0 || ^8.0.0",
"ioredis": "^4.0.0"
}
},
"node_modules/@nestjs/axios": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz",
......@@ -3167,6 +3180,15 @@
"@types/node": "*"
}
},
"node_modules/@types/ioredis": {
"version": "4.28.10",
"resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
"integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/istanbul-lib-coverage": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
......@@ -4490,6 +4512,14 @@
"node": ">=0.8"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
......@@ -4799,17 +4829,6 @@
"clone": "^1.0.2"
}
},
"node_modules/delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
......@@ -4819,6 +4838,14 @@
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
......@@ -6362,6 +6389,31 @@
"node": ">= 0.10"
}
},
"node_modules/ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"dependencies": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
......@@ -7985,6 +8037,16 @@
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
"dev": true
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
},
"node_modules/lodash.flatten": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g=="
},
"node_modules/lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
......@@ -7995,6 +8057,11 @@
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
"integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
},
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
},
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
......@@ -8345,6 +8412,11 @@
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
"integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="
},
"node_modules/node-abort-controller": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz",
"integrity": "sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw=="
},
"node_modules/node-emoji": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz",
......@@ -8594,6 +8666,14 @@
"node": ">=8"
}
},
"node_modules/p-map": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
"integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
"engines": {
"node": ">=6"
}
},
"node_modules/p-queue": {
"version": "6.6.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
......@@ -9179,6 +9259,41 @@
"node": ">= 0.10"
}
},
"node_modules/redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/redlock": {
"version": "5.0.0-beta.2",
"resolved": "https://registry.npmjs.org/redlock/-/redlock-5.0.0-beta.2.tgz",
"integrity": "sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==",
"dependencies": {
"node-abort-controller": "^3.0.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/reflect-metadata": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
......@@ -9704,6 +9819,11 @@
"node": ">=8"
}
},
"node_modules/standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
},
"node_modules/statuses": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
......@@ -13370,6 +13490,12 @@
}
}
},
"@nestjs-modules/ioredis": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@nestjs-modules/ioredis/-/ioredis-1.0.1.tgz",
"integrity": "sha512-UhIOqL5pXeJlCucHMkJONwqecvLFySW/95uw34Umh0ld7v+6tvYRK3JHQYcJqzlmNiq52bfNxf7CUpuHCSdA3A==",
"requires": {}
},
"@nestjs/axios": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz",
......@@ -13925,6 +14051,15 @@
"@types/node": "*"
}
},
"@types/ioredis": {
"version": "4.28.10",
"resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
"integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/istanbul-lib-coverage": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
......@@ -14949,6 +15084,11 @@
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4="
},
"cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw=="
},
"co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
......@@ -15208,17 +15348,17 @@
"clone": "^1.0.2"
}
},
"delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw=="
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true
},
"denque": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw=="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
......@@ -16375,6 +16515,24 @@
"resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
"integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA=="
},
"ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"requires": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
}
},
"ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
......@@ -17611,6 +17769,16 @@
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
"dev": true
},
"lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
},
"lodash.flatten": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g=="
},
"lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
......@@ -17621,6 +17789,11 @@
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
"integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
},
"lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
},
"lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
......@@ -17896,6 +18069,11 @@
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
"integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="
},
"node-abort-controller": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz",
"integrity": "sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw=="
},
"node-emoji": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz",
......@@ -18075,6 +18253,11 @@
"p-limit": "^2.2.0"
}
},
"p-map": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
"integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="
},
"p-queue": {
"version": "6.6.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
......@@ -18511,6 +18694,32 @@
"resolve": "^1.1.6"
}
},
"redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"requires": {
"redis-errors": "^1.0.0"
}
},
"redlock": {
"version": "5.0.0-beta.2",
"resolved": "https://registry.npmjs.org/redlock/-/redlock-5.0.0-beta.2.tgz",
"integrity": "sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==",
"requires": {
"node-abort-controller": "^3.0.1"
}
},
"reflect-metadata": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
......@@ -18913,6 +19122,11 @@
}
}
},
"standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
},
"statuses": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
......
......@@ -24,6 +24,7 @@
"@aws-sdk/client-s3": "^3.26.0",
"@aws-sdk/lib-storage": "^3.26.0",
"@cityssm/map-expire": "^1.1.1",
"@nestjs-modules/ioredis": "^1.0.1",
"@nestjs/axios": "^0.0.1",
"@nestjs/cli": "^8.0.0",
"@nestjs/common": "^8.0.0",
......@@ -37,7 +38,7 @@
"busboy": "^0.2.14",
"class-transformer": "^0.4.0",
"class-validator": "^0.13.1",
"delay": "^5.0.0",
"ioredis": "^4.28.5",
"lodash": "^4.17.21",
"moment": "^2.29.1",
"mustache": "^4.2.0",
......@@ -45,6 +46,7 @@
"p-queue": "6.6.2",
"pg": "^8.7.1",
"readdirp": "^3.6.0",
"redlock": "^5.0.0-beta.2",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0",
......@@ -59,6 +61,7 @@
"@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4",
"@types/express": "^4.17.13",
"@types/ioredis": "^4.28.10",
"@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172",
"@types/multer": "^1.4.7",
......
......@@ -19,19 +19,20 @@ import path from 'path';
import { ArchiveMirror } from './entities/ArchiveMirror.dto';
import { MirrorService } from './mirror/mirror.service';
import { HttpModule } from '@nestjs/axios';
const configModule = ConfigModule.forRoot();
import { RedisModule } from '@nestjs-modules/ioredis';
import { LockService } from './lock/lock.service';
@Module({
imports: [
ServeStaticModule.forRoot({
rootPath: path.join(__dirname, '..', 'public'),
}),
configModule,
ConfigModule.forRoot({
isGlobal: true
}),
HttpModule,
TypeOrmModule.forRootAsync({
name: 'app',
imports: [configModule],
inject: [ConfigService],
useFactory: async (config: ConfigService) => {
return {
......@@ -49,8 +50,16 @@ const configModule = ConfigModule.forRoot();
};
},
}),
RedisModule.forRootAsync({
inject: [ConfigService],
useFactory: async (config: ConfigService) => ({
config: {
url: config.get('REDIS_URL'),
},
}),
}),
],
controllers: [AppController, AdminController, UpdateController],
providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService, MirrorService],
providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService, MirrorService, LockService],
})
export class AppModule {}
import { Test, TestingModule } from '@nestjs/testing';
import { LockService } from './lock.service';
describe('LockService', () => {
let service: LockService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [LockService],
}).compile();
service = module.get<LockService>(LockService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
import { InjectRedis, Redis } from '@nestjs-modules/ioredis';
import { Injectable } from '@nestjs/common';
import Redlock from 'redlock';
@Injectable()
export class LockService extends Redlock {
constructor(@InjectRedis() private readonly redis: Redis) {
super([redis]);
}
}
......@@ -14,9 +14,10 @@ import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
import { Archive, ArchiveType } from '../entities/Archive.entity';
import { AppService } from '../app.service';
import { createHash } from 'crypto';
import delay from 'delay';
import { Cache } from '@cityssm/map-expire';
import PQueue from 'p-queue';
import { LockService } from 'src/lock/lock.service';
import { InjectRedis, Redis } from '@nestjs-modules/ioredis';
export interface FileWithHash {
file: readdirp.EntryInfo;
......@@ -63,17 +64,19 @@ export class ArchiveTask {
@Injectable()
export class PackagerService extends ConsoleLogger {
bucket_max = 10 * 1024 ** 2;
bucket_enter = 1 * 1024 ** 2;
bucket_enter = 1024 ** 2;
noGatherExts = new Set<string>();
packagerWorkingDirectory: string;
private uploadLock = new Set<string>();
private hashCache = new Cache<string, string>();
// private uploadLock = new Set<string>();
// private hashCache = new Cache<string, string>();
constructor(
@Inject(forwardRef(() => AppService)) private readonly appService: AppService,
private s3: PackageS3Service,
config: ConfigService
config: ConfigService,
private redlock: LockService,
@InjectRedis() private readonly redis: Redis
) {
super('packager');
this.bucket_max = (parseInt(config.get('PACKAGE_BUCKET_MAX')) || 10) * 1024 ** 2;
......@@ -85,19 +88,6 @@ export class PackagerService extends ConsoleLogger {
}
}
private async waitForLock(key: string) {
while (this.uploadLock.has(key)) {
await delay(10);
}
this.uploadLock.add(key);
}
private releaseLock(key: string) {
if (this.uploadLock.has(key)) {
this.uploadLock.delete(key);
}
}
async build(
stream: NodeJS.ReadableStream,
pathPrefix?: string,
......@@ -208,21 +198,23 @@ export class PackagerService extends ConsoleLogger {
}
private async lookForExistingArchiveHash(path: string) {
let hash = this.hashCache.get(path);
const hashKey = `hash:${path}`;
let hash = await this.redis.get(hashKey);
if (hash) {
return hash;
}
hash = await this.appService.lookForExistingArchiveHash(path);
if (hash) {
this.hashCache.set(path, hash, 24 * 60 * 60 * 1000);
await this.redis.set(hashKey, hash, 'EX', 60 * 60 * 24);
return hash;
}
return null;
}
async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> {
await this.waitForLock(archiveTask.path);
return this.archiveQueue.add(() => this.archiveProcess(root, archiveTask));
return this.redlock.using([`archive:${archiveTask.path}`], 1800 * 1000, async () =>
this.archiveQueue.add(() => this.archiveProcess(root, archiveTask))
);
}
private archiveQueue = new PQueue({ concurrency: parseInt(process.env.PACKAGE_COCURRENCY) || os.cpus().length });
......@@ -230,55 +222,50 @@ export class PackagerService extends ConsoleLogger {
private async archiveProcess(root: string, archiveTask: ArchiveTask): Promise<Archive> {
const archive = archiveTask.archive;
const archiveName = archiveTask.archiveFullPath;
try {
const existing = await this.s3.fileExists(archiveName);
if (existing) {
const hash = await this.lookForExistingArchiveHash(archiveTask.path);
if (hash) {
archive.size = existing.Size;
this.log(`Archive ${archiveName} exists, skipping.`);
return archive;
}
const existing = await this.s3.fileExists(archiveName);
if (existing) {
const hash = await this.lookForExistingArchiveHash(archiveTask.path);
if (hash) {
archive.size = existing.Size;
this.log(`Archive ${archiveName} exists, skipping.`);
return archive;
}
const files = archiveTask.filePaths;
this.log(`Packaging archive ${archiveName} with ${archiveTask.exactFilePaths.length} files.`);
}
const files = archiveTask.filePaths;
this.log(`Packaging archive ${archiveName} with ${archiveTask.exactFilePaths.length} files.`);
const child = child_process.spawn('tar', ['--zstd', '-cf', '-'].concat(files), {
cwd: root,
});
const childPromise = new Promise<void>((resolve, reject) => {
child.on('exit', (code) => {
if (code == 0) {
resolve();
} else {
reject(code);
}
});
child.on('error', (error) => {
reject(error);
});
const child = child_process.spawn('tar', ['--zstd', '-cf', '-'].concat(files), {
cwd: root,
});
const childPromise = new Promise<void>((resolve, reject) => {
child.on('exit', (code) => {
if (code == 0) {
resolve();
} else {
reject(code);
}
});
const hashObject = createHash('sha256');
child.stdout.on('data', (chunk) => {
hashObject.update(chunk);
child.on('error', (error) => {
reject(error);
});
/* if (existing) {
});
const hashObject = createHash('sha256');
child.stdout.on('data', (chunk) => {
hashObject.update(chunk);
});
/* if (existing) {
await childPromise;
archive.hash = hashObject.digest('hex');
return archive;
}*/
const uploadPromise = this.s3.uploadStream(archiveName, child.stdout, {
ContentType: 'application/tar+zstd',
});
const [, { object }] = await Promise.all([childPromise, uploadPromise]);
archive.hash = hashObject.digest('hex');
this.hashCache.set(archive.path, archive.hash, 24 * 60 * 60 * 1000);
archive.size = object.Size;
} catch (e) {
throw e;
} finally {
this.releaseLock(archiveTask.path);
}
const uploadPromise = this.s3.uploadStream(archiveName, child.stdout, {
ContentType: 'application/tar+zstd',
});
const [, { object }] = await Promise.all([childPromise, uploadPromise]);
archive.hash = hashObject.digest('hex');
await this.redis.set(`hash:${archive.path}`, archive.hash, 'EX', 60 * 60 * 24);
archive.size = object.Size;
return archive;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment