From 6330ebd01dbcc966001fa5057e6751e603c91cf1 Mon Sep 17 00:00:00 2001 From: Julian Krauser Date: Sun, 2 Feb 2025 16:23:44 +0100 Subject: [PATCH] backup serving, storing and restoring --- package-lock.json | 103 ++++++++++++++++++ package.json | 2 + src/controller/admin/club/backupController.ts | 75 +++++++++++++ src/helpers/backupHelper.ts | 6 + src/index.ts | 1 + src/routes/admin/index.ts | 4 +- src/routes/admin/user/backup.ts | 67 ++++++++++++ src/type/permissionTypes.ts | 6 +- 8 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 src/controller/admin/club/backupController.ts create mode 100644 src/routes/admin/user/backup.ts diff --git a/package-lock.json b/package-lock.json index 75ebbc6..6ae2a2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "lodash.uniqby": "^4.7.0", "moment": "^2.30.1", "ms": "^2.1.3", + "multer": "^1.4.5-lts.1", "mysql": "^2.18.1", "node-schedule": "^2.1.1", "nodemailer": "^6.9.14", @@ -38,6 +39,7 @@ "@types/jsonwebtoken": "^9.0.6", "@types/lodash.uniqby": "^4.7.9", "@types/ms": "^0.7.34", + "@types/multer": "^1.4.12", "@types/mysql": "^2.15.21", "@types/node": "^16.18.41", "@types/node-schedule": "^2.1.6", @@ -567,6 +569,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/multer": { + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz", + "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/mysql": { "version": "2.15.26", "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.26.tgz", @@ -781,6 +793,12 @@ "node": ">= 6.0.0" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -1101,6 +1119,23 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1434,6 +1469,21 @@ "license": "MIT", "optional": true }, + "node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, "node_modules/console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", @@ -3203,6 +3253,36 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/multer": { + "version": "1.4.5-lts.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz", + "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/multer/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/mysql": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", @@ -4606,6 +4686,14 @@ "node": ">= 0.8" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/streamx": { "version": "2.21.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.21.1.tgz", @@ -4931,6 +5019,12 @@ "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", "license": "MIT" }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/typeorm": { "version": "0.3.20", "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.3.20.tgz", @@ -5370,6 +5464,15 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", diff --git a/package.json b/package.json index 7c289f7..23d7fa7 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "lodash.uniqby": "^4.7.0", "moment": "^2.30.1", "ms": "^2.1.3", + "multer": "^1.4.5-lts.1", "mysql": "^2.18.1", "node-schedule": "^2.1.1", "nodemailer": "^6.9.14", @@ -53,6 +54,7 @@ "@types/jsonwebtoken": "^9.0.6", "@types/lodash.uniqby": "^4.7.9", "@types/ms": "^0.7.34", + "@types/multer": "^1.4.12", "@types/mysql": "^2.15.21", "@types/node": "^16.18.41", "@types/node-schedule": "^2.1.6", diff --git a/src/controller/admin/club/backupController.ts b/src/controller/admin/club/backupController.ts new file mode 100644 index 0000000..5d36791 --- /dev/null +++ b/src/controller/admin/club/backupController.ts @@ -0,0 +1,75 @@ +import { Request, Response } from "express"; +import { FileSystemHelper } from "../../../helpers/fileSystemHelper"; +import BackupHelper from "../../../helpers/backupHelper"; +import InternalException from "../../../exceptions/internalException"; + +/** + * @description get generated backups + * @param req {Request} Express req object + * @param res {Response} Express res object + * @returns {Promise<*>} + */ +export async function getGeneratedBackups(req: Request, res: Response): Promise { + let filesInFolder = FileSystemHelper.getFilesInDirectory(`backup`); + + res.json(filesInFolder); +} + +/** + * @description download backup file + * @param req {Request} Express req object + * @param res {Response} Express res object + * @returns {Promise<*>} + */ +export async function downloadBackupFile(req: Request, res: Response): Promise { + let filename = req.params.filename; + + let filepath = FileSystemHelper.formatPath("backup", filename); + + res.sendFile(filepath, { + headers: { + "Content-Type": "application/json", + }, + }); +} + +/** + * @description create backup manually + * @param req {Request} Express req object + * @param res {Response} Express res object + * @returns {Promise<*>} + */ +export async function createManualBackup(req: Request, res: Response): Promise { + await BackupHelper.createBackup({}); + + res.sendStatus(204); +} + +/** + * @description restore backup by selected + * @param req {Request} Express req object + * @param res {Response} Express res object + * @returns {Promise<*>} + */ +export async function restoreBackupByLocalFile(req: Request, res: Response): Promise { + let filename = req.body.filename; + let partial = req.body.partial; + let include = req.body.includes; + + await BackupHelper.loadBackup({ filename, include, partial }); + + res.sendStatus(204); +} + +/** + * @description upload backup + * @param req {Request} Express req object + * @param res {Response} Express res object + * @returns {Promise<*>} + */ +export async function uploadBackupFile(req: Request, res: Response): Promise { + if (!req.file) { + throw new InternalException("File upload failed"); + } + res.sendStatus(204); +} diff --git a/src/helpers/backupHelper.ts b/src/helpers/backupHelper.ts index 1471974..f183993 100644 --- a/src/helpers/backupHelper.ts +++ b/src/helpers/backupHelper.ts @@ -96,6 +96,8 @@ export default abstract class BackupHelper { } FileSystemHelper.writeFile(path, filename + ".json", JSON.stringify(json, null, 2)); + + // TODO: delete older backups by copies env } static async loadBackup({ @@ -114,6 +116,10 @@ export default abstract class BackupHelper { let file = FileSystemHelper.readFile(`${path}/${filename}`); let backup: BackupFileContent = JSON.parse(file); + if ((partial && include.length == 0) || (!partial && include.length != 0)) { + throw new InternalException("partial and include have to be set correctly for restoring backup."); + } + dataSource.manager .transaction(async (transaction) => { this.transactionManager = transaction; diff --git a/src/index.ts b/src/index.ts index a189059..fe2fc80 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,4 +40,5 @@ import RefreshCommandHandler from "./command/refreshCommandHandler"; const job = schedule.scheduleJob("0 0 * * *", async () => { console.log(`${new Date().toISOString()}: running Cron`); await RefreshCommandHandler.deleteExpired(); + // TODO: create backup by interval env }); diff --git a/src/routes/admin/index.ts b/src/routes/admin/index.ts index c291c15..59d5990 100644 --- a/src/routes/admin/index.ts +++ b/src/routes/admin/index.ts @@ -1,5 +1,6 @@ import express from "express"; import PermissionHelper from "../../helpers/permissionHelper"; +import preventWebapiAccess from "../../middleware/preventWebApiAccess"; import award from "./settings/award"; import communicationType from "./settings/communicationType"; @@ -23,7 +24,7 @@ import role from "./user/role"; import user from "./user/user"; import invite from "./user/invite"; import api from "./user/webapi"; -import preventWebapiAccess from "../../middleware/preventWebApiAccess"; +import backup from "./user/backup"; var router = express.Router({ mergeParams: true }); @@ -143,5 +144,6 @@ router.use( ); router.use("/invite", PermissionHelper.passCheckMiddleware("read", "user", "user"), invite); router.use("/webapi", preventWebapiAccess, PermissionHelper.passCheckMiddleware("read", "user", "webapi"), api); +router.use("/backup", preventWebapiAccess, PermissionHelper.passCheckMiddleware("read", "user", "backup"), backup); export default router; diff --git a/src/routes/admin/user/backup.ts b/src/routes/admin/user/backup.ts new file mode 100644 index 0000000..e537655 --- /dev/null +++ b/src/routes/admin/user/backup.ts @@ -0,0 +1,67 @@ +import express, { Request, Response } from "express"; +import PermissionHelper from "../../../helpers/permissionHelper"; +import multer from "multer"; +import { + createManualBackup, + downloadBackupFile, + getGeneratedBackups, + restoreBackupByLocalFile, + uploadBackupFile, +} from "../../../controller/admin/club/backupController"; + +const storage = multer.diskStorage({ + destination: (req, file, cb) => { + cb(null, "files/backup/"); + }, + filename: (req, file, cb) => { + cb(null, `${new Date().toISOString().split("T")[0]}-uploaded-${file.originalname}`); + }, +}); + +const upload = multer({ + storage, + fileFilter: (req: Request, file, cb) => { + if (file.mimetype === "application/json") { + cb(null, true); + } else { + cb(new Error("Only JSON files are allowed!")); + } + }, +}); + +var router = express.Router({ mergeParams: true }); + +router.get("/", async (req: Request, res: Response) => { + await getGeneratedBackups(req, res); +}); + +router.get("/:filename", async (req: Request, res: Response) => { + await downloadBackupFile(req, res); +}); + +router.post( + "/", + PermissionHelper.passCheckMiddleware("create", "user", "backup"), + async (req: Request, res: Response) => { + await createManualBackup(req, res); + } +); + +router.post( + "/restore", + PermissionHelper.passCheckMiddleware("admin", "user", "backup"), + async (req: Request, res: Response) => { + await restoreBackupByLocalFile(req, res); + } +); + +router.post( + "/upload", + PermissionHelper.passCheckMiddleware("create", "user", "backup"), + upload.single("file"), + async (req: Request, res: Response) => { + await uploadBackupFile(req, res); + } +); + +export default router; diff --git a/src/type/permissionTypes.ts b/src/type/permissionTypes.ts index cd37e49..e3d3105 100644 --- a/src/type/permissionTypes.ts +++ b/src/type/permissionTypes.ts @@ -19,7 +19,8 @@ export type PermissionModule = | "query" | "query_store" | "template" - | "template_usage"; + | "template_usage" + | "backup"; export type PermissionType = "read" | "create" | "update" | "delete"; @@ -63,6 +64,7 @@ export const permissionModules: Array = [ "query_store", "template", "template_usage", + "backup", ]; export const permissionTypes: Array = ["read", "create", "update", "delete"]; export const sectionsAndModules: SectionsAndModulesObject = { @@ -80,5 +82,5 @@ export const sectionsAndModules: SectionsAndModulesObject = { "template_usage", "newsletter_config", ], - user: ["user", "role", "webapi"], + user: ["user", "role", "webapi", "backup"], };