backup serving, storing and restoring

package-lock.json

@@ -18,6 +18,7 @@
         "lodash.uniqby": "^4.7.0",
         "moment": "^2.30.1",
         "ms": "^2.1.3",
+        "multer": "^1.4.5-lts.1",
         "mysql": "^2.18.1",
         "node-schedule": "^2.1.1",
         "nodemailer": "^6.9.14",
@@ -38,6 +39,7 @@
         "@types/jsonwebtoken": "^9.0.6",
         "@types/lodash.uniqby": "^4.7.9",
         "@types/ms": "^0.7.34",
+        "@types/multer": "^1.4.12",
         "@types/mysql": "^2.15.21",
         "@types/node": "^16.18.41",
         "@types/node-schedule": "^2.1.6",
@@ -567,6 +569,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/multer": {
+      "version": "1.4.12",
+      "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz",
+      "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/express": "*"
+      }
+    },
     "node_modules/@types/mysql": {
       "version": "2.15.26",
       "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.26.tgz",
@@ -781,6 +793,12 @@
         "node": ">= 6.0.0"
       }
     },
+    "node_modules/append-field": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
+      "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
+      "license": "MIT"
+    },
     "node_modules/aproba": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
@@ -1101,6 +1119,23 @@
       "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
       "license": "BSD-3-Clause"
     },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "license": "MIT"
+    },
+    "node_modules/busboy": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
+      "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+      "dependencies": {
+        "streamsearch": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=10.16.0"
+      }
+    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -1434,6 +1469,21 @@
       "license": "MIT",
       "optional": true
     },
+    "node_modules/concat-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+      "engines": [
+        "node >= 0.8"
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.2.2",
+        "typedarray": "^0.0.6"
+      }
+    },
     "node_modules/console-control-strings": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
@@ -3203,6 +3253,36 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
+    "node_modules/multer": {
+      "version": "1.4.5-lts.1",
+      "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz",
+      "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==",
+      "license": "MIT",
+      "dependencies": {
+        "append-field": "^1.0.0",
+        "busboy": "^1.0.0",
+        "concat-stream": "^1.5.2",
+        "mkdirp": "^0.5.4",
+        "object-assign": "^4.1.1",
+        "type-is": "^1.6.4",
+        "xtend": "^4.0.0"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
+    "node_modules/multer/node_modules/mkdirp": {
+      "version": "0.5.6",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
+      "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
+      "license": "MIT",
+      "dependencies": {
+        "minimist": "^1.2.6"
+      },
+      "bin": {
+        "mkdirp": "bin/cmd.js"
+      }
+    },
     "node_modules/mysql": {
       "version": "2.18.1",
       "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
@@ -4606,6 +4686,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/streamsearch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
+      "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
     "node_modules/streamx": {
       "version": "2.21.1",
       "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.21.1.tgz",
@@ -4931,6 +5019,12 @@
       "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==",
       "license": "MIT"
     },
+    "node_modules/typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
+      "license": "MIT"
+    },
     "node_modules/typeorm": {
       "version": "0.3.20",
       "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.3.20.tgz",
@@ -5370,6 +5464,15 @@
         "node": ">=4.0"
       }
     },
+    "node_modules/xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4"
+      }
+    },
     "node_modules/y18n": {
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",

package.json

@@ -33,6 +33,7 @@
     "lodash.uniqby": "^4.7.0",
     "moment": "^2.30.1",
     "ms": "^2.1.3",
+    "multer": "^1.4.5-lts.1",
     "mysql": "^2.18.1",
     "node-schedule": "^2.1.1",
     "nodemailer": "^6.9.14",
@@ -53,6 +54,7 @@
     "@types/jsonwebtoken": "^9.0.6",
     "@types/lodash.uniqby": "^4.7.9",
     "@types/ms": "^0.7.34",
+    "@types/multer": "^1.4.12",
     "@types/mysql": "^2.15.21",
     "@types/node": "^16.18.41",
     "@types/node-schedule": "^2.1.6",

src/controller/admin/club/backupController.ts

@@ -0,0 +1,75 @@
+import { Request, Response } from "express";
+import { FileSystemHelper } from "../../../helpers/fileSystemHelper";
+import BackupHelper from "../../../helpers/backupHelper";
+import InternalException from "../../../exceptions/internalException";
+
+/**
+ * @description get generated backups
+ * @param req {Request} Express req object
+ * @param res {Response} Express res object
+ * @returns {Promise<*>}
+ */
+export async function getGeneratedBackups(req: Request, res: Response): Promise<any> {
+  let filesInFolder = FileSystemHelper.getFilesInDirectory(`backup`);
+
+  res.json(filesInFolder);
+}
+
+/**
+ * @description download backup file
+ * @param req {Request} Express req object
+ * @param res {Response} Express res object
+ * @returns {Promise<*>}
+ */
+export async function downloadBackupFile(req: Request, res: Response): Promise<any> {
+  let filename = req.params.filename;
+
+  let filepath = FileSystemHelper.formatPath("backup", filename);
+
+  res.sendFile(filepath, {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+}
+
+/**
+ * @description create backup manually
+ * @param req {Request} Express req object
+ * @param res {Response} Express res object
+ * @returns {Promise<*>}
+ */
+export async function createManualBackup(req: Request, res: Response): Promise<any> {
+  await BackupHelper.createBackup({});
+
+  res.sendStatus(204);
+}
+
+/**
+ * @description restore backup by selected
+ * @param req {Request} Express req object
+ * @param res {Response} Express res object
+ * @returns {Promise<*>}
+ */
+export async function restoreBackupByLocalFile(req: Request, res: Response): Promise<any> {
+  let filename = req.body.filename;
+  let partial = req.body.partial;
+  let include = req.body.includes;
+
+  await BackupHelper.loadBackup({ filename, include, partial });
+
+  res.sendStatus(204);
+}
+
+/**
+ * @description upload backup
+ * @param req {Request} Express req object
+ * @param res {Response} Express res object
+ * @returns {Promise<*>}
+ */
+export async function uploadBackupFile(req: Request, res: Response): Promise<any> {
+  if (!req.file) {
+    throw new InternalException("File upload failed");
+  }
+  res.sendStatus(204);
+}

src/helpers/backupHelper.ts

@@ -96,6 +96,8 @@ export default abstract class BackupHelper {
     }
 
     FileSystemHelper.writeFile(path, filename + ".json", JSON.stringify(json, null, 2));
+
+    // TODO: delete older backups by copies env
   }
 
   static async loadBackup({
@@ -114,6 +116,10 @@ export default abstract class BackupHelper {
     let file = FileSystemHelper.readFile(`${path}/${filename}`);
     let backup: BackupFileContent = JSON.parse(file);
 
+    if ((partial && include.length == 0) || (!partial && include.length != 0)) {
+      throw new InternalException("partial and include have to be set correctly for restoring backup.");
+    }
+
     dataSource.manager
       .transaction(async (transaction) => {
         this.transactionManager = transaction;

src/index.ts

@@ -40,4 +40,5 @@ import RefreshCommandHandler from "./command/refreshCommandHandler";
 const job = schedule.scheduleJob("0 0 * * *", async () => {
   console.log(`${new Date().toISOString()}: running Cron`);
   await RefreshCommandHandler.deleteExpired();
+  // TODO: create backup by interval env
 });

src/routes/admin/index.ts

@@ -1,5 +1,6 @@
 import express from "express";
 import PermissionHelper from "../../helpers/permissionHelper";
+import preventWebapiAccess from "../../middleware/preventWebApiAccess";
 
 import award from "./settings/award";
 import communicationType from "./settings/communicationType";
@@ -23,7 +24,7 @@ import role from "./user/role";
 import user from "./user/user";
 import invite from "./user/invite";
 import api from "./user/webapi";
-import preventWebapiAccess from "../../middleware/preventWebApiAccess";
+import backup from "./user/backup";
 
 var router = express.Router({ mergeParams: true });
 
@@ -143,5 +144,6 @@ router.use(
 );
 router.use("/invite", PermissionHelper.passCheckMiddleware("read", "user", "user"), invite);
 router.use("/webapi", preventWebapiAccess, PermissionHelper.passCheckMiddleware("read", "user", "webapi"), api);
+router.use("/backup", preventWebapiAccess, PermissionHelper.passCheckMiddleware("read", "user", "backup"), backup);
 
 export default router;

src/routes/admin/user/backup.ts

@@ -0,0 +1,67 @@
+import express, { Request, Response } from "express";
+import PermissionHelper from "../../../helpers/permissionHelper";
+import multer from "multer";
+import {
+  createManualBackup,
+  downloadBackupFile,
+  getGeneratedBackups,
+  restoreBackupByLocalFile,
+  uploadBackupFile,
+} from "../../../controller/admin/club/backupController";
+
+const storage = multer.diskStorage({
+  destination: (req, file, cb) => {
+    cb(null, "files/backup/");
+  },
+  filename: (req, file, cb) => {
+    cb(null, `${new Date().toISOString().split("T")[0]}-uploaded-${file.originalname}`);
+  },
+});
+
+const upload = multer({
+  storage,
+  fileFilter: (req: Request, file, cb) => {
+    if (file.mimetype === "application/json") {
+      cb(null, true);
+    } else {
+      cb(new Error("Only JSON files are allowed!"));
+    }
+  },
+});
+
+var router = express.Router({ mergeParams: true });
+
+router.get("/", async (req: Request, res: Response) => {
+  await getGeneratedBackups(req, res);
+});
+
+router.get("/:filename", async (req: Request, res: Response) => {
+  await downloadBackupFile(req, res);
+});
+
+router.post(
+  "/",
+  PermissionHelper.passCheckMiddleware("create", "user", "backup"),
+  async (req: Request, res: Response) => {
+    await createManualBackup(req, res);
+  }
+);
+
+router.post(
+  "/restore",
+  PermissionHelper.passCheckMiddleware("admin", "user", "backup"),
+  async (req: Request, res: Response) => {
+    await restoreBackupByLocalFile(req, res);
+  }
+);
+
+router.post(
+  "/upload",
+  PermissionHelper.passCheckMiddleware("create", "user", "backup"),
+  upload.single("file"),
+  async (req: Request, res: Response) => {
+    await uploadBackupFile(req, res);
+  }
+);
+
+export default router;

src/type/permissionTypes.ts

@@ -19,7 +19,8 @@ export type PermissionModule =
   | "query"
   | "query_store"
   | "template"
-  | "template_usage";
+  | "template_usage"
+  | "backup";
 
 export type PermissionType = "read" | "create" | "update" | "delete";
 
@@ -63,6 +64,7 @@ export const permissionModules: Array<PermissionModule> = [
   "query_store",
   "template",
   "template_usage",
+  "backup",
 ];
 export const permissionTypes: Array<PermissionType> = ["read", "create", "update", "delete"];
 export const sectionsAndModules: SectionsAndModulesObject = {
@@ -80,5 +82,5 @@ export const sectionsAndModules: SectionsAndModulesObject = {
     "template_usage",
     "newsletter_config",
   ],
-  user: ["user", "role", "webapi"],
+  user: ["user", "role", "webapi", "backup"],
 };