improved backup restore

package-lock.json

@@ -15,6 +15,7 @@
         "handlebars": "^4.7.8",
         "ics": "^3.8.1",
         "jsonwebtoken": "^9.0.2",
+        "lodash.uniqby": "^4.7.0",
         "moment": "^2.30.1",
         "ms": "^2.1.3",
         "mysql": "^2.18.1",
@@ -34,6 +35,7 @@
         "@types/cors": "^2.8.14",
         "@types/express": "^4.17.17",
         "@types/jsonwebtoken": "^9.0.6",
+        "@types/lodash.uniqby": "^4.7.9",
         "@types/ms": "^0.7.34",
         "@types/mysql": "^2.15.21",
         "@types/node": "^16.18.41",
@@ -478,6 +480,23 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/lodash": {
+      "version": "4.17.15",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.15.tgz",
+      "integrity": "sha512-w/P33JFeySuhN6JLkysYUK2gEmy9kHHFN7E8ro0tkfmlDOgxBDzWEZ/J8cWA+fHqFevpswDTFZnDx+R9lbL6xw==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/@types/lodash.uniqby": {
+      "version": "4.7.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.uniqby/-/lodash.uniqby-4.7.9.tgz",
+      "integrity": "sha512-rjrXji/seS6BZJRgXrU2h6FqxRVufsbq/HE0Tx0SdgbtlWr2YmD/M64BlYEYYlaMcpZwy32IYVkMfUMYlPuv0w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/lodash": "*"
+      }
+    },
     "node_modules/@types/mime": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
@@ -2310,6 +2329,12 @@
       "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
       "license": "MIT"
     },
+    "node_modules/lodash.uniqby": {
+      "version": "4.7.0",
+      "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
+      "integrity": "sha512-e/zcLx6CSbmaEgFHCA7BnoQKyCtKMxnuWrJygbwPs/AIn+IMKl66L8/s+wBUn5LRw2pZx3bUHibiV1b6aTWIww==",
+      "license": "MIT"
+    },
     "node_modules/long-timeout": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",

package.json

@@ -30,6 +30,7 @@
     "handlebars": "^4.7.8",
     "ics": "^3.8.1",
     "jsonwebtoken": "^9.0.2",
+    "lodash.uniqby": "^4.7.0",
     "moment": "^2.30.1",
     "ms": "^2.1.3",
     "mysql": "^2.18.1",
@@ -49,6 +50,7 @@
     "@types/cors": "^2.8.14",
     "@types/express": "^4.17.17",
     "@types/jsonwebtoken": "^9.0.6",
+    "@types/lodash.uniqby": "^4.7.9",
     "@types/ms": "^0.7.34",
     "@types/mysql": "^2.15.21",
     "@types/node": "^16.18.41",

src/helpers/backupHelper.ts

@@ -1,6 +1,7 @@
 import { dataSource } from "../data-source";
 import { FileSystemHelper } from "./fileSystemHelper";
 import { EntityManager } from "typeorm";
+import uniqBy from "lodash.uniqby";
 import InternalException from "../exceptions/internalException";
 import UserService from "../service/user/userService";
 
@@ -137,7 +138,7 @@ export default abstract class BackupHelper {
       })
       .catch((err) => {
         this.transactionManager = undefined;
-        throw new InternalException("failed to restore backup - rolling back actions");
+        throw new InternalException("failed to restore backup - rolling back actions", err);
       });
   }
 
@@ -427,27 +428,45 @@ export default abstract class BackupHelper {
 
   private static async setMemberData(data: Array<any>): Promise<void> {
     await this.setMemberBase({
-      award: data
+      award: uniqBy(
-        .map((d) => d.awards.map((c: any) => c.award))
+        data
-        .flat()
+          .map((d) => d.awards.map((c: any) => c.award))
-        .map((d) => ({ ...d, id: undefined })),
+          .flat()
-      communication_type: data
+          .map((d) => ({ ...d, id: undefined })),
-        .map((d) => d.communications.map((c: any) => c.type))
+        "award"
-        .flat()
+      ),
-        .map((d) => ({ ...d, id: undefined })),
+      communication_type: uniqBy(
-      executive_position: data
+        data
-        .map((d) => d.positions.map((c: any) => c.executivePosition))
+          .map((d) => d.communications.map((c: any) => c.type))
-        .flat()
+          .flat()
-        .map((d) => ({ ...d, id: undefined })),
+          .map((d) => ({ ...d, id: undefined })),
-      membership_status: data
+        "type"
-        .map((d) => d.memberships.map((c: any) => c.status))
+      ),
-        .flat()
+      executive_position: uniqBy(
-        .map((d) => ({ ...d, id: undefined })),
+        data
-      salutation: data.map((d) => d.salutation).map((d) => ({ ...d, id: undefined })),
+          .map((d) => d.positions.map((c: any) => c.executivePosition))
-      qualification: data
+          .flat()
-        .map((d) => d.qualifications.map((c: any) => c.qualification))
+          .map((d) => ({ ...d, id: undefined })),
-        .flat()
+        "position"
-        .map((d) => ({ ...d, id: undefined })),
+      ),
+      membership_status: uniqBy(
+        data
+          .map((d) => d.memberships.map((c: any) => c.status))
+          .flat()
+          .map((d) => ({ ...d, id: undefined })),
+        "status"
+      ),
+      salutation: uniqBy(
+        data.map((d) => d.salutation).map((d) => ({ ...d, id: undefined })),
+        "salutation"
+      ),
+      qualification: uniqBy(
+        data
+          .map((d) => d.qualifications.map((c: any) => c.qualification))
+          .flat()
+          .map((d) => ({ ...d, id: undefined })),
+        "qualification"
+      ),
     });
 
     let salutation = await this.transactionManager.getRepository("salutation").find();
@@ -501,46 +520,55 @@ export default abstract class BackupHelper {
     await this.transactionManager.getRepository("member").save(dataWithMappedIds);
   }
   private static async setMemberBase(data: { [key: string]: Array<any> }): Promise<void> {
+    let salutation = await this.transactionManager.getRepository("salutation").find();
+    let communication = await this.transactionManager.getRepository("communication_type").find();
+    let membership = await this.transactionManager.getRepository("membership_status").find();
+    let award = await this.transactionManager.getRepository("award").find();
+    let qualification = await this.transactionManager.getRepository("qualification").find();
+    let position = await this.transactionManager.getRepository("executive_position").find();
+
     await this.transactionManager
       .createQueryBuilder()
       .insert()
       .into("award")
-      .values(data?.["award"] ?? [])
+      .values((data?.["award"] ?? []).filter((d) => !award.map((a) => a.award).includes(d.award)))
       .orIgnore()
       .execute();
     await this.transactionManager
       .createQueryBuilder()
       .insert()
       .into("communication_type")
-      .values(data?.["communication_type"] ?? [])
+      .values((data?.["communication_type"] ?? []).filter((d) => !communication.map((c) => c.type).includes(d.type)))
       .orIgnore()
       .execute();
     await this.transactionManager
       .createQueryBuilder()
       .insert()
       .into("executive_position")
-      .values(data?.["executive_position"] ?? [])
+      .values((data?.["executive_position"] ?? []).filter((d) => !position.map((p) => p.position).includes(d.position)))
       .orIgnore()
       .execute();
     await this.transactionManager
       .createQueryBuilder()
       .insert()
       .into("membership_status")
-      .values(data?.["membership_status"] ?? [])
+      .values((data?.["membership_status"] ?? []).filter((d) => !membership.map((m) => m.status).includes(d.status)))
       .orIgnore()
       .execute();
     await this.transactionManager
       .createQueryBuilder()
       .insert()
       .into("salutation")
-      .values(data?.["salutation"] ?? [])
+      .values((data?.["salutation"] ?? []).filter((d) => !salutation.map((s) => s.salutation).includes(d.salutation)))
       .orIgnore()
       .execute();
     await this.transactionManager
       .createQueryBuilder()
       .insert()
       .into("qualification")
-      .values(data?.["qualification"] ?? [])
+      .values(
+        (data?.["qualification"] ?? []).filter((d) => !qualification.map((q) => q.award).includes(d.qualification))
+      )
       .orIgnore()
       .execute();
   }
@@ -569,7 +597,12 @@ export default abstract class BackupHelper {
     await this.transactionManager.getRepository("protocol").save(dataWithMappedIds);
   }
   private static async setNewsletter(data: Array<any>, collectedIds: boolean): Promise<void> {
-    await this.setQueryStore(data.map((d) => d.recipientsByQueryId).map((d) => ({ ...d, id: undefined })));
+    await this.setQueryStore(
+      uniqBy(
+        data.map((d) => d.recipientsByQueryId).map((d) => ({ ...d, id: undefined })),
+        "query"
+      )
+    );
 
     let queries = await this.transactionManager.getRepository("query").find();
     let members = await this.transactionManager.getRepository("member").find();
@@ -601,7 +634,10 @@ export default abstract class BackupHelper {
   }
   private static async setNewsletterConfig(data: Array<any>): Promise<void> {
     await this.setMemberBase({
-      communication_type: data.map((d) => d.comType).map((d) => ({ ...d, id: undefined })),
+      communication_type: uniqBy(
+        data.map((d) => d.comType).map((d) => ({ ...d, id: undefined })),
+        "type"
+      ),
     });
 
     let types = await this.transactionManager.getRepository("communication_type").find();
@@ -621,7 +657,7 @@ export default abstract class BackupHelper {
       .createQueryBuilder()
       .insert()
       .into("award")
-      .values([...(data?.["calendar_type"] ?? []), ...usedTypes])
+      .values(uniqBy([...(data?.["calendar_type"] ?? []), ...usedTypes], "type"))
       .orIgnore()
       .execute();
 
@@ -636,7 +672,15 @@ export default abstract class BackupHelper {
     await this.transactionManager.getRepository("calendar").save(dataWithMappedIds);
   }
   private static async setQueryStore(data: Array<any>): Promise<void> {
-    await this.transactionManager.createQueryBuilder().insert().into("query").values(data).orIgnore().execute();
+    let query = await this.transactionManager.getRepository("query").find();
+
+    await this.transactionManager
+      .createQueryBuilder()
+      .insert()
+      .into("query")
+      .values(data.filter((d) => !query.map((s) => s.query).includes(d.query)))
+      .orIgnore()
+      .execute();
   }
   private static async setTemplate(data: { [key: string]: Array<any> }): Promise<void> {
     await this.transactionManager
@@ -664,7 +708,7 @@ export default abstract class BackupHelper {
       .createQueryBuilder()
       .insert()
       .into("role")
-      .values([...(data?.["role"] ?? []), ...usedRoles])
+      .values(uniqBy([...(data?.["role"] ?? []), ...usedRoles], "role"))
       .orIgnore()
       .execute();
 

src/migrations/1738166124200-BackupAndResetDatabase.ts

@@ -7,7 +7,11 @@ export class BackupAndResetDatabase1738166124200 implements MigrationInterface {
   name = "BackupAndResetDatabase1738166124200";
 
   public async up(queryRunner: QueryRunner): Promise<void> {
-    if ((await queryRunner.hasTable("user")) && !(await queryRunner.hasTable("salutation"))) {
+    let migrations = await queryRunner.query("SELECT `name` FROM `migrations`");
+    if (
+      (await queryRunner.hasTable("user")) &&
+      migrations.findIndex((m: any) => m.name == "MoveSendNewsletterFlag1737816852011") == -1
+    ) {
       throw new InternalException(
         "Cannot update due to skiped version. Update to v1.2.2 Version first to prevent data loss and get access to the newer Versions."
       );