From ee60f497fa4ce1ca3ee718a690cecbbaa66c6c7a Mon Sep 17 00:00:00 2001 From: Julian Krauser Date: Fri, 31 Jan 2025 13:58:07 +0100 Subject: [PATCH] improved backup restore --- package-lock.json | 25 ++++ package.json | 2 + src/helpers/backupHelper.ts | 110 ++++++++++++------ .../1738166124200-BackupAndResetDatabase.ts | 6 +- 4 files changed, 109 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7b6f159..39f6b73 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "handlebars": "^4.7.8", "ics": "^3.8.1", "jsonwebtoken": "^9.0.2", + "lodash.uniqby": "^4.7.0", "moment": "^2.30.1", "ms": "^2.1.3", "mysql": "^2.18.1", @@ -34,6 +35,7 @@ "@types/cors": "^2.8.14", "@types/express": "^4.17.17", "@types/jsonwebtoken": "^9.0.6", + "@types/lodash.uniqby": "^4.7.9", "@types/ms": "^0.7.34", "@types/mysql": "^2.15.21", "@types/node": "^16.18.41", @@ -478,6 +480,23 @@ "@types/node": "*" } }, + "node_modules/@types/lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-w/P33JFeySuhN6JLkysYUK2gEmy9kHHFN7E8ro0tkfmlDOgxBDzWEZ/J8cWA+fHqFevpswDTFZnDx+R9lbL6xw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/lodash.uniqby": { + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/@types/lodash.uniqby/-/lodash.uniqby-4.7.9.tgz", + "integrity": "sha512-rjrXji/seS6BZJRgXrU2h6FqxRVufsbq/HE0Tx0SdgbtlWr2YmD/M64BlYEYYlaMcpZwy32IYVkMfUMYlPuv0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/lodash": "*" + } + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -2310,6 +2329,12 @@ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "license": "MIT" }, + "node_modules/lodash.uniqby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", + "integrity": "sha512-e/zcLx6CSbmaEgFHCA7BnoQKyCtKMxnuWrJygbwPs/AIn+IMKl66L8/s+wBUn5LRw2pZx3bUHibiV1b6aTWIww==", + "license": "MIT" + }, "node_modules/long-timeout": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz", diff --git a/package.json b/package.json index bb2e766..3eace88 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "handlebars": "^4.7.8", "ics": "^3.8.1", "jsonwebtoken": "^9.0.2", + "lodash.uniqby": "^4.7.0", "moment": "^2.30.1", "ms": "^2.1.3", "mysql": "^2.18.1", @@ -49,6 +50,7 @@ "@types/cors": "^2.8.14", "@types/express": "^4.17.17", "@types/jsonwebtoken": "^9.0.6", + "@types/lodash.uniqby": "^4.7.9", "@types/ms": "^0.7.34", "@types/mysql": "^2.15.21", "@types/node": "^16.18.41", diff --git a/src/helpers/backupHelper.ts b/src/helpers/backupHelper.ts index 4507542..1471974 100644 --- a/src/helpers/backupHelper.ts +++ b/src/helpers/backupHelper.ts @@ -1,6 +1,7 @@ import { dataSource } from "../data-source"; import { FileSystemHelper } from "./fileSystemHelper"; import { EntityManager } from "typeorm"; +import uniqBy from "lodash.uniqby"; import InternalException from "../exceptions/internalException"; import UserService from "../service/user/userService"; @@ -137,7 +138,7 @@ export default abstract class BackupHelper { }) .catch((err) => { this.transactionManager = undefined; - throw new InternalException("failed to restore backup - rolling back actions"); + throw new InternalException("failed to restore backup - rolling back actions", err); }); } @@ -427,27 +428,45 @@ export default abstract class BackupHelper { private static async setMemberData(data: Array): Promise { await this.setMemberBase({ - award: data - .map((d) => d.awards.map((c: any) => c.award)) - .flat() - .map((d) => ({ ...d, id: undefined })), - communication_type: data - .map((d) => d.communications.map((c: any) => c.type)) - .flat() - .map((d) => ({ ...d, id: undefined })), - executive_position: data - .map((d) => d.positions.map((c: any) => c.executivePosition)) - .flat() - .map((d) => ({ ...d, id: undefined })), - membership_status: data - .map((d) => d.memberships.map((c: any) => c.status)) - .flat() - .map((d) => ({ ...d, id: undefined })), - salutation: data.map((d) => d.salutation).map((d) => ({ ...d, id: undefined })), - qualification: data - .map((d) => d.qualifications.map((c: any) => c.qualification)) - .flat() - .map((d) => ({ ...d, id: undefined })), + award: uniqBy( + data + .map((d) => d.awards.map((c: any) => c.award)) + .flat() + .map((d) => ({ ...d, id: undefined })), + "award" + ), + communication_type: uniqBy( + data + .map((d) => d.communications.map((c: any) => c.type)) + .flat() + .map((d) => ({ ...d, id: undefined })), + "type" + ), + executive_position: uniqBy( + data + .map((d) => d.positions.map((c: any) => c.executivePosition)) + .flat() + .map((d) => ({ ...d, id: undefined })), + "position" + ), + membership_status: uniqBy( + data + .map((d) => d.memberships.map((c: any) => c.status)) + .flat() + .map((d) => ({ ...d, id: undefined })), + "status" + ), + salutation: uniqBy( + data.map((d) => d.salutation).map((d) => ({ ...d, id: undefined })), + "salutation" + ), + qualification: uniqBy( + data + .map((d) => d.qualifications.map((c: any) => c.qualification)) + .flat() + .map((d) => ({ ...d, id: undefined })), + "qualification" + ), }); let salutation = await this.transactionManager.getRepository("salutation").find(); @@ -501,46 +520,55 @@ export default abstract class BackupHelper { await this.transactionManager.getRepository("member").save(dataWithMappedIds); } private static async setMemberBase(data: { [key: string]: Array }): Promise { + let salutation = await this.transactionManager.getRepository("salutation").find(); + let communication = await this.transactionManager.getRepository("communication_type").find(); + let membership = await this.transactionManager.getRepository("membership_status").find(); + let award = await this.transactionManager.getRepository("award").find(); + let qualification = await this.transactionManager.getRepository("qualification").find(); + let position = await this.transactionManager.getRepository("executive_position").find(); + await this.transactionManager .createQueryBuilder() .insert() .into("award") - .values(data?.["award"] ?? []) + .values((data?.["award"] ?? []).filter((d) => !award.map((a) => a.award).includes(d.award))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("communication_type") - .values(data?.["communication_type"] ?? []) + .values((data?.["communication_type"] ?? []).filter((d) => !communication.map((c) => c.type).includes(d.type))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("executive_position") - .values(data?.["executive_position"] ?? []) + .values((data?.["executive_position"] ?? []).filter((d) => !position.map((p) => p.position).includes(d.position))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("membership_status") - .values(data?.["membership_status"] ?? []) + .values((data?.["membership_status"] ?? []).filter((d) => !membership.map((m) => m.status).includes(d.status))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("salutation") - .values(data?.["salutation"] ?? []) + .values((data?.["salutation"] ?? []).filter((d) => !salutation.map((s) => s.salutation).includes(d.salutation))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("qualification") - .values(data?.["qualification"] ?? []) + .values( + (data?.["qualification"] ?? []).filter((d) => !qualification.map((q) => q.award).includes(d.qualification)) + ) .orIgnore() .execute(); } @@ -569,7 +597,12 @@ export default abstract class BackupHelper { await this.transactionManager.getRepository("protocol").save(dataWithMappedIds); } private static async setNewsletter(data: Array, collectedIds: boolean): Promise { - await this.setQueryStore(data.map((d) => d.recipientsByQueryId).map((d) => ({ ...d, id: undefined }))); + await this.setQueryStore( + uniqBy( + data.map((d) => d.recipientsByQueryId).map((d) => ({ ...d, id: undefined })), + "query" + ) + ); let queries = await this.transactionManager.getRepository("query").find(); let members = await this.transactionManager.getRepository("member").find(); @@ -601,7 +634,10 @@ export default abstract class BackupHelper { } private static async setNewsletterConfig(data: Array): Promise { await this.setMemberBase({ - communication_type: data.map((d) => d.comType).map((d) => ({ ...d, id: undefined })), + communication_type: uniqBy( + data.map((d) => d.comType).map((d) => ({ ...d, id: undefined })), + "type" + ), }); let types = await this.transactionManager.getRepository("communication_type").find(); @@ -621,7 +657,7 @@ export default abstract class BackupHelper { .createQueryBuilder() .insert() .into("award") - .values([...(data?.["calendar_type"] ?? []), ...usedTypes]) + .values(uniqBy([...(data?.["calendar_type"] ?? []), ...usedTypes], "type")) .orIgnore() .execute(); @@ -636,7 +672,15 @@ export default abstract class BackupHelper { await this.transactionManager.getRepository("calendar").save(dataWithMappedIds); } private static async setQueryStore(data: Array): Promise { - await this.transactionManager.createQueryBuilder().insert().into("query").values(data).orIgnore().execute(); + let query = await this.transactionManager.getRepository("query").find(); + + await this.transactionManager + .createQueryBuilder() + .insert() + .into("query") + .values(data.filter((d) => !query.map((s) => s.query).includes(d.query))) + .orIgnore() + .execute(); } private static async setTemplate(data: { [key: string]: Array }): Promise { await this.transactionManager @@ -664,7 +708,7 @@ export default abstract class BackupHelper { .createQueryBuilder() .insert() .into("role") - .values([...(data?.["role"] ?? []), ...usedRoles]) + .values(uniqBy([...(data?.["role"] ?? []), ...usedRoles], "role")) .orIgnore() .execute(); diff --git a/src/migrations/1738166124200-BackupAndResetDatabase.ts b/src/migrations/1738166124200-BackupAndResetDatabase.ts index 0c07dca..4a190c5 100644 --- a/src/migrations/1738166124200-BackupAndResetDatabase.ts +++ b/src/migrations/1738166124200-BackupAndResetDatabase.ts @@ -7,7 +7,11 @@ export class BackupAndResetDatabase1738166124200 implements MigrationInterface { name = "BackupAndResetDatabase1738166124200"; public async up(queryRunner: QueryRunner): Promise { - if ((await queryRunner.hasTable("user")) && !(await queryRunner.hasTable("salutation"))) { + let migrations = await queryRunner.query("SELECT `name` FROM `migrations`"); + if ( + (await queryRunner.hasTable("user")) && + migrations.findIndex((m: any) => m.name == "MoveSendNewsletterFlag1737816852011") == -1 + ) { throw new InternalException( "Cannot update due to skiped version. Update to v1.2.2 Version first to prevent data loss and get access to the newer Versions." );