import { dataSource } from "../data-source"; import { FileSystemHelper } from "./fileSystemHelper"; import { EntityManager } from "typeorm"; import uniqBy from "lodash.uniqby"; import InternalException from "../exceptions/internalException"; import UserService from "../service/management/userService"; import { BACKUP_COPIES, BACKUP_INTERVAL } from "../env.defaults"; import DatabaseActionException from "../exceptions/databaseActionException"; import { availableTemplates } from "../type/templateTypes"; export type BackupSection = | "member" | "memberBase" | "protocol" | "newsletter" | "newsletter_config" | "calendar" | "query" | "template" | "user" | "webapi"; export type BackupSectionRefered = { [key in BackupSection]?: Array; }; export type BackupFileContent = { [key in BackupSection]?: BackupFileContentSection } & { backup_file_details: { collectIds: boolean; createdAt: Date; version: 1 }; }; export type BackupFileContentSection = Array | { [key: string]: Array }; export default abstract class BackupHelper { // ! Order matters because of foreign keys private static readonly backupSection: Array<{ type: BackupSection; orderOnInsert: number; orderOnClear: number }> = [ { type: "member", orderOnInsert: 2, orderOnClear: 2 }, // CLEAR depends on protcol INSERT depends on Base { type: "memberBase", orderOnInsert: 1, orderOnClear: 3 }, // CLEAR depends on member { type: "protocol", orderOnInsert: 3, orderOnClear: 1 }, // INSERT depends on member { type: "newsletter", orderOnInsert: 3, orderOnClear: 1 }, // INSERT depends on member & query & calendar { type: "newsletter_config", orderOnInsert: 2, orderOnClear: 4 }, // INSERT depends on member com { type: "calendar", orderOnInsert: 1, orderOnClear: 1 }, { type: "query", orderOnInsert: 1, orderOnClear: 2 }, // CLEAR depends on newsletter { type: "template", orderOnInsert: 2, orderOnClear: 1 }, // INSERT depends on member com { type: "user", orderOnInsert: 1, orderOnClear: 1 }, { type: "webapi", orderOnInsert: 1, orderOnClear: 1 }, ]; private static readonly backupSectionRefered: BackupSectionRefered = { member: [ "member", "member_awards", "member_qualifications", "member_executive_positions", "membership", "communication", ], memberBase: [ "award", "qualification", "executive_position", "membership_status", "communication_type", "salutation", ], protocol: [ "protocol", "protocol_agenda", "protocol_decision", "protocol_presence", "protocol_printout", "protocol_voting", ], newsletter: ["newsletter", "newsletter_dates", "newsletter_recipients", "newsletter_config"], newsletter_config: ["newsletter_config"], calendar: ["calendar", "calendar_type"], query: ["query"], template: ["template", "template_usage"], user: ["user", "user_permission", "role", "role_permission", "invite"], webapi: ["webapi", "webapi_permission"], }; private static transactionManager: EntityManager; static async createBackup({ filename, path = "/backup", collectIds = true, }: { filename?: string; path?: string; collectIds?: boolean; }): Promise { if (!filename) { filename = new Date().toISOString().split("T")[0]; } let json: BackupFileContent = { backup_file_details: { collectIds, createdAt: new Date(), version: 1 } }; for (const section of this.backupSection) { json[section.type] = await this.getSectionData(section.type, collectIds); } FileSystemHelper.writeFile(path, filename + ".json", JSON.stringify(json, null, 2)); let files = FileSystemHelper.getFilesInDirectory("backup", ".json"); let sorted = files.sort((a, b) => new Date(b.split(".")[0]).getTime() - new Date(a.split(".")[0]).getTime()); const filesToDelete = sorted.slice(BACKUP_COPIES); for (const file of filesToDelete) { FileSystemHelper.deleteFile("backup", file); } } static async createBackupOnInterval() { let files = FileSystemHelper.getFilesInDirectory("backup", ".json"); let newestFile = files.sort((a, b) => new Date(b.split(".")[0]).getTime() - new Date(a.split(".")[0]).getTime())[0]; let lastBackup = new Date(newestFile.split(".")[0]); let diffInMs = new Date().getTime() - lastBackup.getTime(); let diffInDays = diffInMs / (1000 * 60 * 60 * 24); if (diffInDays >= BACKUP_INTERVAL) { await this.createBackup({}); } } static async loadBackup({ filename, path = "/backup", include = [], partial = false, overwrite = false, }: { filename: string; path?: string; partial?: boolean; include?: Array; overwrite?: boolean; }): Promise { this.transactionManager = undefined; let file = FileSystemHelper.readFile(`${path}/${filename}`); let backup: BackupFileContent = JSON.parse(file); if ((partial && include.length == 0) || (!partial && include.length != 0)) { throw new InternalException("partial and include have to be set correctly for restoring backup."); } await dataSource.manager .transaction(async (transaction) => { this.transactionManager = transaction; const sections = this.backupSection .filter((bs) => (partial ? include.includes(bs.type) : true)) .sort((a, b) => a.orderOnClear - b.orderOnClear); if (!overwrite) { for (const section of sections.filter((s) => Object.keys(backup).includes(s.type))) { let refered = this.backupSectionRefered[section.type]; for (const ref of refered) { await this.transactionManager.getRepository(ref).delete({}); } } } for (const section of sections .filter((s) => Object.keys(backup).includes(s.type)) .sort((a, b) => a.orderOnInsert - b.orderOnInsert)) { await this.setSectionData(section.type, backup[section.type], backup.backup_file_details.collectIds ?? false); } this.transactionManager = undefined; }) .catch((err) => { console.log(err); this.transactionManager = undefined; throw new DatabaseActionException("BACKUP RESTORE", include.join(", ") || "FULL", err); }); } public static async autoRestoreBackup() { let count = await UserService.count(); if (count == 0) { let files = FileSystemHelper.getFilesInDirectory("/backup", ".json"); let newestFile = files.sort( (a, b) => new Date(b.split(".")[0]).getTime() - new Date(a.split(".")[0]).getTime() )[0]; if (newestFile) { console.log(`${new Date().toISOString()}: auto-restoring ${newestFile}`); await this.loadBackup({ filename: newestFile }); console.log(`${new Date().toISOString()}: finished auto-restore`); } else { console.log(`${new Date().toISOString()}: skip auto-restore as no backup was found`); } } else { console.log(`${new Date().toISOString()}: skip auto-restore as users exist`); } } private static async getSectionData( section: BackupSection, collectIds: boolean ): Promise | { [key: string]: any }> { switch (section) { case "member": return await this.getMemberData(collectIds); case "memberBase": return await this.getMemberBase(); case "protocol": return await this.getProtocol(collectIds); case "newsletter": return await this.getNewsletter(collectIds); case "newsletter_config": return await this.getNewsletterConfig(); case "calendar": return await this.getCalendar(); case "query": return await this.getQueryStore(collectIds); case "template": return await this.getTemplate(); case "user": return await this.getUser(collectIds); case "webapi": return await this.getWebapi(); default: return []; } } private static async getMemberData(collectIds: boolean): Promise> { return await dataSource .getRepository("member") .createQueryBuilder("member") .leftJoin("member.salutation", "salutation") .leftJoin("member.communications", "communication") .leftJoin("communication.type", "communicationType") .leftJoin("member.memberships", "memberships") .leftJoin("memberships.status", "membershipStatus") .leftJoin("member.awards", "awards") .leftJoin("awards.award", "award") .leftJoin("member.positions", "positions") .leftJoin("positions.executivePosition", "executivePosition") .leftJoin("member.qualifications", "qualifications") .leftJoin("qualifications.qualification", "qualification") .select([ ...(collectIds ? ["member.id"] : []), "member.firstname", "member.lastname", "member.nameaffix", "member.birthdate", "member.internalId", ]) .addSelect(["salutation.salutation"]) .addSelect([ "communication.preferred", "communication.isSMSAlarming", "communication.isSendNewsletter", "communication.mobile", "communication.email", "communication.postalCode", "communication.city", "communication.street", "communication.streetNumber", "communication.streetNumberAddition", "communicationType.type", "communicationType.useColumns", ]) .addSelect(["memberships.start", "memberships.end", "memberships.terminationReason", "membershipStatus.status"]) .addSelect(["awards.given", "awards.note", "awards.note", "awards.date", "award.award"]) .addSelect(["positions.note", "positions.start", "positions.end", "executivePosition.position"]) .addSelect([ "qualifications.note", "qualifications.start", "qualifications.end", "qualifications.terminationReason", "qualification.qualification", "qualification.description", ]) .getMany(); } private static async getMemberBase(): Promise<{ [key: string]: Array }> { return { award: await dataSource.getRepository("award").find({ select: { award: true } }), communication_type: await dataSource .getRepository("communication_type") .find({ select: { type: true, useColumns: true } }), executive_position: await dataSource.getRepository("executive_position").find({ select: { position: true } }), membership_status: await dataSource.getRepository("membership_status").find({ select: { status: true } }), salutation: await dataSource.getRepository("salutation").find({ select: { salutation: true } }), qualification: await dataSource .getRepository("qualification") .find({ select: { qualification: true, description: true } }), }; } private static async getProtocol(collectIds: boolean): Promise> { return await dataSource .getRepository("protocol") .createQueryBuilder("protocol") .leftJoin("protocol.agendas", "agendas") .leftJoin("protocol.decisions", "decisions") .leftJoin("protocol.presences", "presences") .leftJoin("presences.member", "member") .leftJoin("protocol.printouts", "printouts") .leftJoin("protocol.votings", "votings") .select(["protocol.title", "protocol.date", "protocol.starttime", "protocol.endtime", "protocol.summary"]) .addSelect(["agendas.topic", "agendas.context", "agendas.sort"]) .addSelect(["decisions.topic", "decisions.context", "decisions.sort"]) .addSelect(["presences.absent", "presences.excused"]) .addSelect([ ...(collectIds ? ["member.id"] : []), "member.firstname", "member.lastname", "member.nameaffix", "member.birthdate", "member.internalId", ]) .addSelect(["printouts.title", "printouts.iteration", "printouts.filename", "printouts.createdAt"]) .addSelect([ "votings.topic", "votings.context", "votings.favour", "votings.abstain", "votings.against", "votings.sort", ]) .getMany(); } private static async getNewsletter(collectIds: boolean): Promise> { return await dataSource .getRepository("newsletter") .createQueryBuilder("newsletter") .leftJoin("newsletter.dates", "dates") .leftJoin("newsletter.recipients", "recipients") .leftJoin("recipients.member", "member") .leftJoin("newsletter.recipientsByQuery", "recipientsByQuery") .select([ "newsletter.title", "newsletter.description", "newsletter.newsletterTitle", "newsletter.newsletterText", "newsletter.newsletterSignatur", "newsletter.isSent", ]) .addSelect(["dates.calendarId", "dates.diffTitle", "dates.diffDescription"]) .addSelect(["recipients.memberId"]) .addSelect([ ...(collectIds ? ["member.id"] : []), "member.firstname", "member.lastname", "member.nameaffix", "member.birthdate", "member.internalId", ]) .addSelect([...(collectIds ? ["query.id"] : []), "recipientsByQuery.title", "recipientsByQuery.query"]) .getMany() .then((res: any) => res.map((n: any) => ({ ...n, recipients: n.recipients.map((r: any) => ({ ...r, ...(false ? {} : { memberId: undefined }) })), })) ); } private static async getNewsletterConfig(): Promise> { return await dataSource .getRepository("newsletter_config") .createQueryBuilder("newsletter_config") .leftJoin("newsletter_config.comType", "comType") .select(["newsletter_config.config"]) .addSelect(["comType.type", "comType.useColumns"]) .getMany(); } private static async getCalendar(): Promise<{ [key: string]: Array }> { return { calendar: await dataSource .getRepository("calendar") .createQueryBuilder("calendar") .leftJoin("calendar.type", "type") .select([ "calendar.id", "calendar.starttime", "calendar.endtime", "calendar.title", "calendar.content", "calendar.location", "calendar.allDay", "calendar.sequence", "calendar.createdAt", "calendar.updatedAt", ]) .addSelect(["type.type", "type.nscdr", "type.color", "type.passphrase"]) .getMany(), calendar_type: await dataSource .getRepository("calendar_type") .createQueryBuilder("calendar_type") .select(["calendar_type.type", "calendar_type.nscdr", "calendar_type.color", "calendar_type.passphrase"]) .getMany(), }; } private static async getQueryStore(collectIds: boolean): Promise> { return await dataSource.getRepository("query").find({ select: { id: collectIds, title: true, query: true } }); } private static async getTemplate(): Promise<{ [key: string]: Array }> { return { template: await dataSource .getRepository("template") .find({ select: { template: true, description: true, design: true, html: true } }), template_usage: await dataSource .getRepository("template_usage") .createQueryBuilder("template_usage") .leftJoin("template_usage.header", "header") .leftJoin("template_usage.body", "body") .leftJoin("template_usage.footer", "footer") .select(["template_usage.scope", "template_usage.headerHeight", "template_usage.footerHeight"]) .addSelect(["header.template", "header.description", "header.design", "header.html"]) .addSelect(["body.template", "body.description", "body.design", "body.html"]) .addSelect(["footer.template", "footer.description", "footer.design", "footer.html"]) .getMany(), }; } private static async getUser(collectIds: boolean): Promise<{ [key: string]: Array }> { return { user: await dataSource .getRepository("user") .createQueryBuilder("user") .leftJoin("user.roles", "roles") .leftJoin("roles.permissions", "role_permissions") .leftJoin("user.permissions", "permissions") .select([ ...(collectIds ? ["user.id"] : []), "user.mail", "user.username", "user.firstname", "user.lastname", "user.secret", "user.isOwner", ]) .addSelect(["permissions.permission"]) .addSelect(["roles.role"]) .addSelect(["role_permissions.permission"]) .getMany(), role: await dataSource .getRepository("role") .createQueryBuilder("role") .leftJoin("role.permissions", "permissions") .addSelect(["role.role"]) .addSelect(["permissions.permission"]) .getMany(), invite: await dataSource.getRepository("invite").find(), }; } private static async getWebapi(): Promise> { return await dataSource .getRepository("webapi") .createQueryBuilder("webapi") .leftJoin("webapi.permissions", "permissions") .select(["webapi.token", "webapi.title", "webapi.createdAt", "webapi.lastUsage", "webapi.expiry"]) .addSelect(["permissions.permission"]) .getMany(); } private static async setSectionData( section: BackupSection, data: BackupFileContentSection, collectedIds: boolean ): Promise { if (section == "member" && Array.isArray(data)) await this.setMemberData(data); if (section == "memberBase" && !Array.isArray(data)) await this.setMemberBase(data); if (section == "protocol" && Array.isArray(data)) await this.setProtocol(data, collectedIds); if (section == "newsletter" && Array.isArray(data)) await this.setNewsletter(data, collectedIds); if (section == "newsletter_config" && Array.isArray(data)) await this.setNewsletterConfig(data); if (section == "calendar" && !Array.isArray(data)) await this.setCalendar(data); if (section == "query" && Array.isArray(data)) await this.setQueryStore(data); if (section == "template" && !Array.isArray(data)) await this.setTemplate(data); if (section == "user" && !Array.isArray(data)) await this.setUser(data); if (section == "webapi" && Array.isArray(data)) await this.setWebapi(data); } private static async setMemberData(data: Array): Promise { await this.setMemberBase({ award: uniqBy( data .map((d) => d.awards.map((c: any) => c.award)) .flat() .map((d) => ({ ...d, id: undefined })), "award" ), communication_type: uniqBy( data .map((d) => d.communications.map((c: any) => c.type)) .flat() .map((d) => ({ ...d, id: undefined })), "type" ), executive_position: uniqBy( data .map((d) => d.positions.map((c: any) => c.executivePosition)) .flat() .map((d) => ({ ...d, id: undefined })), "position" ), membership_status: uniqBy( data .map((d) => d.memberships.map((c: any) => c.status)) .flat() .map((d) => ({ ...d, id: undefined })), "status" ), salutation: uniqBy( data.map((d) => d.salutation).map((d) => ({ ...d, id: undefined })), "salutation" ), qualification: uniqBy( data .map((d) => d.qualifications.map((c: any) => c.qualification)) .flat() .map((d) => ({ ...d, id: undefined })), "qualification" ), }); let salutation = await this.transactionManager.getRepository("salutation").find(); let communication = await this.transactionManager.getRepository("communication_type").find(); let membership = await this.transactionManager.getRepository("membership_status").find(); let award = await this.transactionManager.getRepository("award").find(); let qualification = await this.transactionManager.getRepository("qualification").find(); let position = await this.transactionManager.getRepository("executive_position").find(); let dataWithMappedIds = data.map((d) => ({ ...d, salutation: { ...d.salutation, id: salutation.find((s) => s.salutation == d.salutation.salutation)?.id ?? undefined, }, communications: d.communications.map((c: any) => ({ ...c, type: { ...c.type, id: communication.find((s) => s.type == c.type.type)?.id ?? undefined, }, })), memberships: d.memberships.map((m: any) => ({ ...m, status: { ...m.status, id: membership.find((ms) => ms.status == m.status.status)?.id ?? undefined, }, })), awards: d.awards.map((a: any) => ({ ...a, award: { ...a.award, id: award.find((ia) => ia.award == a.award.award)?.id ?? undefined, }, })), positions: d.positions.map((p: any) => ({ ...p, executivePosition: { ...p.executivePosition, id: position.find((ip) => ip.position == p.executivePosition.position)?.id ?? undefined, }, })), qualifications: d.qualifications.map((q: any) => ({ ...q, qualification: { ...q.qualification, id: qualification.find((iq) => iq.qualification == q.qualification.qualification)?.id ?? undefined, }, })), })); await this.transactionManager.getRepository("member").save(dataWithMappedIds); } private static async setMemberBase(data: { [key: string]: Array }): Promise { let salutation = await this.transactionManager.getRepository("salutation").find(); let communication = await this.transactionManager.getRepository("communication_type").find(); let membership = await this.transactionManager.getRepository("membership_status").find(); let award = await this.transactionManager.getRepository("award").find(); let qualification = await this.transactionManager.getRepository("qualification").find(); let position = await this.transactionManager.getRepository("executive_position").find(); await this.transactionManager .createQueryBuilder() .insert() .into("award") .values((data?.["award"] ?? []).filter((d) => !award.map((a) => a.award).includes(d.award))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("communication_type") .values((data?.["communication_type"] ?? []).filter((d) => !communication.map((c) => c.type).includes(d.type))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("executive_position") .values((data?.["executive_position"] ?? []).filter((d) => !position.map((p) => p.position).includes(d.position))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("membership_status") .values((data?.["membership_status"] ?? []).filter((d) => !membership.map((m) => m.status).includes(d.status))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("salutation") .values((data?.["salutation"] ?? []).filter((d) => !salutation.map((s) => s.salutation).includes(d.salutation))) .orIgnore() .execute(); await this.transactionManager .createQueryBuilder() .insert() .into("qualification") .values( (data?.["qualification"] ?? []).filter((d) => !qualification.map((q) => q.award).includes(d.qualification)) ) .orIgnore() .execute(); } private static async setProtocol(data: Array, collectedIds: boolean): Promise { let members = await this.transactionManager.getRepository("member").find(); let dataWithMappedIds = data.map((d) => ({ ...d, ...(!collectedIds ? { presences: d.presences.map((p: any) => ({ ...p, memberId: members.find( (m) => m.firstname == p.member.firstname && m.lastname == p.member.lastname && m.nameaffix == p.member.nameaffix && m.birthdate == p.member.birthdate && m.internalId == p.member.internalId )?.id ?? undefined, member: null, })), } : {}), })); await this.transactionManager.getRepository("protocol").save(dataWithMappedIds); } private static async setNewsletter(data: Array, collectedIds: boolean): Promise { await this.setQueryStore( uniqBy( data .map((d) => d.recipientsByQuery) .filter((q) => q != null) .map((d) => ({ ...d, id: undefined })), "query" ) ); let queries = await this.transactionManager.getRepository("query").find(); let members = await this.transactionManager.getRepository("member").find(); let dataWithMappedIds = data.map((d) => ({ ...d, ...(d.recipientsByQuery != null ? { recipientsByQuery: { ...d.recipientsByQuery, id: queries.find((s) => s.title == d.recipientsByQuery.title)?.id ?? undefined, }, } : {}), ...(!collectedIds ? { recipients: d.recipients.map((r: any) => ({ ...r, memberId: members.find( (m) => m.firstname == r.member.firstname && m.lastname == r.member.lastname && m.nameaffix == r.member.nameaffix && m.birthdate == r.member.birthdate && m.internalId == r.member.internalId )?.id ?? undefined, member: null, })), } : {}), })); await this.transactionManager.getRepository("newsletter").save(dataWithMappedIds); } private static async setNewsletterConfig(data: Array): Promise { await this.setMemberBase({ communication_type: uniqBy( data.map((d) => d.comType).map((d) => ({ ...d, id: undefined })), "type" ), }); let types = await this.transactionManager.getRepository("communication_type").find(); let dataWithMappedIds = data.map((d) => ({ ...d, comType: { ...d.comType, id: types.find((type) => type.type == d.comType.type)?.id ?? undefined, }, })); await this.transactionManager.getRepository("newsletter_config").save(dataWithMappedIds); } private static async setCalendar(data: { [key: string]: Array }): Promise { let usedTypes = (data?.["calendar"] ?? []).map((d) => d.type).map((d) => ({ ...d, id: undefined })); await this.transactionManager .createQueryBuilder() .insert() .into("calendar_type") .values(uniqBy([...(data?.["calendar_type"] ?? []), ...usedTypes], "type")) .orIgnore() .execute(); let types = await this.transactionManager.getRepository("calendar_type").find(); let dataWithMappedIds = (data?.["calendar"] ?? []).map((c) => ({ ...c, type: { ...c.type, id: types.find((type) => type.type == c.type.type)?.id ?? undefined, }, })); await this.transactionManager.getRepository("calendar").save(dataWithMappedIds); } private static async setQueryStore(data: Array): Promise { let query = await this.transactionManager.getRepository("query").find(); await this.transactionManager .createQueryBuilder() .insert() .into("query") .values(data.filter((d) => !query.map((s) => s.query).includes(d.query))) .orIgnore() .execute(); } private static async setTemplate(data: { [key: string]: Array }): Promise { await this.transactionManager .createQueryBuilder() .insert() .into("template") .values(data?.["template"] ?? []) .orIgnore() .execute(); let templates = await this.transactionManager.getRepository("template").find(); let dataWithMappedId = (data?.["template_usage"] ?? []) .filter((d) => availableTemplates.includes(d.scope)) .map((d) => ({ ...d, headerHeightId: templates.find((template) => template.template == d.headerHeight.template)?.id ?? null, footerHeightId: templates.find((template) => template.template == d.footerHeight.template)?.id ?? null, headerId: templates.find((template) => template.template == d.header.template)?.id ?? null, bodyId: templates.find((template) => template.template == d.body.template)?.id ?? null, footerId: templates.find((template) => template.template == d.footer.template)?.id ?? null, })); availableTemplates.forEach((at) => { if (!dataWithMappedId.some((d) => d.scope == at)) { dataWithMappedId.push({ scope: at, }); } }); await this.transactionManager .createQueryBuilder() .insert() .into("template_usage") .values(dataWithMappedId) .orIgnore() .execute(); } private static async setUser(data: { [key: string]: Array }): Promise { let usedRoles = (data?.["user"] ?? []) .map((d) => d.roles) .flat() .map((d) => ({ ...d, id: undefined })); await this.transactionManager .createQueryBuilder() .insert() .into("role") .values(uniqBy([...(data?.["role"] ?? []), ...usedRoles], "role")) .orIgnore() .execute(); let roles = await this.transactionManager.getRepository("role").find(); let dataWithMappedIds = (data?.["user"] ?? []).map((u) => ({ ...u, roles: u.roles.map((r: any) => ({ ...r, id: roles.find((role) => role.role == r.role)?.id ?? undefined, })), })); await this.transactionManager.getRepository("user").save(dataWithMappedIds); await this.transactionManager .createQueryBuilder() .insert() .into("invite") .values(data["invite"]) .orIgnore() .execute(); } private static async setWebapi(data: Array): Promise { await this.transactionManager.getRepository("webapi").save(data); } }