split uploaded and generated backups

This commit is contained in:
Julian Krauser 2025-02-03 11:03:31 +01:00
parent 542a77fbef
commit 0d6103170a
6 changed files with 122 additions and 15 deletions

View file

@ -12,7 +12,9 @@ import InternalException from "../../../exceptions/internalException";
export async function getGeneratedBackups(req: Request, res: Response): Promise<any> { export async function getGeneratedBackups(req: Request, res: Response): Promise<any> {
let filesInFolder = FileSystemHelper.getFilesInDirectory(`backup`); let filesInFolder = FileSystemHelper.getFilesInDirectory(`backup`);
res.json(filesInFolder); let sorted = filesInFolder.sort((a, b) => new Date(b.split(".")[0]).getTime() - new Date(a.split(".")[0]).getTime());
res.json(sorted);
} }
/** /**
@ -33,6 +35,38 @@ export async function downloadBackupFile(req: Request, res: Response): Promise<a
}); });
} }
/**
* @description get uploaded backups
* @param req {Request} Express req object
* @param res {Response} Express res object
* @returns {Promise<*>}
*/
export async function getUploadedBackups(req: Request, res: Response): Promise<any> {
let filesInFolder = FileSystemHelper.getFilesInDirectory("uploaded-backup");
let sorted = filesInFolder.sort((a, b) => new Date(b.split("_")[0]).getTime() - new Date(a.split("_")[0]).getTime());
res.json(sorted);
}
/**
* @description download uploaded backup file
* @param req {Request} Express req object
* @param res {Response} Express res object
* @returns {Promise<*>}
*/
export async function downloadUploadedBackupFile(req: Request, res: Response): Promise<any> {
let filename = req.params.filename;
let filepath = FileSystemHelper.formatPath("uploaded-backup", filename);
res.sendFile(filepath, {
headers: {
"Content-Type": "application/json",
},
});
}
/** /**
* @description create backup manually * @description create backup manually
* @param req {Request} Express req object * @param req {Request} Express req object
@ -55,8 +89,25 @@ export async function restoreBackupByLocalFile(req: Request, res: Response): Pro
let filename = req.body.filename; let filename = req.body.filename;
let partial = req.body.partial; let partial = req.body.partial;
let include = req.body.include; let include = req.body.include;
let overwrite = req.body.overwrite;
await BackupHelper.loadBackup({ filename, include, partial }); await BackupHelper.loadBackup({ filename, include, partial, overwrite });
res.sendStatus(204);
}
/**
* @description restore uploaded backup by selected
* @param req {Request} Express req object
* @param res {Response} Express res object
* @returns {Promise<*>}
*/
export async function restoreBackupByUploadedFile(req: Request, res: Response): Promise<any> {
let filename = req.body.filename;
let partial = req.body.partial;
let include = req.body.include;
await BackupHelper.loadBackup({ filename, path: "uploaded-backup", include, partial });
res.sendStatus(204); res.sendStatus(204);
} }

View file

@ -40,7 +40,7 @@ export default abstract class MembershipFactory {
return { return {
durationInDays: record.durationInDays, durationInDays: record.durationInDays,
durationInYears: record.durationInYears, durationInYears: record.durationInYears,
exactDuration: record.exactDuration, exactDuration: record.exactDuration.toString(),
status: record.status, status: record.status,
statusId: record.statusId, statusId: record.statusId,
memberId: record.memberId, memberId: record.memberId,

View file

@ -4,6 +4,8 @@ import { EntityManager } from "typeorm";
import uniqBy from "lodash.uniqby"; import uniqBy from "lodash.uniqby";
import InternalException from "../exceptions/internalException"; import InternalException from "../exceptions/internalException";
import UserService from "../service/user/userService"; import UserService from "../service/user/userService";
import { BACKUP_COPIES, BACKUP_INTERVAL } from "../env.defaults";
import DatabaseActionException from "../exceptions/databaseActionException";
export type BackupSection = export type BackupSection =
| "member" | "member"
@ -97,7 +99,26 @@ export default abstract class BackupHelper {
FileSystemHelper.writeFile(path, filename + ".json", JSON.stringify(json, null, 2)); FileSystemHelper.writeFile(path, filename + ".json", JSON.stringify(json, null, 2));
// TODO: delete older backups by copies env let files = FileSystemHelper.getFilesInDirectory("backup", ".json");
let sorted = files.sort((a, b) => new Date(b.split(".")[0]).getTime() - new Date(a.split(".")[0]).getTime());
const filesToDelete = sorted.slice(BACKUP_COPIES);
for (const file of filesToDelete) {
FileSystemHelper.deleteFile("backup", file);
}
}
static async createBackupOnInterval() {
let files = FileSystemHelper.getFilesInDirectory("backup", ".json");
let newestFile = files.sort((a, b) => new Date(b.split(".")[0]).getTime() - new Date(a.split(".")[0]).getTime())[0];
let lastBackup = new Date(newestFile.split(".")[0]);
let diffInMs = new Date().getTime() - lastBackup.getTime();
let diffInDays = diffInMs / (1000 * 60 * 60 * 24);
if (diffInDays >= BACKUP_INTERVAL) {
await this.createBackup({});
}
} }
static async loadBackup({ static async loadBackup({
@ -105,11 +126,13 @@ export default abstract class BackupHelper {
path = "/backup", path = "/backup",
include = [], include = [],
partial = false, partial = false,
overwrite = false,
}: { }: {
filename: string; filename: string;
path?: string; path?: string;
partial?: boolean; partial?: boolean;
include?: Array<BackupSection>; include?: Array<BackupSection>;
overwrite?: boolean;
}): Promise<void> { }): Promise<void> {
this.transactionManager = undefined; this.transactionManager = undefined;
@ -127,10 +150,12 @@ export default abstract class BackupHelper {
const sections = this.backupSection const sections = this.backupSection
.filter((bs) => (partial ? include.includes(bs.type) : true)) .filter((bs) => (partial ? include.includes(bs.type) : true))
.sort((a, b) => a.orderOnClear - b.orderOnClear); .sort((a, b) => a.orderOnClear - b.orderOnClear);
for (const section of sections.filter((s) => Object.keys(backup).includes(s.type))) { if (!overwrite) {
let refered = this.backupSectionRefered[section.type]; for (const section of sections.filter((s) => Object.keys(backup).includes(s.type))) {
for (const ref of refered) { let refered = this.backupSectionRefered[section.type];
await this.transactionManager.getRepository(ref).delete({}); for (const ref of refered) {
await this.transactionManager.getRepository(ref).delete({});
}
} }
} }
@ -144,7 +169,7 @@ export default abstract class BackupHelper {
}) })
.catch((err) => { .catch((err) => {
this.transactionManager = undefined; this.transactionManager = undefined;
throw new InternalException("failed to restore backup - rolling back actions", err); throw new DatabaseActionException("BACKUP RESTORE", include.join(", "), err);
}); });
} }

View file

@ -11,14 +11,17 @@ export abstract class FileSystemHelper {
} }
static readFile(...filePath: string[]) { static readFile(...filePath: string[]) {
this.createFolder(...filePath);
return readFileSync(this.formatPath(...filePath), "utf8"); return readFileSync(this.formatPath(...filePath), "utf8");
} }
static readFileasBase64(...filePath: string[]) { static readFileasBase64(...filePath: string[]) {
this.createFolder(...filePath);
return readFileSync(this.formatPath(...filePath), "base64"); return readFileSync(this.formatPath(...filePath), "base64");
} }
static readTemplateFile(filePath: string) { static readTemplateFile(filePath: string) {
this.createFolder(filePath);
return readFileSync(process.cwd() + filePath, "utf8"); return readFileSync(process.cwd() + filePath, "utf8");
} }
@ -28,6 +31,13 @@ export abstract class FileSystemHelper {
writeFileSync(path, file); writeFileSync(path, file);
} }
static deleteFile(...filePath: string[]) {
const path = this.formatPath(...filePath);
if (existsSync(path)) {
unlinkSync(path);
}
}
static formatPath(...args: string[]) { static formatPath(...args: string[]) {
return join(process.cwd(), "files", ...args); return join(process.cwd(), "files", ...args);
} }

View file

@ -40,5 +40,5 @@ import RefreshCommandHandler from "./command/refreshCommandHandler";
const job = schedule.scheduleJob("0 0 * * *", async () => { const job = schedule.scheduleJob("0 0 * * *", async () => {
console.log(`${new Date().toISOString()}: running Cron`); console.log(`${new Date().toISOString()}: running Cron`);
await RefreshCommandHandler.deleteExpired(); await RefreshCommandHandler.deleteExpired();
// TODO: create backup by interval env await BackupHelper.createBackupOnInterval();
}); });

View file

@ -4,17 +4,22 @@ import multer from "multer";
import { import {
createManualBackup, createManualBackup,
downloadBackupFile, downloadBackupFile,
downloadUploadedBackupFile,
getGeneratedBackups, getGeneratedBackups,
getUploadedBackups,
restoreBackupByLocalFile, restoreBackupByLocalFile,
restoreBackupByUploadedFile,
uploadBackupFile, uploadBackupFile,
} from "../../../controller/admin/user/backupController"; } from "../../../controller/admin/user/backupController";
import { FileSystemHelper } from "../../../helpers/fileSystemHelper";
const storage = multer.diskStorage({ const storage = multer.diskStorage({
destination: (req, file, cb) => { destination: (req, file, cb) => {
cb(null, "files/backup/"); FileSystemHelper.createFolder("uploaded-backup");
cb(null, "files/uploaded-backup/");
}, },
filename: (req, file, cb) => { filename: (req, file, cb) => {
cb(null, `${new Date().toISOString().split("T")[0]}-uploaded-${file.originalname}`); cb(null, `${new Date().toISOString().split("T")[0]}_${file.originalname}`);
}, },
}); });
@ -31,14 +36,22 @@ const upload = multer({
var router = express.Router({ mergeParams: true }); var router = express.Router({ mergeParams: true });
router.get("/", async (req: Request, res: Response) => { router.get("/generated", async (req: Request, res: Response) => {
await getGeneratedBackups(req, res); await getGeneratedBackups(req, res);
}); });
router.get("/:filename", async (req: Request, res: Response) => { router.get("/generated/:filename", async (req: Request, res: Response) => {
await downloadBackupFile(req, res); await downloadBackupFile(req, res);
}); });
router.get("/uploaded", async (req: Request, res: Response) => {
await getUploadedBackups(req, res);
});
router.get("/uploaded/:filename", async (req: Request, res: Response) => {
await downloadUploadedBackupFile(req, res);
});
router.post( router.post(
"/", "/",
PermissionHelper.passCheckMiddleware("create", "user", "backup"), PermissionHelper.passCheckMiddleware("create", "user", "backup"),
@ -48,13 +61,21 @@ router.post(
); );
router.post( router.post(
"/restore", "/generated/restore",
PermissionHelper.passCheckMiddleware("admin", "user", "backup"), PermissionHelper.passCheckMiddleware("admin", "user", "backup"),
async (req: Request, res: Response) => { async (req: Request, res: Response) => {
await restoreBackupByLocalFile(req, res); await restoreBackupByLocalFile(req, res);
} }
); );
router.post(
"/uploaded/restore",
PermissionHelper.passCheckMiddleware("admin", "user", "backup"),
async (req: Request, res: Response) => {
await restoreBackupByUploadedFile(req, res);
}
);
router.post( router.post(
"/upload", "/upload",
PermissionHelper.passCheckMiddleware("create", "user", "backup"), PermissionHelper.passCheckMiddleware("create", "user", "backup"),