+ A lot of functionality added

This commit is contained in:
Daniel Legt 2021-08-21 20:46:43 +03:00
parent 0513652e0a
commit 8612b80e9f
7 changed files with 218 additions and 31 deletions

1
.gitignore vendored
View File

@ -122,5 +122,6 @@ dist/*
!dist/.keep !dist/.keep
release/* release/*
!release/.keep !release/.keep
exports
package-lock.json package-lock.json
config.json config.json

View File

@ -8,9 +8,9 @@ This supports mailing for logging what is happening to the databases, future pla
### Features ### Features
- [ ] Database Exporting - [x] Database Exporting
- [ ] Log all errors to a final log - [ ] Log all errors to a final log
- [ ] Email the log through email - [x] Email the log through email
- [ ] Discord WebHook - [ ] Discord WebHook
- [ ] Telegram Hook - [ ] Telegram Hook
- [ ] Docker Support - [ ] Docker Support

View File

@ -3,7 +3,9 @@
"parallel_nodes": 2, "parallel_nodes": 2,
"parallel_node_tasks": 5, "parallel_node_tasks": 5,
"separate_into_tables": false, "separate_into_tables": false,
"mysql_dump_settings": "--no-comments" "mysql_dump_settings": "--no-comments",
"database_export_path": "./exports",
"logs_path": "./logs"
}, },
"smtp": { "smtp": {
@ -12,7 +14,8 @@
"ssl": true, "ssl": true,
"username": "", "username": "",
"password": "", "password": "",
"email_from": "" "email_from": "",
"to": ""
}, },
"nodes": [ "nodes": [

View File

@ -4,6 +4,11 @@ import { getConfig, NodeConnection } from "./helper/config";
import { SystemControl } from "./models/SystemController"; import { SystemControl } from "./models/SystemController";
// Import the worker processes // Import the worker processes
import { DataWorker } from "./models/WorkerProcess"; import { DataWorker } from "./models/WorkerProcess";
// Import the mail model
import { sendMail } from "./models/MailModel";
// Import the file system library
import * as fs from "fs";
import * as path from "path";
// Global system control singleton // Global system control singleton
((global as any).logger as SystemControl) = new SystemControl(); ((global as any).logger as SystemControl) = new SystemControl();
@ -21,7 +26,7 @@ async function main() {
// Spawn in the DataWorkers // Spawn in the DataWorkers
for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) { for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
// Create a new data worker // Create a new data worker
workerNodes.push(new DataWorker(config.options.parallel_node_tasks)); workerNodes.push(new DataWorker(config.options.parallel_node_tasks, config.options.database_export_path));
} }
/** /**
@ -30,7 +35,6 @@ async function main() {
let workerIndex = 0; let workerIndex = 0;
// Start assigning nodes to the DataWorkers // Start assigning nodes to the DataWorkers
for ( let node of config.nodes ) { for ( let node of config.nodes ) {
console.log(`Adding workerIndex[${workerIndex}]`);
// Add the task to the node // Add the task to the node
workerNodes[workerIndex].addNodeTask(node); workerNodes[workerIndex].addNodeTask(node);
// Increment the worker index // Increment the worker index
@ -38,18 +42,49 @@ async function main() {
if ( workerIndex >= workerNodes.length ) workerIndex = 0; if ( workerIndex >= workerNodes.length ) workerIndex = 0;
} }
console.log(`Succesfully added ${config.nodes.length} Tasks to ${workerNodes.length} Workers`); const responseLogs: Map<string, string[]> = new Map<string, string[]>();
for( let w of workerNodes ) { // @TODO: Actually make these run in parallel
let resolves: PromiseSettledResult<any>[] = await w.startProcessing(); for ( let w of workerNodes ) {
for( let r of resolves ) { const workerStartTime = new Date().getTime();
const reason = (r as any).reason; let resolves: PromiseSettledResult<any>[] | string = await w.startProcessing();
const workerEndTime: string = Number((workerStartTime - new Date().getTime()) / 1000).toFixed(2) + "s"
let myLog: string[] = [
`runTime: ${workerEndTime}`,
];
console.log(w.getID(), r); for ( let r of resolves ) {
myLog.push(JSON.stringify(r, null, 2));
} }
responseLogs.set(w.getID(), myLog);
}
// Log to file
if ( !!config.options.logs_path ) {
// Log to a JSON file path
const logFilePath: string = path.join(config.options.logs_path, new Date().getTime() + ".json");
const dirPath = path.dirname(logFilePath);
// Create the directory if it don't exist.
if ( !fs.existsSync(dirPath) ) { fs.mkdirSync(dirPath); }
// Write to file
fs.writeFileSync(logFilePath, JSON.stringify(responseLogs as Object));
}
// Send the log via email
if ( !!config.smtp ) {
let htmlContent = `<h3>Database Log Report</h3><br>`;
responseLogs.forEach( (value, key) => {
htmlContent += `<h6>${value}<h6>`
htmlContent += `<samp>${JSON.stringify(key, null, 4)}</samp>`
htmlContent += "<hr>";
})
sendMail(htmlContent, config.smtp.to);
} }
} }

View File

@ -27,6 +27,16 @@ export interface ConfigOptions {
* A string of CLI options for the mysql dump command * A string of CLI options for the mysql dump command
*/ */
mysql_dump_settings: string, mysql_dump_settings: string,
/**
* Path to where to export the database to
*/
database_export_path: string,
/**
* Path to the logs file
*/
logs_path?: string
} }
/** /**
@ -58,6 +68,11 @@ export interface SMTPConfig {
* Secure Connection * Secure Connection
*/ */
ssl : boolean, ssl : boolean,
/**
* Main target email address
*/
to : string,
} }
export interface NodeConnectionMail { export interface NodeConnectionMail {
@ -139,7 +154,7 @@ export interface ConfigObject {
* SMTP details for sending out email notifications * SMTP details for sending out email notifications
* for real-time updates on erros, health, and backups. * for real-time updates on erros, health, and backups.
*/ */
smtp: SMTPConfig, smtp?: SMTPConfig,
/** /**
* An array of Node connections that we should backup * An array of Node connections that we should backup

View File

@ -11,12 +11,12 @@ function getTransporter() : nodemailer.Transporter {
// Generate the transporter // Generate the transporter
const transporter = nodemailer.createTransport({ const transporter = nodemailer.createTransport({
host: config?.smtp.host, host: config?.smtp?.host,
port: config?.smtp.port, port: config?.smtp?.port,
secure: config?.smtp.ssl, secure: config?.smtp?.ssl,
auth: { auth: {
user: config?.smtp.username, user: config?.smtp?.username,
pass: config?.smtp.password, pass: config?.smtp?.password,
} }
}); });
@ -27,22 +27,28 @@ function getTransporter() : nodemailer.Transporter {
/** /**
* Send out an email with the config details. * Send out an email with the config details.
*/ */
export async function sendMail(mailContent: string, toAddress: string, nodeConfig: NodeConnectionMail, ccAddresses?: Array<string>) { export async function sendMail(mailContent: string, toAddress: string, ccAddresses: Array<string> = []) {
// Get the transporter // Get the transporter
const transporter = getTransporter(); const transporter = getTransporter();
// Generate the mail options // Generate the mail options
const mailOptions : nodemailer.SendMailOptions = { const mailOptions : nodemailer.SendMailOptions = {
from: config?.smtp.email_from, subject: "Database Backup Update",
to: nodeConfig.email_to, from: config?.smtp?.email_from,
cc: nodeConfig.email_cc, to: toAddress,
cc: ccAddresses,
html: mailContent, html: mailContent,
} }
await new Promise((_r, _e) => { await new Promise((_r, _e) => {
transporter.sendMail(mailOptions, (err, info) => { transporter.sendMail(mailOptions, (err, info) => {
if ( err ) {
console.error(err);
}
console.log(info);
}) })
}) })

View File

@ -1,7 +1,20 @@
import * as mysql from "mysql"; import * as mysql from "mysql";
import { NodeConnection } from "../helper/config"; import { NodeConnection } from "../helper/config";
import { uuid } from "../helper/mainHelper"; import { uuid } from "../helper/mainHelper";
import * as child from 'child_process';
import * as path from "path";
import * as fs from "fs";
export enum databaseExportLogType {
ERROR = "Error",
SUCCESS = "Success",
}
export interface databaseExportLog {
data: any,
type: databaseExportLogType,
error?: Error,
}
export class DataWorker { export class DataWorker {
@ -15,15 +28,20 @@ export class DataWorker {
*/ */
private parallelTasksCount: number; private parallelTasksCount: number;
private exportPath: string;
private id: string; private id: string;
/** /**
* @param parallelTasks The amount of parallel tasks that this Worker can run at once * @param parallelTasks The amount of parallel tasks that this Worker can run at once
*/ */
constructor(parallelTasks: number) { constructor(parallelTasks: number, exportPath: string) {
// Initialize the amount of paralel tasks to run at the same time // Initialize the amount of paralel tasks to run at the same time
this.parallelTasksCount = parallelTasks; this.parallelTasksCount = parallelTasks;
// Assign the export path
this.exportPath = exportPath;
// Generate an ID for the worker // Generate an ID for the worker
this.id = uuid(); this.id = uuid();
} }
@ -41,9 +59,14 @@ export class DataWorker {
/** /**
* Start processing the node tasks. * Start processing the node tasks.
*/ */
public startProcessing() : Promise<PromiseSettledResult<any>[]> { public startProcessing() : Promise<PromiseSettledResult<any>[] | string> {
return new Promise( async (_r, _e) => { return new Promise( async (_r, _e) => {
try { try {
if ( this.nodeTasks.length <= 0 ) {
return _r("No tasks to run, skipping");
}
// Go through all of the tasks to run // Go through all of the tasks to run
for ( let i = 0; i < this.nodeTasks.length; i++ ) { for ( let i = 0; i < this.nodeTasks.length; i++ ) {
// Spawn in the task queue // Spawn in the task queue
@ -77,7 +100,7 @@ export class DataWorker {
* @param task The task to process * @param task The task to process
*/ */
private processTask(task: NodeConnection) { private processTask(task: NodeConnection) {
return new Promise<string>( async (_r, _e) => { return new Promise<databaseExportLog[]>( async (_r, _e) => {
// Connect to the mysql database to test out the connection // Connect to the mysql database to test out the connection
const conn : mysql.Connection = mysql.createConnection({ const conn : mysql.Connection = mysql.createConnection({
host: task.hostname, host: task.hostname,
@ -102,9 +125,11 @@ export class DataWorker {
return _e(error); return _e(error);
} }
let results: databaseExportLog[] = [];
try { try {
// Dump the database // Dump the database
await this.dumpDatabase(conn, task); results = await this.dumpDatabase(conn, task);
} catch ( err ) { } catch ( err ) {
conn.destroy(); conn.destroy();
return _e(err); return _e(err);
@ -120,18 +145,120 @@ export class DataWorker {
} catch ( err ) {}; } catch ( err ) {};
// Stop the code and mark as success // Stop the code and mark as success
_r(`Succesfully ran task ${task.name}`); _r(results);
}) })
} }
private dumpDatabase(conn : mysql.Connection, task: NodeConnection) { private dumpDatabase(conn : mysql.Connection, task: NodeConnection) {
return new Promise<void>((_r, _e) => { return new Promise<databaseExportLog[]>( async (_r, _e) => {
// Get all of the databases from the connection // Get all of the databases from the connection
let databaseList: string[] = []; const databaseList: string[] = [];
let rawDatabaseList: string[] = [];
try {
rawDatabaseList = await new Promise<string[]>((res, er) => {
conn.query("SHOW DATABASES", (err, rez, fields) => {
if ( err ) {
// Could not list databases, there's definitely something wrong going to happen, do a forceful stop.
return _e(err);
}
let databaseList: string[] = [];
for ( let db of rez ) {
databaseList.push(db['Database']);
}
res(databaseList);
})
});
} catch ( ex ) {
return _e(ex);
}
// Filter the database list based on the blacklist/whitelist
for ( let db of rawDatabaseList ) {
if ( !!task.databases ) {
// Check the whitelist
if ( !!task.databases.whitelist ) {
if ( !task.databases.whitelist?.includes(db) ) {
continue;
}
}
// Check the blacklist
if ( !!task.databases.blacklist ) {
if ( task.databases.blacklist?.includes(db) ) {
continue;
}
}
}
// The database passed our filters, add it to the final list!
databaseList.push(db);
}
// List of logs for the different database dumps
const resultLogs: databaseExportLog[] = [];
// Run the mysqldump for every database
for ( let db of databaseList ) {
let startTime = new Date().getTime();
try {
await new Promise<void>((res, err) => {
// Get the path to export to
const exportPath: string = path.join( this.exportPath , `${db}.sql`);
// Make sure the folder exists
if ( !fs.existsSync(path.dirname(exportPath)) ) {
fs.mkdirSync(path.dirname(exportPath));
}
// Generate the export command
const exportCommand: string = `mysqldump -u${task.username} -p${task.password} -h${task.hostname} -P${task.port} ${db} > ${exportPath}`;
// Execute the export process
child.exec(exportCommand, (execErr, stdout, stderr) => {
if ( execErr ) {
resultLogs.push({
type: databaseExportLogType.ERROR,
data: {
runTime: (new Date().getTime() - startTime) + "ms",
stderr: stderr
},
error: execErr,
});
return res();
}
resultLogs.push({
type: databaseExportLogType.SUCCESS,
data: {
runTime: (new Date().getTime() - startTime) + "ms",
stdout: stdout,
},
});
res();
});
});
} catch ( error ) {
resultLogs.push({
type: databaseExportLogType.ERROR,
data: {
runTime: (new Date().getTime() - startTime) + "ms",
},
error: error
});
}
}
// Resolve the promise
_r(resultLogs);
}) })
} }