+ A lot of functionality added
This commit is contained in:
parent
0513652e0a
commit
8612b80e9f
|
|
@ -122,5 +122,6 @@ dist/*
|
|||
!dist/.keep
|
||||
release/*
|
||||
!release/.keep
|
||||
exports
|
||||
package-lock.json
|
||||
config.json
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ This supports mailing for logging what is happening to the databases, future pla
|
|||
|
||||
### Features
|
||||
|
||||
- [ ] Database Exporting
|
||||
- [x] Database Exporting
|
||||
- [ ] Log all errors to a final log
|
||||
- [ ] Email the log through email
|
||||
- [x] Email the log through email
|
||||
- [ ] Discord WebHook
|
||||
- [ ] Telegram Hook
|
||||
- [ ] Docker Support
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
"parallel_nodes": 2,
|
||||
"parallel_node_tasks": 5,
|
||||
"separate_into_tables": false,
|
||||
"mysql_dump_settings": "--no-comments"
|
||||
"mysql_dump_settings": "--no-comments",
|
||||
"database_export_path": "./exports",
|
||||
"logs_path": "./logs"
|
||||
},
|
||||
|
||||
"smtp": {
|
||||
|
|
@ -12,7 +14,8 @@
|
|||
"ssl": true,
|
||||
"username": "",
|
||||
"password": "",
|
||||
"email_from": ""
|
||||
"email_from": "",
|
||||
"to": ""
|
||||
},
|
||||
|
||||
"nodes": [
|
||||
|
|
|
|||
51
src/app.ts
51
src/app.ts
|
|
@ -4,6 +4,11 @@ import { getConfig, NodeConnection } from "./helper/config";
|
|||
import { SystemControl } from "./models/SystemController";
|
||||
// Import the worker processes
|
||||
import { DataWorker } from "./models/WorkerProcess";
|
||||
// Import the mail model
|
||||
import { sendMail } from "./models/MailModel";
|
||||
// Import the file system library
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
// Global system control singleton
|
||||
((global as any).logger as SystemControl) = new SystemControl();
|
||||
|
|
@ -21,7 +26,7 @@ async function main() {
|
|||
// Spawn in the DataWorkers
|
||||
for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
|
||||
// Create a new data worker
|
||||
workerNodes.push(new DataWorker(config.options.parallel_node_tasks));
|
||||
workerNodes.push(new DataWorker(config.options.parallel_node_tasks, config.options.database_export_path));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -30,7 +35,6 @@ async function main() {
|
|||
let workerIndex = 0;
|
||||
// Start assigning nodes to the DataWorkers
|
||||
for ( let node of config.nodes ) {
|
||||
console.log(`Adding workerIndex[${workerIndex}]`);
|
||||
// Add the task to the node
|
||||
workerNodes[workerIndex].addNodeTask(node);
|
||||
// Increment the worker index
|
||||
|
|
@ -38,18 +42,49 @@ async function main() {
|
|||
if ( workerIndex >= workerNodes.length ) workerIndex = 0;
|
||||
}
|
||||
|
||||
console.log(`Succesfully added ${config.nodes.length} Tasks to ${workerNodes.length} Workers`);
|
||||
const responseLogs: Map<string, string[]> = new Map<string, string[]>();
|
||||
|
||||
for( let w of workerNodes ) {
|
||||
let resolves: PromiseSettledResult<any>[] = await w.startProcessing();
|
||||
// @TODO: Actually make these run in parallel
|
||||
for ( let w of workerNodes ) {
|
||||
|
||||
for( let r of resolves ) {
|
||||
const reason = (r as any).reason;
|
||||
const workerStartTime = new Date().getTime();
|
||||
let resolves: PromiseSettledResult<any>[] | string = await w.startProcessing();
|
||||
const workerEndTime: string = Number((workerStartTime - new Date().getTime()) / 1000).toFixed(2) + "s"
|
||||
|
||||
let myLog: string[] = [
|
||||
`runTime: ${workerEndTime}`,
|
||||
];
|
||||
|
||||
console.log(w.getID(), r);
|
||||
for ( let r of resolves ) {
|
||||
myLog.push(JSON.stringify(r, null, 2));
|
||||
}
|
||||
|
||||
responseLogs.set(w.getID(), myLog);
|
||||
}
|
||||
|
||||
// Log to file
|
||||
if ( !!config.options.logs_path ) {
|
||||
// Log to a JSON file path
|
||||
const logFilePath: string = path.join(config.options.logs_path, new Date().getTime() + ".json");
|
||||
const dirPath = path.dirname(logFilePath);
|
||||
// Create the directory if it don't exist.
|
||||
if ( !fs.existsSync(dirPath) ) { fs.mkdirSync(dirPath); }
|
||||
|
||||
// Write to file
|
||||
fs.writeFileSync(logFilePath, JSON.stringify(responseLogs as Object));
|
||||
}
|
||||
|
||||
// Send the log via email
|
||||
if ( !!config.smtp ) {
|
||||
let htmlContent = `<h3>Database Log Report</h3><br>`;
|
||||
|
||||
responseLogs.forEach( (value, key) => {
|
||||
htmlContent += `<h6>${value}<h6>`
|
||||
htmlContent += `<samp>${JSON.stringify(key, null, 4)}</samp>`
|
||||
htmlContent += "<hr>";
|
||||
})
|
||||
|
||||
sendMail(htmlContent, config.smtp.to);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,16 @@ export interface ConfigOptions {
|
|||
* A string of CLI options for the mysql dump command
|
||||
*/
|
||||
mysql_dump_settings: string,
|
||||
|
||||
/**
|
||||
* Path to where to export the database to
|
||||
*/
|
||||
database_export_path: string,
|
||||
|
||||
/**
|
||||
* Path to the logs file
|
||||
*/
|
||||
logs_path?: string
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -58,6 +68,11 @@ export interface SMTPConfig {
|
|||
* Secure Connection
|
||||
*/
|
||||
ssl : boolean,
|
||||
|
||||
/**
|
||||
* Main target email address
|
||||
*/
|
||||
to : string,
|
||||
}
|
||||
|
||||
export interface NodeConnectionMail {
|
||||
|
|
@ -139,7 +154,7 @@ export interface ConfigObject {
|
|||
* SMTP details for sending out email notifications
|
||||
* for real-time updates on erros, health, and backups.
|
||||
*/
|
||||
smtp: SMTPConfig,
|
||||
smtp?: SMTPConfig,
|
||||
|
||||
/**
|
||||
* An array of Node connections that we should backup
|
||||
|
|
|
|||
|
|
@ -11,12 +11,12 @@ function getTransporter() : nodemailer.Transporter {
|
|||
|
||||
// Generate the transporter
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: config?.smtp.host,
|
||||
port: config?.smtp.port,
|
||||
secure: config?.smtp.ssl,
|
||||
host: config?.smtp?.host,
|
||||
port: config?.smtp?.port,
|
||||
secure: config?.smtp?.ssl,
|
||||
auth: {
|
||||
user: config?.smtp.username,
|
||||
pass: config?.smtp.password,
|
||||
user: config?.smtp?.username,
|
||||
pass: config?.smtp?.password,
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -27,21 +27,27 @@ function getTransporter() : nodemailer.Transporter {
|
|||
/**
|
||||
* Send out an email with the config details.
|
||||
*/
|
||||
export async function sendMail(mailContent: string, toAddress: string, nodeConfig: NodeConnectionMail, ccAddresses?: Array<string>) {
|
||||
export async function sendMail(mailContent: string, toAddress: string, ccAddresses: Array<string> = []) {
|
||||
|
||||
// Get the transporter
|
||||
const transporter = getTransporter();
|
||||
|
||||
// Generate the mail options
|
||||
const mailOptions : nodemailer.SendMailOptions = {
|
||||
from: config?.smtp.email_from,
|
||||
to: nodeConfig.email_to,
|
||||
cc: nodeConfig.email_cc,
|
||||
subject: "Database Backup Update",
|
||||
from: config?.smtp?.email_from,
|
||||
to: toAddress,
|
||||
cc: ccAddresses,
|
||||
html: mailContent,
|
||||
}
|
||||
|
||||
await new Promise((_r, _e) => {
|
||||
transporter.sendMail(mailOptions, (err, info) => {
|
||||
if ( err ) {
|
||||
console.error(err);
|
||||
}
|
||||
|
||||
console.log(info);
|
||||
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,7 +1,20 @@
|
|||
import * as mysql from "mysql";
|
||||
import { NodeConnection } from "../helper/config";
|
||||
import { uuid } from "../helper/mainHelper";
|
||||
import * as child from 'child_process';
|
||||
import * as path from "path";
|
||||
import * as fs from "fs";
|
||||
|
||||
export enum databaseExportLogType {
|
||||
ERROR = "Error",
|
||||
SUCCESS = "Success",
|
||||
}
|
||||
|
||||
export interface databaseExportLog {
|
||||
data: any,
|
||||
type: databaseExportLogType,
|
||||
error?: Error,
|
||||
}
|
||||
|
||||
export class DataWorker {
|
||||
|
||||
|
|
@ -15,15 +28,20 @@ export class DataWorker {
|
|||
*/
|
||||
private parallelTasksCount: number;
|
||||
|
||||
private exportPath: string;
|
||||
|
||||
private id: string;
|
||||
|
||||
/**
|
||||
* @param parallelTasks The amount of parallel tasks that this Worker can run at once
|
||||
*/
|
||||
constructor(parallelTasks: number) {
|
||||
constructor(parallelTasks: number, exportPath: string) {
|
||||
// Initialize the amount of paralel tasks to run at the same time
|
||||
this.parallelTasksCount = parallelTasks;
|
||||
|
||||
// Assign the export path
|
||||
this.exportPath = exportPath;
|
||||
|
||||
// Generate an ID for the worker
|
||||
this.id = uuid();
|
||||
}
|
||||
|
|
@ -41,9 +59,14 @@ export class DataWorker {
|
|||
/**
|
||||
* Start processing the node tasks.
|
||||
*/
|
||||
public startProcessing() : Promise<PromiseSettledResult<any>[]> {
|
||||
public startProcessing() : Promise<PromiseSettledResult<any>[] | string> {
|
||||
return new Promise( async (_r, _e) => {
|
||||
try {
|
||||
|
||||
if ( this.nodeTasks.length <= 0 ) {
|
||||
return _r("No tasks to run, skipping");
|
||||
}
|
||||
|
||||
// Go through all of the tasks to run
|
||||
for ( let i = 0; i < this.nodeTasks.length; i++ ) {
|
||||
// Spawn in the task queue
|
||||
|
|
@ -77,7 +100,7 @@ export class DataWorker {
|
|||
* @param task The task to process
|
||||
*/
|
||||
private processTask(task: NodeConnection) {
|
||||
return new Promise<string>( async (_r, _e) => {
|
||||
return new Promise<databaseExportLog[]>( async (_r, _e) => {
|
||||
// Connect to the mysql database to test out the connection
|
||||
const conn : mysql.Connection = mysql.createConnection({
|
||||
host: task.hostname,
|
||||
|
|
@ -102,9 +125,11 @@ export class DataWorker {
|
|||
return _e(error);
|
||||
}
|
||||
|
||||
let results: databaseExportLog[] = [];
|
||||
|
||||
try {
|
||||
// Dump the database
|
||||
await this.dumpDatabase(conn, task);
|
||||
results = await this.dumpDatabase(conn, task);
|
||||
} catch ( err ) {
|
||||
conn.destroy();
|
||||
return _e(err);
|
||||
|
|
@ -120,18 +145,120 @@ export class DataWorker {
|
|||
} catch ( err ) {};
|
||||
|
||||
// Stop the code and mark as success
|
||||
_r(`Succesfully ran task ${task.name}`);
|
||||
_r(results);
|
||||
})
|
||||
}
|
||||
|
||||
private dumpDatabase(conn : mysql.Connection, task: NodeConnection) {
|
||||
return new Promise<void>((_r, _e) => {
|
||||
return new Promise<databaseExportLog[]>( async (_r, _e) => {
|
||||
|
||||
// Get all of the databases from the connection
|
||||
let databaseList: string[] = [];
|
||||
const databaseList: string[] = [];
|
||||
let rawDatabaseList: string[] = [];
|
||||
|
||||
try {
|
||||
rawDatabaseList = await new Promise<string[]>((res, er) => {
|
||||
conn.query("SHOW DATABASES", (err, rez, fields) => {
|
||||
if ( err ) {
|
||||
// Could not list databases, there's definitely something wrong going to happen, do a forceful stop.
|
||||
return _e(err);
|
||||
}
|
||||
|
||||
let databaseList: string[] = [];
|
||||
|
||||
for ( let db of rez ) {
|
||||
databaseList.push(db['Database']);
|
||||
}
|
||||
|
||||
res(databaseList);
|
||||
})
|
||||
|
||||
});
|
||||
} catch ( ex ) {
|
||||
return _e(ex);
|
||||
}
|
||||
|
||||
// Filter the database list based on the blacklist/whitelist
|
||||
for ( let db of rawDatabaseList ) {
|
||||
if ( !!task.databases ) {
|
||||
|
||||
// Check the whitelist
|
||||
if ( !!task.databases.whitelist ) {
|
||||
if ( !task.databases.whitelist?.includes(db) ) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Check the blacklist
|
||||
if ( !!task.databases.blacklist ) {
|
||||
if ( task.databases.blacklist?.includes(db) ) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The database passed our filters, add it to the final list!
|
||||
databaseList.push(db);
|
||||
}
|
||||
|
||||
// List of logs for the different database dumps
|
||||
const resultLogs: databaseExportLog[] = [];
|
||||
|
||||
// Run the mysqldump for every database
|
||||
for ( let db of databaseList ) {
|
||||
let startTime = new Date().getTime();
|
||||
try {
|
||||
await new Promise<void>((res, err) => {
|
||||
// Get the path to export to
|
||||
const exportPath: string = path.join( this.exportPath , `${db}.sql`);
|
||||
// Make sure the folder exists
|
||||
if ( !fs.existsSync(path.dirname(exportPath)) ) {
|
||||
fs.mkdirSync(path.dirname(exportPath));
|
||||
}
|
||||
|
||||
// Generate the export command
|
||||
const exportCommand: string = `mysqldump -u${task.username} -p${task.password} -h${task.hostname} -P${task.port} ${db} > ${exportPath}`;
|
||||
// Execute the export process
|
||||
child.exec(exportCommand, (execErr, stdout, stderr) => {
|
||||
if ( execErr ) {
|
||||
|
||||
resultLogs.push({
|
||||
type: databaseExportLogType.ERROR,
|
||||
data: {
|
||||
runTime: (new Date().getTime() - startTime) + "ms",
|
||||
stderr: stderr
|
||||
},
|
||||
error: execErr,
|
||||
});
|
||||
|
||||
return res();
|
||||
}
|
||||
|
||||
resultLogs.push({
|
||||
type: databaseExportLogType.SUCCESS,
|
||||
data: {
|
||||
runTime: (new Date().getTime() - startTime) + "ms",
|
||||
stdout: stdout,
|
||||
},
|
||||
});
|
||||
|
||||
res();
|
||||
});
|
||||
|
||||
});
|
||||
} catch ( error ) {
|
||||
resultLogs.push({
|
||||
type: databaseExportLogType.ERROR,
|
||||
data: {
|
||||
runTime: (new Date().getTime() - startTime) + "ms",
|
||||
},
|
||||
error: error
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the promise
|
||||
_r(resultLogs);
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
Reference in New Issue