.gitignore

@@ -122,6 +122,5 @@ dist/*
 !dist/.keep
 release/*
 !release/.keep
-exports
 package-lock.json
 config.json

README.md

@@ -8,9 +8,9 @@ This supports mailing for logging what is happening to the databases, future pla
 
 ### Features
 
-- [x] Database Exporting
+- [ ] Database Exporting
 - [ ] Log all errors to a final log
-- [x] Email the log through email
+- [ ] Email the log through email
 - [ ] Discord WebHook
 - [ ] Telegram Hook
 - [ ] Docker Support

config.example.json

@@ -3,9 +3,7 @@
         "parallel_nodes": 2,
         "parallel_node_tasks": 5,
         "separate_into_tables": false,
-        "mysql_dump_settings": "--no-comments",
+        "mysql_dump_settings": "--no-comments"
-        "database_export_path": "./exports",
-        "logs_path": "./logs"
     },
 
     "smtp": {
@@ -14,8 +12,7 @@
         "ssl":  true,
         "username": "",
         "password": "",
-        "email_from": "",
+        "email_from": ""
-        "to": ""
     },
 
     "nodes": [

src/app.ts

@@ -4,11 +4,6 @@ import { getConfig, NodeConnection } from "./helper/config";
 import { SystemControl } from "./models/SystemController";
 // Import the worker processes
 import { DataWorker } from "./models/WorkerProcess";
-// Import the mail model
-import { sendMail } from "./models/MailModel";
-// Import the file system library
-import * as fs from "fs";
-import * as path from "path";
 
 // Global system control singleton
 ((global as any).logger as SystemControl) = new SystemControl();
@@ -26,7 +21,7 @@ async function main() {
     // Spawn in the DataWorkers
     for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
         // Create a new data worker
-        workerNodes.push(new DataWorker(config.options.parallel_node_tasks, config.options.database_export_path));
+        workerNodes.push(new DataWorker(config.options.parallel_node_tasks));
     }
 
     /**
@@ -35,6 +30,7 @@ async function main() {
     let workerIndex = 0;
     // Start assigning nodes to the DataWorkers
     for ( let node of config.nodes ) {
+        console.log(`Adding workerIndex[${workerIndex}]`);
         // Add the task to the node
         workerNodes[workerIndex].addNodeTask(node);
         // Increment the worker index
@@ -42,49 +38,18 @@ async function main() {
         if ( workerIndex >= workerNodes.length ) workerIndex = 0;
     }
 
-    const responseLogs: Map<string, string[]> = new Map<string, string[]>();
+    console.log(`Succesfully added ${config.nodes.length} Tasks to ${workerNodes.length} Workers`);
 
-    // @TODO: Actually make these run in parallel
+    for( let w of workerNodes ) {
-    for ( let w of workerNodes ) {
+        let resolves: PromiseSettledResult<any>[] = await w.startProcessing();
 
-        const workerStartTime = new Date().getTime();
+        for( let r of resolves ) {
-        let resolves: PromiseSettledResult<any>[] | string = await w.startProcessing();
+            const reason = (r as any).reason;
-        const workerEndTime: string = Number((workerStartTime - new Date().getTime()) / 1000).toFixed(2) + "s"
 
-        let myLog: string[] = [
-            `runTime: ${workerEndTime}`,
-        ];
 
-        for ( let r of resolves ) {
+            console.log(w.getID(), r);
-            myLog.push(JSON.stringify(r, null, 2));
         }
 
-        responseLogs.set(w.getID(), myLog);
-    }
-
-    // Log to file
-    if ( !!config.options.logs_path ) {
-        // Log to a JSON file path
-        const logFilePath: string = path.join(config.options.logs_path, new Date().getTime() + ".json");
-        const dirPath = path.dirname(logFilePath);
-        // Create the directory if it don't exist.
-        if ( !fs.existsSync(dirPath) ) { fs.mkdirSync(dirPath); }
-
-        // Write to file
-        fs.writeFileSync(logFilePath, JSON.stringify(responseLogs as Object));
-    }
-
-    // Send the log via email
-    if ( !!config.smtp ) {
-        let htmlContent = `<h3>Database Log Report</h3><br>`;
-
-        responseLogs.forEach( (value, key) => {
-            htmlContent += `<h6>${value}<h6>`
-            htmlContent += `<samp>${JSON.stringify(key, null, 4)}</samp>`
-            htmlContent += "<hr>";
-        })
-
-        sendMail(htmlContent, config.smtp.to);
     }
 
 }

src/helper/config.ts

@@ -27,16 +27,6 @@ export interface ConfigOptions {
      * A string of CLI options for the mysql dump command
      */
     mysql_dump_settings: string,
-
-    /**
-     * Path to where to export the database to
-     */
-    database_export_path: string,
-
-    /**
-     * Path to the logs file
-     */
-    logs_path?: string
 }
 
 /**
@@ -68,11 +58,6 @@ export interface SMTPConfig {
      * Secure Connection
      */
     ssl        : boolean,
-
-    /**
-     * Main target email address
-     */
-    to         : string,
 }
 
 export interface NodeConnectionMail {
@@ -154,7 +139,7 @@ export interface ConfigObject {
      * SMTP details for sending out email notifications
      * for real-time updates on erros, health, and backups.
      */
-    smtp?:    SMTPConfig,
+    smtp:    SMTPConfig,
 
     /**
      * An array of Node connections that we should backup

src/models/MailModel.ts

@@ -11,12 +11,12 @@ function getTransporter() : nodemailer.Transporter {
     
     // Generate the transporter
     const transporter = nodemailer.createTransport({
-        host: config?.smtp?.host,
+        host: config?.smtp.host,
-        port: config?.smtp?.port,
+        port: config?.smtp.port,
-        secure: config?.smtp?.ssl,
+        secure: config?.smtp.ssl,
         auth: {
-            user: config?.smtp?.username,
+            user: config?.smtp.username,
-            pass: config?.smtp?.password,
+            pass: config?.smtp.password,
         }
     });
 
@@ -27,27 +27,21 @@ function getTransporter() : nodemailer.Transporter {
 /**
  * Send out an email with the config details.
  */
-export async function sendMail(mailContent: string, toAddress: string, ccAddresses: Array<string> = []) {
+export async function sendMail(mailContent: string, toAddress: string, nodeConfig: NodeConnectionMail, ccAddresses?: Array<string>) {
 
     // Get the transporter
     const transporter = getTransporter();
 
     // Generate the mail options
     const mailOptions : nodemailer.SendMailOptions = {
-        subject: "Database Backup Update",
+        from: config?.smtp.email_from,
-        from: config?.smtp?.email_from,
+        to: nodeConfig.email_to,
-        to: toAddress,
+        cc: nodeConfig.email_cc,
-        cc: ccAddresses,
         html: mailContent,
     }
 
     await new Promise((_r, _e) => {
         transporter.sendMail(mailOptions, (err, info) => {
-            if ( err ) {
-                console.error(err);
-            }
-
-            console.log(info);
             
         })
     })

src/models/WorkerProcess.ts

@@ -1,20 +1,7 @@
 import * as mysql from "mysql";
 import { NodeConnection } from "../helper/config";
 import {      uuid      } from "../helper/mainHelper";
-import * as child from 'child_process';
-import * as path from "path";
-import * as fs from "fs";
 
-export enum databaseExportLogType {
-    ERROR   = "Error",
-    SUCCESS = "Success",
-}
-
-export interface databaseExportLog {
-    data:   any,
-    type:   databaseExportLogType,
-    error?: Error,
-}
 
 export class DataWorker {
 
@@ -28,20 +15,15 @@ export class DataWorker {
      */
     private parallelTasksCount: number;
 
-    private exportPath: string;
-
     private id: string;
 
     /**
      * @param parallelTasks The amount of parallel tasks that this Worker can run at once
      */
-    constructor(parallelTasks: number, exportPath: string) {
+    constructor(parallelTasks: number) {
         // Initialize the amount of paralel tasks to run at the same time
         this.parallelTasksCount = parallelTasks;
 
-        // Assign the export path
-        this.exportPath = exportPath;
-
         // Generate an ID for the worker
         this.id = uuid();
     }
@@ -59,14 +41,9 @@ export class DataWorker {
     /**
      * Start processing the node tasks.
      */
-    public startProcessing() : Promise<PromiseSettledResult<any>[] | string> {
+    public startProcessing() : Promise<PromiseSettledResult<any>[]> {
         return new Promise( async (_r, _e) => {
             try {
-
-                if ( this.nodeTasks.length <= 0 ) {
-                    return _r("No tasks to run, skipping");
-                }
-
                 // Go through all of the tasks to run
                 for ( let i = 0; i < this.nodeTasks.length; i++ ) {
                     // Spawn in the task queue
@@ -100,7 +77,7 @@ export class DataWorker {
      * @param task The task to process
      */
     private processTask(task: NodeConnection) {
-        return new Promise<databaseExportLog[]>( async (_r, _e) => {
+        return new Promise<string>( async (_r, _e) => {
             // Connect to the mysql database to test out the connection
             const conn : mysql.Connection = mysql.createConnection({
                 host:     task.hostname,
@@ -125,15 +102,8 @@ export class DataWorker {
                 return _e(error);
             }
 
-            let results: databaseExportLog[] = [];
+            // Check if we have target 
 
-            try {
-                // Dump the database
-                results = await this.dumpDatabase(conn, task);
-            } catch ( err ) {
-                conn.destroy();
-                return _e(err);
-            }
             
             try { // Close the connection
                 await new Promise<void>((re, er) => {
@@ -145,120 +115,7 @@ export class DataWorker {
             } catch ( err ) {};
 
             // Stop the code and mark as success
-            _r(results);
+            _r(`Succesfully ran task ${task.name}`);
-        })
-    }
-
-    private dumpDatabase(conn : mysql.Connection, task: NodeConnection) {
-        return new Promise<databaseExportLog[]>( async (_r, _e) => {
-
-            // Get all of the databases from the connection
-            const databaseList:    string[] = [];
-            let   rawDatabaseList: string[] = [];
-
-            try {
-                rawDatabaseList = await new Promise<string[]>((res, er) => {
-                    conn.query("SHOW DATABASES", (err, rez, fields) => {
-                        if ( err ) {
-                            // Could not list databases, there's definitely something wrong going to happen, do a forceful stop.
-                            return _e(err);
-                        }
-
-                        let databaseList: string[] = [];
-
-                        for ( let db of rez ) {
-                            databaseList.push(db['Database']);
-                        }
-
-                        res(databaseList);
-                    })
-
-                });
-            } catch ( ex ) {
-                return _e(ex);
-            }
-
-            // Filter the database list based on the blacklist/whitelist
-            for ( let db of rawDatabaseList ) {
-                if ( !!task.databases ) {
-
-                    // Check the whitelist
-                    if ( !!task.databases.whitelist ) {
-                        if ( !task.databases.whitelist?.includes(db) ) {
-                            continue;
-                        }
-                    }
-
-                    // Check the blacklist
-                    if ( !!task.databases.blacklist ) {
-                        if ( task.databases.blacklist?.includes(db) ) {
-                            continue;
-                        }
-                    }
-                }
-
-                // The database passed our filters, add it to the final list!
-                databaseList.push(db);
-            }
-
-            // List of logs for the different database dumps
-            const resultLogs: databaseExportLog[] = [];
-
-            // Run the mysqldump for every database
-            for ( let db of databaseList ) {
-                let startTime = new Date().getTime();
-                try {
-                    await new Promise<void>((res, err) => {
-                        // Get the path to export to
-                        const exportPath:    string = path.join( this.exportPath , `${db}.sql`);
-                        // Make sure the folder exists
-                        if ( !fs.existsSync(path.dirname(exportPath)) ) {
-                            fs.mkdirSync(path.dirname(exportPath));
-                        }
-
-                        // Generate the export command
-                        const exportCommand: string = `mysqldump -u${task.username} -p${task.password} -h${task.hostname} -P${task.port} ${db} > ${exportPath}`;
-                        // Execute the export process
-                        child.exec(exportCommand, (execErr, stdout, stderr) => {
-                            if ( execErr ) {
-
-                                resultLogs.push({
-                                    type: databaseExportLogType.ERROR,
-                                    data: {
-                                        runTime: (new Date().getTime() - startTime) + "ms",
-                                        stderr: stderr
-                                    },
-                                    error: execErr,
-                                });
-
-                                return res();
-                            }
-
-                            resultLogs.push({
-                                type: databaseExportLogType.SUCCESS,
-                                data: {
-                                    runTime: (new Date().getTime() - startTime) + "ms",
-                                    stdout: stdout,
-                                },
-                            });
-    
-                            res();
-                        });
-
-                    });
-                } catch ( error ) {
-                    resultLogs.push({
-                        type: databaseExportLogType.ERROR,
-                        data: {
-                            runTime: (new Date().getTime() - startTime) + "ms",
-                        },
-                        error: error
-                    });
-                }
-            }
-
-            // Resolve the promise
-            _r(resultLogs);
         })
     }