+ A lot of functionality added

+ Upped JS Version as I don't really see a reason why not.

.gitignore

@@ -120,5 +120,8 @@ out
 
 dist/*
 !dist/.keep
+release/*
+!release/.keep
+exports
 package-lock.json
 config.json

README.md

@@ -8,11 +8,12 @@ This supports mailing for logging what is happening to the databases, future pla
 
 ### Features
 
-- [ ] Database Exporting
+- [x] Database Exporting
 - [ ] Log all errors to a final log
-- [ ] Email the log through email
+- [x] Email the log through email
 - [ ] Discord WebHook
 - [ ] Telegram Hook
+- [ ] Docker Support
 
 ### How to Use
 1. Setup the configuration file which is [This file](./config.example.json) and rename it into `config.json` instead of `config.example.json`.

config.example.json

@@ -3,7 +3,9 @@
         "parallel_nodes": 2,
         "parallel_node_tasks": 5,
         "separate_into_tables": false,
-        "mysql_dump_settings": "--no-comments"
+        "mysql_dump_settings": "--no-comments",
+        "database_export_path": "./exports",
+        "logs_path": "./logs"
     },
 
     "smtp": {
@@ -12,7 +14,8 @@
         "ssl":  true,
         "username": "",
         "password": "",
-        "email_from": ""
+        "email_from": "",
+        "to": ""
     },
 
     "nodes": [

package.json

@@ -8,7 +8,9 @@
   "description": "Make sure your database backups run and keep track of performance",
   "version": "0.1.0",
   "main": "dist/app.js",
-  "devDependencies": {},
+  "devDependencies": {
+    "@types/mysql": "^2.15.19"
+  },
   "scripts": {
     "start": "tsc && node dist/app.js"
   },

src/app.ts

@@ -4,6 +4,11 @@ import { getConfig, NodeConnection } from "./helper/config";
 import { SystemControl } from "./models/SystemController";
 // Import the worker processes
 import { DataWorker } from "./models/WorkerProcess";
+// Import the mail model
+import { sendMail } from "./models/MailModel";
+// Import the file system library
+import * as fs from "fs";
+import * as path from "path";
 
 // Global system control singleton
 ((global as any).logger as SystemControl) = new SystemControl();
@@ -21,7 +26,7 @@ async function main() {
     // Spawn in the DataWorkers
     for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
         // Create a new data worker
-        workerNodes.push(new DataWorker(config.options.parallel_node_tasks));
+        workerNodes.push(new DataWorker(config.options.parallel_node_tasks, config.options.database_export_path));
     }
 
     /**
@@ -30,7 +35,6 @@ async function main() {
     let workerIndex = 0;
     // Start assigning nodes to the DataWorkers
     for ( let node of config.nodes ) {
-        console.log(`Adding workerIndex[${workerIndex}]`);
         // Add the task to the node
         workerNodes[workerIndex].addNodeTask(node);
         // Increment the worker index
@@ -38,10 +42,49 @@ async function main() {
         if ( workerIndex >= workerNodes.length ) workerIndex = 0;
     }
 
-    console.log(`Succesfully added ${config.nodes.length} Tasks to ${workerNodes.length} Workers`);
+    const responseLogs: Map<string, string[]> = new Map<string, string[]>();
 
-    for( let w of workerNodes ) {
+    // @TODO: Actually make these run in parallel
-        await w.startProcessing();
+    for ( let w of workerNodes ) {
+
+        const workerStartTime = new Date().getTime();
+        let resolves: PromiseSettledResult<any>[] | string = await w.startProcessing();
+        const workerEndTime: string = Number((workerStartTime - new Date().getTime()) / 1000).toFixed(2) + "s"
+
+        let myLog: string[] = [
+            `runTime: ${workerEndTime}`,
+        ];
+
+        for ( let r of resolves ) {
+            myLog.push(JSON.stringify(r, null, 2));
+        }
+
+        responseLogs.set(w.getID(), myLog);
+    }
+
+    // Log to file
+    if ( !!config.options.logs_path ) {
+        // Log to a JSON file path
+        const logFilePath: string = path.join(config.options.logs_path, new Date().getTime() + ".json");
+        const dirPath = path.dirname(logFilePath);
+        // Create the directory if it don't exist.
+        if ( !fs.existsSync(dirPath) ) { fs.mkdirSync(dirPath); }
+
+        // Write to file
+        fs.writeFileSync(logFilePath, JSON.stringify(responseLogs as Object));
+    }
+
+    // Send the log via email
+    if ( !!config.smtp ) {
+        let htmlContent = `<h3>Database Log Report</h3><br>`;
+
+        responseLogs.forEach( (value, key) => {
+            htmlContent += `<h6>${value}<h6>`
+            htmlContent += `<samp>${JSON.stringify(key, null, 4)}</samp>`
+            htmlContent += "<hr>";
+        })
+
+        sendMail(htmlContent, config.smtp.to);
     }
 
 }

src/helper/config.ts

@@ -27,6 +27,16 @@ export interface ConfigOptions {
      * A string of CLI options for the mysql dump command
      */
     mysql_dump_settings: string,
+
+    /**
+     * Path to where to export the database to
+     */
+    database_export_path: string,
+
+    /**
+     * Path to the logs file
+     */
+    logs_path?: string
 }
 
 /**
@@ -58,6 +68,11 @@ export interface SMTPConfig {
      * Secure Connection
      */
     ssl        : boolean,
+
+    /**
+     * Main target email address
+     */
+    to         : string,
 }
 
 export interface NodeConnectionMail {
@@ -139,7 +154,7 @@ export interface ConfigObject {
      * SMTP details for sending out email notifications
      * for real-time updates on erros, health, and backups.
      */
-    smtp:    SMTPConfig,
+    smtp?:    SMTPConfig,
 
     /**
      * An array of Node connections that we should backup

src/models/MailModel.ts

@@ -11,12 +11,12 @@ function getTransporter() : nodemailer.Transporter {
     
     // Generate the transporter
     const transporter = nodemailer.createTransport({
-        host: config?.smtp.host,
+        host: config?.smtp?.host,
-        port: config?.smtp.port,
+        port: config?.smtp?.port,
-        secure: config?.smtp.ssl,
+        secure: config?.smtp?.ssl,
         auth: {
-            user: config?.smtp.username,
+            user: config?.smtp?.username,
-            pass: config?.smtp.password,
+            pass: config?.smtp?.password,
         }
     });
 
@@ -27,21 +27,27 @@ function getTransporter() : nodemailer.Transporter {
 /**
  * Send out an email with the config details.
  */
-export async function sendMail(mailContent: string, toAddress: string, nodeConfig: NodeConnectionMail, ccAddresses?: Array<string>) {
+export async function sendMail(mailContent: string, toAddress: string, ccAddresses: Array<string> = []) {
 
     // Get the transporter
     const transporter = getTransporter();
 
     // Generate the mail options
     const mailOptions : nodemailer.SendMailOptions = {
-        from: config?.smtp.email_from,
+        subject: "Database Backup Update",
-        to: nodeConfig.email_to,
+        from: config?.smtp?.email_from,
-        cc: nodeConfig.email_cc,
+        to: toAddress,
+        cc: ccAddresses,
         html: mailContent,
     }
 
     await new Promise((_r, _e) => {
         transporter.sendMail(mailOptions, (err, info) => {
+            if ( err ) {
+                console.error(err);
+            }
+
+            console.log(info);
 
         })
     })

src/models/WorkerProcess.ts

@@ -1,6 +1,20 @@
+import * as mysql from "mysql";
 import { NodeConnection } from "../helper/config";
 import {      uuid      } from "../helper/mainHelper";
+import * as child from 'child_process';
+import * as path from "path";
+import * as fs from "fs";
 
+export enum databaseExportLogType {
+    ERROR   = "Error",
+    SUCCESS = "Success",
+}
+
+export interface databaseExportLog {
+    data:   any,
+    type:   databaseExportLogType,
+    error?: Error,
+}
 
 export class DataWorker {
 
@@ -14,19 +28,29 @@ export class DataWorker {
      */
     private parallelTasksCount: number;
 
+    private exportPath: string;
+
     private id: string;
 
     /**
      * @param parallelTasks The amount of parallel tasks that this Worker can run at once
      */
-    constructor(parallelTasks: number) {
+    constructor(parallelTasks: number, exportPath: string) {
         // Initialize the amount of paralel tasks to run at the same time
         this.parallelTasksCount = parallelTasks;
 
+        // Assign the export path
+        this.exportPath = exportPath;
+
         // Generate an ID for the worker
         this.id = uuid();
     }
 
+    public getID() : string {
+        return this.id;
+    }
+    
+
     public addNodeTask(node: NodeConnection) {
         // Add the task to the list
         this.nodeTasks.push(node);
@@ -35,31 +59,39 @@ export class DataWorker {
     /**
      * Start processing the node tasks.
      */
-    public async startProcessing() : Promise<void> {
+    public startProcessing() : Promise<PromiseSettledResult<any>[] | string> {
+        return new Promise( async (_r, _e) => {
+            try {
 
-        // Go through all of the tasks to run
+                if ( this.nodeTasks.length <= 0 ) {
-        for ( let i = 0; i < this.nodeTasks.length; i++ ) {
+                    return _r("No tasks to run, skipping");
-            // Spawn in the task queue
+                }
-            let taskQueue : Array<Promise<any>> = [];
 
-            // Start running the parallel tasks
+                // Go through all of the tasks to run
-            for ( let x = 0; x < this.parallelTasksCount; x++ ) {
+                for ( let i = 0; i < this.nodeTasks.length; i++ ) {
-                // Check if we have stept out of bounds
+                    // Spawn in the task queue
-                if ( i >= this.nodeTasks.length ) // Ignore out-of-bounds
+                    let taskQueue : Array<Promise<any>> = [];
-                    break;
         
-                // Add the process to the queue
+                    // Start running the parallel tasks
-                taskQueue.push(this.processTask(this.nodeTasks[i]))
+                    for ( let x = 0; x < this.parallelTasksCount; x++ ) {
-                // Add to the index
+                        // Check if we have stept out of bounds
-                i += x;
+                        if ( i >= this.nodeTasks.length ) // Ignore out-of-bounds
+                            break;
+        
+                        // Add the process to the queue
+                        taskQueue.push(this.processTask(this.nodeTasks[i]))
+                        // Add to the index
+                        i++;
+                    }
+
+                    // Return the resulting resolves
+                    _r(await Promise.allSettled(taskQueue));
+                }
+            } catch ( ex ) {
+                _e(ex);
             }
+        })
 
-            // Get the results of the running
-            const runResults = await Promise.all(taskQueue);
-
-            console.log(runResults);
-
-        }
 
     }
 
@@ -67,8 +99,167 @@ export class DataWorker {
      * Process a task
      * @param task The task to process
      */
-    private async processTask(task: NodeConnection) {
+    private processTask(task: NodeConnection) {
-        return `[${this.id}]: Succesfully ran task ${task.name}`;
+        return new Promise<databaseExportLog[]>( async (_r, _e) => {
+            // Connect to the mysql database to test out the connection
+            const conn : mysql.Connection = mysql.createConnection({
+                host:     task.hostname,
+                port:     task.port,
+                user:     task.username,
+                password: task.password,
+            });
+    
+            try { // Try and connect to the database
+                // Connect to the database
+                await new Promise<void>( (rez, err) => {
+                    conn.connect(e => {
+                        if ( !!e ) {
+                            // Mark the results as failure
+                            err(e);
+                        }
+                        // Mark the promise as done
+                        rez();
+                    })
+                })
+            } catch ( error ) { // Return the connection error upstream
+                return _e(error);
+            }
+
+            let results: databaseExportLog[] = [];
+
+            try {
+                // Dump the database
+                results = await this.dumpDatabase(conn, task);
+            } catch ( err ) {
+                conn.destroy();
+                return _e(err);
+            }
+            
+            try { // Close the connection
+                await new Promise<void>((re, er) => {
+                    conn.end( err => {
+                        // Just ignore it, whatever.
+                        re();
+                    });
+                })
+            } catch ( err ) {};
+
+            // Stop the code and mark as success
+            _r(results);
+        })
+    }
+
+    private dumpDatabase(conn : mysql.Connection, task: NodeConnection) {
+        return new Promise<databaseExportLog[]>( async (_r, _e) => {
+
+            // Get all of the databases from the connection
+            const databaseList:    string[] = [];
+            let   rawDatabaseList: string[] = [];
+
+            try {
+                rawDatabaseList = await new Promise<string[]>((res, er) => {
+                    conn.query("SHOW DATABASES", (err, rez, fields) => {
+                        if ( err ) {
+                            // Could not list databases, there's definitely something wrong going to happen, do a forceful stop.
+                            return _e(err);
+                        }
+
+                        let databaseList: string[] = [];
+
+                        for ( let db of rez ) {
+                            databaseList.push(db['Database']);
+                        }
+
+                        res(databaseList);
+                    })
+
+                });
+            } catch ( ex ) {
+                return _e(ex);
+            }
+
+            // Filter the database list based on the blacklist/whitelist
+            for ( let db of rawDatabaseList ) {
+                if ( !!task.databases ) {
+
+                    // Check the whitelist
+                    if ( !!task.databases.whitelist ) {
+                        if ( !task.databases.whitelist?.includes(db) ) {
+                            continue;
+                        }
+                    }
+
+                    // Check the blacklist
+                    if ( !!task.databases.blacklist ) {
+                        if ( task.databases.blacklist?.includes(db) ) {
+                            continue;
+                        }
+                    }
+                }
+
+                // The database passed our filters, add it to the final list!
+                databaseList.push(db);
+            }
+
+            // List of logs for the different database dumps
+            const resultLogs: databaseExportLog[] = [];
+
+            // Run the mysqldump for every database
+            for ( let db of databaseList ) {
+                let startTime = new Date().getTime();
+                try {
+                    await new Promise<void>((res, err) => {
+                        // Get the path to export to
+                        const exportPath:    string = path.join( this.exportPath , `${db}.sql`);
+                        // Make sure the folder exists
+                        if ( !fs.existsSync(path.dirname(exportPath)) ) {
+                            fs.mkdirSync(path.dirname(exportPath));
+                        }
+
+                        // Generate the export command
+                        const exportCommand: string = `mysqldump -u${task.username} -p${task.password} -h${task.hostname} -P${task.port} ${db} > ${exportPath}`;
+                        // Execute the export process
+                        child.exec(exportCommand, (execErr, stdout, stderr) => {
+                            if ( execErr ) {
+
+                                resultLogs.push({
+                                    type: databaseExportLogType.ERROR,
+                                    data: {
+                                        runTime: (new Date().getTime() - startTime) + "ms",
+                                        stderr: stderr
+                                    },
+                                    error: execErr,
+                                });
+
+                                return res();
+                            }
+
+                            resultLogs.push({
+                                type: databaseExportLogType.SUCCESS,
+                                data: {
+                                    runTime: (new Date().getTime() - startTime) + "ms",
+                                    stdout: stdout,
+                                },
+                            });
+    
+                            res();
+                        });
+
+                    });
+                } catch ( error ) {
+                    resultLogs.push({
+                        type: databaseExportLogType.ERROR,
+                        data: {
+                            runTime: (new Date().getTime() - startTime) + "ms",
+                        },
+                        error: error
+                    });
+                }
+            }
+
+            // Resolve the promise
+            _r(resultLogs);
+        })
     }
 
 }

tsconfig.json

@@ -4,9 +4,9 @@
 
     /* Basic Options */
     // "incremental": true,                         /* Enable incremental compilation */
-    "target": "ES2015",                                /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
+    "target": "ES2020",                                /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
     "module": "commonjs",                           /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
-    "lib": ["DOM", "ES2015", "ES2015.Promise"],                                   /* Specify library files to be included in the compilation. */
+    "lib": ["ES2020", "ES2020.Promise"],                                   /* Specify library files to be included in the compilation. */
     // "allowJs": true,                             /* Allow javascript files to be compiled. */
     // "checkJs": true,                             /* Report errors in .js files. */
     // "jsx": "preserve",                           /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
@@ -18,7 +18,7 @@
     "rootDir": "./src",                             /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
     // "composite": true,                           /* Enable project compilation */
     // "tsBuildInfoFile": "./",                     /* Specify file to store incremental compilation information */
-    // "removeComments": true,                      /* Do not emit comments to output. */
+    "removeComments": true,                      /* Do not emit comments to output. */
     // "noEmit": true,                              /* Do not emit outputs. */
     // "importHelpers": true,                       /* Import emit helpers from 'tslib'. */
     // "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
@@ -32,7 +32,7 @@
     // "strictBindCallApply": true,                 /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
     // "strictPropertyInitialization": true,        /* Enable strict checking of property initialization in classes. */
     // "noImplicitThis": true,                      /* Raise error on 'this' expressions with an implied 'any' type. */
-    // "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
+    "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
 
     /* Additional Checks */
     // "noUnusedLocals": true,                      /* Report errors on unused locals. */