+ A lot of functionality added

+ Upped JS Version as I don't really see a reason why not.

.gitignore

@@ -120,5 +120,8 @@ out
 
 dist/*
 !dist/.keep
+release/*
+!release/.keep
+exports
 package-lock.json
 config.json

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch Program",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/src/app.ts",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/dist/**/*.js"
+            ]
+        }
+    ]
+}

README.md

@@ -8,11 +8,12 @@ This supports mailing for logging what is happening to the databases, future pla
 
 ### Features
 
-- [ ] Database Exporting
+- [x] Database Exporting
 - [ ] Log all errors to a final log
-- [ ] Email the log through email
+- [x] Email the log through email
 - [ ] Discord WebHook
 - [ ] Telegram Hook
+- [ ] Docker Support
 
 ### How to Use
 1. Setup the configuration file which is [This file](./config.example.json) and rename it into `config.json` instead of `config.example.json`.

config.example.json

@@ -3,7 +3,9 @@
         "parallel_nodes": 2,
         "parallel_node_tasks": 5,
         "separate_into_tables": false,
-        "mysql_dump_settings": "--no-comments"
+        "mysql_dump_settings": "--no-comments",
+        "database_export_path": "./exports",
+        "logs_path": "./logs"
     },
 
     "smtp": {
@@ -12,7 +14,8 @@
         "ssl":  true,
         "username": "",
         "password": "",
-        "email_from": ""
+        "email_from": "",
+        "to": ""
     },
 
     "nodes": [
@@ -29,6 +32,7 @@
             "hostname": "localhost",
             "username": "root",
             "password": "",
+            "port"    : 3306,
             "databases": {
                 "blacklist": null,
                 "whitelist": null

package.json

@@ -3,5 +3,21 @@
     "@types/nodemailer": "^6.4.4",
     "mysql": "^2.18.1",
     "nodemailer": "^6.6.3"
-  }
+  },
+  "name": "datahoard",
+  "description": "Make sure your database backups run and keep track of performance",
+  "version": "0.1.0",
+  "main": "dist/app.js",
+  "devDependencies": {
+    "@types/mysql": "^2.15.19"
+  },
+  "scripts": {
+    "start": "tsc && node dist/app.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://tea.chunkbyte.com/kato/DataHoard.git"
+  },
+  "author": "Kato Twofold",
+  "license": "SEE LICENSE IN LICENSE"
 }

src/app.ts

@@ -1,13 +1,94 @@
+// Import the configuration file
+import { getConfig, NodeConnection } from "./helper/config";
 // Import SystemControl
 import { SystemControl } from "./models/SystemController";
+// Import the worker processes
+import { DataWorker } from "./models/WorkerProcess";
+// Import the mail model
+import { sendMail } from "./models/MailModel";
+// Import the file system library
+import * as fs from "fs";
+import * as path from "path";
 
 // Global system control singleton
-(global as any).logger = new SystemControl();
+((global as any).logger as SystemControl) = new SystemControl();
 
 /**
  * The main function to run the program with
  */
 async function main() {
+    // Get the configuration file
+    const config = getConfig();
 
+    // Initialize the Worker Nodes array
+    let workerNodes : Array<DataWorker> = [];
+
+    // Spawn in the DataWorkers
+    for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
+        // Create a new data worker
+        workerNodes.push(new DataWorker(config.options.parallel_node_tasks, config.options.database_export_path));
+    }
+
+    /**
+     * Keep track of the worker that is assigned this node
+     */
+    let workerIndex = 0;
+    // Start assigning nodes to the DataWorkers
+    for ( let node of config.nodes ) {
+        // Add the task to the node
+        workerNodes[workerIndex].addNodeTask(node);
+        // Increment the worker index
+        workerIndex++;
+        if ( workerIndex >= workerNodes.length ) workerIndex = 0;
+    }
+
+    const responseLogs: Map<string, string[]> = new Map<string, string[]>();
+
+    // @TODO: Actually make these run in parallel
+    for ( let w of workerNodes ) {
+
+        const workerStartTime = new Date().getTime();
+        let resolves: PromiseSettledResult<any>[] | string = await w.startProcessing();
+        const workerEndTime: string = Number((workerStartTime - new Date().getTime()) / 1000).toFixed(2) + "s"
+
+        let myLog: string[] = [
+            `runTime: ${workerEndTime}`,
+        ];
+
+        for ( let r of resolves ) {
+            myLog.push(JSON.stringify(r, null, 2));
+        }
+
+        responseLogs.set(w.getID(), myLog);
+    }
+
+    // Log to file
+    if ( !!config.options.logs_path ) {
+        // Log to a JSON file path
+        const logFilePath: string = path.join(config.options.logs_path, new Date().getTime() + ".json");
+        const dirPath = path.dirname(logFilePath);
+        // Create the directory if it don't exist.
+        if ( !fs.existsSync(dirPath) ) { fs.mkdirSync(dirPath); }
+
+        // Write to file
+        fs.writeFileSync(logFilePath, JSON.stringify(responseLogs as Object));
+    }
+
+    // Send the log via email
+    if ( !!config.smtp ) {
+        let htmlContent = `<h3>Database Log Report</h3><br>`;
+
+        responseLogs.forEach( (value, key) => {
+            htmlContent += `<h6>${value}<h6>`
+            htmlContent += `<samp>${JSON.stringify(key, null, 4)}</samp>`
+            htmlContent += "<hr>";
+        })
+
+        sendMail(htmlContent, config.smtp.to);
+    }
 
 }
+
+
+// Run the main program
+main();

src/helper/config.ts

@@ -1,3 +1,5 @@
+import * as fs from "fs";
+
 /**
  * A Collection of settings
  */
@@ -25,6 +27,16 @@ export interface ConfigOptions {
      * A string of CLI options for the mysql dump command
      */
     mysql_dump_settings: string,
+
+    /**
+     * Path to where to export the database to
+     */
+    database_export_path: string,
+
+    /**
+     * Path to the logs file
+     */
+    logs_path?: string
 }
 
 /**
@@ -56,6 +68,11 @@ export interface SMTPConfig {
      * Secure Connection
      */
     ssl        : boolean,
+
+    /**
+     * Main target email address
+     */
+    to         : string,
 }
 
 export interface NodeConnectionMail {
@@ -116,6 +133,10 @@ export interface NodeConnection {
      * The password to connect to the database with
      */
     password  : string,
+    /**
+     * The port for the database connection
+     */
+    port      : number,
     /**
      * The database filter, this can contain a blacklist or a whitelist.
      */
@@ -133,7 +154,7 @@ export interface ConfigObject {
      * SMTP details for sending out email notifications
      * for real-time updates on erros, health, and backups.
      */
-    smtp:    SMTPConfig,
+    smtp?:    SMTPConfig,
 
     /**
      * An array of Node connections that we should backup
@@ -141,13 +162,54 @@ export interface ConfigObject {
     nodes:   Array<NodeConnection>
 }
 
+/**
+ * Get the name of the config file
+ */
+export function getConfigFileName() : string {
+    return "config.json";
+}
+
+/**
+ * The config file can really be in 2 places, in the project or above it, so we're going to take a peak in both cases.
+ */
+function findConfigFile() : string {
+    const fileName : string = getConfigFileName();
+
+    // Look in the same folder
+    if ( fs.existsSync(fileName) ) {
+        return fileName;
+    }
+    
+    // Check outside
+    if ( fs.existsSync(fileName) ) {
+        return `../${fileName}`;
+    }
+
+    // Throw an error because the file does not exist.
+    throw new Error("Failed to find the config file, please create a `config.json` file and complete it similar to the provided example file `config.example.json`");
+}
+
 /**
  * Read the config file from the files of the project
  * @returns The configuration object of the project
  */
-export function getConfig() : ConfigObject | null {
+export function getConfig() : ConfigObject {
 
+    // Check for the location of the config file
+    const fileName : string = findConfigFile();
 
+    // Initialize the config data object
+    let configData : ConfigObject;
 
-    return null;
+    // Load in the configuration data from the json file
+    const rawFileData : string = fs.readFileSync(fileName, `utf-8`);
+    // Parse the data
+    configData = JSON.parse(rawFileData);
+
+    if ( !!configData ) {
+        return configData;
+    }
+
+    // Throw an error because it has been a failure
+    throw new Error("Failed to load the configuration file.");
 }

src/helper/mainHelper.ts

@@ -0,0 +1,10 @@
+/**
+ * Get an unique UUID V4
+ * @returns An unique UUID
+ */
+export function uuid(): string {
+    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+        let r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8);
+        return v.toString(16);
+    });
+}

src/models/MailModel.ts

@@ -11,12 +11,12 @@ function getTransporter() : nodemailer.Transporter {
     
     // Generate the transporter
     const transporter = nodemailer.createTransport({
-        host: config?.smtp.host,
-        port: config?.smtp.port,
-        secure: config?.smtp.ssl,
+        host: config?.smtp?.host,
+        port: config?.smtp?.port,
+        secure: config?.smtp?.ssl,
         auth: {
-            user: config?.smtp.username,
-            pass: config?.smtp.password,
+            user: config?.smtp?.username,
+            pass: config?.smtp?.password,
         }
     });
 
@@ -27,21 +27,27 @@ function getTransporter() : nodemailer.Transporter {
 /**
  * Send out an email with the config details.
  */
-export async function sendMail(mailContent: string, toAddress: string, nodeConfig: NodeConnectionMail, ccAddresses?: Array<string>) {
+export async function sendMail(mailContent: string, toAddress: string, ccAddresses: Array<string> = []) {
 
     // Get the transporter
     const transporter = getTransporter();
 
     // Generate the mail options
     const mailOptions : nodemailer.SendMailOptions = {
-        from: config?.smtp.email_from,
-        to: nodeConfig.email_to,
-        cc: nodeConfig.email_cc,
+        subject: "Database Backup Update",
+        from: config?.smtp?.email_from,
+        to: toAddress,
+        cc: ccAddresses,
         html: mailContent,
     }
 
     await new Promise((_r, _e) => {
         transporter.sendMail(mailOptions, (err, info) => {
+            if ( err ) {
+                console.error(err);
+            }
+
+            console.log(info);
 
         })
     })

src/models/WorkerProcess.ts

@@ -0,0 +1,265 @@
+import * as mysql from "mysql";
+import { NodeConnection } from "../helper/config";
+import {      uuid      } from "../helper/mainHelper";
+import * as child from 'child_process';
+import * as path from "path";
+import * as fs from "fs";
+
+export enum databaseExportLogType {
+    ERROR   = "Error",
+    SUCCESS = "Success",
+}
+
+export interface databaseExportLog {
+    data:   any,
+    type:   databaseExportLogType,
+    error?: Error,
+}
+
+export class DataWorker {
+
+    /**
+     * An array of tasks that should be taken care of
+     */
+    private nodeTasks : Array<NodeConnection> = [];
+
+    /**
+     * The amount of parallel tasks to run at once
+     */
+    private parallelTasksCount: number;
+
+    private exportPath: string;
+
+    private id: string;
+
+    /**
+     * @param parallelTasks The amount of parallel tasks that this Worker can run at once
+     */
+    constructor(parallelTasks: number, exportPath: string) {
+        // Initialize the amount of paralel tasks to run at the same time
+        this.parallelTasksCount = parallelTasks;
+
+        // Assign the export path
+        this.exportPath = exportPath;
+
+        // Generate an ID for the worker
+        this.id = uuid();
+    }
+
+    public getID() : string {
+        return this.id;
+    }
+    
+
+    public addNodeTask(node: NodeConnection) {
+        // Add the task to the list
+        this.nodeTasks.push(node);
+    }
+
+    /**
+     * Start processing the node tasks.
+     */
+    public startProcessing() : Promise<PromiseSettledResult<any>[] | string> {
+        return new Promise( async (_r, _e) => {
+            try {
+
+                if ( this.nodeTasks.length <= 0 ) {
+                    return _r("No tasks to run, skipping");
+                }
+
+                // Go through all of the tasks to run
+                for ( let i = 0; i < this.nodeTasks.length; i++ ) {
+                    // Spawn in the task queue
+                    let taskQueue : Array<Promise<any>> = [];
+        
+                    // Start running the parallel tasks
+                    for ( let x = 0; x < this.parallelTasksCount; x++ ) {
+                        // Check if we have stept out of bounds
+                        if ( i >= this.nodeTasks.length ) // Ignore out-of-bounds
+                            break;
+        
+                        // Add the process to the queue
+                        taskQueue.push(this.processTask(this.nodeTasks[i]))
+                        // Add to the index
+                        i++;
+                    }
+
+                    // Return the resulting resolves
+                    _r(await Promise.allSettled(taskQueue));
+                }
+            } catch ( ex ) {
+                _e(ex);
+            }
+        })
+
+
+    }
+
+    /**
+     * Process a task
+     * @param task The task to process
+     */
+    private processTask(task: NodeConnection) {
+        return new Promise<databaseExportLog[]>( async (_r, _e) => {
+            // Connect to the mysql database to test out the connection
+            const conn : mysql.Connection = mysql.createConnection({
+                host:     task.hostname,
+                port:     task.port,
+                user:     task.username,
+                password: task.password,
+            });
+    
+            try { // Try and connect to the database
+                // Connect to the database
+                await new Promise<void>( (rez, err) => {
+                    conn.connect(e => {
+                        if ( !!e ) {
+                            // Mark the results as failure
+                            err(e);
+                        }
+                        // Mark the promise as done
+                        rez();
+                    })
+                })
+            } catch ( error ) { // Return the connection error upstream
+                return _e(error);
+            }
+
+            let results: databaseExportLog[] = [];
+
+            try {
+                // Dump the database
+                results = await this.dumpDatabase(conn, task);
+            } catch ( err ) {
+                conn.destroy();
+                return _e(err);
+            }
+            
+            try { // Close the connection
+                await new Promise<void>((re, er) => {
+                    conn.end( err => {
+                        // Just ignore it, whatever.
+                        re();
+                    });
+                })
+            } catch ( err ) {};
+
+            // Stop the code and mark as success
+            _r(results);
+        })
+    }
+
+    private dumpDatabase(conn : mysql.Connection, task: NodeConnection) {
+        return new Promise<databaseExportLog[]>( async (_r, _e) => {
+
+            // Get all of the databases from the connection
+            const databaseList:    string[] = [];
+            let   rawDatabaseList: string[] = [];
+
+            try {
+                rawDatabaseList = await new Promise<string[]>((res, er) => {
+                    conn.query("SHOW DATABASES", (err, rez, fields) => {
+                        if ( err ) {
+                            // Could not list databases, there's definitely something wrong going to happen, do a forceful stop.
+                            return _e(err);
+                        }
+
+                        let databaseList: string[] = [];
+
+                        for ( let db of rez ) {
+                            databaseList.push(db['Database']);
+                        }
+
+                        res(databaseList);
+                    })
+
+                });
+            } catch ( ex ) {
+                return _e(ex);
+            }
+
+            // Filter the database list based on the blacklist/whitelist
+            for ( let db of rawDatabaseList ) {
+                if ( !!task.databases ) {
+
+                    // Check the whitelist
+                    if ( !!task.databases.whitelist ) {
+                        if ( !task.databases.whitelist?.includes(db) ) {
+                            continue;
+                        }
+                    }
+
+                    // Check the blacklist
+                    if ( !!task.databases.blacklist ) {
+                        if ( task.databases.blacklist?.includes(db) ) {
+                            continue;
+                        }
+                    }
+                }
+
+                // The database passed our filters, add it to the final list!
+                databaseList.push(db);
+            }
+
+            // List of logs for the different database dumps
+            const resultLogs: databaseExportLog[] = [];
+
+            // Run the mysqldump for every database
+            for ( let db of databaseList ) {
+                let startTime = new Date().getTime();
+                try {
+                    await new Promise<void>((res, err) => {
+                        // Get the path to export to
+                        const exportPath:    string = path.join( this.exportPath , `${db}.sql`);
+                        // Make sure the folder exists
+                        if ( !fs.existsSync(path.dirname(exportPath)) ) {
+                            fs.mkdirSync(path.dirname(exportPath));
+                        }
+
+                        // Generate the export command
+                        const exportCommand: string = `mysqldump -u${task.username} -p${task.password} -h${task.hostname} -P${task.port} ${db} > ${exportPath}`;
+                        // Execute the export process
+                        child.exec(exportCommand, (execErr, stdout, stderr) => {
+                            if ( execErr ) {
+
+                                resultLogs.push({
+                                    type: databaseExportLogType.ERROR,
+                                    data: {
+                                        runTime: (new Date().getTime() - startTime) + "ms",
+                                        stderr: stderr
+                                    },
+                                    error: execErr,
+                                });
+
+                                return res();
+                            }
+
+                            resultLogs.push({
+                                type: databaseExportLogType.SUCCESS,
+                                data: {
+                                    runTime: (new Date().getTime() - startTime) + "ms",
+                                    stdout: stdout,
+                                },
+                            });
+    
+                            res();
+                        });
+
+                    });
+                } catch ( error ) {
+                    resultLogs.push({
+                        type: databaseExportLogType.ERROR,
+                        data: {
+                            runTime: (new Date().getTime() - startTime) + "ms",
+                        },
+                        error: error
+                    });
+                }
+            }
+
+            // Resolve the promise
+            _r(resultLogs);
+        })
+    }
+
+}

tsconfig.json

@@ -4,9 +4,9 @@
 
     /* Basic Options */
     // "incremental": true,                         /* Enable incremental compilation */
-    "target": "ES2015",                                /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
+    "target": "ES2020",                                /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
     "module": "commonjs",                           /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
-    "lib": ["DOM", "ES2015", "ES2015.Promise"],                                   /* Specify library files to be included in the compilation. */
+    "lib": ["ES2020", "ES2020.Promise"],                                   /* Specify library files to be included in the compilation. */
     // "allowJs": true,                             /* Allow javascript files to be compiled. */
     // "checkJs": true,                             /* Report errors in .js files. */
     // "jsx": "preserve",                           /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
@@ -18,7 +18,7 @@
     "rootDir": "./src",                             /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
     // "composite": true,                           /* Enable project compilation */
     // "tsBuildInfoFile": "./",                     /* Specify file to store incremental compilation information */
-    // "removeComments": true,                      /* Do not emit comments to output. */
+    "removeComments": true,                      /* Do not emit comments to output. */
     // "noEmit": true,                              /* Do not emit outputs. */
     // "importHelpers": true,                       /* Import emit helpers from 'tslib'. */
     // "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
@@ -32,7 +32,7 @@
     // "strictBindCallApply": true,                 /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
     // "strictPropertyInitialization": true,        /* Enable strict checking of property initialization in classes. */
     // "noImplicitThis": true,                      /* Raise error on 'this' expressions with an implied 'any' type. */
-    // "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
+    "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
 
     /* Additional Checks */
     // "noUnusedLocals": true,                      /* Report errors on unused locals. */
@@ -58,7 +58,7 @@
     /* Source Map Options */
     // "sourceRoot": "",                            /* Specify the location where debugger should locate TypeScript files instead of source locations. */
     // "mapRoot": "",                               /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSourceMap": true,                     /* Emit a single file with source maps instead of having a separate file. */
+    "inlineSourceMap": true,                     /* Emit a single file with source maps instead of having a separate file. */
     // "inlineSources": true,                       /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
 
     /* Experimental Options */