Compare commits
No commits in common. "7296410b1252aaa5996d6f7a998f3c772b9c0f1c" and "36b1e37370757796e50b8e02f70af7236bb40f34" have entirely different histories.
7296410b12
...
36b1e37370
|
@ -1,21 +0,0 @@
|
||||||
{
|
|
||||||
// Use IntelliSense to learn about possible attributes.
|
|
||||||
// Hover to view descriptions of existing attributes.
|
|
||||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
|
||||||
"version": "0.2.0",
|
|
||||||
"configurations": [
|
|
||||||
{
|
|
||||||
"type": "node",
|
|
||||||
"request": "launch",
|
|
||||||
"name": "Launch Program",
|
|
||||||
"skipFiles": [
|
|
||||||
"<node_internals>/**"
|
|
||||||
],
|
|
||||||
"program": "${workspaceFolder}/src/app.ts",
|
|
||||||
"preLaunchTask": "tsc: build - tsconfig.json",
|
|
||||||
"outFiles": [
|
|
||||||
"${workspaceFolder}/dist/**/*.js"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
|
@ -29,7 +29,6 @@
|
||||||
"hostname": "localhost",
|
"hostname": "localhost",
|
||||||
"username": "root",
|
"username": "root",
|
||||||
"password": "",
|
"password": "",
|
||||||
"port" : 3306,
|
|
||||||
"databases": {
|
"databases": {
|
||||||
"blacklist": null,
|
"blacklist": null,
|
||||||
"whitelist": null
|
"whitelist": null
|
||||||
|
|
16
package.json
16
package.json
|
@ -3,19 +3,5 @@
|
||||||
"@types/nodemailer": "^6.4.4",
|
"@types/nodemailer": "^6.4.4",
|
||||||
"mysql": "^2.18.1",
|
"mysql": "^2.18.1",
|
||||||
"nodemailer": "^6.6.3"
|
"nodemailer": "^6.6.3"
|
||||||
},
|
}
|
||||||
"name": "datahoard",
|
|
||||||
"description": "Make sure your database backups run and keep track of performance",
|
|
||||||
"version": "0.1.0",
|
|
||||||
"main": "dist/app.js",
|
|
||||||
"devDependencies": {},
|
|
||||||
"scripts": {
|
|
||||||
"start": "tsc && node dist/app.js"
|
|
||||||
},
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "https://tea.chunkbyte.com/kato/DataHoard.git"
|
|
||||||
},
|
|
||||||
"author": "Kato Twofold",
|
|
||||||
"license": "SEE LICENSE IN LICENSE"
|
|
||||||
}
|
}
|
||||||
|
|
40
src/app.ts
40
src/app.ts
|
@ -1,51 +1,13 @@
|
||||||
// Import the configuration file
|
|
||||||
import { getConfig, NodeConnection } from "./helper/config";
|
|
||||||
// Import SystemControl
|
// Import SystemControl
|
||||||
import { SystemControl } from "./models/SystemController";
|
import { SystemControl } from "./models/SystemController";
|
||||||
// Import the worker processes
|
|
||||||
import { DataWorker } from "./models/WorkerProcess";
|
|
||||||
|
|
||||||
// Global system control singleton
|
// Global system control singleton
|
||||||
((global as any).logger as SystemControl) = new SystemControl();
|
(global as any).logger = new SystemControl();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The main function to run the program with
|
* The main function to run the program with
|
||||||
*/
|
*/
|
||||||
async function main() {
|
async function main() {
|
||||||
// Get the configuration file
|
|
||||||
const config = getConfig();
|
|
||||||
|
|
||||||
// Initialize the Worker Nodes array
|
|
||||||
let workerNodes : Array<DataWorker> = [];
|
|
||||||
|
|
||||||
// Spawn in the DataWorkers
|
|
||||||
for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
|
|
||||||
// Create a new data worker
|
|
||||||
workerNodes.push(new DataWorker(config.options.parallel_node_tasks));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Keep track of the worker that is assigned this node
|
|
||||||
*/
|
|
||||||
let workerIndex = 0;
|
|
||||||
// Start assigning nodes to the DataWorkers
|
|
||||||
for ( let node of config.nodes ) {
|
|
||||||
console.log(`Adding workerIndex[${workerIndex}]`);
|
|
||||||
// Add the task to the node
|
|
||||||
workerNodes[workerIndex].addNodeTask(node);
|
|
||||||
// Increment the worker index
|
|
||||||
workerIndex++;
|
|
||||||
if ( workerIndex >= workerNodes.length ) workerIndex = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Succesfully added ${config.nodes.length} Tasks to ${workerNodes.length} Workers`);
|
|
||||||
|
|
||||||
for( let w of workerNodes ) {
|
|
||||||
await w.startProcessing();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Run the main program
|
|
||||||
main();
|
|
|
@ -1,5 +1,3 @@
|
||||||
import * as fs from "fs";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A Collection of settings
|
* A Collection of settings
|
||||||
*/
|
*/
|
||||||
|
@ -118,10 +116,6 @@ export interface NodeConnection {
|
||||||
* The password to connect to the database with
|
* The password to connect to the database with
|
||||||
*/
|
*/
|
||||||
password : string,
|
password : string,
|
||||||
/**
|
|
||||||
* The port for the database connection
|
|
||||||
*/
|
|
||||||
port : number,
|
|
||||||
/**
|
/**
|
||||||
* The database filter, this can contain a blacklist or a whitelist.
|
* The database filter, this can contain a blacklist or a whitelist.
|
||||||
*/
|
*/
|
||||||
|
@ -147,54 +141,13 @@ export interface ConfigObject {
|
||||||
nodes: Array<NodeConnection>
|
nodes: Array<NodeConnection>
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the name of the config file
|
|
||||||
*/
|
|
||||||
export function getConfigFileName() : string {
|
|
||||||
return "config.json";
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The config file can really be in 2 places, in the project or above it, so we're going to take a peak in both cases.
|
|
||||||
*/
|
|
||||||
function findConfigFile() : string {
|
|
||||||
const fileName : string = getConfigFileName();
|
|
||||||
|
|
||||||
// Look in the same folder
|
|
||||||
if ( fs.existsSync(fileName) ) {
|
|
||||||
return fileName;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check outside
|
|
||||||
if ( fs.existsSync(fileName) ) {
|
|
||||||
return `../${fileName}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Throw an error because the file does not exist.
|
|
||||||
throw new Error("Failed to find the config file, please create a `config.json` file and complete it similar to the provided example file `config.example.json`");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read the config file from the files of the project
|
* Read the config file from the files of the project
|
||||||
* @returns The configuration object of the project
|
* @returns The configuration object of the project
|
||||||
*/
|
*/
|
||||||
export function getConfig() : ConfigObject {
|
export function getConfig() : ConfigObject | null {
|
||||||
|
|
||||||
// Check for the location of the config file
|
|
||||||
const fileName : string = findConfigFile();
|
|
||||||
|
|
||||||
// Initialize the config data object
|
|
||||||
let configData : ConfigObject;
|
|
||||||
|
|
||||||
// Load in the configuration data from the json file
|
return null;
|
||||||
const rawFileData : string = fs.readFileSync(fileName, `utf-8`);
|
|
||||||
// Parse the data
|
|
||||||
configData = JSON.parse(rawFileData);
|
|
||||||
|
|
||||||
if ( !!configData ) {
|
|
||||||
return configData;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Throw an error because it has been a failure
|
|
||||||
throw new Error("Failed to load the configuration file.");
|
|
||||||
}
|
}
|
|
@ -1,10 +0,0 @@
|
||||||
/**
|
|
||||||
* Get an unique UUID V4
|
|
||||||
* @returns An unique UUID
|
|
||||||
*/
|
|
||||||
export function uuid(): string {
|
|
||||||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
|
|
||||||
let r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8);
|
|
||||||
return v.toString(16);
|
|
||||||
});
|
|
||||||
}
|
|
|
@ -1,74 +0,0 @@
|
||||||
import { NodeConnection } from "../helper/config";
|
|
||||||
import { uuid } from "../helper/mainHelper";
|
|
||||||
|
|
||||||
|
|
||||||
export class DataWorker {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An array of tasks that should be taken care of
|
|
||||||
*/
|
|
||||||
private nodeTasks : Array<NodeConnection> = [];
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The amount of parallel tasks to run at once
|
|
||||||
*/
|
|
||||||
private parallelTasksCount: number;
|
|
||||||
|
|
||||||
private id: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param parallelTasks The amount of parallel tasks that this Worker can run at once
|
|
||||||
*/
|
|
||||||
constructor(parallelTasks: number) {
|
|
||||||
// Initialize the amount of paralel tasks to run at the same time
|
|
||||||
this.parallelTasksCount = parallelTasks;
|
|
||||||
|
|
||||||
// Generate an ID for the worker
|
|
||||||
this.id = uuid();
|
|
||||||
}
|
|
||||||
|
|
||||||
public addNodeTask(node: NodeConnection) {
|
|
||||||
// Add the task to the list
|
|
||||||
this.nodeTasks.push(node);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start processing the node tasks.
|
|
||||||
*/
|
|
||||||
public async startProcessing() : Promise<void> {
|
|
||||||
|
|
||||||
// Go through all of the tasks to run
|
|
||||||
for ( let i = 0; i < this.nodeTasks.length; i++ ) {
|
|
||||||
// Spawn in the task queue
|
|
||||||
let taskQueue : Array<Promise<any>> = [];
|
|
||||||
|
|
||||||
// Start running the parallel tasks
|
|
||||||
for ( let x = 0; x < this.parallelTasksCount; x++ ) {
|
|
||||||
// Check if we have stept out of bounds
|
|
||||||
if ( i >= this.nodeTasks.length ) // Ignore out-of-bounds
|
|
||||||
break;
|
|
||||||
|
|
||||||
// Add the process to the queue
|
|
||||||
taskQueue.push(this.processTask(this.nodeTasks[i]))
|
|
||||||
// Add to the index
|
|
||||||
i += x;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the results of the running
|
|
||||||
const runResults = await Promise.all(taskQueue);
|
|
||||||
|
|
||||||
console.log(runResults);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process a task
|
|
||||||
* @param task The task to process
|
|
||||||
*/
|
|
||||||
private async processTask(task: NodeConnection) {
|
|
||||||
return `[${this.id}]: Succesfully ran task ${task.name}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -58,7 +58,7 @@
|
||||||
/* Source Map Options */
|
/* Source Map Options */
|
||||||
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
|
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
|
||||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||||
"inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
|
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
|
||||||
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
|
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
|
||||||
|
|
||||||
/* Experimental Options */
|
/* Experimental Options */
|
||||||
|
|
Reference in New Issue