Compare commits

..

No commits in common. "7296410b1252aaa5996d6f7a998f3c772b9c0f1c" and "36b1e37370757796e50b8e02f70af7236bb40f34" have entirely different histories.

8 changed files with 9 additions and 214 deletions

21
.vscode/launch.json vendored
View File

@ -1,21 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "Launch Program",
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder}/src/app.ts",
"preLaunchTask": "tsc: build - tsconfig.json",
"outFiles": [
"${workspaceFolder}/dist/**/*.js"
]
}
]
}

View File

@ -29,7 +29,6 @@
"hostname": "localhost", "hostname": "localhost",
"username": "root", "username": "root",
"password": "", "password": "",
"port" : 3306,
"databases": { "databases": {
"blacklist": null, "blacklist": null,
"whitelist": null "whitelist": null

View File

@ -3,19 +3,5 @@
"@types/nodemailer": "^6.4.4", "@types/nodemailer": "^6.4.4",
"mysql": "^2.18.1", "mysql": "^2.18.1",
"nodemailer": "^6.6.3" "nodemailer": "^6.6.3"
}, }
"name": "datahoard",
"description": "Make sure your database backups run and keep track of performance",
"version": "0.1.0",
"main": "dist/app.js",
"devDependencies": {},
"scripts": {
"start": "tsc && node dist/app.js"
},
"repository": {
"type": "git",
"url": "https://tea.chunkbyte.com/kato/DataHoard.git"
},
"author": "Kato Twofold",
"license": "SEE LICENSE IN LICENSE"
} }

View File

@ -1,51 +1,13 @@
// Import the configuration file
import { getConfig, NodeConnection } from "./helper/config";
// Import SystemControl // Import SystemControl
import { SystemControl } from "./models/SystemController"; import { SystemControl } from "./models/SystemController";
// Import the worker processes
import { DataWorker } from "./models/WorkerProcess";
// Global system control singleton // Global system control singleton
((global as any).logger as SystemControl) = new SystemControl(); (global as any).logger = new SystemControl();
/** /**
* The main function to run the program with * The main function to run the program with
*/ */
async function main() { async function main() {
// Get the configuration file
const config = getConfig();
// Initialize the Worker Nodes array
let workerNodes : Array<DataWorker> = [];
// Spawn in the DataWorkers
for ( let i: number = 0; i < config.options.parallel_nodes; i++ ) {
// Create a new data worker
workerNodes.push(new DataWorker(config.options.parallel_node_tasks));
}
/**
* Keep track of the worker that is assigned this node
*/
let workerIndex = 0;
// Start assigning nodes to the DataWorkers
for ( let node of config.nodes ) {
console.log(`Adding workerIndex[${workerIndex}]`);
// Add the task to the node
workerNodes[workerIndex].addNodeTask(node);
// Increment the worker index
workerIndex++;
if ( workerIndex >= workerNodes.length ) workerIndex = 0;
}
console.log(`Succesfully added ${config.nodes.length} Tasks to ${workerNodes.length} Workers`);
for( let w of workerNodes ) {
await w.startProcessing();
}
} }
// Run the main program
main();

View File

@ -1,5 +1,3 @@
import * as fs from "fs";
/** /**
* A Collection of settings * A Collection of settings
*/ */
@ -118,10 +116,6 @@ export interface NodeConnection {
* The password to connect to the database with * The password to connect to the database with
*/ */
password : string, password : string,
/**
* The port for the database connection
*/
port : number,
/** /**
* The database filter, this can contain a blacklist or a whitelist. * The database filter, this can contain a blacklist or a whitelist.
*/ */
@ -147,54 +141,13 @@ export interface ConfigObject {
nodes: Array<NodeConnection> nodes: Array<NodeConnection>
} }
/**
* Get the name of the config file
*/
export function getConfigFileName() : string {
return "config.json";
}
/**
* The config file can really be in 2 places, in the project or above it, so we're going to take a peak in both cases.
*/
function findConfigFile() : string {
const fileName : string = getConfigFileName();
// Look in the same folder
if ( fs.existsSync(fileName) ) {
return fileName;
}
// Check outside
if ( fs.existsSync(fileName) ) {
return `../${fileName}`;
}
// Throw an error because the file does not exist.
throw new Error("Failed to find the config file, please create a `config.json` file and complete it similar to the provided example file `config.example.json`");
}
/** /**
* Read the config file from the files of the project * Read the config file from the files of the project
* @returns The configuration object of the project * @returns The configuration object of the project
*/ */
export function getConfig() : ConfigObject { export function getConfig() : ConfigObject | null {
// Check for the location of the config file
const fileName : string = findConfigFile();
// Initialize the config data object
let configData : ConfigObject;
// Load in the configuration data from the json file return null;
const rawFileData : string = fs.readFileSync(fileName, `utf-8`);
// Parse the data
configData = JSON.parse(rawFileData);
if ( !!configData ) {
return configData;
}
// Throw an error because it has been a failure
throw new Error("Failed to load the configuration file.");
} }

View File

@ -1,10 +0,0 @@
/**
* Get an unique UUID V4
* @returns An unique UUID
*/
export function uuid(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
let r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8);
return v.toString(16);
});
}

View File

@ -1,74 +0,0 @@
import { NodeConnection } from "../helper/config";
import { uuid } from "../helper/mainHelper";
export class DataWorker {
/**
* An array of tasks that should be taken care of
*/
private nodeTasks : Array<NodeConnection> = [];
/**
* The amount of parallel tasks to run at once
*/
private parallelTasksCount: number;
private id: string;
/**
* @param parallelTasks The amount of parallel tasks that this Worker can run at once
*/
constructor(parallelTasks: number) {
// Initialize the amount of paralel tasks to run at the same time
this.parallelTasksCount = parallelTasks;
// Generate an ID for the worker
this.id = uuid();
}
public addNodeTask(node: NodeConnection) {
// Add the task to the list
this.nodeTasks.push(node);
}
/**
* Start processing the node tasks.
*/
public async startProcessing() : Promise<void> {
// Go through all of the tasks to run
for ( let i = 0; i < this.nodeTasks.length; i++ ) {
// Spawn in the task queue
let taskQueue : Array<Promise<any>> = [];
// Start running the parallel tasks
for ( let x = 0; x < this.parallelTasksCount; x++ ) {
// Check if we have stept out of bounds
if ( i >= this.nodeTasks.length ) // Ignore out-of-bounds
break;
// Add the process to the queue
taskQueue.push(this.processTask(this.nodeTasks[i]))
// Add to the index
i += x;
}
// Get the results of the running
const runResults = await Promise.all(taskQueue);
console.log(runResults);
}
}
/**
* Process a task
* @param task The task to process
*/
private async processTask(task: NodeConnection) {
return `[${this.id}]: Succesfully ran task ${task.name}`;
}
}

View File

@ -58,7 +58,7 @@
/* Source Map Options */ /* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
"inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */ /* Experimental Options */