Started moving to a sqlite3 implementation

This commit is contained in:
Daniel Legt 2024-01-20 18:35:50 +02:00
parent c4aef27eda
commit c0f1ed6879
14 changed files with 195 additions and 331 deletions

View File

@ -1,3 +1,3 @@
DISK_FETCH_FREQUENCY=5
MEMORY_DUMP_FREQUENCY=60
MAX_HISTORY_AGE=2592000
LISTEN=":8080"

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
snapshots.dat
.env
dist
data.db

5
go.mod
View File

@ -14,11 +14,14 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.17.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.19 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
@ -31,4 +34,6 @@ require (
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/sqlite v1.5.4 // indirect
gorm.io/gorm v1.25.5 // indirect
)

10
go.sum
View File

@ -30,6 +30,10 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@ -42,6 +46,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -86,5 +92,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0=
gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

View File

@ -9,10 +9,10 @@ import (
)
type DHConfig struct {
DiskFetchFrequency int `json:"diskFetchFrequency"`
MemoryDumpFrequency int `json:"memoryDumpFrequency"`
MaxHistoryAge int `json:"maxHistoryAge"`
Listen string
DiskFetchFrequency int `json:"diskFetchFrequency"`
MaxHistoryAge int `json:"maxHistoryAge"`
DatabaseFilePath string
Listen string
}
func GetConfiguration() DHConfig {
@ -22,9 +22,9 @@ func GetConfiguration() DHConfig {
}
config := DHConfig{
DiskFetchFrequency: 5, // default value
MemoryDumpFrequency: 60, // default value
MaxHistoryAge: 2592000, // default value
DiskFetchFrequency: 5, // default value
MaxHistoryAge: 2592000, // default value
DatabaseFilePath: "./data.db",
Listen: ":8080",
}
@ -35,12 +35,6 @@ func GetConfiguration() DHConfig {
}
}
if val, exists := os.LookupEnv("MEMORY_DUMP_FREQUENCY"); exists {
if intValue, err := strconv.Atoi(val); err == nil {
config.MemoryDumpFrequency = intValue
}
}
if val, exists := os.LookupEnv("MAX_HISTORY_AGE"); exists {
if intValue, err := strconv.Atoi(val); err == nil {
config.MaxHistoryAge = intValue
@ -51,5 +45,9 @@ func GetConfiguration() DHConfig {
config.Listen = val
}
if val, exists := os.LookupEnv("DATABASE_FILE_PATH"); exists {
config.DatabaseFilePath = val
}
return config
}

View File

@ -6,9 +6,12 @@ import (
"fmt"
"os/exec"
"strings"
"time"
"gorm.io/gorm"
)
func GetSystemHardDrives() ([]*HardDrive, error) {
func GetSystemHardDrives(db *gorm.DB, olderThan *time.Time, newerThan *time.Time) ([]*HardDrive, error) {
// Execute the lsblk command to get detailed block device information
cmd := exec.Command("lsblk", "-d", "-o", "NAME,TRAN,SIZE,MODEL,SERIAL,TYPE")
@ -20,7 +23,7 @@ func GetSystemHardDrives() ([]*HardDrive, error) {
return nil, err
}
var hardDrives []*HardDrive
var systemHardDrives []*HardDrive
// Scan the output line by line
scanner := bufio.NewScanner(&out)
@ -39,17 +42,16 @@ func GetSystemHardDrives() ([]*HardDrive, error) {
}
// Filter out nvme drives (M.2)
if cols[1] != "nvme" && cols[5] != "Device" {
if cols[1] != "nvme" && cols[5] != "Device" && cols[1] != "usb" {
hd := &HardDrive{
Name: cols[0],
Transport: cols[1],
Size: cols[2],
Model: cols[3],
Serial: cols[4],
Type: cols[5],
Temperature: 0,
Name: cols[0],
Transport: cols[1],
Size: cols[2],
Model: cols[3],
Serial: cols[4],
Type: cols[5],
}
hardDrives = append(hardDrives, hd)
systemHardDrives = append(systemHardDrives, hd)
}
}
@ -58,5 +60,35 @@ func GetSystemHardDrives() ([]*HardDrive, error) {
return nil, err
}
return hardDrives, nil
var updatedHardDrives []*HardDrive
for _, sysHDD := range systemHardDrives {
var existingHD HardDrive
q := db.Where("serial = ?", sysHDD.Serial)
if newerThan != nil && olderThan != nil {
fmt.Printf("\nNewer Than: %s\n", newerThan)
fmt.Printf("Older Than: %s\n\n", olderThan)
q = q.Preload("Temperatures", "time_stamp < ? AND time_stamp > ?", newerThan, olderThan)
}
result := q.First(&existingHD)
if result.Error == gorm.ErrRecordNotFound {
// Hard drive not found, create new
db.Create(&sysHDD)
updatedHardDrives = append(updatedHardDrives, sysHDD)
} else {
// Hard drive found, update existing
existingHD.Name = sysHDD.Name
existingHD.Transport = sysHDD.Transport
existingHD.Size = sysHDD.Size
existingHD.Model = sysHDD.Model
existingHD.Type = sysHDD.Type
db.Save(&existingHD)
updatedHardDrives = append(updatedHardDrives, &existingHD)
}
}
return updatedHardDrives, nil
}

64
lib/hardware/models.go Normal file
View File

@ -0,0 +1,64 @@
package hardware
import (
"fmt"
"time"
"github.com/anatol/smart.go"
"gorm.io/gorm"
)
type HardDrive struct {
ID uint `gorm:"primarykey"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt `gorm:"index"`
Name string
Transport string
Size string
Model string
Serial string
Type string
Temperatures []HardDriveTemperature `gorm:"foreignKey:HardDriveID"`
}
type HardDriveTemperature struct {
gorm.Model
HardDriveID uint
TimeStamp time.Time
Temperature int
}
// A snapshot in time of the current state of the harddrives
type HardwareSnapshot struct {
TimeStamp time.Time
HDD []*HardDrive
}
type Snapshots struct {
List []*HardwareSnapshot
}
// Fetch the temperature of the device, optinally update the reference object
func (h *HardDrive) GetTemperature() int {
// Fetch the device by name
disk, err := smart.Open("/dev/" + h.Name)
if err != nil {
fmt.Printf("Failed to open device %s: %s\n", h.Name, err)
return -1
}
defer disk.Close()
// Fetch SMART data
smartInfo, err := disk.ReadGenericAttributes()
if err != nil {
fmt.Printf("Failed to get SMART data for %s: %s\n", h.Name, err)
return -1
}
// Parse the temperature
temperature := int(smartInfo.Temperature)
// Return the found value
return temperature
}

View File

@ -1,46 +0,0 @@
package hardware
import (
"fmt"
"github.com/anatol/smart.go"
)
type HardDrive struct {
Name string
Transport string
Size string
Model string
Serial string
Type string
Temperature int
}
// Fetch the temperature of the device, optinally update the reference object
func (h *HardDrive) GetTemperature(updateTemp bool) int {
// Fetch the device by name
disk, err := smart.Open("/dev/" + h.Name)
if err != nil {
fmt.Printf("Failed to open device %s: %s\n", h.Name, err)
return -1
}
defer disk.Close()
// Fetch SMART data
smartInfo, err := disk.ReadGenericAttributes()
if err != nil {
fmt.Printf("Failed to get SMART data for %s: %s\n", h.Name, err)
return -1
}
// Parse the temperature
temperature := int(smartInfo.Temperature)
// Optionally update the reference object's temperature
if updateTemp {
h.Temperature = temperature
}
// Return the found value
return temperature
}

View File

@ -1,127 +1,67 @@
package svc
import (
"encoding/gob"
"fmt"
"io"
"os"
"time"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"tea.chunkbyte.com/kato/drive-health/lib/config"
"tea.chunkbyte.com/kato/drive-health/lib/hardware"
)
// The path to where the snapshot database is located
const SNAPSHOT_LIST_PATH = "./snapshots.dat"
var db *gorm.DB
// A simple in-memory buffer for the history of snapshots
var snapShotBuffer []*HardwareSnapshot
// A snapshot in time of the current state of the harddrives
type HardwareSnapshot struct {
TimeStamp time.Time
HDD []*hardware.HardDrive
}
type Snapshots struct {
List []*HardwareSnapshot
}
// The function itterates through all hard disks and takes a snapshot of their state,
// returns a struct which contains metadata as well as the harddrives themselves.
func TakeHardwareSnapshot() (*HardwareSnapshot, error) {
drives, err := hardware.GetSystemHardDrives()
if err != nil {
return nil, err
func InitDB() {
var err error
dbPath := config.GetConfiguration().DatabaseFilePath
if dbPath == "" {
dbPath = "./data.db"
}
snapShot := &HardwareSnapshot{
TimeStamp: time.Now(),
HDD: []*hardware.HardDrive{},
db, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
if err != nil {
panic("failed to connect database")
}
// Migrate the schema
db.AutoMigrate(&hardware.HardDrive{}, &hardware.HardDriveTemperature{})
}
func GetDatabaseRef() *gorm.DB {
return db
}
func LogDriveTemps() error {
drives, err := hardware.GetSystemHardDrives(db, nil, nil)
if err != nil {
return err
}
for _, hdd := range drives {
hdd.GetTemperature(true)
snapShot.HDD = append(snapShot.HDD, hdd)
}
// Append to the in-memory listing
snapShotBuffer = append(snapShotBuffer, snapShot)
// Return the snapshot just in case there is any need to modify it,
// any modification to it will also affect the current buffer from memory.
return snapShot, nil
}
// The function wil check if the `.dat` file is present, if it is then it will load it into memory
func UpdateHardwareSnapshotsFromFile() {
file, err := os.Open(SNAPSHOT_LIST_PATH)
if err != nil {
if os.IsNotExist(err) {
return // File does not exist, no snapshots to load
}
panic(err) // Handle error according to your error handling policy
}
defer file.Close()
decoder := gob.NewDecoder(file)
var snapshots Snapshots
if err := decoder.Decode(&snapshots); err != nil {
if err == io.EOF {
return // End of file reached
}
panic(err) // Handle error according to your error handling policy
}
snapShotBuffer = snapshots.List
fmt.Printf("Loaded %v snapshots from .dat", len(snapShotBuffer))
}
// Get the list of snapshots that have been buffered in memory
func GetHardwareSnapshot() []*HardwareSnapshot {
return snapShotBuffer
}
// Dump the current snapshot history from memory to file
func SaveSnapshotsToFile() error {
file, err := os.Create(SNAPSHOT_LIST_PATH)
if err != nil {
return err
}
defer file.Close()
encoder := gob.NewEncoder(file)
snapshots := Snapshots{List: snapShotBuffer}
if err := encoder.Encode(snapshots); err != nil {
return err
temp := hdd.GetTemperature()
db.Create(&hardware.HardDriveTemperature{
HardDriveID: hdd.ID,
TimeStamp: time.Now(),
Temperature: temp,
})
}
return nil
}
func RunService() {
func RunLoggerService() {
fmt.Println("Initializing Temperature Logging Service...")
tickTime := time.Duration(config.GetConfiguration().DiskFetchFrequency) * time.Second
// Snapshot taking routine
go func() {
waitTime := time.Duration(config.GetConfiguration().DiskFetchFrequency) * time.Second
for {
time.Sleep(waitTime)
_, err := TakeHardwareSnapshot()
time.Sleep(tickTime)
err := LogDriveTemps()
if err != nil {
fmt.Printf("Hardware Fetch Error: %s", err)
}
}
}()
// Periodic saving routine
go func() {
for {
waitTime := time.Duration(config.GetConfiguration().MemoryDumpFrequency) * time.Second
time.Sleep(waitTime)
err := SaveSnapshotsToFile()
if err != nil {
fmt.Printf("Memory Dump Error: %s", err)
fmt.Printf("🛑 Temperature logging failed: %s\n", err)
}
}
}()

View File

@ -2,6 +2,7 @@ package web
import (
"net/http"
"time"
"github.com/gin-gonic/gin"
"tea.chunkbyte.com/kato/drive-health/lib/hardware"
@ -9,18 +10,22 @@ import (
)
func setupApi(r *gin.Engine) {
api := r.Group("/v1/api")
api := r.Group("/api/v1")
api.GET("/disks", func(ctx *gin.Context) {
olderThan := time.Now().Add(time.Minute * time.Duration(10) * -1)
newerThan := time.Now()
// Fetch the disk list
disks, err := hardware.GetSystemHardDrives()
disks, err := hardware.GetSystemHardDrives(svc.GetDatabaseRef(), &olderThan, &newerThan)
if err != nil {
ctx.Error(err)
}
if ctx.Request.URL.Query().Get("temp") != "" {
for _, d := range disks {
d.GetTemperature(true)
d.GetTemperature()
}
}
@ -30,10 +35,4 @@ func setupApi(r *gin.Engine) {
})
})
api.GET("/snapshots", func(ctx *gin.Context) {
snapshots := svc.GetHardwareSnapshot()
ctx.JSON(http.StatusOK, snapshots)
})
}

View File

@ -2,9 +2,11 @@ package web
import (
"net/http"
"time"
"github.com/gin-gonic/gin"
"tea.chunkbyte.com/kato/drive-health/lib/hardware"
"tea.chunkbyte.com/kato/drive-health/lib/svc"
)
func setupFrontend(r *gin.Engine) {
@ -13,14 +15,16 @@ func setupFrontend(r *gin.Engine) {
// Set up a route for the root URL
r.GET("/", func(c *gin.Context) {
olderThan := time.Now().Add(time.Minute * time.Duration(10) * -1)
newerThan := time.Now()
hardDrives, err := hardware.GetSystemHardDrives()
hardDrives, err := hardware.GetSystemHardDrives(svc.GetDatabaseRef(), &olderThan, &newerThan)
if err != nil {
c.AbortWithStatus(500)
}
for _, hdd := range hardDrives {
hdd.GetTemperature(true)
hdd.GetTemperature()
}
// Render the HTML template

View File

@ -15,8 +15,8 @@ import (
)
func main() {
// Load existing snapshots from file
svc.UpdateHardwareSnapshotsFromFile()
// Init the database
svc.InitDB()
router := web.SetupRouter()
@ -33,7 +33,7 @@ func main() {
}()
// Run the hardware service
svc.RunService()
svc.RunLoggerService()
// Setting up signal capturing
quit := make(chan os.Signal, 1)

View File

@ -1,134 +0,0 @@
function stringToColor(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
}
let color = '#';
for (let i = 0; i < 3; i++) {
const value = (hash >> (i * 8)) & 0xFF;
color += ('00' + value.toString(16)).substr(-2);
}
return color;
}
document.addEventListener('DOMContentLoaded', function() {
const diskTableBody = document.getElementById('disk-table-body');
const ctx = document.getElementById('temperatureChart').getContext('2d');
let temperatureChart = new Chart(ctx, {
type: 'line',
data: {
datasets: []
},
options: {
scales: {
x: {
type: 'time',
time: {
unit: 'second',
displayFormats: {
second: 'HH:mm:ss'
}
},
title: {
display: true,
text: 'Time'
}
},
y: {
beginAtZero: true,
title: {
display: true,
text: 'Temperature (°C)'
}
}
}
}
});
function fetchAndUpdateDisks() {
fetch('/v1/api/disks?temp=true')
.then(response => response.json())
.then(data => {
updateDiskTable(data.disks);
})
.catch(error => console.error('Error fetching disk data:', error));
}
function updateDiskTable(disks) {
let tableHTML = '';
disks.forEach(disk => {
tableHTML += `
<tr>
<td>${disk.Name}</td>
<td>${disk.Transport}</td>
<td>${disk.Size}</td>
<td>${disk.Model}</td>
<td>${disk.Serial}</td>
<td>${disk.Type}</td>
<td>${disk.Temperature}</td>
</tr>
`;
});
diskTableBody.innerHTML = tableHTML;
}
function fetchAndUpdateTemperatureChart() {
fetch('/v1/api/snapshots')
.then(response => response.json())
.then(snapshots => {
updateTemperatureChart(snapshots);
})
.catch(error => console.error('Error fetching temperature data:', error));
}
function updateTemperatureChart(snapshots) {
// Clear existing datasets
temperatureChart.data.datasets = [];
snapshots.forEach(snapshot => {
const time = new Date(snapshot.TimeStamp);
snapshot.HDD.forEach(disk => {
let dataset = temperatureChart.data.datasets.find(d => d.label === disk.Name);
if (!dataset) {
dataset = {
label: disk.Name,
data: [],
fill: false,
borderColor: stringToColor(disk.Name),
borderWidth: 1
};
temperatureChart.data.datasets.push(dataset);
}
dataset.data.push({
x: time,
y: disk.Temperature
});
});
});
temperatureChart.update();
}
// Chart.js zoom and pan configuration
temperatureChart.options.plugins.zoom = {
zoom: {
wheel: {
enabled: true,
},
pinch: {
enabled: true
},
mode: 'x',
},
pan: {
enabled: true,
mode: 'x',
}
};
fetchAndUpdateDisks();
fetchAndUpdateTemperatureChart();
setInterval(fetchAndUpdateDisks, 5000);
setInterval(fetchAndUpdateTemperatureChart, 5000);
});

View File

@ -11,26 +11,17 @@
<h1>Drive Health Dashboard</h1>
<table>
<thead>
<!-- ... table headers ... -->
</thead>
<tbody id="disk-table-body">
<!-- Data will be populated here by JavaScript -->
</tbody>
</table>
<hr>
<div class="chart-container" style="position: relative; height:40vh; width:80vw; overflow-x: scroll;">
<canvas id="temperatureChart"></canvas>
</div>
</div>
<script src="https://cdn.jsdelivr.net/npm/chart.js@latest"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns@latest"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-zoom@latest"></script>
<script src="/static/main.js"></script>
</body>
</html>