57 lines
946 B
Go
57 lines
946 B
Go
package app
|
|
|
|
import (
|
|
"encoding/json"
|
|
"math/rand"
|
|
"os"
|
|
"time"
|
|
|
|
"scrappr/internal/logx"
|
|
"scrappr/internal/model"
|
|
"scrappr/internal/scraper"
|
|
)
|
|
|
|
const outputPath = "outward_data.json"
|
|
|
|
func Run() error {
|
|
rand.Seed(time.Now().UnixNano())
|
|
|
|
logx.Eventf("start", "Outward scraper booting")
|
|
|
|
cfg := scraper.DefaultConfig()
|
|
s := scraper.New(cfg)
|
|
|
|
dataset, err := s.Run()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
logx.Eventf("write", "writing dataset to %s", outputPath)
|
|
if err := writeDataset(outputPath, dataset); err != nil {
|
|
return err
|
|
}
|
|
|
|
logx.Eventf(
|
|
"success",
|
|
"wrote %d items and %d effects to %s",
|
|
len(dataset.Items),
|
|
len(dataset.Effects),
|
|
outputPath,
|
|
)
|
|
|
|
return nil
|
|
}
|
|
|
|
func writeDataset(outputPath string, dataset model.Dataset) error {
|
|
file, err := os.Create(outputPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
encoder := json.NewEncoder(file)
|
|
encoder.SetIndent("", " ")
|
|
|
|
return encoder.Encode(dataset)
|
|
}
|