outward-scrapper/internal/app/run.go

package app

import (
	"encoding/json"
	"math/rand"
	"os"
	"os/signal"
	"syscall"
	"time"

	"scrappr/internal/logx"
	"scrappr/internal/model"
	"scrappr/internal/scraper"
)

const outputPath = "outward_data.json"

func Run() error {
	rand.Seed(time.Now().UnixNano())

	logx.Eventf("start", "Outward scraper booting")

	cfg := scraper.DefaultConfig()
	s := scraper.New(cfg)
	installSignalCheckpoint(s)

	dataset, err := s.Run()
	if err != nil {
		return err
	}

	logx.Eventf("write", "writing dataset to %s", outputPath)
	if err := writeDataset(outputPath, dataset); err != nil {
		return err
	}

	logx.Eventf(
		"success",
		"wrote %d items and %d effects to %s",
		len(dataset.Items),
		len(dataset.Effects),
		outputPath,
	)

	if err := s.ClearCheckpoint(); err != nil {
		logx.Eventf("warn", "failed to clear checkpoint: %v", err)
	}

	return nil
}

func installSignalCheckpoint(s *scraper.Scraper) {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, syscall.SIGTERM)

	go func() {
		sig := <-signals
		logx.Eventf("warn", "received %s, saving checkpoint before exit", sig.String())
		if err := s.SaveCheckpoint("signal"); err != nil {
			logx.Eventf("error", "failed to save checkpoint on signal: %v", err)
		}
		os.Exit(130)
	}()
}

func writeDataset(outputPath string, dataset model.Dataset) error {
	file, err := os.Create(outputPath)
	if err != nil {
		return err
	}
	defer file.Close()

	encoder := json.NewEncoder(file)
	encoder.SetIndent("", "  ")

	return encoder.Encode(dataset)
}