outward-scrapper/internal/webui/server.go

package webui

import (
	"crypto/sha256"
	"embed"
	"encoding/json"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"scrappr/internal/logx"
	"scrappr/internal/model"
)

//go:embed static/*
var staticFiles embed.FS

type Catalog struct {
	ItemNames  []string         `json:"item_names"`
	Items      []CatalogItem    `json:"items"`
	Craftables []CraftableEntry `json:"craftables"`
}

type CatalogItem struct {
	Name       string         `json:"name"`
	URL        string         `json:"url"`
	ImageURL   string         `json:"image_url,omitempty"`
	Categories []string       `json:"categories,omitempty"`
	ItemType   string         `json:"item_type,omitempty"`
}

type CraftableEntry struct {
	CatalogItem
	Recipes    []model.Recipe `json:"recipes"`
}

type imageProxy struct {
	cacheDir string
	client   *http.Client
}

type cachedImageMeta struct {
	ContentType string    `json:"content_type"`
	SourceURL   string    `json:"source_url"`
	SavedAt     time.Time `json:"saved_at"`
}

func Run(addr, dataPath, imageCacheDir string) error {
	catalog, err := loadCatalog(dataPath)
	if err != nil {
		return err
	}

	images, err := newImageProxy(imageCacheDir)
	if err != nil {
		return err
	}

	staticFS, err := fs.Sub(staticFiles, "static")
	if err != nil {
		return err
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/", serveStaticFile(staticFS, "index.html", "text/html; charset=utf-8"))
	mux.HandleFunc("/styles.css", serveStaticFile(staticFS, "styles.css", "text/css; charset=utf-8"))
	mux.HandleFunc("/app.js", serveStaticFile(staticFS, "app.js", "application/javascript; charset=utf-8"))
	mux.HandleFunc("/api/catalog", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}

		setNoCache(w)
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		if err := json.NewEncoder(w).Encode(catalog); err != nil {
			logx.Eventf("error", "catalog encode failed: %v", err)
		}
	})
	mux.HandleFunc("/api/image", images.handle)

	logx.Eventf(
		"start",
		"web UI listening on %s using %s (%d craftable items, image cache %s)",
		displayAddr(addr),
		dataPath,
		len(catalog.Craftables),
		imageCacheDir,
	)

	return http.ListenAndServe(addr, mux)
}

func loadCatalog(dataPath string) (Catalog, error) {
	file, err := os.Open(dataPath)
	if err != nil {
		return Catalog{}, fmt.Errorf("open dataset %s: %w", dataPath, err)
	}
	defer file.Close()

	var dataset model.Dataset
	if err := json.NewDecoder(file).Decode(&dataset); err != nil {
		return Catalog{}, fmt.Errorf("decode dataset %s: %w", dataPath, err)
	}

	return buildCatalog(dataset), nil
}

func buildCatalog(dataset model.Dataset) Catalog {
	itemNames := make([]string, 0, len(dataset.Items))
	items := make([]CatalogItem, 0, len(dataset.Items))
	craftableByName := map[string]*CraftableEntry{}
	itemByName := map[string]CatalogItem{}
	seen := map[string]bool{}

	for _, item := range dataset.Items {
		name := strings.TrimSpace(item.Name)
		if name != "" && !seen[name] {
			seen[name] = true
			itemNames = append(itemNames, name)
		}

		catalogItem := CatalogItem{
			Name:       item.Name,
			URL:        item.URL,
			ImageURL:   item.ImageURL,
			Categories: item.Categories,
			ItemType:   item.Infobox["Type"],
		}

		items = append(items, catalogItem)
		itemByName[strings.ToLower(strings.TrimSpace(item.Name))] = catalogItem
	}

	for _, item := range dataset.Items {
		for _, recipe := range item.Recipes {
			resultKey := strings.ToLower(strings.TrimSpace(recipe.Result))
			if resultKey == "" {
				continue
			}

			entry := craftableByName[resultKey]
			if entry == nil {
				base := itemByName[resultKey]
				if strings.TrimSpace(base.Name) == "" {
					base = CatalogItem{
						Name:     recipe.Result,
						URL:      item.URL,
						ImageURL: item.ImageURL,
					}
				}

				entry = &CraftableEntry{
					CatalogItem: base,
				}
				craftableByName[resultKey] = entry
			}

			if !hasRecipe(entry.Recipes, recipe) {
				entry.Recipes = append(entry.Recipes, recipe)
			}
		}
	}

	sort.Strings(itemNames)
	sort.Slice(items, func(i, j int) bool {
		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
	})

	craftables := make([]CraftableEntry, 0, len(craftableByName))
	for _, entry := range craftableByName {
		craftables = append(craftables, *entry)
	}
	sort.Slice(craftables, func(i, j int) bool {
		return strings.ToLower(craftables[i].Name) < strings.ToLower(craftables[j].Name)
	})

	return Catalog{
		ItemNames:  itemNames,
		Items:      items,
		Craftables: craftables,
	}
}

func hasRecipe(existing []model.Recipe, target model.Recipe) bool {
	for _, recipe := range existing {
		if recipe.Result != target.Result || recipe.ResultCount != target.ResultCount || recipe.Station != target.Station {
			continue
		}
		if len(recipe.Ingredients) != len(target.Ingredients) {
			continue
		}

		match := true
		for i := range recipe.Ingredients {
			if recipe.Ingredients[i] != target.Ingredients[i] {
				match = false
				break
			}
		}

		if match {
			return true
		}
	}

	return false
}

func displayAddr(addr string) string {
	if strings.HasPrefix(addr, ":") {
		return "http://localhost" + addr
	}
	if strings.HasPrefix(addr, "http://") || strings.HasPrefix(addr, "https://") {
		return addr
	}
	return "http://" + addr
}

func serveStaticFile(staticFS fs.FS, name, contentType string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" && "/"+name != r.URL.Path {
			http.NotFound(w, r)
			return
		}

		content, err := fs.ReadFile(staticFS, name)
		if err != nil {
			http.Error(w, "asset missing", http.StatusInternalServerError)
			return
		}

		setNoCache(w)
		w.Header().Set("Content-Type", contentType)
		_, _ = w.Write(content)
	}
}

func setNoCache(w http.ResponseWriter) {
	w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate")
	w.Header().Set("Pragma", "no-cache")
	w.Header().Set("Expires", "0")
}

func newImageProxy(cacheDir string) (*imageProxy, error) {
	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
		return nil, err
	}

	return &imageProxy{
		cacheDir: cacheDir,
		client: &http.Client{
			Timeout: 20 * time.Second,
		},
	}, nil
}

func (p *imageProxy) handle(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}

	rawURL := strings.TrimSpace(r.URL.Query().Get("url"))
	if rawURL == "" {
		http.Error(w, "missing url", http.StatusBadRequest)
		return
	}

	if !isAllowedImageURL(rawURL) {
		http.Error(w, "forbidden image host", http.StatusForbidden)
		return
	}

	cacheKey := cacheKeyFor(rawURL)
	if p.serveCached(w, cacheKey) {
		return
	}

	contentType, body, sourceURL, err := p.fetchRemote(rawURL)
	if err != nil {
		logx.Eventf("warn", "image proxy failed for %s: %v", rawURL, err)
		http.Error(w, "image unavailable", http.StatusBadGateway)
		return
	}

	if err := p.storeCached(cacheKey, contentType, body, sourceURL); err != nil {
		logx.Eventf("warn", "image cache store failed for %s: %v", rawURL, err)
	}

	w.Header().Set("Content-Type", contentType)
	w.Header().Set("Cache-Control", "public, max-age=86400")
	_, _ = w.Write(body)
}

func (p *imageProxy) serveCached(w http.ResponseWriter, cacheKey string) bool {
	bodyPath := filepath.Join(p.cacheDir, cacheKey+".bin")
	metaPath := filepath.Join(p.cacheDir, cacheKey+".json")

	metaBytes, err := os.ReadFile(metaPath)
	if err != nil {
		return false
	}

	bodyBytes, err := os.ReadFile(bodyPath)
	if err != nil {
		return false
	}

	var meta cachedImageMeta
	if err := json.Unmarshal(metaBytes, &meta); err != nil {
		return false
	}

	if meta.ContentType == "" {
		meta.ContentType = "image/png"
	}

	w.Header().Set("Content-Type", meta.ContentType)
	w.Header().Set("Cache-Control", "public, max-age=86400")
	_, _ = w.Write(bodyBytes)
	return true
}

func (p *imageProxy) storeCached(cacheKey, contentType string, body []byte, sourceURL string) error {
	bodyPath := filepath.Join(p.cacheDir, cacheKey+".bin")
	metaPath := filepath.Join(p.cacheDir, cacheKey+".json")

	meta := cachedImageMeta{
		ContentType: contentType,
		SourceURL:   sourceURL,
		SavedAt:     time.Now(),
	}

	metaBytes, err := json.Marshal(meta)
	if err != nil {
		return err
	}

	if err := os.WriteFile(bodyPath, body, 0o644); err != nil {
		return err
	}

	return os.WriteFile(metaPath, metaBytes, 0o644)
}

func (p *imageProxy) fetchRemote(rawURL string) (string, []byte, string, error) {
	var lastErr error
	for _, candidate := range imageCandidates(rawURL) {
		req, err := http.NewRequest(http.MethodGet, candidate, nil)
		if err != nil {
			lastErr = err
			continue
		}

		req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36")
		req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
		req.Header.Set("Referer", "https://outward.fandom.com/")
		req.Header.Set("Sec-Fetch-Dest", "image")
		req.Header.Set("Sec-Fetch-Mode", "no-cors")
		req.Header.Set("Sec-Fetch-Site", "cross-site")

		resp, err := p.client.Do(req)
		if err != nil {
			lastErr = err
			continue
		}

		body, readErr := io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		if readErr != nil {
			lastErr = readErr
			continue
		}

		if resp.StatusCode != http.StatusOK {
			lastErr = fmt.Errorf("status %d from %s", resp.StatusCode, candidate)
			continue
		}

		contentType := resp.Header.Get("Content-Type")
		if !strings.HasPrefix(contentType, "image/") {
			contentType = "image/png"
		}

		logx.Eventf("cache", "fetched image %s", candidate)
		return contentType, body, candidate, nil
	}

	if lastErr == nil {
		lastErr = fmt.Errorf("no usable candidate")
	}
	return "", nil, "", lastErr
}

func imageCandidates(raw string) []string {
	seen := map[string]bool{}
	var out []string

	push := func(value string) {
		value = strings.TrimSpace(value)
		if value == "" || seen[value] {
			return
		}
		seen[value] = true
		out = append(out, value)
	}

	normalized := normalizeRemoteImageURL(raw)
	push(normalized)
	push(removeQuery(normalized))

	withoutRevision := strings.Replace(normalized, "/revision/latest", "", 1)
	push(withoutRevision)
	push(removeQuery(withoutRevision))

	push(raw)
	push(removeQuery(raw))

	return out
}

func normalizeRemoteImageURL(raw string) string {
	raw = strings.TrimSpace(raw)
	if strings.HasPrefix(raw, "//") {
		raw = "https:" + raw
	}

	query := ""
	if idx := strings.Index(raw, "?"); idx >= 0 {
		query = raw[idx:]
		raw = raw[:idx]
	}

	const scaledMarker = "/revision/latest/scale-to-width-down/"
	if idx := strings.Index(raw, scaledMarker); idx >= 0 {
		raw = raw[:idx] + "/revision/latest"
	}

	return raw + query
}

func removeQuery(raw string) string {
	if idx := strings.Index(raw, "?"); idx >= 0 {
		return raw[:idx]
	}
	return raw
}

func cacheKeyFor(raw string) string {
	sum := sha256.Sum256([]byte(strings.TrimSpace(raw)))
	return fmt.Sprintf("%x", sum[:])
}

func isAllowedImageURL(raw string) bool {
	parsed, err := url.Parse(strings.TrimSpace(raw))
	if err != nil {
		return false
	}

	host := strings.ToLower(parsed.Host)
	if host != "static.wikia.nocookie.net" {
		return false
	}

	return strings.Contains(parsed.Path, "/outward_gamepedia/images/")
}