package webui import ( "crypto/sha256" "embed" "encoding/json" "fmt" "io" "io/fs" "net/http" "net/url" "os" "path/filepath" "sort" "strings" "time" "scrappr/internal/logx" "scrappr/internal/model" ) //go:embed static/* var staticFiles embed.FS type Catalog struct { ItemNames []string `json:"item_names"` Items []CatalogItem `json:"items"` Craftables []CraftableEntry `json:"craftables"` } type CatalogItem struct { Name string `json:"name"` URL string `json:"url"` ImageURL string `json:"image_url,omitempty"` Categories []string `json:"categories,omitempty"` ItemType string `json:"item_type,omitempty"` } type CraftableEntry struct { CatalogItem Recipes []model.Recipe `json:"recipes"` } type imageProxy struct { cacheDir string client *http.Client } type cachedImageMeta struct { ContentType string `json:"content_type"` SourceURL string `json:"source_url"` SavedAt time.Time `json:"saved_at"` } func Run(addr, dataPath, imageCacheDir string) error { catalog, err := loadCatalog(dataPath) if err != nil { return err } images, err := newImageProxy(imageCacheDir) if err != nil { return err } staticFS, err := fs.Sub(staticFiles, "static") if err != nil { return err } mux := http.NewServeMux() mux.HandleFunc("/", serveStaticFile(staticFS, "index.html", "text/html; charset=utf-8")) mux.HandleFunc("/styles.css", serveStaticFile(staticFS, "styles.css", "text/css; charset=utf-8")) mux.HandleFunc("/app.js", serveStaticFile(staticFS, "app.js", "application/javascript; charset=utf-8")) mux.HandleFunc("/api/catalog", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } setNoCache(w) w.Header().Set("Content-Type", "application/json; charset=utf-8") if err := json.NewEncoder(w).Encode(catalog); err != nil { logx.Eventf("error", "catalog encode failed: %v", err) } }) mux.HandleFunc("/api/image", images.handle) logx.Eventf( "start", "web UI listening on %s using %s (%d craftable items, image cache %s)", displayAddr(addr), dataPath, len(catalog.Craftables), imageCacheDir, ) return http.ListenAndServe(addr, mux) } func loadCatalog(dataPath string) (Catalog, error) { file, err := os.Open(dataPath) if err != nil { return Catalog{}, fmt.Errorf("open dataset %s: %w", dataPath, err) } defer file.Close() var dataset model.Dataset if err := json.NewDecoder(file).Decode(&dataset); err != nil { return Catalog{}, fmt.Errorf("decode dataset %s: %w", dataPath, err) } return buildCatalog(dataset), nil } func buildCatalog(dataset model.Dataset) Catalog { itemNames := make([]string, 0, len(dataset.Items)) items := make([]CatalogItem, 0, len(dataset.Items)) craftableByName := map[string]*CraftableEntry{} itemByName := map[string]CatalogItem{} seen := map[string]bool{} for _, item := range dataset.Items { name := strings.TrimSpace(item.Name) if name != "" && !seen[name] { seen[name] = true itemNames = append(itemNames, name) } catalogItem := CatalogItem{ Name: item.Name, URL: item.URL, ImageURL: item.ImageURL, Categories: item.Categories, ItemType: item.Infobox["Type"], } items = append(items, catalogItem) itemByName[strings.ToLower(strings.TrimSpace(item.Name))] = catalogItem } for _, item := range dataset.Items { for _, recipe := range item.Recipes { resultKey := strings.ToLower(strings.TrimSpace(recipe.Result)) if resultKey == "" { continue } entry := craftableByName[resultKey] if entry == nil { base := itemByName[resultKey] if strings.TrimSpace(base.Name) == "" { base = CatalogItem{ Name: recipe.Result, URL: item.URL, ImageURL: item.ImageURL, } } entry = &CraftableEntry{ CatalogItem: base, } craftableByName[resultKey] = entry } if !hasRecipe(entry.Recipes, recipe) { entry.Recipes = append(entry.Recipes, recipe) } } } sort.Strings(itemNames) sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) }) craftables := make([]CraftableEntry, 0, len(craftableByName)) for _, entry := range craftableByName { craftables = append(craftables, *entry) } sort.Slice(craftables, func(i, j int) bool { return strings.ToLower(craftables[i].Name) < strings.ToLower(craftables[j].Name) }) return Catalog{ ItemNames: itemNames, Items: items, Craftables: craftables, } } func hasRecipe(existing []model.Recipe, target model.Recipe) bool { for _, recipe := range existing { if recipe.Result != target.Result || recipe.ResultCount != target.ResultCount || recipe.Station != target.Station { continue } if len(recipe.Ingredients) != len(target.Ingredients) { continue } match := true for i := range recipe.Ingredients { if recipe.Ingredients[i] != target.Ingredients[i] { match = false break } } if match { return true } } return false } func displayAddr(addr string) string { if strings.HasPrefix(addr, ":") { return "http://localhost" + addr } if strings.HasPrefix(addr, "http://") || strings.HasPrefix(addr, "https://") { return addr } return "http://" + addr } func serveStaticFile(staticFS fs.FS, name, contentType string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" && "/"+name != r.URL.Path { http.NotFound(w, r) return } content, err := fs.ReadFile(staticFS, name) if err != nil { http.Error(w, "asset missing", http.StatusInternalServerError) return } setNoCache(w) w.Header().Set("Content-Type", contentType) _, _ = w.Write(content) } } func setNoCache(w http.ResponseWriter) { w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate") w.Header().Set("Pragma", "no-cache") w.Header().Set("Expires", "0") } func newImageProxy(cacheDir string) (*imageProxy, error) { if err := os.MkdirAll(cacheDir, 0o755); err != nil { return nil, err } return &imageProxy{ cacheDir: cacheDir, client: &http.Client{ Timeout: 20 * time.Second, }, }, nil } func (p *imageProxy) handle(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } rawURL := strings.TrimSpace(r.URL.Query().Get("url")) if rawURL == "" { http.Error(w, "missing url", http.StatusBadRequest) return } if !isAllowedImageURL(rawURL) { http.Error(w, "forbidden image host", http.StatusForbidden) return } cacheKey := cacheKeyFor(rawURL) if p.serveCached(w, cacheKey) { return } contentType, body, sourceURL, err := p.fetchRemote(rawURL) if err != nil { logx.Eventf("warn", "image proxy failed for %s: %v", rawURL, err) http.Error(w, "image unavailable", http.StatusBadGateway) return } if err := p.storeCached(cacheKey, contentType, body, sourceURL); err != nil { logx.Eventf("warn", "image cache store failed for %s: %v", rawURL, err) } w.Header().Set("Content-Type", contentType) w.Header().Set("Cache-Control", "public, max-age=86400") _, _ = w.Write(body) } func (p *imageProxy) serveCached(w http.ResponseWriter, cacheKey string) bool { bodyPath := filepath.Join(p.cacheDir, cacheKey+".bin") metaPath := filepath.Join(p.cacheDir, cacheKey+".json") metaBytes, err := os.ReadFile(metaPath) if err != nil { return false } bodyBytes, err := os.ReadFile(bodyPath) if err != nil { return false } var meta cachedImageMeta if err := json.Unmarshal(metaBytes, &meta); err != nil { return false } if meta.ContentType == "" { meta.ContentType = "image/png" } w.Header().Set("Content-Type", meta.ContentType) w.Header().Set("Cache-Control", "public, max-age=86400") _, _ = w.Write(bodyBytes) return true } func (p *imageProxy) storeCached(cacheKey, contentType string, body []byte, sourceURL string) error { bodyPath := filepath.Join(p.cacheDir, cacheKey+".bin") metaPath := filepath.Join(p.cacheDir, cacheKey+".json") meta := cachedImageMeta{ ContentType: contentType, SourceURL: sourceURL, SavedAt: time.Now(), } metaBytes, err := json.Marshal(meta) if err != nil { return err } if err := os.WriteFile(bodyPath, body, 0o644); err != nil { return err } return os.WriteFile(metaPath, metaBytes, 0o644) } func (p *imageProxy) fetchRemote(rawURL string) (string, []byte, string, error) { var lastErr error for _, candidate := range imageCandidates(rawURL) { req, err := http.NewRequest(http.MethodGet, candidate, nil) if err != nil { lastErr = err continue } req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36") req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.9") req.Header.Set("Referer", "https://outward.fandom.com/") req.Header.Set("Sec-Fetch-Dest", "image") req.Header.Set("Sec-Fetch-Mode", "no-cors") req.Header.Set("Sec-Fetch-Site", "cross-site") resp, err := p.client.Do(req) if err != nil { lastErr = err continue } body, readErr := io.ReadAll(resp.Body) _ = resp.Body.Close() if readErr != nil { lastErr = readErr continue } if resp.StatusCode != http.StatusOK { lastErr = fmt.Errorf("status %d from %s", resp.StatusCode, candidate) continue } contentType := resp.Header.Get("Content-Type") if !strings.HasPrefix(contentType, "image/") { contentType = "image/png" } logx.Eventf("cache", "fetched image %s", candidate) return contentType, body, candidate, nil } if lastErr == nil { lastErr = fmt.Errorf("no usable candidate") } return "", nil, "", lastErr } func imageCandidates(raw string) []string { seen := map[string]bool{} var out []string push := func(value string) { value = strings.TrimSpace(value) if value == "" || seen[value] { return } seen[value] = true out = append(out, value) } normalized := normalizeRemoteImageURL(raw) push(normalized) push(removeQuery(normalized)) withoutRevision := strings.Replace(normalized, "/revision/latest", "", 1) push(withoutRevision) push(removeQuery(withoutRevision)) push(raw) push(removeQuery(raw)) return out } func normalizeRemoteImageURL(raw string) string { raw = strings.TrimSpace(raw) if strings.HasPrefix(raw, "//") { raw = "https:" + raw } query := "" if idx := strings.Index(raw, "?"); idx >= 0 { query = raw[idx:] raw = raw[:idx] } const scaledMarker = "/revision/latest/scale-to-width-down/" if idx := strings.Index(raw, scaledMarker); idx >= 0 { raw = raw[:idx] + "/revision/latest" } return raw + query } func removeQuery(raw string) string { if idx := strings.Index(raw, "?"); idx >= 0 { return raw[:idx] } return raw } func cacheKeyFor(raw string) string { sum := sha256.Sum256([]byte(strings.TrimSpace(raw))) return fmt.Sprintf("%x", sum[:]) } func isAllowedImageURL(raw string) bool { parsed, err := url.Parse(strings.TrimSpace(raw)) if err != nil { return false } host := strings.ToLower(parsed.Host) if host != "static.wikia.nocookie.net" { return false } return strings.Contains(parsed.Path, "/outward_gamepedia/images/") }