Initial COmmit
This commit is contained in:
487
internal/scraper/parse.go
Normal file
487
internal/scraper/parse.go
Normal file
@@ -0,0 +1,487 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gocolly/colly/v2"
|
||||
|
||||
"scrappr/internal/model"
|
||||
)
|
||||
|
||||
func (s *Scraper) shouldVisit(raw string) bool {
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if parsed.Host != s.cfg.AllowedDomain {
|
||||
return false
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(parsed.Path, "/wiki/") {
|
||||
return false
|
||||
}
|
||||
|
||||
if s.cfg.IgnoredExact[parsed.Path] {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, prefix := range s.cfg.IgnoredPrefixes {
|
||||
if strings.HasPrefix(parsed.Path, prefix) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Scraper) applyBrowserHeaders(r *colly.Request) {
|
||||
r.Headers.Set("User-Agent", s.randomUserAgent())
|
||||
r.Headers.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
|
||||
r.Headers.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
r.Headers.Set("Cache-Control", "max-age=0")
|
||||
r.Headers.Set("DNT", "1")
|
||||
r.Headers.Set("Sec-Fetch-Dest", "document")
|
||||
r.Headers.Set("Sec-Fetch-Mode", "navigate")
|
||||
r.Headers.Set("Sec-Fetch-Site", "same-origin")
|
||||
r.Headers.Set("Sec-Fetch-User", "?1")
|
||||
r.Headers.Set("Upgrade-Insecure-Requests", "1")
|
||||
|
||||
if r.Headers.Get("Referer") == "" {
|
||||
r.Headers.Set("Referer", s.cfg.BrowserReferrer)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scraper) randomUserAgent() string {
|
||||
return s.cfg.BrowserAgents[rand.Intn(len(s.cfg.BrowserAgents))]
|
||||
}
|
||||
|
||||
func (s *Scraper) shouldQueueFromPage(fromPath, toURL string) bool {
|
||||
parsed, err := url.Parse(toURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
toPath := parsed.Path
|
||||
|
||||
if s.cfg.ItemListPathRe.MatchString(fromPath) {
|
||||
return strings.HasPrefix(toPath, "/wiki/") &&
|
||||
!s.cfg.ItemListPathRe.MatchString(toPath) &&
|
||||
!s.cfg.CraftingPathRe.MatchString(toPath)
|
||||
}
|
||||
|
||||
if s.cfg.CraftingPathRe.MatchString(fromPath) {
|
||||
return strings.HasPrefix(toPath, "/wiki/") &&
|
||||
!s.cfg.CraftingPathRe.MatchString(toPath)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Scraper) shouldQueueTableLink(e *colly.HTMLElement) bool {
|
||||
label := s.clean(e.Text)
|
||||
if label == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
cell := e.DOM.Closest("td, th")
|
||||
if cell.Length() == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
columnIndex := 0
|
||||
for prev := cell.Prev(); prev.Length() > 0; prev = prev.Prev() {
|
||||
switch goquery.NodeName(prev) {
|
||||
case "td", "th":
|
||||
columnIndex++
|
||||
}
|
||||
}
|
||||
|
||||
return columnIndex <= 1
|
||||
}
|
||||
|
||||
func (s *Scraper) pageKindForPath(pagePath string) string {
|
||||
switch {
|
||||
case s.cfg.ItemListPathRe.MatchString(pagePath):
|
||||
return "item-list"
|
||||
case s.cfg.CraftingPathRe.MatchString(pagePath):
|
||||
return "crafting"
|
||||
default:
|
||||
return "article"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scraper) isLikelyItemPage(pagePath, title string, lcCats []string, doc *goquery.Document) bool {
|
||||
if s.cfg.ItemListPathRe.MatchString(pagePath) || s.cfg.CraftingPathRe.MatchString(pagePath) {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(strings.ToLower(title), "/") {
|
||||
return false
|
||||
}
|
||||
if doc.Find("aside.portable-infobox").Length() == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, category := range lcCats {
|
||||
if strings.Contains(category, "items") || strings.Contains(category, "food") ||
|
||||
strings.Contains(category, "consumables") || strings.Contains(category, "ingredients") ||
|
||||
strings.Contains(category, "equipment") || strings.Contains(category, "weapons") ||
|
||||
strings.Contains(category, "deployables") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
infoboxText := strings.ToLower(s.clean(doc.Find("aside.portable-infobox").Text()))
|
||||
return strings.Contains(infoboxText, "item details") ||
|
||||
strings.Contains(infoboxText, "consumable details") ||
|
||||
strings.Contains(infoboxText, "equipment details") ||
|
||||
strings.Contains(infoboxText, "weapon")
|
||||
}
|
||||
|
||||
func (s *Scraper) isEffectPage(pagePath, title string, lcCats []string) bool {
|
||||
if strings.Contains(strings.ToLower(title), "effect") {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, category := range lcCats {
|
||||
if strings.Contains(category, "effects") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Contains(strings.ToLower(pagePath), "/effect")
|
||||
}
|
||||
|
||||
func (s *Scraper) parseItemPage(doc *goquery.Document, title, pageURL string, categories []string) model.Item {
|
||||
item := model.Item{
|
||||
Name: title,
|
||||
URL: pageURL,
|
||||
Categories: categories,
|
||||
Infobox: s.parsePortableInfobox(doc),
|
||||
}
|
||||
|
||||
item.Description = s.parseDescription(doc)
|
||||
item.Effects, item.EffectLinks = s.parseEffectsSection(doc)
|
||||
item.Recipes = s.parseRecipesFromPage(doc, title)
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
func (s *Scraper) parseEffectPage(doc *goquery.Document, title, pageURL string, categories []string) model.Effect {
|
||||
return model.Effect{
|
||||
Name: title,
|
||||
URL: pageURL,
|
||||
Categories: categories,
|
||||
Infobox: s.parsePortableInfobox(doc),
|
||||
Description: s.parseDescription(doc),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scraper) parsePortableInfobox(doc *goquery.Document) map[string]string {
|
||||
out := map[string]string{}
|
||||
|
||||
doc.Find("aside.portable-infobox .pi-item").Each(func(_ int, section *goquery.Selection) {
|
||||
label := s.clean(section.Find(".pi-data-label").First().Text())
|
||||
value := s.clean(section.Find(".pi-data-value").First().Text())
|
||||
|
||||
if label == "" {
|
||||
label = s.clean(section.Find("h3").First().Text())
|
||||
}
|
||||
if value == "" {
|
||||
value = s.clean(section.Find("div").Last().Text())
|
||||
}
|
||||
|
||||
if label != "" && value != "" {
|
||||
out[label] = value
|
||||
}
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Scraper) parseCategories(doc *goquery.Document) []string {
|
||||
var categories []string
|
||||
seen := map[string]bool{}
|
||||
|
||||
doc.Find(".page-header__categories a, .category-page__member-link, .wds-tabs__tab-label a").Each(func(_ int, node *goquery.Selection) {
|
||||
text := s.clean(node.Text())
|
||||
if text != "" && !seen[text] {
|
||||
seen[text] = true
|
||||
categories = append(categories, text)
|
||||
}
|
||||
})
|
||||
|
||||
if len(categories) == 0 {
|
||||
headerText := s.clean(doc.Find("body").Text())
|
||||
if idx := strings.Index(headerText, "in:"); idx >= 0 {
|
||||
line := strings.TrimPrefix(headerText[idx:], "in:")
|
||||
for _, part := range strings.Split(line, ",") {
|
||||
part = s.clean(part)
|
||||
if part != "" && !seen[part] {
|
||||
seen[part] = true
|
||||
categories = append(categories, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return categories
|
||||
}
|
||||
|
||||
func (s *Scraper) parseDescription(doc *goquery.Document) string {
|
||||
var description string
|
||||
|
||||
doc.Find(".mw-parser-output > p").EachWithBreak(func(_ int, paragraph *goquery.Selection) bool {
|
||||
text := s.clean(paragraph.Text())
|
||||
if text == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
lower := strings.ToLower(text)
|
||||
if strings.Contains(lower, "is a") || strings.Contains(lower, "is an") || len(text) > 30 {
|
||||
description = text
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
return description
|
||||
}
|
||||
|
||||
func (s *Scraper) parseEffectsSection(doc *goquery.Document) ([]string, []string) {
|
||||
var effects []string
|
||||
var effectLinks []string
|
||||
|
||||
seenText := map[string]bool{}
|
||||
seenLink := map[string]bool{}
|
||||
section := s.findSection(doc, "Effects")
|
||||
if section.Length() == 0 {
|
||||
return effects, effectLinks
|
||||
}
|
||||
|
||||
s.walkSectionUntilNextHeading(section, func(node *goquery.Selection) {
|
||||
node.Find("li").Each(func(_ int, item *goquery.Selection) {
|
||||
text := s.clean(item.Text())
|
||||
if text != "" && !seenText[text] {
|
||||
seenText[text] = true
|
||||
effects = append(effects, text)
|
||||
}
|
||||
|
||||
item.Find("a[href^='/wiki/']").Each(func(_ int, anchor *goquery.Selection) {
|
||||
href, _ := anchor.Attr("href")
|
||||
label := s.clean(anchor.Text())
|
||||
if href != "" && label != "" && s.looksLikeEffectLink(href, label) && !seenLink[href] {
|
||||
seenLink[href] = true
|
||||
effectLinks = append(effectLinks, href)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return effects, effectLinks
|
||||
}
|
||||
|
||||
func (s *Scraper) parseRecipesFromPage(doc *goquery.Document, pageTitle string) []model.Recipe {
|
||||
var recipes []model.Recipe
|
||||
|
||||
doc.Find("table").Each(func(_ int, table *goquery.Selection) {
|
||||
headerText := strings.ToLower(s.clean(table.Find("tr").First().Text()))
|
||||
if !(strings.Contains(headerText, "result") && strings.Contains(headerText, "ingredient")) {
|
||||
return
|
||||
}
|
||||
|
||||
table.Find("tr").Each(func(i int, row *goquery.Selection) {
|
||||
if i == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
cells := row.Find("th, td")
|
||||
if cells.Length() < 2 {
|
||||
return
|
||||
}
|
||||
|
||||
resultCell := cells.Eq(0)
|
||||
ingredientCell := cells.Eq(1)
|
||||
stationCell := cells.Eq(2)
|
||||
|
||||
resultText := s.clean(resultCell.Text())
|
||||
if resultText == "" {
|
||||
return
|
||||
}
|
||||
|
||||
resultCount := ""
|
||||
if matches := s.cfg.AmountPrefixRe.FindStringSubmatch(resultText); len(matches) > 1 {
|
||||
resultCount = matches[1]
|
||||
resultText = s.clean(strings.TrimPrefix(resultText, matches[0]))
|
||||
}
|
||||
|
||||
var ingredients []string
|
||||
ingredientCell.Find("li").Each(func(_ int, ingredient *goquery.Selection) {
|
||||
text := s.clean(ingredient.Text())
|
||||
if text != "" {
|
||||
ingredients = append(ingredients, text)
|
||||
}
|
||||
})
|
||||
|
||||
if len(ingredients) == 0 {
|
||||
for _, line := range s.splitLines(ingredientCell.Text()) {
|
||||
if line != "" {
|
||||
ingredients = append(ingredients, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
recipes = append(recipes, model.Recipe{
|
||||
Result: resultText,
|
||||
ResultCount: resultCount,
|
||||
Ingredients: ingredients,
|
||||
Station: s.clean(stationCell.Text()),
|
||||
SourcePage: pageTitle,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return recipes
|
||||
}
|
||||
|
||||
func (s *Scraper) findSection(doc *goquery.Document, title string) *goquery.Selection {
|
||||
var found *goquery.Selection
|
||||
|
||||
doc.Find(".mw-parser-output h2, .mw-parser-output h3, .mw-parser-output h4").EachWithBreak(func(_ int, section *goquery.Selection) bool {
|
||||
text := strings.TrimSpace(strings.TrimSuffix(s.clean(section.Text()), "[]"))
|
||||
if strings.EqualFold(text, title) {
|
||||
found = section
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if found == nil {
|
||||
return &goquery.Selection{}
|
||||
}
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
func (s *Scraper) walkSectionUntilNextHeading(start *goquery.Selection, fn func(*goquery.Selection)) {
|
||||
for section := start.Next(); section.Length() > 0; section = section.Next() {
|
||||
if goquery.NodeName(section) == "h2" || goquery.NodeName(section) == "h3" || goquery.NodeName(section) == "h4" {
|
||||
break
|
||||
}
|
||||
fn(section)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scraper) looksLikeEffectLink(href, label string) bool {
|
||||
lowerHref := strings.ToLower(href)
|
||||
lowerLabel := strings.ToLower(label)
|
||||
|
||||
if strings.Contains(lowerHref, "/wiki/") && (strings.Contains(lowerHref, "effect") || strings.Contains(lowerLabel, "effect")) {
|
||||
return true
|
||||
}
|
||||
|
||||
knownHints := []string{
|
||||
"bleeding", "burning", "poisoned", "possessed", "discipline", "rage",
|
||||
"sapped", "scorched", "curse", "doomed", "chill", "warm", "cool",
|
||||
"barrier", "protection", "imbue", "energized", "shimmer",
|
||||
}
|
||||
|
||||
for _, hint := range knownHints {
|
||||
if strings.Contains(lowerLabel, hint) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Scraper) absoluteWikiURL(href string) string {
|
||||
if href == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
|
||||
if strings.HasPrefix(href, "/wiki/") {
|
||||
return s.cfg.BaseURL + href
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Scraper) flattenItems() []model.Item {
|
||||
out := make([]model.Item, 0, len(s.items))
|
||||
for _, item := range s.items {
|
||||
out = append(out, *item)
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Scraper) flattenEffects() []model.Effect {
|
||||
out := make([]model.Effect, 0, len(s.effects))
|
||||
for _, effect := range s.effects {
|
||||
out = append(out, *effect)
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Scraper) clean(value string) string {
|
||||
value = strings.ReplaceAll(value, "\u00a0", " ")
|
||||
value = strings.ReplaceAll(value, "\n", " ")
|
||||
value = strings.ReplaceAll(value, "\t", " ")
|
||||
value = strings.TrimSpace(value)
|
||||
return s.cfg.WhitespaceRe.ReplaceAllString(value, " ")
|
||||
}
|
||||
|
||||
func (s *Scraper) splitLines(value string) []string {
|
||||
value = strings.ReplaceAll(value, "\u00a0", " ")
|
||||
raw := strings.Split(value, "\n")
|
||||
out := make([]string, 0, len(raw))
|
||||
|
||||
for _, line := range raw {
|
||||
line = s.clean(line)
|
||||
if line != "" {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Scraper) lowerSlice(in []string) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
for _, value := range in {
|
||||
out = append(out, strings.ToLower(value))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Scraper) debugURLName(raw string) string {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return path.Base(parsed.Path)
|
||||
}
|
||||
Reference in New Issue
Block a user