feat(scraper): add checkpointing and richer page extraction
Add resumable checkpoint support so long scrapes can recover from interruptions instead of restarting from scratch. - introduce autosave/load/clear checkpoint flow in `.cache/scrape-state.json`, including SIGINT/SIGTERM save-on-exit handling - expand parsing/model output to capture legacy and portable infobox fields, primary image URLs, effects, recipes, raw tables, and improved category extraction - skip infobox tables during recipe parsing to avoid false recipe matches - add cache log event type, ignore cache/output artifacts, and document new autosave tuning options in READMEfeat(scraper): add checkpointing and richer page extraction Add resumable checkpoint support so long scrapes can recover from interruptions instead of restarting from scratch. - introduce autosave/load/clear checkpoint flow in `.cache/scrape-state.json`, including SIGINT/SIGTERM save-on-exit handling - expand parsing/model output to capture legacy and portable infobox fields, primary image URLs, effects, recipes, raw tables, and improved category extraction - skip infobox tables during recipe parsing to avoid false recipe matches - add cache log event type, ignore cache/output artifacts, and document new autosave tuning options in README
This commit is contained in:
@@ -8,14 +8,23 @@ type Recipe struct {
|
||||
SourcePage string `json:"source_page,omitempty"`
|
||||
}
|
||||
|
||||
type Table struct {
|
||||
Title string `json:"title,omitempty"`
|
||||
Headers []string `json:"headers,omitempty"`
|
||||
Rows []map[string]string `json:"rows,omitempty"`
|
||||
RawRows [][]string `json:"raw_rows,omitempty"`
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
Infobox map[string]string `json:"infobox,omitempty"`
|
||||
ImageURL string `json:"image_url,omitempty"`
|
||||
Effects []string `json:"effects,omitempty"`
|
||||
EffectLinks []string `json:"effect_links,omitempty"`
|
||||
Recipes []Recipe `json:"recipes,omitempty"`
|
||||
Tables []Table `json:"tables,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user