package handlers import ( "fmt" "net/http" "strings" "time" ) // RobotsTxt serves /robots.txt dynamically so the Sitemap URL reflects the // configured base URL rather than a hard-coded placeholder. func (a *App) RobotsTxt(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Cache-Control", "public, max-age=86400") fmt.Fprintf(w, `User-agent: * Allow: / # Private routes — do not crawl Disallow: /admin/ Disallow: /api/ Disallow: /app/ Disallow: /account/ Disallow: /d/*/f/*/download Disallow: /d/*/zip Disallow: /d/*/thumb/ Disallow: /d/*/scene/ Disallow: /d/*/archive/ Disallow: /d/*/og-image.jpg Disallow: /d/*/unlock Disallow: /d/*/manage/ Sitemap: %s/sitemap.xml `, strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/")) } // SitemapXML serves a minimal /sitemap.xml containing only the public, // indexable homepage. Box/file pages are noindex and deliberately excluded. func (a *App) SitemapXML(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml; charset=utf-8") w.Header().Set("Cache-Control", "public, max-age=3600") baseURL := strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/") lastMod := time.Now().UTC().Format("2006-01-02") fmt.Fprintf(w, ` %s/ %s weekly 1.0 `, baseURL, lastMod) } func siteBaseURL(r *http.Request, configured string) string { if configured != "" { return configured } return absoluteURL(r, "/") }