Daniel Legt
e17c5e92a7
All checks were successful
Build and Publish Docker Image / deploy (push) Successful in 2m2s
Register routes for robots.txt and sitemap.xml, and implement search engine indexing controls to protect user privacy. Specifically: - Set `X-Robots-Tag: noindex, nofollow, noarchive` headers on file downloads, thumbnails, and zip generation. - Configure `Robots: web.RobotsNone` on download and preview pages to prevent indexing of temporary user uploads. - Add canonical URLs, improved descriptions, and image alt tags to page metadata for better social sharing.
59 lines
1.6 KiB
Go
59 lines
1.6 KiB
Go
package handlers
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// RobotsTxt serves /robots.txt dynamically so the Sitemap URL reflects the
|
|
// configured base URL rather than a hard-coded placeholder.
|
|
func (a *App) RobotsTxt(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "public, max-age=86400")
|
|
fmt.Fprintf(w, `User-agent: *
|
|
Allow: /
|
|
|
|
# Private routes — do not crawl
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
Disallow: /app/
|
|
Disallow: /account/
|
|
Disallow: /d/*/f/*/download
|
|
Disallow: /d/*/zip
|
|
Disallow: /d/*/thumb/
|
|
Disallow: /d/*/og-image.jpg
|
|
Disallow: /d/*/unlock
|
|
Disallow: /d/*/manage/
|
|
|
|
Sitemap: %s/sitemap.xml
|
|
`, strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/"))
|
|
}
|
|
|
|
// SitemapXML serves a minimal /sitemap.xml containing only the public,
|
|
// indexable homepage. Box/file pages are noindex and deliberately excluded.
|
|
func (a *App) SitemapXML(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "public, max-age=3600")
|
|
baseURL := strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/")
|
|
lastMod := time.Now().UTC().Format("2006-01-02")
|
|
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
|
<url>
|
|
<loc>%s/</loc>
|
|
<lastmod>%s</lastmod>
|
|
<changefreq>weekly</changefreq>
|
|
<priority>1.0</priority>
|
|
</url>
|
|
</urlset>
|
|
`, baseURL, lastMod)
|
|
}
|
|
|
|
func siteBaseURL(r *http.Request, configured string) string {
|
|
if configured != "" {
|
|
return configured
|
|
}
|
|
return absoluteURL(r, "/")
|
|
}
|