feat(seo): add robots.txt, sitemap, and noindex tags for downloads

Register routes for robots.txt and sitemap.xml, and implement search engine indexing controls to protect user privacy.

Specifically:
- Set `X-Robots-Tag: noindex, nofollow, noarchive` headers on file downloads, thumbnails, and zip generation.
- Configure `Robots: web.RobotsNone` on download and preview pages to prevent indexing of temporary user uploads.
- Add canonical URLs, improved descriptions, and image alt tags to page metadata for better social sharing.

backend/libs/handlers/app.go

@@ -134,6 +134,8 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("GET /d/{boxID}/f/{fileID}/download", a.DownloadFileContent)
 	mux.HandleFunc("GET /d/{boxID}/thumb/{fileID}", a.Thumbnail)
 	mux.HandleFunc("GET /d/{boxID}/og-image.jpg", a.BoxOGImage)
+	mux.HandleFunc("GET /robots.txt", a.RobotsTxt)
+	mux.HandleFunc("GET /sitemap.xml", a.SitemapXML)
 	mux.HandleFunc("GET /health", a.Health)
 	mux.HandleFunc("GET /healthz", notFound)
 	mux.HandleFunc("GET /api/v1/health", notFound)

backend/libs/handlers/download.go

@@ -136,10 +136,19 @@ func (a *App) DownloadPage(w http.ResponseWriter, r *http.Request) {
 		description = "This shared box is password protected."
 	}
 
+	pageURL := absoluteURL(r, fmt.Sprintf("/d/%s", box.ID))
+	ogImage := absoluteURL(r, fmt.Sprintf("/d/%s/og-image.jpg", box.ID))
+
+	// All user uploads are private/temporary — noindex by default.
+	robots := web.RobotsNone
+
 	a.renderPage(w, r, http.StatusOK, "download.html", web.PageData{
-		Title:       title,
-		Description: description,
-		ImageURL:    absoluteURL(r, fmt.Sprintf("/d/%s/og-image.jpg", box.ID)),
+		Title:        title,
+		Description:  description,
+		CanonicalURL: pageURL,
+		Robots:       robots,
+		ImageURL:     ogImage,
+		ImageAlt:     fmt.Sprintf("%d shared file%s on Warp Box", len(box.Files), plural(len(box.Files))),
 		Data: downloadPageData{
 			Box:           boxView{ID: box.ID},
 			Files:         files,
@@ -179,19 +188,27 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	view := a.fileView(box, file)
+	fileSize := helpers.FormatBytes(file.Size)
 	title := file.Name
-	description := fmt.Sprintf("%s shared via Warpbox", helpers.FormatBytes(file.Size))
+	description := fmt.Sprintf("%s · %s file shared via Warp Box", fileSize, file.ContentType)
 	imageURL := absoluteURL(r, view.ThumbnailURL)
+	imageAlt := fmt.Sprintf("Preview of %s", file.Name)
 	if locked && box.Obfuscate {
 		title = "Protected Warpbox file"
 		description = "This shared file is password protected."
 		imageURL = absoluteURL(r, "/static/img/file-placeholder.webp")
+		imageAlt = "Password protected file on Warp Box"
 	}
 
+	pageURL := absoluteURL(r, fmt.Sprintf("/d/%s/f/%s", box.ID, file.ID))
+
 	a.renderPage(w, r, http.StatusOK, "preview.html", web.PageData{
-		Title:       title,
-		Description: description,
-		ImageURL:    imageURL,
+		Title:        title,
+		Description:  description,
+		CanonicalURL: pageURL,
+		Robots:       web.RobotsNone,
+		ImageURL:     imageURL,
+		ImageAlt:     imageAlt,
 		Data: previewPageData{
 			Box:         boxView{ID: box.ID},
 			File:        view,
@@ -203,6 +220,7 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, file, ok := a.loadFileForRequest(w, r)
 	if !ok {
 		return
@@ -222,6 +240,7 @@ func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *App) Thumbnail(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, file, ok := a.loadFileForRequest(w, r)
 	if !ok {
 		return
@@ -342,6 +361,7 @@ func readSeekCloser(source io.ReadCloser) io.ReadSeeker {
 }
 
 func (a *App) DownloadZip(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, err := a.uploadService.GetBox(r.PathValue("boxID"))
 	if err != nil {
 		a.logger.Warn("zip request missing box", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4044, "box_id", r.PathValue("boxID"))...)

backend/libs/handlers/meta.go

@@ -0,0 +1,58 @@
+package handlers
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// RobotsTxt serves /robots.txt dynamically so the Sitemap URL reflects the
+// configured base URL rather than a hard-coded placeholder.
+func (a *App) RobotsTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=86400")
+	fmt.Fprintf(w, `User-agent: *
+Allow: /
+
+# Private routes — do not crawl
+Disallow: /admin/
+Disallow: /api/
+Disallow: /app/
+Disallow: /account/
+Disallow: /d/*/f/*/download
+Disallow: /d/*/zip
+Disallow: /d/*/thumb/
+Disallow: /d/*/og-image.jpg
+Disallow: /d/*/unlock
+Disallow: /d/*/manage/
+
+Sitemap: %s/sitemap.xml
+`, strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/"))
+}
+
+// SitemapXML serves a minimal /sitemap.xml containing only the public,
+// indexable homepage. Box/file pages are noindex and deliberately excluded.
+func (a *App) SitemapXML(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=3600")
+	baseURL := strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/")
+	lastMod := time.Now().UTC().Format("2006-01-02")
+	fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>%s/</loc>
+    <lastmod>%s</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>1.0</priority>
+  </url>
+</urlset>
+`, baseURL, lastMod)
+}
+
+func siteBaseURL(r *http.Request, configured string) string {
+	if configured != "" {
+		return configured
+	}
+	return absoluteURL(r, "/")
+}

backend/libs/handlers/pages.go

@@ -60,9 +60,12 @@ func (a *App) Home(w http.ResponseWriter, r *http.Request) {
 	maxUploadSize, limitSummary := a.homeUploadPolicyLabels(settings, user, loggedIn, isAdmin)
 	expiryOptions, defaultExpiry := a.homeExpiryOptions(settings, user, loggedIn, isAdmin)
 	a.renderPage(w, r, http.StatusOK, "home.html", web.PageData{
-		Title:       "Upload your files",
-		Description: "Upload and share files through a self-hosted Warpbox instance.",
-		CurrentUser: currentUser,
+		Title:        "Upload your files",
+		Description:  "Upload and share files fast. Drop a file, get a link — private, temporary transfers that expire on your terms.",
+		CanonicalURL: absoluteURL(r, "/"),
+		ImageURL:     absoluteURL(r, "/static/og-default.png"),
+		ImageAlt:     "Warp Box — simple file sharing and fast downloads",
+		CurrentUser:  currentUser,
 		Data: homeData{
 			MaxUploadSize:        maxUploadSize,
 			LimitSummary:         limitSummary,

backend/libs/web/renderer.go

@@ -7,6 +7,9 @@ import (
 	"time"
 )
 
+// RobotsNone is used for private, protected, expired, or temporary pages.
+const RobotsNone = "noindex,nofollow,noarchive"
+
 type Renderer struct {
 	templates  map[string]*template.Template
 	appName    string
@@ -15,16 +18,19 @@ type Renderer struct {
 }
 
 type PageData struct {
-	AppName     string
-	AppVersion  string
-	BaseURL     string
-	Title       string
-	Description string
-	ImageURL    string
-	CurrentYear int
-	CurrentUser any
-	CSRFToken   string
-	Data        any
+	AppName      string
+	AppVersion   string
+	BaseURL      string
+	CanonicalURL string
+	Robots       string
+	Title        string
+	Description  string
+	ImageURL     string
+	ImageAlt     string
+	CurrentYear  int
+	CurrentUser  any
+	CSRFToken    string
+	Data         any
 }
 
 func NewRenderer(templateDir, appName, appVersion, baseURL string) (*Renderer, error) {