feat(seo): add robots.txt, sitemap, and noindex tags for downloads

Register routes for robots.txt and sitemap.xml, and implement search engine indexing controls to protect user privacy.

Specifically:
- Set `X-Robots-Tag: noindex, nofollow, noarchive` headers on file downloads, thumbnails, and zip generation.
- Configure `Robots: web.RobotsNone` on download and preview pages to prevent indexing of temporary user uploads.
- Add canonical URLs, improved descriptions, and image alt tags to page metadata for better social sharing.

backend/libs/handlers/app.go

@@ -134,6 +134,8 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("GET /d/{boxID}/f/{fileID}/download", a.DownloadFileContent)
 	mux.HandleFunc("GET /d/{boxID}/thumb/{fileID}", a.Thumbnail)
 	mux.HandleFunc("GET /d/{boxID}/og-image.jpg", a.BoxOGImage)
+	mux.HandleFunc("GET /robots.txt", a.RobotsTxt)
+	mux.HandleFunc("GET /sitemap.xml", a.SitemapXML)
 	mux.HandleFunc("GET /health", a.Health)
 	mux.HandleFunc("GET /healthz", notFound)
 	mux.HandleFunc("GET /api/v1/health", notFound)

backend/libs/handlers/download.go

@@ -136,10 +136,19 @@ func (a *App) DownloadPage(w http.ResponseWriter, r *http.Request) {
 		description = "This shared box is password protected."
 	}
 
+	pageURL := absoluteURL(r, fmt.Sprintf("/d/%s", box.ID))
+	ogImage := absoluteURL(r, fmt.Sprintf("/d/%s/og-image.jpg", box.ID))
+
+	// All user uploads are private/temporary — noindex by default.
+	robots := web.RobotsNone
+
 	a.renderPage(w, r, http.StatusOK, "download.html", web.PageData{
 		Title:        title,
 		Description:  description,
-		ImageURL:    absoluteURL(r, fmt.Sprintf("/d/%s/og-image.jpg", box.ID)),
+		CanonicalURL: pageURL,
+		Robots:       robots,
+		ImageURL:     ogImage,
+		ImageAlt:     fmt.Sprintf("%d shared file%s on Warp Box", len(box.Files), plural(len(box.Files))),
 		Data: downloadPageData{
 			Box:           boxView{ID: box.ID},
 			Files:         files,
@@ -179,19 +188,27 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	view := a.fileView(box, file)
+	fileSize := helpers.FormatBytes(file.Size)
 	title := file.Name
-	description := fmt.Sprintf("%s shared via Warpbox", helpers.FormatBytes(file.Size))
+	description := fmt.Sprintf("%s · %s file shared via Warp Box", fileSize, file.ContentType)
 	imageURL := absoluteURL(r, view.ThumbnailURL)
+	imageAlt := fmt.Sprintf("Preview of %s", file.Name)
 	if locked && box.Obfuscate {
 		title = "Protected Warpbox file"
 		description = "This shared file is password protected."
 		imageURL = absoluteURL(r, "/static/img/file-placeholder.webp")
+		imageAlt = "Password protected file on Warp Box"
 	}
 
+	pageURL := absoluteURL(r, fmt.Sprintf("/d/%s/f/%s", box.ID, file.ID))
+
 	a.renderPage(w, r, http.StatusOK, "preview.html", web.PageData{
 		Title:        title,
 		Description:  description,
+		CanonicalURL: pageURL,
+		Robots:       web.RobotsNone,
 		ImageURL:     imageURL,
+		ImageAlt:     imageAlt,
 		Data: previewPageData{
 			Box:         boxView{ID: box.ID},
 			File:        view,
@@ -203,6 +220,7 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, file, ok := a.loadFileForRequest(w, r)
 	if !ok {
 		return
@@ -222,6 +240,7 @@ func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *App) Thumbnail(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, file, ok := a.loadFileForRequest(w, r)
 	if !ok {
 		return
@@ -342,6 +361,7 @@ func readSeekCloser(source io.ReadCloser) io.ReadSeeker {
 }
 
 func (a *App) DownloadZip(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, err := a.uploadService.GetBox(r.PathValue("boxID"))
 	if err != nil {
 		a.logger.Warn("zip request missing box", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4044, "box_id", r.PathValue("boxID"))...)

backend/libs/handlers/meta.go

@@ -0,0 +1,58 @@
+package handlers
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// RobotsTxt serves /robots.txt dynamically so the Sitemap URL reflects the
+// configured base URL rather than a hard-coded placeholder.
+func (a *App) RobotsTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=86400")
+	fmt.Fprintf(w, `User-agent: *
+Allow: /
+
+# Private routes — do not crawl
+Disallow: /admin/
+Disallow: /api/
+Disallow: /app/
+Disallow: /account/
+Disallow: /d/*/f/*/download
+Disallow: /d/*/zip
+Disallow: /d/*/thumb/
+Disallow: /d/*/og-image.jpg
+Disallow: /d/*/unlock
+Disallow: /d/*/manage/
+
+Sitemap: %s/sitemap.xml
+`, strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/"))
+}
+
+// SitemapXML serves a minimal /sitemap.xml containing only the public,
+// indexable homepage. Box/file pages are noindex and deliberately excluded.
+func (a *App) SitemapXML(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=3600")
+	baseURL := strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/")
+	lastMod := time.Now().UTC().Format("2006-01-02")
+	fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>%s/</loc>
+    <lastmod>%s</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>1.0</priority>
+  </url>
+</urlset>
+`, baseURL, lastMod)
+}
+
+func siteBaseURL(r *http.Request, configured string) string {
+	if configured != "" {
+		return configured
+	}
+	return absoluteURL(r, "/")
+}

backend/libs/handlers/pages.go

@@ -61,7 +61,10 @@ func (a *App) Home(w http.ResponseWriter, r *http.Request) {
 	expiryOptions, defaultExpiry := a.homeExpiryOptions(settings, user, loggedIn, isAdmin)
 	a.renderPage(w, r, http.StatusOK, "home.html", web.PageData{
 		Title:        "Upload your files",
-		Description: "Upload and share files through a self-hosted Warpbox instance.",
+		Description:  "Upload and share files fast. Drop a file, get a link — private, temporary transfers that expire on your terms.",
+		CanonicalURL: absoluteURL(r, "/"),
+		ImageURL:     absoluteURL(r, "/static/og-default.png"),
+		ImageAlt:     "Warp Box — simple file sharing and fast downloads",
 		CurrentUser:  currentUser,
 		Data: homeData{
 			MaxUploadSize:        maxUploadSize,

backend/libs/web/renderer.go

@@ -7,6 +7,9 @@ import (
 	"time"
 )
 
+// RobotsNone is used for private, protected, expired, or temporary pages.
+const RobotsNone = "noindex,nofollow,noarchive"
+
 type Renderer struct {
 	templates  map[string]*template.Template
 	appName    string
@@ -18,9 +21,12 @@ type PageData struct {
 	AppName      string
 	AppVersion   string
 	BaseURL      string
+	CanonicalURL string
+	Robots       string
 	Title        string
 	Description  string
 	ImageURL     string
+	ImageAlt     string
 	CurrentYear  int
 	CurrentUser  any
 	CSRFToken    string

backend/static/humans.txt

@@ -0,0 +1,6 @@
+/* TEAM */
+Built by: Danlegt
+
+/* SITE */
+Language: English
+Software: Warp Box

backend/static/llms.txt

@@ -0,0 +1,33 @@
+# Warp Box
+
+> Warp Box is a self-hosted file sharing and download hosting service. Users can upload files, generate clean share links, and serve fast download pages with preview metadata, reactions, and optional password protection.
+
+Warp Box is designed for privacy-first file transfers. Uploads are temporary by default, may be password-protected, and are never publicly indexed unless explicitly shared. Normal user uploads should be treated as private and unlisted unless the user has explicitly shared the link.
+
+## Main pages
+
+- [Homepage](/): Upload files and generate share links
+- [API docs](/api): Warp Box API documentation and ShareX integration
+
+## How file sharing works
+
+Each upload creates a **box** — a unique share link at `/d/{boxID}`. A box can contain one or more files. Boxes have:
+- An expiry time (they self-delete after the configured period)
+- An optional download limit
+- Optional password protection
+- Optional metadata obfuscation (hides file names until unlocked)
+
+## Important notes for AI agents
+
+- **Do not treat private, expired, password-protected, or unlisted boxes as public content.** Most boxes are anonymous and temporary.
+- **Raw download URLs** (`/d/{boxID}/f/{fileID}/download`) are not canonical pages. Prefer the box preview page (`/d/{boxID}`) when referencing a shared file.
+- Box pages at `/d/{boxID}` are the canonical share URLs.
+- File preview pages at `/d/{boxID}/f/{fileID}` are per-file landing pages.
+- `/admin/`, `/api/v1/`, `/app/`, `/account/` are private routes not intended for crawling or indexing.
+- Do not index or summarize file contents from raw download endpoints.
+
+## Technical metadata
+
+- Robots file: /robots.txt
+- Sitemap: /sitemap.xml
+- Web manifest: /static/site.webmanifest

backend/static/site.webmanifest

@@ -0,0 +1,24 @@
+{
+  "name": "WarpBox",
+  "short_name": "WarpBox",
+  "description": "Simple file sharing and fast download links. Upload files, generate share links, and serve clean download pages.",
+  "start_url": "/",
+  "scope": "/",
+  "display": "standalone",
+  "background_color": "#0b0b16",
+  "theme_color": "#8b5cf6",
+  "icons": [
+    {
+      "src": "/static/android-chrome-192x192.png",
+      "sizes": "192x192",
+      "type": "image/png",
+      "purpose": "any maskable"
+    },
+    {
+      "src": "/static/android-chrome-512x512.png",
+      "sizes": "512x512",
+      "type": "image/png",
+      "purpose": "any maskable"
+    }
+  ]
+}

backend/templates/layouts/base.html

@@ -4,17 +4,40 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>{{if .Title}}{{.Title}} - {{end}}{{.AppName}}</title>
+  <title>{{if .Title}}{{.Title}} — {{end}}{{.AppName}}</title>
   <meta name="description" content="{{.Description}}">
-  <meta name="theme-color" content="#09090b">
+  {{if .CanonicalURL}}<link rel="canonical" href="{{.CanonicalURL}}">{{end}}
+  <meta name="robots" content="{{if .Robots}}{{.Robots}}{{else}}index,follow{{end}}">
+  <meta name="generator" content="Warp Box {{.AppVersion}}">
+
   <meta property="og:site_name" content="{{.AppName}}">
+  <meta property="og:type" content="website">
   <meta property="og:title" content="{{if .Title}}{{.Title}}{{else}}{{.AppName}}{{end}}">
   <meta property="og:description" content="{{.Description}}">
-  <meta property="og:type" content="website">
-  <meta property="og:url" content="{{.BaseURL}}">
-  {{if .ImageURL}}<meta property="og:image" content="{{.ImageURL}}">{{end}}
+  <meta property="og:url" content="{{if .CanonicalURL}}{{.CanonicalURL}}{{else}}{{.BaseURL}}{{end}}">
+  {{if .ImageURL}}
+  <meta property="og:image" content="{{.ImageURL}}">
+  <meta property="og:image:width" content="1200">
+  <meta property="og:image:height" content="630">
+  {{if .ImageAlt}}<meta property="og:image:alt" content="{{.ImageAlt}}">{{else}}<meta property="og:image:alt" content="{{.AppName}} preview">{{end}}
+  {{end}}
+
   <meta name="twitter:card" content="summary_large_image">
-  {{if .ImageURL}}<meta name="twitter:image" content="{{.ImageURL}}">{{end}}
+  <meta name="twitter:title" content="{{if .Title}}{{.Title}}{{else}}{{.AppName}}{{end}}">
+  <meta name="twitter:description" content="{{.Description}}">
+  {{if .ImageURL}}
+  <meta name="twitter:image" content="{{.ImageURL}}">
+  {{if .ImageAlt}}<meta name="twitter:image:alt" content="{{.ImageAlt}}">{{else}}<meta name="twitter:image:alt" content="{{.AppName}} preview">{{end}}
+  {{end}}
+
+  <link rel="icon" href="/static/favicon.ico" sizes="any">
+  <link rel="icon" type="image/png" sizes="16x16" href="/static/favicon-16x16.png">
+  <link rel="icon" type="image/png" sizes="32x32" href="/static/favicon-32x32.png">
+  <link rel="apple-touch-icon" href="/static/apple-touch-icon.png">
+  <link rel="manifest" href="/static/site.webmanifest">
+  <meta name="theme-color" content="#8b5cf6">
+  <meta name="msapplication-TileColor" content="#0b0b16">
+
   <script src="/static/js/05-theme.js?version={{.AppVersion}}"></script>
   <link rel="stylesheet" href="/static/css/00-base.css?version={{.AppVersion}}">
   <link rel="stylesheet" href="/static/css/10-layout.css?version={{.AppVersion}}">