feat(seo): add robots.txt, sitemap, and noindex tags for downloads

Register routes for robots.txt and sitemap.xml, and implement search engine indexing controls to protect user privacy.

Specifically:
- Set `X-Robots-Tag: noindex, nofollow, noarchive` headers on file downloads, thumbnails, and zip generation.
- Configure `Robots: web.RobotsNone` on download and preview pages to prevent indexing of temporary user uploads.
- Add canonical URLs, improved descriptions, and image alt tags to page metadata for better social sharing.

backend/libs/handlers/app.go

@@ -134,6 +134,8 @@ func (a *App) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("GET /d/{boxID}/f/{fileID}/download", a.DownloadFileContent)
 	mux.HandleFunc("GET /d/{boxID}/thumb/{fileID}", a.Thumbnail)
 	mux.HandleFunc("GET /d/{boxID}/og-image.jpg", a.BoxOGImage)
+	mux.HandleFunc("GET /robots.txt", a.RobotsTxt)
+	mux.HandleFunc("GET /sitemap.xml", a.SitemapXML)
 	mux.HandleFunc("GET /health", a.Health)
 	mux.HandleFunc("GET /healthz", notFound)
 	mux.HandleFunc("GET /api/v1/health", notFound)

backend/libs/handlers/download.go

@@ -136,10 +136,19 @@ func (a *App) DownloadPage(w http.ResponseWriter, r *http.Request) {
 		description = "This shared box is password protected."
 	}
 
+	pageURL := absoluteURL(r, fmt.Sprintf("/d/%s", box.ID))
+	ogImage := absoluteURL(r, fmt.Sprintf("/d/%s/og-image.jpg", box.ID))
+
+	// All user uploads are private/temporary — noindex by default.
+	robots := web.RobotsNone
+
 	a.renderPage(w, r, http.StatusOK, "download.html", web.PageData{
-		Title:       title,
-		Description: description,
-		ImageURL:    absoluteURL(r, fmt.Sprintf("/d/%s/og-image.jpg", box.ID)),
+		Title:        title,
+		Description:  description,
+		CanonicalURL: pageURL,
+		Robots:       robots,
+		ImageURL:     ogImage,
+		ImageAlt:     fmt.Sprintf("%d shared file%s on Warp Box", len(box.Files), plural(len(box.Files))),
 		Data: downloadPageData{
 			Box:           boxView{ID: box.ID},
 			Files:         files,
@@ -179,19 +188,27 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	view := a.fileView(box, file)
+	fileSize := helpers.FormatBytes(file.Size)
 	title := file.Name
-	description := fmt.Sprintf("%s shared via Warpbox", helpers.FormatBytes(file.Size))
+	description := fmt.Sprintf("%s · %s file shared via Warp Box", fileSize, file.ContentType)
 	imageURL := absoluteURL(r, view.ThumbnailURL)
+	imageAlt := fmt.Sprintf("Preview of %s", file.Name)
 	if locked && box.Obfuscate {
 		title = "Protected Warpbox file"
 		description = "This shared file is password protected."
 		imageURL = absoluteURL(r, "/static/img/file-placeholder.webp")
+		imageAlt = "Password protected file on Warp Box"
 	}
 
+	pageURL := absoluteURL(r, fmt.Sprintf("/d/%s/f/%s", box.ID, file.ID))
+
 	a.renderPage(w, r, http.StatusOK, "preview.html", web.PageData{
-		Title:       title,
-		Description: description,
-		ImageURL:    imageURL,
+		Title:        title,
+		Description:  description,
+		CanonicalURL: pageURL,
+		Robots:       web.RobotsNone,
+		ImageURL:     imageURL,
+		ImageAlt:     imageAlt,
 		Data: previewPageData{
 			Box:         boxView{ID: box.ID},
 			File:        view,
@@ -203,6 +220,7 @@ func (a *App) DownloadFile(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, file, ok := a.loadFileForRequest(w, r)
 	if !ok {
 		return
@@ -222,6 +240,7 @@ func (a *App) DownloadFileContent(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *App) Thumbnail(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, file, ok := a.loadFileForRequest(w, r)
 	if !ok {
 		return
@@ -342,6 +361,7 @@ func readSeekCloser(source io.ReadCloser) io.ReadSeeker {
 }
 
 func (a *App) DownloadZip(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Robots-Tag", "noindex, nofollow, noarchive")
 	box, err := a.uploadService.GetBox(r.PathValue("boxID"))
 	if err != nil {
 		a.logger.Warn("zip request missing box", withRequestLogAttrs(r, "source", "download", "severity", "warn", "code", 4044, "box_id", r.PathValue("boxID"))...)

backend/libs/handlers/meta.go

@@ -0,0 +1,58 @@
+package handlers
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// RobotsTxt serves /robots.txt dynamically so the Sitemap URL reflects the
+// configured base URL rather than a hard-coded placeholder.
+func (a *App) RobotsTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=86400")
+	fmt.Fprintf(w, `User-agent: *
+Allow: /
+
+# Private routes — do not crawl
+Disallow: /admin/
+Disallow: /api/
+Disallow: /app/
+Disallow: /account/
+Disallow: /d/*/f/*/download
+Disallow: /d/*/zip
+Disallow: /d/*/thumb/
+Disallow: /d/*/og-image.jpg
+Disallow: /d/*/unlock
+Disallow: /d/*/manage/
+
+Sitemap: %s/sitemap.xml
+`, strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/"))
+}
+
+// SitemapXML serves a minimal /sitemap.xml containing only the public,
+// indexable homepage. Box/file pages are noindex and deliberately excluded.
+func (a *App) SitemapXML(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=3600")
+	baseURL := strings.TrimRight(siteBaseURL(r, a.cfg.BaseURL), "/")
+	lastMod := time.Now().UTC().Format("2006-01-02")
+	fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>%s/</loc>
+    <lastmod>%s</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>1.0</priority>
+  </url>
+</urlset>
+`, baseURL, lastMod)
+}
+
+func siteBaseURL(r *http.Request, configured string) string {
+	if configured != "" {
+		return configured
+	}
+	return absoluteURL(r, "/")
+}

backend/libs/handlers/pages.go

@@ -60,9 +60,12 @@ func (a *App) Home(w http.ResponseWriter, r *http.Request) {
 	maxUploadSize, limitSummary := a.homeUploadPolicyLabels(settings, user, loggedIn, isAdmin)
 	expiryOptions, defaultExpiry := a.homeExpiryOptions(settings, user, loggedIn, isAdmin)
 	a.renderPage(w, r, http.StatusOK, "home.html", web.PageData{
-		Title:       "Upload your files",
-		Description: "Upload and share files through a self-hosted Warpbox instance.",
-		CurrentUser: currentUser,
+		Title:        "Upload your files",
+		Description:  "Upload and share files fast. Drop a file, get a link — private, temporary transfers that expire on your terms.",
+		CanonicalURL: absoluteURL(r, "/"),
+		ImageURL:     absoluteURL(r, "/static/og-default.png"),
+		ImageAlt:     "Warp Box — simple file sharing and fast downloads",
+		CurrentUser:  currentUser,
 		Data: homeData{
 			MaxUploadSize:        maxUploadSize,
 			LimitSummary:         limitSummary,
@@ -95,7 +98,7 @@ func (a *App) homeExpiryOptions(settings services.UploadPolicySettings, user ser
 }
 
 func buildExpiryOptions(maxDays int, unlimited bool) ([]expiryOption, int) {
-	ladder := []int{60, 720, 1440, 2880, 4320, 7200, 10080, 14400, 20160, 43200, 86400, 129600, 259200, 525600}
+	ladder := []int{60, 360, 720, 1440, 2880, 4320, 7200, 10080, 14400, 20160, 43200, 86400, 129600, 259200, 525600}
 
 	capMinutes := maxDays * 24 * 60
 	if unlimited || capMinutes <= 0 {

backend/libs/web/renderer.go

@@ -7,6 +7,9 @@ import (
 	"time"
 )
 
+// RobotsNone is used for private, protected, expired, or temporary pages.
+const RobotsNone = "noindex,nofollow,noarchive"
+
 type Renderer struct {
 	templates  map[string]*template.Template
 	appName    string
@@ -15,16 +18,19 @@ type Renderer struct {
 }
 
 type PageData struct {
-	AppName     string
-	AppVersion  string
-	BaseURL     string
-	Title       string
-	Description string
-	ImageURL    string
-	CurrentYear int
-	CurrentUser any
-	CSRFToken   string
-	Data        any
+	AppName      string
+	AppVersion   string
+	BaseURL      string
+	CanonicalURL string
+	Robots       string
+	Title        string
+	Description  string
+	ImageURL     string
+	ImageAlt     string
+	CurrentYear  int
+	CurrentUser  any
+	CSRFToken    string
+	Data         any
 }
 
 func NewRenderer(templateDir, appName, appVersion, baseURL string) (*Renderer, error) {

backend/static/css/20-upload.css

@@ -52,6 +52,10 @@
   margin-top: -0.25rem;
 }
 
+.upload-options .form-footer .upload-new-button[hidden] {
+  display: none !important;
+}
+
 .hero-copy {
   text-align: center;
 }

backend/static/humans.txt

@@ -0,0 +1,6 @@
+/* TEAM */
+Built by: Danlegt
+
+/* SITE */
+Language: English
+Software: Warp Box

backend/static/js/40-upload.js

@@ -117,9 +117,7 @@
         uploadQueue.hidden = true;
         uploadQueue.replaceChildren();
       }
-      if (newUpload) {
-        newUpload.hidden = true;
-      }
+      updateNewUploadVisibility();
       if (fileSummary) {
         fileSummary.textContent = "Upload complete.";
       }
@@ -189,9 +187,16 @@
       uploadQueue.hidden = true;
       uploadQueue.replaceChildren();
     }
-    if (newUpload) {
-      newUpload.hidden = !(resumeMode && recoveredDraft);
+    updateNewUploadVisibility();
+  }
+
+  function updateNewUploadVisibility() {
+    if (!newUpload) {
+      return;
     }
+    const visible = Boolean(resumeMode && recoveredDraft);
+    newUpload.hidden = !visible;
+    newUpload.style.display = visible ? "" : "none";
   }
 
   function setLoading(isLoading, submit) {
@@ -216,6 +221,41 @@
     }
   }
 
+  function updateUploadProgress(percent, bytesPerSecond) {
+    const clamped = Math.max(0, Math.min(100, Math.round(percent || 0)));
+    const rate = formatTransferRate(bytesPerSecond);
+    updateStatus(rate ? `${clamped}% · ${rate}` : `${clamped}%`);
+  }
+
+  function createTransferRateTracker(initialBytes) {
+    const startedAt = performance.now();
+    const baseline = Math.max(0, initialBytes || 0);
+    let lastRate = 0;
+    return function track(currentBytes) {
+      const elapsedSeconds = (performance.now() - startedAt) / 1000;
+      const transferred = Math.max(0, (currentBytes || 0) - baseline);
+      if (elapsedSeconds < 0.25 || transferred <= 0) {
+        return lastRate;
+      }
+      lastRate = transferred / elapsedSeconds;
+      return lastRate;
+    };
+  }
+
+  function formatTransferRate(bytesPerSecond) {
+    if (!Number.isFinite(bytesPerSecond) || bytesPerSecond <= 0) {
+      return "";
+    }
+    const units = ["b/s", "Kb/s", "Mb/s", "Gb/s"];
+    let value = bytesPerSecond * 8;
+    let unit = 0;
+    while (value >= 1000 && unit < units.length - 1) {
+      value /= 1000;
+      unit += 1;
+    }
+    return `${value >= 10 || unit === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}`;
+  }
+
   function renderResult(payload) {
     if (!result || !resultList || !resultMeta || !openBox) {
       return;
@@ -248,16 +288,18 @@
   function uploadWithProgress(url, formData, files) {
     return new Promise((resolve, reject) => {
       const request = new XMLHttpRequest();
+      const rateTracker = createTransferRateTracker(0);
       request.open("POST", url);
       request.setRequestHeader("Accept", "application/json");
 
       request.upload.addEventListener("progress", (event) => {
+        const rate = rateTracker(event.loaded || 0);
         if (!event.lengthComputable) {
-          updateStatus("Uploading...");
+          updateStatus(rate > 0 ? `Uploading · ${formatTransferRate(rate)}` : "Uploading...");
           return;
         }
         const percent = Math.round((event.loaded / event.total) * 100);
-        updateStatus(`${percent}%`);
+        updateUploadProgress(percent, rate);
         setTotalProgress(percent);
         setFileProgress(files, percent);
       });
@@ -348,7 +390,9 @@
       completedByFile[index] = uploadedBytesForSessionFile(sessionFile, session.chunkSize);
       setSingleFileProgress(index, files[index], percentForBytes(completedByFile[index], files[index].size));
     });
-    setTotalProgress(percentForBytes(completedByFile.reduce((sum, bytes) => sum + bytes, 0), totalBytes));
+    const initiallyUploadedBytes = completedByFile.reduce((sum, bytes) => sum + bytes, 0);
+    const rateTracker = createTransferRateTracker(initiallyUploadedBytes);
+    setTotalProgress(percentForBytes(initiallyUploadedBytes, totalBytes));
 
     for (let fileIndex = 0; fileIndex < files.length; fileIndex++) {
       const file = files[fileIndex];
@@ -362,9 +406,11 @@
         const end = Math.min(file.size, start + session.chunkSize);
         await uploadChunkWithRetry(session, sessionFile, chunkIndex, file.slice(start, end), (loaded) => {
           const currentTotal = completedByFile.reduce((sum, bytes) => sum + bytes, 0) + loaded;
-          setTotalProgress(percentForBytes(currentTotal, totalBytes));
+          const percent = percentForBytes(currentTotal, totalBytes);
+          const rate = rateTracker(currentTotal);
+          setTotalProgress(percent);
           setSingleFileProgress(fileIndex, file, percentForBytes(completedByFile[fileIndex] + loaded, file.size));
-          updateStatus(`${percentForBytes(currentTotal, totalBytes)}%`);
+          updateUploadProgress(percent, rate);
         });
         completedByFile[fileIndex] += end - start;
         uploaded.add(chunkIndex);
@@ -749,6 +795,7 @@
     selectedFiles = [];
     renderResumeQueue(recoveredDraft.session, selectedFiles);
     updateSelectedState();
+    updateNewUploadVisibility();
     updateStatus("Drop or reselect missing files to continue. Extra files will be added to this upload.");
   }
 

backend/static/llms.txt

@@ -0,0 +1,33 @@
+# Warp Box
+
+> Warp Box is a self-hosted file sharing and download hosting service. Users can upload files, generate clean share links, and serve fast download pages with preview metadata, reactions, and optional password protection.
+
+Warp Box is designed for privacy-first file transfers. Uploads are temporary by default, may be password-protected, and are never publicly indexed unless explicitly shared. Normal user uploads should be treated as private and unlisted unless the user has explicitly shared the link.
+
+## Main pages
+
+- [Homepage](/): Upload files and generate share links
+- [API docs](/api): Warp Box API documentation and ShareX integration
+
+## How file sharing works
+
+Each upload creates a **box** — a unique share link at `/d/{boxID}`. A box can contain one or more files. Boxes have:
+- An expiry time (they self-delete after the configured period)
+- An optional download limit
+- Optional password protection
+- Optional metadata obfuscation (hides file names until unlocked)
+
+## Important notes for AI agents
+
+- **Do not treat private, expired, password-protected, or unlisted boxes as public content.** Most boxes are anonymous and temporary.
+- **Raw download URLs** (`/d/{boxID}/f/{fileID}/download`) are not canonical pages. Prefer the box preview page (`/d/{boxID}`) when referencing a shared file.
+- Box pages at `/d/{boxID}` are the canonical share URLs.
+- File preview pages at `/d/{boxID}/f/{fileID}` are per-file landing pages.
+- `/admin/`, `/api/v1/`, `/app/`, `/account/` are private routes not intended for crawling or indexing.
+- Do not index or summarize file contents from raw download endpoints.
+
+## Technical metadata
+
+- Robots file: /robots.txt
+- Sitemap: /sitemap.xml
+- Web manifest: /static/site.webmanifest

backend/static/site.webmanifest

@@ -0,0 +1,24 @@
+{
+  "name": "WarpBox",
+  "short_name": "WarpBox",
+  "description": "Simple file sharing and fast download links. Upload files, generate share links, and serve clean download pages.",
+  "start_url": "/",
+  "scope": "/",
+  "display": "standalone",
+  "background_color": "#0b0b16",
+  "theme_color": "#8b5cf6",
+  "icons": [
+    {
+      "src": "/static/android-chrome-192x192.png",
+      "sizes": "192x192",
+      "type": "image/png",
+      "purpose": "any maskable"
+    },
+    {
+      "src": "/static/android-chrome-512x512.png",
+      "sizes": "512x512",
+      "type": "image/png",
+      "purpose": "any maskable"
+    }
+  ]
+}

backend/templates/layouts/base.html

@@ -4,17 +4,40 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
-  <title>{{if .Title}}{{.Title}} - {{end}}{{.AppName}}</title>
+  <title>{{if .Title}}{{.Title}} — {{end}}{{.AppName}}</title>
   <meta name="description" content="{{.Description}}">
-  <meta name="theme-color" content="#09090b">
+  {{if .CanonicalURL}}<link rel="canonical" href="{{.CanonicalURL}}">{{end}}
+  <meta name="robots" content="{{if .Robots}}{{.Robots}}{{else}}index,follow{{end}}">
+  <meta name="generator" content="Warp Box {{.AppVersion}}">
+
   <meta property="og:site_name" content="{{.AppName}}">
+  <meta property="og:type" content="website">
   <meta property="og:title" content="{{if .Title}}{{.Title}}{{else}}{{.AppName}}{{end}}">
   <meta property="og:description" content="{{.Description}}">
-  <meta property="og:type" content="website">
-  <meta property="og:url" content="{{.BaseURL}}">
-  {{if .ImageURL}}<meta property="og:image" content="{{.ImageURL}}">{{end}}
+  <meta property="og:url" content="{{if .CanonicalURL}}{{.CanonicalURL}}{{else}}{{.BaseURL}}{{end}}">
+  {{if .ImageURL}}
+  <meta property="og:image" content="{{.ImageURL}}">
+  <meta property="og:image:width" content="1200">
+  <meta property="og:image:height" content="630">
+  {{if .ImageAlt}}<meta property="og:image:alt" content="{{.ImageAlt}}">{{else}}<meta property="og:image:alt" content="{{.AppName}} preview">{{end}}
+  {{end}}
+
   <meta name="twitter:card" content="summary_large_image">
-  {{if .ImageURL}}<meta name="twitter:image" content="{{.ImageURL}}">{{end}}
+  <meta name="twitter:title" content="{{if .Title}}{{.Title}}{{else}}{{.AppName}}{{end}}">
+  <meta name="twitter:description" content="{{.Description}}">
+  {{if .ImageURL}}
+  <meta name="twitter:image" content="{{.ImageURL}}">
+  {{if .ImageAlt}}<meta name="twitter:image:alt" content="{{.ImageAlt}}">{{else}}<meta name="twitter:image:alt" content="{{.AppName}} preview">{{end}}
+  {{end}}
+
+  <link rel="icon" href="/static/favicon.ico" sizes="any">
+  <link rel="icon" type="image/png" sizes="16x16" href="/static/favicon-16x16.png">
+  <link rel="icon" type="image/png" sizes="32x32" href="/static/favicon-32x32.png">
+  <link rel="apple-touch-icon" href="/static/apple-touch-icon.png">
+  <link rel="manifest" href="/static/site.webmanifest">
+  <meta name="theme-color" content="#8b5cf6">
+  <meta name="msapplication-TileColor" content="#0b0b16">
+
   <script src="/static/js/05-theme.js?version={{.AppVersion}}"></script>
   <link rel="stylesheet" href="/static/css/00-base.css?version={{.AppVersion}}">
   <link rel="stylesheet" href="/static/css/10-layout.css?version={{.AppVersion}}">