feat(config): support large uploads with read header timeout
All checks were successful
Build and Publish Docker Image / deploy (push) Successful in 1m40s
All checks were successful
Build and Publish Docker Image / deploy (push) Successful in 1m40s
Disable default read and write timeouts (set to 0s) to prevent Go from prematurely closing connections during large multi-GB uploads. Introduce `WARPBOX_READ_HEADER_TIMEOUT` (defaulting to 15s) to protect against slowloris-style attacks while still allowing long-running uploads to complete. Update documentation and example configurations accordingly.
This commit is contained in:
@@ -27,7 +27,8 @@ WARPBOX_SHORT_WINDOW_REQUESTS=60
|
|||||||
WARPBOX_SHORT_WINDOW_SECONDS=60
|
WARPBOX_SHORT_WINDOW_SECONDS=60
|
||||||
WARPBOX_ANONYMOUS_STORAGE_BACKEND=local
|
WARPBOX_ANONYMOUS_STORAGE_BACKEND=local
|
||||||
WARPBOX_USER_STORAGE_BACKEND=local
|
WARPBOX_USER_STORAGE_BACKEND=local
|
||||||
WARPBOX_READ_TIMEOUT=15s
|
WARPBOX_READ_HEADER_TIMEOUT=15s
|
||||||
WARPBOX_WRITE_TIMEOUT=60s
|
WARPBOX_READ_TIMEOUT=0s
|
||||||
|
WARPBOX_WRITE_TIMEOUT=0s
|
||||||
WARPBOX_IDLE_TIMEOUT=120s
|
WARPBOX_IDLE_TIMEOUT=120s
|
||||||
WARPBOX_TRUSTED_PROXIES=
|
WARPBOX_TRUSTED_PROXIES=
|
||||||
|
|||||||
@@ -38,6 +38,11 @@ Upload policy defaults are also configured in megabytes and can later be changed
|
|||||||
Runtime data is configured with `WARPBOX_DATA_DIR` and defaults to `./data` in the dev environment.
|
Runtime data is configured with `WARPBOX_DATA_DIR` and defaults to `./data` in the dev environment.
|
||||||
The dev script resolves that path from the repository root.
|
The dev script resolves that path from the repository root.
|
||||||
|
|
||||||
|
Large uploads are expected to take minutes on normal home/server connections. Keep
|
||||||
|
`WARPBOX_READ_TIMEOUT=0s` and `WARPBOX_WRITE_TIMEOUT=0s` so Go does not close the connection
|
||||||
|
mid-upload; `WARPBOX_READ_HEADER_TIMEOUT=15s` still protects header reads from slowloris-style
|
||||||
|
connections.
|
||||||
|
|
||||||
Background jobs are enabled with `WARPBOX_JOBS_ENABLED=true`. Individual jobs can be toggled with
|
Background jobs are enabled with `WARPBOX_JOBS_ENABLED=true`. Individual jobs can be toggled with
|
||||||
`WARPBOX_CLEANUP_ENABLED` and `WARPBOX_THUMBNAIL_ENABLED`, and their schedules are configured with
|
`WARPBOX_CLEANUP_ENABLED` and `WARPBOX_THUMBNAIL_ENABLED`, and their schedules are configured with
|
||||||
`WARPBOX_CLEANUP_EVERY` and `WARPBOX_THUMBNAIL_EVERY`.
|
`WARPBOX_CLEANUP_EVERY` and `WARPBOX_THUMBNAIL_EVERY`.
|
||||||
@@ -106,6 +111,9 @@ WARPBOX_DATA_DIR=/var/lib/warpbox
|
|||||||
WARPBOX_STATIC_DIR=/opt/warpbox-dev/backend/static
|
WARPBOX_STATIC_DIR=/opt/warpbox-dev/backend/static
|
||||||
WARPBOX_TEMPLATE_DIR=/opt/warpbox-dev/backend/templates
|
WARPBOX_TEMPLATE_DIR=/opt/warpbox-dev/backend/templates
|
||||||
WARPBOX_TRUSTED_PROXIES=127.0.0.1,::1
|
WARPBOX_TRUSTED_PROXIES=127.0.0.1,::1
|
||||||
|
WARPBOX_READ_HEADER_TIMEOUT=15s
|
||||||
|
WARPBOX_READ_TIMEOUT=0s
|
||||||
|
WARPBOX_WRITE_TIMEOUT=0s
|
||||||
```
|
```
|
||||||
|
|
||||||
Example `/etc/systemd/system/warpbox.service`:
|
Example `/etc/systemd/system/warpbox.service`:
|
||||||
|
|||||||
@@ -54,6 +54,24 @@ network edge, or set it to a value that does not include public clients. Direct
|
|||||||
public exposure is not recommended; use a reverse proxy for TLS and request
|
public exposure is not recommended; use a reverse proxy for TLS and request
|
||||||
normalization.
|
normalization.
|
||||||
|
|
||||||
|
## Large Uploads
|
||||||
|
|
||||||
|
Multi-GB uploads must not use whole-body read/write deadlines. Keep these
|
||||||
|
Warpbox values for production unless you intentionally want a hard wall-clock
|
||||||
|
upload limit:
|
||||||
|
|
||||||
|
```env
|
||||||
|
WARPBOX_READ_HEADER_TIMEOUT=15s
|
||||||
|
WARPBOX_READ_TIMEOUT=0s
|
||||||
|
WARPBOX_WRITE_TIMEOUT=0s
|
||||||
|
```
|
||||||
|
|
||||||
|
`WARPBOX_READ_HEADER_TIMEOUT` protects request headers. `WARPBOX_READ_TIMEOUT`
|
||||||
|
and `WARPBOX_WRITE_TIMEOUT` cover the whole upload/response lifetime in Go, so
|
||||||
|
small values can cause browser errors such as `NS_ERROR_NET_INTERRUPT` during
|
||||||
|
large transfers. Upload size, daily, storage, and box limits still enforce abuse
|
||||||
|
controls independently of these timeout values.
|
||||||
|
|
||||||
## Ban Behavior
|
## Ban Behavior
|
||||||
|
|
||||||
Active bans return:
|
Active bans return:
|
||||||
|
|||||||
@@ -11,26 +11,27 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
AppName string
|
AppName string
|
||||||
AppVersion string
|
AppVersion string
|
||||||
Environment string
|
Environment string
|
||||||
Addr string
|
Addr string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
DataDir string
|
DataDir string
|
||||||
AdminToken string
|
AdminToken string
|
||||||
StaticDir string
|
StaticDir string
|
||||||
TemplateDir string
|
TemplateDir string
|
||||||
ReadTimeout time.Duration
|
ReadHeaderTimeout time.Duration
|
||||||
WriteTimeout time.Duration
|
ReadTimeout time.Duration
|
||||||
IdleTimeout time.Duration
|
WriteTimeout time.Duration
|
||||||
TrustedProxies []string
|
IdleTimeout time.Duration
|
||||||
JobsEnabled bool
|
TrustedProxies []string
|
||||||
CleanupEnabled bool
|
JobsEnabled bool
|
||||||
CleanupEvery time.Duration
|
CleanupEnabled bool
|
||||||
ThumbnailEnabled bool
|
CleanupEvery time.Duration
|
||||||
ThumbnailEvery time.Duration
|
ThumbnailEnabled bool
|
||||||
MaxUploadSize int64
|
ThumbnailEvery time.Duration
|
||||||
DefaultSettings SettingsDefaults
|
MaxUploadSize int64
|
||||||
|
DefaultSettings SettingsDefaults
|
||||||
}
|
}
|
||||||
|
|
||||||
type SettingsDefaults struct {
|
type SettingsDefaults struct {
|
||||||
@@ -55,25 +56,26 @@ type SettingsDefaults struct {
|
|||||||
|
|
||||||
func Load() (Config, error) {
|
func Load() (Config, error) {
|
||||||
cfg := Config{
|
cfg := Config{
|
||||||
AppName: envString("WARPBOX_APP_NAME", "warpbox.dev"),
|
AppName: envString("WARPBOX_APP_NAME", "warpbox.dev"),
|
||||||
AppVersion: envString("APP_VERSION", "dev"),
|
AppVersion: envString("APP_VERSION", "dev"),
|
||||||
Environment: envString("WARPBOX_ENV", "development"),
|
Environment: envString("WARPBOX_ENV", "development"),
|
||||||
Addr: envString("WARPBOX_ADDR", ":8080"),
|
Addr: envString("WARPBOX_ADDR", ":8080"),
|
||||||
BaseURL: strings.TrimRight(envString("WARPBOX_BASE_URL", "http://localhost:8080"), "/"),
|
BaseURL: strings.TrimRight(envString("WARPBOX_BASE_URL", "http://localhost:8080"), "/"),
|
||||||
DataDir: envString("WARPBOX_DATA_DIR", defaultPath("data")),
|
DataDir: envString("WARPBOX_DATA_DIR", defaultPath("data")),
|
||||||
AdminToken: envString("WARPBOX_ADMIN_TOKEN", ""),
|
AdminToken: envString("WARPBOX_ADMIN_TOKEN", ""),
|
||||||
StaticDir: envString("WARPBOX_STATIC_DIR", defaultPath("static")),
|
StaticDir: envString("WARPBOX_STATIC_DIR", defaultPath("static")),
|
||||||
TemplateDir: envString("WARPBOX_TEMPLATE_DIR", defaultPath("templates")),
|
TemplateDir: envString("WARPBOX_TEMPLATE_DIR", defaultPath("templates")),
|
||||||
ReadTimeout: envDuration("WARPBOX_READ_TIMEOUT", 15*time.Second),
|
ReadHeaderTimeout: envDuration("WARPBOX_READ_HEADER_TIMEOUT", 15*time.Second),
|
||||||
WriteTimeout: envDuration("WARPBOX_WRITE_TIMEOUT", 60*time.Second),
|
ReadTimeout: envDuration("WARPBOX_READ_TIMEOUT", 0),
|
||||||
IdleTimeout: envDuration("WARPBOX_IDLE_TIMEOUT", 120*time.Second),
|
WriteTimeout: envDuration("WARPBOX_WRITE_TIMEOUT", 0),
|
||||||
TrustedProxies: envCSV("WARPBOX_TRUSTED_PROXIES"),
|
IdleTimeout: envDuration("WARPBOX_IDLE_TIMEOUT", 120*time.Second),
|
||||||
JobsEnabled: envBool("WARPBOX_JOBS_ENABLED", true),
|
TrustedProxies: envCSV("WARPBOX_TRUSTED_PROXIES"),
|
||||||
CleanupEnabled: envBool("WARPBOX_CLEANUP_ENABLED", true),
|
JobsEnabled: envBool("WARPBOX_JOBS_ENABLED", true),
|
||||||
CleanupEvery: envDuration("WARPBOX_CLEANUP_EVERY", time.Hour),
|
CleanupEnabled: envBool("WARPBOX_CLEANUP_ENABLED", true),
|
||||||
ThumbnailEnabled: envBool("WARPBOX_THUMBNAIL_ENABLED", true),
|
CleanupEvery: envDuration("WARPBOX_CLEANUP_EVERY", time.Hour),
|
||||||
ThumbnailEvery: envDuration("WARPBOX_THUMBNAIL_EVERY", time.Minute),
|
ThumbnailEnabled: envBool("WARPBOX_THUMBNAIL_ENABLED", true),
|
||||||
MaxUploadSize: envMegabytes("WARPBOX_MAX_UPLOAD_SIZE_MB", 2048), // 2 GiB default.
|
ThumbnailEvery: envDuration("WARPBOX_THUMBNAIL_EVERY", time.Minute),
|
||||||
|
MaxUploadSize: envMegabytes("WARPBOX_MAX_UPLOAD_SIZE_MB", 2048), // 2 GiB default.
|
||||||
DefaultSettings: SettingsDefaults{
|
DefaultSettings: SettingsDefaults{
|
||||||
AnonymousUploadsEnabled: envBool("WARPBOX_ANONYMOUS_UPLOADS_ENABLED", true),
|
AnonymousUploadsEnabled: envBool("WARPBOX_ANONYMOUS_UPLOADS_ENABLED", true),
|
||||||
AnonymousMaxUploadMB: envMegabytesLimitFloat("WARPBOX_ANONYMOUS_MAX_UPLOAD_MB", 512),
|
AnonymousMaxUploadMB: envMegabytesLimitFloat("WARPBOX_ANONYMOUS_MAX_UPLOAD_MB", 512),
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
func TestParseMegabytes(t *testing.T) {
|
func TestParseMegabytes(t *testing.T) {
|
||||||
tests := map[string]int64{
|
tests := map[string]int64{
|
||||||
@@ -49,3 +52,20 @@ func TestEnvBool(t *testing.T) {
|
|||||||
t.Fatalf("envBool() did not fall back to true")
|
t.Fatalf("envBool() did not fall back to true")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadDefaultsUseLargeUploadFriendlyTimeouts(t *testing.T) {
|
||||||
|
t.Setenv("WARPBOX_BASE_URL", "http://example.test")
|
||||||
|
cfg, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load returned error: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.ReadHeaderTimeout != 15*time.Second {
|
||||||
|
t.Fatalf("ReadHeaderTimeout = %s, want 15s", cfg.ReadHeaderTimeout)
|
||||||
|
}
|
||||||
|
if cfg.ReadTimeout != 0 {
|
||||||
|
t.Fatalf("ReadTimeout = %s, want 0 for long uploads", cfg.ReadTimeout)
|
||||||
|
}
|
||||||
|
if cfg.WriteTimeout != 0 {
|
||||||
|
t.Fatalf("WriteTimeout = %s, want 0 for long uploads", cfg.WriteTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -54,11 +54,12 @@ func New(cfg config.Config, logger *slog.Logger) (*http.Server, error) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
server := &http.Server{
|
server := &http.Server{
|
||||||
Addr: cfg.Addr,
|
Addr: cfg.Addr,
|
||||||
Handler: handler,
|
Handler: handler,
|
||||||
ReadTimeout: cfg.ReadTimeout,
|
ReadHeaderTimeout: cfg.ReadHeaderTimeout,
|
||||||
WriteTimeout: cfg.WriteTimeout,
|
ReadTimeout: cfg.ReadTimeout,
|
||||||
IdleTimeout: cfg.IdleTimeout,
|
WriteTimeout: cfg.WriteTimeout,
|
||||||
|
IdleTimeout: cfg.IdleTimeout,
|
||||||
}
|
}
|
||||||
server.RegisterOnShutdown(func() {
|
server.RegisterOnShutdown(func() {
|
||||||
stopJobs()
|
stopJobs()
|
||||||
|
|||||||
5
scripts/env/dev.env.example
vendored
5
scripts/env/dev.env.example
vendored
@@ -27,7 +27,8 @@ WARPBOX_SHORT_WINDOW_REQUESTS=60
|
|||||||
WARPBOX_SHORT_WINDOW_SECONDS=60
|
WARPBOX_SHORT_WINDOW_SECONDS=60
|
||||||
WARPBOX_ANONYMOUS_STORAGE_BACKEND=local
|
WARPBOX_ANONYMOUS_STORAGE_BACKEND=local
|
||||||
WARPBOX_USER_STORAGE_BACKEND=local
|
WARPBOX_USER_STORAGE_BACKEND=local
|
||||||
WARPBOX_READ_TIMEOUT=15s
|
WARPBOX_READ_HEADER_TIMEOUT=15s
|
||||||
WARPBOX_WRITE_TIMEOUT=60s
|
WARPBOX_READ_TIMEOUT=0s
|
||||||
|
WARPBOX_WRITE_TIMEOUT=0s
|
||||||
WARPBOX_IDLE_TIMEOUT=120s
|
WARPBOX_IDLE_TIMEOUT=120s
|
||||||
WARPBOX_TRUSTED_PROXIES=
|
WARPBOX_TRUSTED_PROXIES=
|
||||||
|
|||||||
Reference in New Issue
Block a user