perf(backend): optimize ban lookups and prune upload group map
- Optimize the ban matching middleware by using a read-only transaction (`db.View`) for the initial scan, avoiding the single bbolt write lock on every request when no ban matches. - Implement periodic pruning of stale entries in the upload grouper map to prevent unbounded memory growth over time. - Avoid redundant parsing of the `max_days` form value in the upload handler.
This commit is contained in:
@@ -16,13 +16,18 @@ const uploadGroupWindow = 20 * time.Second
|
||||
// value (per account/IP) within uploadGroupWindow are grouped into one box.
|
||||
const uploadBatchHeader = "X-Warpbox-Batch"
|
||||
|
||||
// uploadGroupPruneInterval is how often entryFor drops stale entries so the map
|
||||
// can't grow without bound (one key per account/IP + batch value otherwise).
|
||||
const uploadGroupPruneInterval = 5 * time.Minute
|
||||
|
||||
// uploadGrouper tracks the most recent box per batch key so opt-in batched
|
||||
// uploads land in a single box. Each key has its own lock, which also serialises
|
||||
// that key's concurrent uploads so they append to the same box instead of racing
|
||||
// to create several.
|
||||
type uploadGrouper struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]*uploadGroupEntry
|
||||
mu sync.Mutex
|
||||
entries map[string]*uploadGroupEntry
|
||||
lastPrune time.Time
|
||||
}
|
||||
|
||||
type uploadGroupEntry struct {
|
||||
@@ -40,6 +45,7 @@ func newUploadGrouper() *uploadGrouper {
|
||||
func (g *uploadGrouper) entryFor(key string) *uploadGroupEntry {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.pruneLocked(time.Now())
|
||||
entry, ok := g.entries[key]
|
||||
if !ok {
|
||||
entry = &uploadGroupEntry{}
|
||||
@@ -47,3 +53,24 @@ func (g *uploadGrouper) entryFor(key string) *uploadGroupEntry {
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
// pruneLocked drops entries whose last use is well past the grouping window so
|
||||
// the map stays bounded to recently-active keys. Callers must hold g.mu. Entries
|
||||
// currently in use, or freshly created but not yet used (zero timestamp), are
|
||||
// kept to avoid removing one a request is about to populate.
|
||||
func (g *uploadGrouper) pruneLocked(now time.Time) {
|
||||
if now.Sub(g.lastPrune) < uploadGroupPruneInterval {
|
||||
return
|
||||
}
|
||||
g.lastPrune = now
|
||||
for key, entry := range g.entries {
|
||||
if !entry.mu.TryLock() {
|
||||
continue
|
||||
}
|
||||
stale := !entry.at.IsZero() && now.Sub(entry.at) > 2*uploadGroupWindow
|
||||
entry.mu.Unlock()
|
||||
if stale {
|
||||
delete(g.entries, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user