viaprog/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-06-13 23:36:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(filer): object size distribution metric and dashboard panels (#9902)

Browse Source 
* feat(filer): record object size distribution histogram

Add SeaweedFS_filer_object_size_bytes, a histogram sampled when an
object is first created in the filer namespace, covering every write
protocol (S3, WebDAV, FUSE mount, direct HTTP). Buckets follow the
1KB/100KB/1MB/100MB/1GB ranges operators use to size collections.
Directories, overwrites, and metadata-only updates are not sampled, so
the bucket counts track the size distribution of distinct objects.

* feat(metrics): add filer object size distribution dashboard panels

Add a write-rate-by-size-range graph and a size-distribution bar gauge,
driven by SeaweedFS_filer_object_size_bytes, to the standalone and Helm
Grafana dashboards. Per-range subtractions are clamped at zero so
transient negative rate() samples do not render below the axis.
This commit is contained in:
Chris Lu
2026-06-09 10:41:11 -07:00
committed by GitHub
parent 7b07d8177a
commit 8776b9d311
5 changed files with 606 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
+285
View File
@@ -3666,6 +3666,291 @@
],
"title": "S3 Bucket Object Count",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 130
},
"id": 92,
"panels": [],
"title": "Filer Object Size Distribution",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Rate of new objects created, split by size range.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "objects/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 25,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 131
},
"id": 93,
"links": [],
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.1.2",
"targets": [
{
"exemplar": true,
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "< 1KB",
"refId": "A",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1KB - 100KB",
"refId": "B",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100KB - 1MB",
"refId": "C",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1MB - 100MB",
"refId": "D",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100MB - 1GB",
"refId": "E",
"step": 60
},
{
"exemplar": true,
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "> 1GB",
"refId": "F",
"step": 60
}
],
"title": "Filer Object Write Rate by Size Range",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Objects created per size range over the selected time window.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 131
},
"id": 94,
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"valueMode": "color"
},
"pluginVersion": "8.1.2",
"targets": [
{
"exemplar": true,
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "< 1KB",
"refId": "A"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1KB - 100KB",
"refId": "B"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100KB - 1MB",
"refId": "C"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "1MB - 100MB",
"refId": "D"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "100MB - 1GB",
"refId": "E"
},
{
"exemplar": true,
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 2,
"legendFormat": "> 1GB",
"refId": "F"
}
],
"title": "Filer Object Size Distribution",
"type": "bargauge"
}
],
"refresh": "",
other/metrics/grafana_seaweedfs.json
+231
View File
@@ -623,6 +623,237 @@
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 5,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 31
},
"hiddenSeries": false,
"id": 91,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": false,
"hideZero": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.1.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "< 1KB",
"refId": "A"
},
{
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "1KB - 100KB",
"refId": "B"
},
{
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "100KB - 1MB",
"refId": "C"
},
{
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "1MB - 100MB",
"refId": "D"
},
{
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "100MB - 1GB",
"refId": "E"
},
{
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "> 1GB",
"refId": "F"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Filer Object Write Rate by Size Range",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "objects/s",
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"datasource": "${DS_PROMETHEUS}",
"description": "Objects created per size range over the selected time window.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 31
},
"id": 92,
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"valueMode": "color"
},
"pluginVersion": "8.1.2",
"targets": [
{
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "< 1KB",
"refId": "A",
"instant": true
},
{
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "1KB - 100KB",
"refId": "B",
"instant": true
},
{
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "100KB - 1MB",
"refId": "C",
"instant": true
},
{
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "1MB - 100MB",
"refId": "D",
"instant": true
},
{
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "100MB - 1GB",
"refId": "E",
"instant": true
},
{
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])), 0)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "> 1GB",
"refId": "F",
"instant": true
}
],
"title": "Filer Object Size Distribution",
"type": "bargauge"
}
],
"repeat": null,
weed/filer/filer.go
+4
View File
@@ -22,6 +22,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
@@ -256,6 +257,9 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
glog.ErrorfCtx(ctx, "insert entry %s: %v", entry.FullPath, err)
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
}
if !entry.IsDirectory() {
stats.FilerObjectSizeBytesHistogram.Observe(float64(entry.Size()))
}
} else {
if o_excl {
glog.V(3).InfofCtx(ctx, "EEXIST: entry %s already exists", entry.FullPath)
weed/filer/leveldb/object_size_metric_test.go
+75
View File
@@ -0,0 +1,75 @@
package leveldb
import (
"context"
"os"
"testing"
dto "github.com/prometheus/client_model/go"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func histogramState(t *testing.T) (count uint64, perBucket map[float64]uint64) {
t.Helper()
m := &dto.Metric{}
if err := stats.FilerObjectSizeBytesHistogram.Write(m); err != nil {
t.Fatalf("write histogram: %v", err)
}
perBucket = make(map[float64]uint64)
for _, b := range m.GetHistogram().GetBucket() {
perBucket[b.GetUpperBound()] = b.GetCumulativeCount()
}
return m.GetHistogram().GetSampleCount(), perBucket
}
func TestCreateEntryRecordsObjectSize(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
store := &LevelDBStore{}
if err := store.initialize(t.TempDir()); err != nil {
t.Fatalf("init store: %v", err)
}
testFiler.SetStore(store)
defer testFiler.Shutdown()
ctx := context.Background()
file := func(path string, size uint64) *filer.Entry {
return &filer.Entry{
FullPath: util.FullPath(path),
Attr: filer.Attr{Mode: 0640, FileSize: size},
}
}
before, _ := histogramState(t)
if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 500), nil, false, false, nil, false, 255); err != nil {
t.Fatalf("create small: %v", err)
}
if err := testFiler.CreateEntry(ctx, file("/data/big.bin", 5*1024*1024), nil, false, false, nil, false, 255); err != nil {
t.Fatalf("create big: %v", err)
}
dir := &filer.Entry{FullPath: util.FullPath("/data/sub"), Attr: filer.Attr{Mode: os.ModeDir | 0755}}
if err := testFiler.CreateEntry(ctx, dir, nil, false, false, nil, false, 255); err != nil {
t.Fatalf("create dir: %v", err)
}
// overwrite is an update, not a new object
if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 800), nil, false, false, nil, false, 255); err != nil {
t.Fatalf("overwrite small: %v", err)
}
after, buckets := histogramState(t)
// only the two new files are sampled, into their respective ranges
if got := after - before; got != 2 {
t.Fatalf("expected 2 sampled objects, got %d", got)
}
if buckets[1024] < 1 {
t.Errorf("expected the 500-byte object in the <=1024 bucket, buckets=%v", buckets)
}
if buckets[104857600]-buckets[1048576] < 1 {
t.Errorf("expected the 5MB object in the (1MB,100MB] range, buckets=%v", buckets)
}
}
weed/stats/metrics.go
+11
View File
@@ -199,6 +199,16 @@ var (
Help: "The last send timestamp of the filer subscription.",
}, []string{"sourceFiler", "clientName", "path"})
// Sampled only on first creation, so counts track distinct objects.
FilerObjectSizeBytesHistogram = prometheus.NewHistogram(
prometheus.HistogramOpts{
Namespace: Namespace,
Subsystem: "filer",
Name: "object_size_bytes",
Help: "Distribution of object sizes in bytes, sampled when an object is first created.",
Buckets: []float64{1024, 102400, 1048576, 104857600, 1073741824},
})
FilerStoreCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,
@@ -751,6 +761,7 @@ func init() {
Gather.MustRegister(FilerStoreHistogram)
Gather.MustRegister(FilerSyncOffsetGauge)
Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
Gather.MustRegister(FilerObjectSizeBytesHistogram)
Gather.MustRegister(collectors.NewGoCollector())
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))