diff --git a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json index 8556ed1e2..25fb8b2d5 100644 --- a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json +++ b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json @@ -3666,6 +3666,291 @@ ], "title": "S3 Bucket Object Count", "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 130 + }, + "id": 92, + "panels": [], + "title": "Filer Object Size Distribution", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of new objects created, split by size range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "objects/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 131 + }, + "id": 93, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "< 1KB", + "refId": "A", + "step": 60 + }, + { + "exemplar": true, + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "1KB - 100KB", + "refId": "B", + "step": 60 + }, + { + "exemplar": true, + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "100KB - 1MB", + "refId": "C", + "step": 60 + }, + { + "exemplar": true, + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "1MB - 100MB", + "refId": "D", + "step": 60 + }, + { + "exemplar": true, + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "100MB - 1GB", + "refId": "E", + "step": 60 + }, + { + "exemplar": true, + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "> 1GB", + "refId": "F", + "step": 60 + } + ], + "title": "Filer Object Write Rate by Size Range", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Objects created per size range over the selected time window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 131 + }, + "id": 94, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "< 1KB", + "refId": "A" + }, + { + "exemplar": true, + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "1KB - 100KB", + "refId": "B" + }, + { + "exemplar": true, + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "100KB - 1MB", + "refId": "C" + }, + { + "exemplar": true, + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "1MB - 100MB", + "refId": "D" + }, + { + "exemplar": true, + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "100MB - 1GB", + "refId": "E" + }, + { + "exemplar": true, + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "> 1GB", + "refId": "F" + } + ], + "title": "Filer Object Size Distribution", + "type": "bargauge" } ], "refresh": "", diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index 39cb5399d..67c2fea13 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -623,6 +623,237 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 5, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "< 1KB", + "refId": "A" + }, + { + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1KB - 100KB", + "refId": "B" + }, + { + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100KB - 1MB", + "refId": "C" + }, + { + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1MB - 100MB", + "refId": "D" + }, + { + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100MB - 1GB", + "refId": "E" + }, + { + "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "> 1GB", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Filer Object Write Rate by Size Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "objects/s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Objects created per size range over the selected time window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 92, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "< 1KB", + "refId": "A", + "instant": true + }, + { + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1KB - 100KB", + "refId": "B", + "instant": true + }, + { + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100KB - 1MB", + "refId": "C", + "instant": true + }, + { + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1MB - 100MB", + "refId": "D", + "instant": true + }, + { + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "100MB - 1GB", + "refId": "E", + "instant": true + }, + { + "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])), 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "> 1GB", + "refId": "F", + "instant": true + } + ], + "title": "Filer Object Size Distribution", + "type": "bargauge" } ], "repeat": null, diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 6b39317a9..750862751 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -22,6 +22,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/log_buffer" "github.com/seaweedfs/seaweedfs/weed/wdclient" @@ -256,6 +257,9 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry, glog.ErrorfCtx(ctx, "insert entry %s: %v", entry.FullPath, err) return fmt.Errorf("insert entry %s: %v", entry.FullPath, err) } + if !entry.IsDirectory() { + stats.FilerObjectSizeBytesHistogram.Observe(float64(entry.Size())) + } } else { if o_excl { glog.V(3).InfofCtx(ctx, "EEXIST: entry %s already exists", entry.FullPath) diff --git a/weed/filer/leveldb/object_size_metric_test.go b/weed/filer/leveldb/object_size_metric_test.go new file mode 100644 index 000000000..4e4d2c525 --- /dev/null +++ b/weed/filer/leveldb/object_size_metric_test.go @@ -0,0 +1,75 @@ +package leveldb + +import ( + "context" + "os" + "testing" + + dto "github.com/prometheus/client_model/go" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func histogramState(t *testing.T) (count uint64, perBucket map[float64]uint64) { + t.Helper() + m := &dto.Metric{} + if err := stats.FilerObjectSizeBytesHistogram.Write(m); err != nil { + t.Fatalf("write histogram: %v", err) + } + perBucket = make(map[float64]uint64) + for _, b := range m.GetHistogram().GetBucket() { + perBucket[b.GetUpperBound()] = b.GetCumulativeCount() + } + return m.GetHistogram().GetSampleCount(), perBucket +} + +func TestCreateEntryRecordsObjectSize(t *testing.T) { + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil) + store := &LevelDBStore{} + if err := store.initialize(t.TempDir()); err != nil { + t.Fatalf("init store: %v", err) + } + testFiler.SetStore(store) + defer testFiler.Shutdown() + ctx := context.Background() + + file := func(path string, size uint64) *filer.Entry { + return &filer.Entry{ + FullPath: util.FullPath(path), + Attr: filer.Attr{Mode: 0640, FileSize: size}, + } + } + + before, _ := histogramState(t) + + if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 500), nil, false, false, nil, false, 255); err != nil { + t.Fatalf("create small: %v", err) + } + if err := testFiler.CreateEntry(ctx, file("/data/big.bin", 5*1024*1024), nil, false, false, nil, false, 255); err != nil { + t.Fatalf("create big: %v", err) + } + dir := &filer.Entry{FullPath: util.FullPath("/data/sub"), Attr: filer.Attr{Mode: os.ModeDir | 0755}} + if err := testFiler.CreateEntry(ctx, dir, nil, false, false, nil, false, 255); err != nil { + t.Fatalf("create dir: %v", err) + } + // overwrite is an update, not a new object + if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 800), nil, false, false, nil, false, 255); err != nil { + t.Fatalf("overwrite small: %v", err) + } + + after, buckets := histogramState(t) + + // only the two new files are sampled, into their respective ranges + if got := after - before; got != 2 { + t.Fatalf("expected 2 sampled objects, got %d", got) + } + if buckets[1024] < 1 { + t.Errorf("expected the 500-byte object in the <=1024 bucket, buckets=%v", buckets) + } + if buckets[104857600]-buckets[1048576] < 1 { + t.Errorf("expected the 5MB object in the (1MB,100MB] range, buckets=%v", buckets) + } +} diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index a30463504..c75fae447 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -199,6 +199,16 @@ var ( Help: "The last send timestamp of the filer subscription.", }, []string{"sourceFiler", "clientName", "path"}) + // Sampled only on first creation, so counts track distinct objects. + FilerObjectSizeBytesHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: "filer", + Name: "object_size_bytes", + Help: "Distribution of object sizes in bytes, sampled when an object is first created.", + Buckets: []float64{1024, 102400, 1048576, 104857600, 1073741824}, + }) + FilerStoreCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -751,6 +761,7 @@ func init() { Gather.MustRegister(FilerStoreHistogram) Gather.MustRegister(FilerSyncOffsetGauge) Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge) + Gather.MustRegister(FilerObjectSizeBytesHistogram) Gather.MustRegister(collectors.NewGoCollector()) Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))