mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-06-13 23:36:45 +03:00
Chris Lu
2386fa550a
A Canceled/DeadlineExceeded from the caller's per-request context was treated like a dead channel: it closed the shared cached ClientConn and cancelled every other in-flight RPC on it with "the client connection is closing". Under a burst of concurrent chunk assigns (e.g. a large S3 multipart upload) one slow assign hitting its 10s attempt timeout could poison the connection for all the rest, cascading into a flood of 500s. Thread the caller's context into shouldInvalidateConnection and only invalidate on Canceled/DeadlineExceeded while that context is still live, which isolates the genuine stale-channel signal (a peer restart behind a k8s Service VIP). To carry the context, add a ctx parameter to the existing WithGrpcClient, WithMasterClient, and WithMasterServerClient; the master assign and volume-lookup paths pass their per-attempt context and every other caller passes context.Background().
46 lines
1.8 KiB
Go
46 lines
1.8 KiB
Go
package shell
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/metadata"
|
|
)
|
|
|
|
// iamRequestTimeout caps every shell-originated IAM gRPC call so the shell
|
|
// can't hang on an unresponsive filer.
|
|
const iamRequestTimeout = 30 * time.Second
|
|
|
|
// withIamClient invokes fn against the filer's IAM gRPC service. When
|
|
// jwt.filer_signing.key is configured in security.toml, a freshly minted admin
|
|
// Bearer token is attached to the outgoing context so the filer's
|
|
// IamGrpcServer.checkAdminAuth passes; with no key configured the filer
|
|
// accepts unauthenticated calls. The context already has the iamRequestTimeout
|
|
// applied — callers can derive child contexts but should not need their own
|
|
// timeout boilerplate.
|
|
func (ce *CommandEnv) withIamClient(fn func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error) error {
|
|
return pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error {
|
|
ctx, cancel := context.WithTimeout(iamAdminAuthContext(context.Background()), iamRequestTimeout)
|
|
defer cancel()
|
|
return fn(ctx, iam_pb.NewSeaweedIdentityAccessManagementClient(conn))
|
|
}, ce.option.FilerAddress.ToGrpcAddress(), false, ce.option.GrpcDialOption)
|
|
}
|
|
|
|
func iamAdminAuthContext(ctx context.Context) context.Context {
|
|
signingKey := util.GetViper().GetString("jwt.filer_signing.key")
|
|
if signingKey == "" {
|
|
return ctx
|
|
}
|
|
expiresAfterSec := util.GetViper().GetInt("jwt.filer_signing.expires_after_seconds")
|
|
token := security.GenJwtForFilerAdmin(security.SigningKey(signingKey), expiresAfterSec)
|
|
if token == "" {
|
|
return ctx
|
|
}
|
|
return metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+string(token))
|
|
}
|