You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
208 lines
6.0 KiB
208 lines
6.0 KiB
package pb
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/keepalive"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
|
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
|
|
)
|
|
|
|
const (
|
|
Max_Message_Size = 1 << 30 // 1 GB
|
|
)
|
|
|
|
var (
|
|
// cache grpc connections
|
|
grpcClients = make(map[string]*grpc.ClientConn)
|
|
grpcClientsLock sync.Mutex
|
|
)
|
|
|
|
func init() {
|
|
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
|
|
}
|
|
|
|
func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
|
|
var options []grpc.ServerOption
|
|
options = append(options,
|
|
grpc.KeepaliveParams(keepalive.ServerParameters{
|
|
Time: 10 * time.Second, // wait time before ping if no activity
|
|
Timeout: 20 * time.Second, // ping timeout
|
|
}),
|
|
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
|
|
MinTime: 60 * time.Second, // min time a client should wait before sending a ping
|
|
PermitWithoutStream: false,
|
|
}),
|
|
grpc.MaxRecvMsgSize(Max_Message_Size),
|
|
grpc.MaxSendMsgSize(Max_Message_Size),
|
|
)
|
|
for _, opt := range opts {
|
|
if opt != nil {
|
|
options = append(options, opt)
|
|
}
|
|
}
|
|
return grpc.NewServer(options...)
|
|
}
|
|
|
|
func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
|
// opts = append(opts, grpc.WithBlock())
|
|
// opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
|
|
var options []grpc.DialOption
|
|
options = append(options,
|
|
// grpc.WithInsecure(),
|
|
grpc.WithDefaultCallOptions(
|
|
grpc.MaxCallSendMsgSize(Max_Message_Size),
|
|
grpc.MaxCallRecvMsgSize(Max_Message_Size),
|
|
),
|
|
grpc.WithKeepaliveParams(keepalive.ClientParameters{
|
|
Time: 30 * time.Second, // client ping server if no activity for this long
|
|
Timeout: 20 * time.Second,
|
|
PermitWithoutStream: false,
|
|
}))
|
|
for _, opt := range opts {
|
|
if opt != nil {
|
|
options = append(options, opt)
|
|
}
|
|
}
|
|
return grpc.DialContext(ctx, address, options...)
|
|
}
|
|
|
|
func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {
|
|
|
|
grpcClientsLock.Lock()
|
|
|
|
existingConnection, found := grpcClients[address]
|
|
if found {
|
|
grpcClientsLock.Unlock()
|
|
err := fn(existingConnection)
|
|
if err != nil {
|
|
grpcClientsLock.Lock()
|
|
// delete(grpcClients, address)
|
|
grpcClientsLock.Unlock()
|
|
// println("closing existing connection to", existingConnection.Target())
|
|
// existingConnection.Close()
|
|
}
|
|
return err
|
|
}
|
|
|
|
println(" dialing to", address, "...")
|
|
grpcConnection, err := GrpcDial(context.Background(), address, opts...)
|
|
if err != nil {
|
|
grpcClientsLock.Unlock()
|
|
return fmt.Errorf("fail to dial %s: %v", address, err)
|
|
}
|
|
|
|
grpcClients[address] = grpcConnection
|
|
grpcClientsLock.Unlock()
|
|
|
|
err = fn(grpcConnection)
|
|
if err != nil {
|
|
grpcClientsLock.Lock()
|
|
// delete(grpcClients, address)
|
|
grpcClientsLock.Unlock()
|
|
// println("closing created new connection to", grpcConnection.Target())
|
|
// grpcConnection.Close()
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
|
|
colonIndex := strings.LastIndex(server, ":")
|
|
if colonIndex < 0 {
|
|
return "", fmt.Errorf("server should have hostname:port format: %v", server)
|
|
}
|
|
|
|
port, parseErr := strconv.ParseUint(server[colonIndex+1:], 10, 64)
|
|
if parseErr != nil {
|
|
return "", fmt.Errorf("server port parse error: %v", parseErr)
|
|
}
|
|
|
|
grpcPort := int(port) + 10000
|
|
|
|
return fmt.Sprintf("%s:%d", server[:colonIndex], grpcPort), nil
|
|
}
|
|
|
|
func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
|
|
hostnameAndPort := strings.Split(server, ":")
|
|
if len(hostnameAndPort) != 2 {
|
|
return fmt.Sprintf("unexpected server address: %s", server)
|
|
}
|
|
|
|
port, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
|
|
if parseErr != nil {
|
|
return fmt.Sprintf("failed to parse port for %s:%s", hostnameAndPort[0], hostnameAndPort[1])
|
|
}
|
|
|
|
grpcPort := int(port) + 10000
|
|
|
|
return fmt.Sprintf("%s:%d", hostnameAndPort[0], grpcPort)
|
|
}
|
|
|
|
func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {
|
|
|
|
masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
|
|
if parseErr != nil {
|
|
return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
|
|
}
|
|
|
|
return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
|
|
client := master_pb.NewSeaweedClient(grpcConnection)
|
|
return fn(client)
|
|
}, masterGrpcAddress, grpcDialOption)
|
|
|
|
}
|
|
|
|
func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {
|
|
|
|
return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
|
|
client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
|
|
return fn(client)
|
|
}, brokerGrpcAddress, grpcDialOption)
|
|
|
|
}
|
|
|
|
func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
|
|
|
filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
|
|
if parseErr != nil {
|
|
return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
|
|
}
|
|
|
|
return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)
|
|
|
|
}
|
|
|
|
func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
|
|
|
return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
|
|
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
|
|
return fn(client)
|
|
}, filerGrpcAddress, grpcDialOption)
|
|
|
|
}
|
|
|
|
func ParseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
|
|
hostnameAndPort := strings.Split(filer, ":")
|
|
if len(hostnameAndPort) != 2 {
|
|
return "", fmt.Errorf("filer should have hostname:port format: %v", hostnameAndPort)
|
|
}
|
|
|
|
filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
|
|
if parseErr != nil {
|
|
return "", fmt.Errorf("filer port parse error: %v", parseErr)
|
|
}
|
|
|
|
filerGrpcPort := int(filerPort) + 10000
|
|
|
|
return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
|
|
}
|