From 756c82482978498c8adb047183c237358d6c7fe9 Mon Sep 17 00:00:00 2001 From: finalt Date: Fri, 12 Jan 2024 22:38:16 +0800 Subject: [PATCH 1/6] add internal service extension --- common/constant/key.go | 10 ++ protocol/triple/health/healthServer.go | 15 ++- .../triple/reflection/serverreflection.go | 15 ++- server/server.go | 102 +++++++++++++++--- 4 files changed, 115 insertions(+), 27 deletions(-) diff --git a/common/constant/key.go b/common/constant/key.go index e3dfa87606..5dd9dcb0cd 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -17,6 +17,8 @@ package constant +import "math" + type DubboCtxKey string const ( @@ -439,3 +441,11 @@ const ( DefaultMetaFileName = "dubbo.metadata." DefaultEntrySize = 100 ) + +// priority +const ( + DefaultPriority = 0 + HighestPriority = math.MinInt32 + // LowestPriority for metadata service + LowestPriority = math.MaxInt32 +) diff --git a/protocol/triple/health/healthServer.go b/protocol/triple/health/healthServer.go index af2e69e777..e077aafa89 100644 --- a/protocol/triple/health/healthServer.go +++ b/protocol/triple/health/healthServer.go @@ -171,11 +171,16 @@ func (srv *HealthTripleServer) Resume() { func init() { healthServer = NewServer() - server.SetProServices(&server.ServiceDefinition{ - Handler: healthServer, - Info: &triple_health.Health_ServiceInfo, - Opts: []server.ServiceOption{server.WithNotRegister(), - server.WithInterface(constant.HealthCheckServiceInterface)}, + server.SetProServices(&server.InternalService{ + Init: func(options *server.ServiceOptions) (*server.ServiceDefinition, bool) { + return &server.ServiceDefinition{ + Handler: healthServer, + Info: &triple_health.Health_ServiceInfo, + Opts: []server.ServiceOption{server.WithNotRegister(), + server.WithInterface(constant.HealthCheckServiceInterface)}, + }, true + }, + Priority: constant.DefaultPriority, }) // In order to adapt config.Load diff --git a/protocol/triple/reflection/serverreflection.go b/protocol/triple/reflection/serverreflection.go index ae7bd09f57..5b139f67b1 100644 --- a/protocol/triple/reflection/serverreflection.go +++ b/protocol/triple/reflection/serverreflection.go @@ -279,11 +279,16 @@ var reflectionServer *ReflectionServer func init() { reflectionServer = NewServer() - server.SetProServices(&server.ServiceDefinition{ - Handler: reflectionServer, - Info: &rpb.ServerReflection_ServiceInfo, - Opts: []server.ServiceOption{server.WithNotRegister(), - server.WithInterface(constant.ReflectionServiceInterface)}, + server.SetProServices(&server.InternalService{ + Init: func(options *server.ServiceOptions) (*server.ServiceDefinition, bool) { + return &server.ServiceDefinition{ + Handler: reflectionServer, + Info: &rpb.ServerReflection_ServiceInfo, + Opts: []server.ServiceOption{server.WithNotRegister(), + server.WithInterface(constant.ReflectionServiceInterface)}, + }, true + }, + Priority: constant.DefaultPriority, }) // In order to adapt config.Load // Plans for future removal diff --git a/server/server.go b/server/server.go index a76d9cd59a..bbb07c0b88 100644 --- a/server/server.go +++ b/server/server.go @@ -21,6 +21,7 @@ package server import ( "context" "fmt" + "sort" "sync" ) @@ -38,7 +39,8 @@ import ( ) // proServices are for internal services -var proServices = map[string]*ServiceDefinition{} +var proServices = make([]*InternalService, 0, 16) +var proLock sync.Mutex type Server struct { invoker protocol.Invoker @@ -123,8 +125,17 @@ func newInfoInvoker(url *common.URL, info *ServiceInfo, svc common.RPCService) p // Register assemble invoker chains like ProviderConfig.Load, init a service per call func (s *Server) Register(handler interface{}, info *ServiceInfo, opts ...ServiceOption) error { + newSvcOpts, err := s.genSvcOpts(handler, opts...) + if err != nil { + return err + } + s.svcOptsMap.Store(newSvcOpts, info) + return nil +} + +func (s *Server) genSvcOpts(handler interface{}, opts ...ServiceOption) (*ServiceOptions, error) { if s.cfg == nil { - return errors.New("Server has not been initialized, please use NewServer() to create Server") + return nil, errors.New("Server has not been initialized, please use NewServer() to create Server") } var svcOpts []ServiceOption appCfg := s.cfg.Application @@ -153,16 +164,13 @@ func (s *Server) Register(handler interface{}, info *ServiceInfo, opts ...Servic SetRegistries(regsCfg), ) } - // options passed by users have higher priority svcOpts = append(svcOpts, opts...) if err := newSvcOpts.init(s, svcOpts...); err != nil { - return err + return nil, err } newSvcOpts.Implement(handler) - s.svcOptsMap.Store(newSvcOpts, info) - - return nil + return newSvcOpts, nil } func (s *Server) exportServices() (err error) { @@ -187,11 +195,77 @@ func (s *Server) Serve() error { if err := s.exportServices(); err != nil { return err } + if err := s.exportInternalServices(); err != nil { + return err + } metadata.ExportMetadataService() registry_exposed.RegisterServiceInstance(s.cfg.Application.Name, s.cfg.Application.Tag, s.cfg.Application.MetadataType) select {} } +// In order to expose internal services +func (s *Server) exportInternalServices() error { + cfg := &ServiceOptions{} + cfg.Application = s.cfg.Application + cfg.Provider = s.cfg.Provider + cfg.Protocols = s.cfg.Protocols + cfg.Registries = s.cfg.Registries + + services := make([]*InternalService, 0, len(proServices)) + + proLock.Lock() + defer proLock.Unlock() + for _, service := range proServices { + sd, ok := service.Init(cfg) + if !ok { + continue + } + newSvcOpts, err := s.genSvcOpts(sd.Handler, sd.Opts...) + if err != nil { + return err + } + service.svcOpts = newSvcOpts + service.info = sd.Info + services = append(services, service) + } + + sort.Slice(services, func(i, j int) bool { + return services[i].Priority < services[j].Priority + }) + + for _, service := range services { + if service.BeforeExport != nil { + service.BeforeExport(service.svcOpts) + } + err := service.svcOpts.ExportWithInfo(service.info) + if service.AfterExport != nil { + service.AfterExport(service.svcOpts, err) + } + if err != nil { + logger.Errorf("export %s internal service failed, err: %s", service.svcOpts.Service.Interface, err) + return err + } + } + + return nil +} + +// InternalService for dubbo internal services +type InternalService struct { + svcOpts *ServiceOptions + info *ServiceInfo + // Return serviceDefinition and bool, where bool indicates whether it is exported + Init func(options *ServiceOptions) (*ServiceDefinition, bool) + BeforeExport func(options *ServiceOptions) + AfterExport func(options *ServiceOptions, err error) + // Priority of service exposure + // Lower numbers have the higher priority + // The default priority is 0 + // The metadata service is exposed at the end + // If you have no requirements for the order of service exposure, you can use the default priority or not set + Priority int +} + type MethodInfo struct { Name string Type string @@ -210,17 +284,11 @@ func NewServer(opts ...ServerOption) (*Server, error) { srv := &Server{ cfg: newSrvOpts, } - - for _, service := range proServices { - err := srv.Register(service.Handler, service.Info, service.Opts...) - if err != nil { - return nil, err - } - } - return srv, nil } -func SetProServices(sd *ServiceDefinition) { - proServices[sd.Info.InterfaceName] = sd +func SetProServices(sd *InternalService) { + proLock.Lock() + defer proLock.Unlock() + proServices = append(proServices, sd) } From db7e50b56a9f901052c0ced21818f2e0effbd689 Mon Sep 17 00:00:00 2001 From: finalt Date: Sat, 13 Jan 2024 12:38:31 +0800 Subject: [PATCH 2/6] add errors --- server/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/server.go b/server/server.go index bbb07c0b88..598e8fcccc 100644 --- a/server/server.go +++ b/server/server.go @@ -216,6 +216,9 @@ func (s *Server) exportInternalServices() error { proLock.Lock() defer proLock.Unlock() for _, service := range proServices { + if service.Init == nil { + return errors.New("internal service init func is empty, please set the init func correctly ") + } sd, ok := service.Init(cfg) if !ok { continue @@ -254,6 +257,7 @@ func (s *Server) exportInternalServices() error { type InternalService struct { svcOpts *ServiceOptions info *ServiceInfo + // This is required // Return serviceDefinition and bool, where bool indicates whether it is exported Init func(options *ServiceOptions) (*ServiceDefinition, bool) BeforeExport func(options *ServiceOptions) From 78bc28d494b2faeddafef1495a083ea36e496fc3 Mon Sep 17 00:00:00 2001 From: finalt Date: Sun, 14 Jan 2024 17:25:05 +0800 Subject: [PATCH 3/6] modify comment --- server/server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/server.go b/server/server.go index 598e8fcccc..0130a2f445 100644 --- a/server/server.go +++ b/server/server.go @@ -258,10 +258,13 @@ type InternalService struct { svcOpts *ServiceOptions info *ServiceInfo // This is required + // This options is service configuration // Return serviceDefinition and bool, where bool indicates whether it is exported - Init func(options *ServiceOptions) (*ServiceDefinition, bool) + Init func(options *ServiceOptions) (*ServiceDefinition, bool) + // This options is InternalService.svcOpts itself BeforeExport func(options *ServiceOptions) - AfterExport func(options *ServiceOptions, err error) + // This options is InternalService.svcOpts itself + AfterExport func(options *ServiceOptions, err error) // Priority of service exposure // Lower numbers have the higher priority // The default priority is 0 From a8fbcd526b1f94cd12fe8c39e09c9407a12f77ef Mon Sep 17 00:00:00 2001 From: finalt Date: Sun, 14 Jan 2024 17:32:44 +0800 Subject: [PATCH 4/6] modify comment --- protocol/triple/health/healthServer.go | 1 + protocol/triple/reflection/serverreflection.go | 1 + server/server.go | 11 +++++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/protocol/triple/health/healthServer.go b/protocol/triple/health/healthServer.go index e077aafa89..315b7aeca3 100644 --- a/protocol/triple/health/healthServer.go +++ b/protocol/triple/health/healthServer.go @@ -172,6 +172,7 @@ func (srv *HealthTripleServer) Resume() { func init() { healthServer = NewServer() server.SetProServices(&server.InternalService{ + Name: "healthCheck", Init: func(options *server.ServiceOptions) (*server.ServiceDefinition, bool) { return &server.ServiceDefinition{ Handler: healthServer, diff --git a/protocol/triple/reflection/serverreflection.go b/protocol/triple/reflection/serverreflection.go index 5b139f67b1..d1794348c5 100644 --- a/protocol/triple/reflection/serverreflection.go +++ b/protocol/triple/reflection/serverreflection.go @@ -280,6 +280,7 @@ var reflectionServer *ReflectionServer func init() { reflectionServer = NewServer() server.SetProServices(&server.InternalService{ + Name: "reflection", Init: func(options *server.ServiceOptions) (*server.ServiceDefinition, bool) { return &server.ServiceDefinition{ Handler: reflectionServer, diff --git a/server/server.go b/server/server.go index 0130a2f445..bc17ff4de7 100644 --- a/server/server.go +++ b/server/server.go @@ -217,10 +217,11 @@ func (s *Server) exportInternalServices() error { defer proLock.Unlock() for _, service := range proServices { if service.Init == nil { - return errors.New("internal service init func is empty, please set the init func correctly ") + return errors.New("[internal service]internal service init func is empty, please set the init func correctly") } sd, ok := service.Init(cfg) if !ok { + logger.Infof("[internal service]%s service will not expose", service.Name) continue } newSvcOpts, err := s.genSvcOpts(sd.Handler, sd.Opts...) @@ -245,7 +246,7 @@ func (s *Server) exportInternalServices() error { service.AfterExport(service.svcOpts, err) } if err != nil { - logger.Errorf("export %s internal service failed, err: %s", service.svcOpts.Service.Interface, err) + logger.Errorf("[internal service]export %s service failed, err: %s", service.Name, err) return err } } @@ -255,6 +256,9 @@ func (s *Server) exportInternalServices() error { // InternalService for dubbo internal services type InternalService struct { + // This is required + // internal service name + Name string svcOpts *ServiceOptions info *ServiceInfo // This is required @@ -295,6 +299,9 @@ func NewServer(opts ...ServerOption) (*Server, error) { } func SetProServices(sd *InternalService) { + if sd.Name == "" { + logger.Warnf("[internal service]internal name is empty, please set internal name") + } proLock.Lock() defer proLock.Unlock() proServices = append(proServices, sd) From 72376ceb9cf4f94cb43739f29333336e03fac8c0 Mon Sep 17 00:00:00 2001 From: finalt Date: Sun, 14 Jan 2024 15:32:47 +0800 Subject: [PATCH 5/6] feat:Solving Circular Dependency in Dubbo-Go (#2576) * Solving Circular Dependency in Dubbo-Go * fix test * modify comment * modify comment * fix test --- internal/internal.go | 35 +++++++++++++++++ internal/reflection/api.go | 38 +++++++++++++++++++ protocol/triple/health/healthServer.go | 2 + .../triple/reflection/serverreflection.go | 24 ++---------- protocol/triple/server.go | 4 +- protocol/triple/triple.go | 6 +-- 6 files changed, 84 insertions(+), 25 deletions(-) create mode 100644 internal/internal.go create mode 100644 internal/reflection/api.go diff --git a/internal/internal.go b/internal/internal.go new file mode 100644 index 0000000000..983a28ddbd --- /dev/null +++ b/internal/internal.go @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package internal contains dubbo-go-internal code, to avoid polluting +// the top-level dubbo-go package. It must not import any dubbo-go symbols +// except internal symbols to avoid circular dependencies. +package internal + +import ( + "dubbo.apache.org/dubbo-go/v3/internal/reflection" +) + +var ( + // HealthSetServingStatusServing is used to set service serving status + // the initialization place is in /protocol/triple/health/healthServer.go + HealthSetServingStatusServing = func(service string) {} + // ReflectionRegister is used to register reflection service provider + // the initialization place is in /protocol/triple/reflection/serverreflection.go + ReflectionRegister = func(reflection reflection.ServiceInfoProvider) {} + // todo: add metadata func +) diff --git a/internal/reflection/api.go b/internal/reflection/api.go new file mode 100644 index 0000000000..0f7bfd7aad --- /dev/null +++ b/internal/reflection/api.go @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package reflection + +import ( + "google.golang.org/grpc" +) + +// ServiceInfoProvider is an interface used to retrieve metadata about the +// services to expose. +// +// The reflection service is only interested in the service names, but the +// signature is this way so that *grpc.Server implements it. So it is okay +// for a custom implementation to return zero values for the +// grpc.ServiceInfo values in the map. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ServiceInfoProvider interface { + GetServiceInfo() map[string]grpc.ServiceInfo +} diff --git a/protocol/triple/health/healthServer.go b/protocol/triple/health/healthServer.go index 315b7aeca3..5bd89063e3 100644 --- a/protocol/triple/health/healthServer.go +++ b/protocol/triple/health/healthServer.go @@ -32,6 +32,7 @@ import ( import ( "dubbo.apache.org/dubbo-go/v3/common/constant" "dubbo.apache.org/dubbo-go/v3/config" + "dubbo.apache.org/dubbo-go/v3/internal" "dubbo.apache.org/dubbo-go/v3/protocol/triple/health/triple_health" "dubbo.apache.org/dubbo-go/v3/server" ) @@ -171,6 +172,7 @@ func (srv *HealthTripleServer) Resume() { func init() { healthServer = NewServer() + internal.HealthSetServingStatusServing = SetServingStatusServing server.SetProServices(&server.InternalService{ Name: "healthCheck", Init: func(options *server.ServiceOptions) (*server.ServiceDefinition, bool) { diff --git a/protocol/triple/reflection/serverreflection.go b/protocol/triple/reflection/serverreflection.go index d1794348c5..fc79e1f9a5 100644 --- a/protocol/triple/reflection/serverreflection.go +++ b/protocol/triple/reflection/serverreflection.go @@ -25,12 +25,9 @@ import ( ) import ( - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/protoregistry" @@ -39,26 +36,12 @@ import ( import ( "dubbo.apache.org/dubbo-go/v3/common/constant" "dubbo.apache.org/dubbo-go/v3/config" + "dubbo.apache.org/dubbo-go/v3/internal" + "dubbo.apache.org/dubbo-go/v3/internal/reflection" rpb "dubbo.apache.org/dubbo-go/v3/protocol/triple/reflection/triple_reflection" "dubbo.apache.org/dubbo-go/v3/server" ) -// ServiceInfoProvider is an interface used to retrieve metadata about the -// services to expose. -// -// The reflection service is only interested in the service names, but the -// signature is this way so that *grpc.Server implements it. So it is okay -// for a custom implementation to return zero values for the -// grpc.ServiceInfo values in the map. -// -// # Experimental -// -// Notice: This type is EXPERIMENTAL and may be changed or removed in a -// later release. -type ServiceInfoProvider interface { - GetServiceInfo() map[string]grpc.ServiceInfo -} - // ExtensionResolver is the interface used to query details about extensions. // This interface is satisfied by protoregistry.GlobalTypes. // @@ -79,7 +62,7 @@ func NewServer() *ReflectionServer { } type ReflectionServer struct { - s ServiceInfoProvider + s reflection.ServiceInfoProvider descResolver protodesc.Resolver extResolver ExtensionResolver } @@ -279,6 +262,7 @@ var reflectionServer *ReflectionServer func init() { reflectionServer = NewServer() + internal.ReflectionRegister = Register server.SetProServices(&server.InternalService{ Name: "reflection", Init: func(options *server.ServiceOptions) (*server.ServiceDefinition, bool) { diff --git a/protocol/triple/server.go b/protocol/triple/server.go index 70ca805a85..1a403cc8c7 100644 --- a/protocol/triple/server.go +++ b/protocol/triple/server.go @@ -38,10 +38,10 @@ import ( "dubbo.apache.org/dubbo-go/v3/common" "dubbo.apache.org/dubbo-go/v3/common/constant" "dubbo.apache.org/dubbo-go/v3/config" + "dubbo.apache.org/dubbo-go/v3/internal" "dubbo.apache.org/dubbo-go/v3/protocol" "dubbo.apache.org/dubbo-go/v3/protocol/dubbo3" "dubbo.apache.org/dubbo-go/v3/protocol/invocation" - "dubbo.apache.org/dubbo-go/v3/protocol/triple/reflection" tri "dubbo.apache.org/dubbo-go/v3/protocol/triple/triple_protocol" "dubbo.apache.org/dubbo-go/v3/server" ) @@ -94,7 +94,7 @@ func (s *Server) Start(invoker protocol.Invoker, info *server.ServiceInfo) { // old triple idl mode and non-idl mode s.compatHandleService(intfName, URL.Group(), URL.Version(), hanOpts...) } - reflection.Register(s) + internal.ReflectionRegister(s) go func() { if runErr := s.triServer.Run(); runErr != nil { diff --git a/protocol/triple/triple.go b/protocol/triple/triple.go index 74bd772b83..9ee503ff96 100644 --- a/protocol/triple/triple.go +++ b/protocol/triple/triple.go @@ -29,8 +29,8 @@ import ( "dubbo.apache.org/dubbo-go/v3/common" "dubbo.apache.org/dubbo-go/v3/common/constant" "dubbo.apache.org/dubbo-go/v3/common/extension" + "dubbo.apache.org/dubbo-go/v3/internal" "dubbo.apache.org/dubbo-go/v3/protocol" - "dubbo.apache.org/dubbo-go/v3/protocol/triple/health" "dubbo.apache.org/dubbo-go/v3/server" ) @@ -66,7 +66,7 @@ func (tp *TripleProtocol) Export(invoker protocol.Invoker) protocol.Exporter { tp.SetExporterMap(serviceKey, exporter) logger.Infof("[TRIPLE Protocol] Export service: %s", url.String()) tp.openServer(invoker, info) - health.SetServingStatusServing(url.Service()) + internal.HealthSetServingStatusServing(url.Service()) return exporter } @@ -80,7 +80,7 @@ func (tp *TripleProtocol) exportForTest(invoker protocol.Invoker, info *server.S tp.SetExporterMap(serviceKey, exporter) logger.Infof("[TRIPLE Protocol] Export service: %s", url.String()) tp.openServer(invoker, info) - health.SetServingStatusServing(url.Service()) + internal.HealthSetServingStatusServing(url.Service()) return exporter } From c5353a321e97d9adbb68dc48b642aaec10a76f0c Mon Sep 17 00:00:00 2001 From: finalt Date: Sun, 14 Jan 2024 17:42:38 +0800 Subject: [PATCH 6/6] modify comment --- server/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/server.go b/server/server.go index bc17ff4de7..0258a940d3 100644 --- a/server/server.go +++ b/server/server.go @@ -301,6 +301,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { func SetProServices(sd *InternalService) { if sd.Name == "" { logger.Warnf("[internal service]internal name is empty, please set internal name") + return } proLock.Lock() defer proLock.Unlock()