Merge branch 'dev' into stakers_model

Author: abi87

.github/workflows/test.upgrade.yml

@@ -26,7 +26,6 @@ jobs:
       - name: Build the avalanchego binary
         shell: bash
         run: ./scripts/build.sh
-      # TODO: re-activate this test after there is a compatible tag to use
-      # - name: Run upgrade tests
-      #   shell: bash
-      #   run: scripts/tests.upgrade.sh 1.9.0 ./build/avalanchego
+      - name: Run upgrade tests
+        shell: bash
+        run: scripts/tests.upgrade.sh 1.10.1 ./build/avalanchego

api/health/health_test.go

@@ -245,7 +245,7 @@ func TestDeadlockRegression(t *testing.T) {
 	h.Start(context.Background(), time.Nanosecond)
 	defer h.Stop()
 
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 100; i++ {
 		lock.Lock()
 		require.NoError(h.RegisterHealthCheck(fmt.Sprintf("check-%d", i), check))
 		lock.Unlock()

api/server/allowed_hosts.go

@@ -0,0 +1,76 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package server
+
+import (
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/ava-labs/avalanchego/utils/set"
+)
+
+const wildcard = "*"
+
+var _ http.Handler = (*allowedHostsHandler)(nil)
+
+func filterInvalidHosts(
+	handler http.Handler,
+	allowed []string,
+) http.Handler {
+	s := set.Set[string]{}
+
+	for _, host := range allowed {
+		if host == wildcard {
+			// wildcards match all hostnames, so just return the base handler
+			return handler
+		}
+		s.Add(strings.ToLower(host))
+	}
+
+	return &allowedHostsHandler{
+		handler: handler,
+		hosts:   s,
+	}
+}
+
+// allowedHostsHandler is an implementation of http.Handler that validates the
+// http host header of incoming requests. This can prevent DNS rebinding attacks
+// which do not utilize CORS-headers. Http request host headers are validated
+// against a whitelist to determine whether the request should be dropped or
+// not.
+type allowedHostsHandler struct {
+	handler http.Handler
+	hosts   set.Set[string]
+}
+
+func (a *allowedHostsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// if the host header is missing we can serve this request because dns
+	// rebinding attacks rely on this header
+	if r.Host == "" {
+		a.handler.ServeHTTP(w, r)
+		return
+	}
+
+	host, _, err := net.SplitHostPort(r.Host)
+	if err != nil {
+		// either invalid (too many colons) or no port specified
+		host = r.Host
+	}
+
+	if ipAddr := net.ParseIP(host); ipAddr != nil {
+		// accept requests from ips
+		a.handler.ServeHTTP(w, r)
+		return
+	}
+
+	// a specific hostname - we need to check the whitelist to see if we should
+	// accept this r
+	if a.hosts.Contains(strings.ToLower(host)) {
+		a.handler.ServeHTTP(w, r)
+		return
+	}
+
+	http.Error(w, "invalid host specified", http.StatusForbidden)
+}

api/server/allowed_hosts_test.go

@@ -0,0 +1,77 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package server
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAllowedHostsHandler_ServeHTTP(t *testing.T) {
+	tests := []struct {
+		name    string
+		allowed []string
+		host    string
+		serve   bool
+	}{
+		{
+			name:    "no host header",
+			allowed: []string{"www.foobar.com"},
+			host:    "",
+			serve:   true,
+		},
+		{
+			name:    "ip",
+			allowed: []string{"www.foobar.com"},
+			host:    "192.168.1.1",
+			serve:   true,
+		},
+		{
+			name:    "hostname not allowed",
+			allowed: []string{"www.foobar.com"},
+			host:    "www.evil.com",
+		},
+		{
+			name:    "hostname allowed",
+			allowed: []string{"www.foobar.com"},
+			host:    "www.foobar.com",
+			serve:   true,
+		},
+		{
+			name:    "wildcard",
+			allowed: []string{"*"},
+			host:    "www.foobar.com",
+			serve:   true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+
+			baseHandler := &testHandler{}
+
+			httpAllowedHostsHandler := filterInvalidHosts(
+				baseHandler,
+				test.allowed,
+			)
+
+			w := &httptest.ResponseRecorder{}
+			r := httptest.NewRequest("", "/", nil)
+			r.Host = test.host
+
+			httpAllowedHostsHandler.ServeHTTP(w, r)
+
+			if test.serve {
+				require.True(baseHandler.called)
+				return
+			}
+
+			require.Equal(http.StatusForbidden, w.Code)
+		})
+	}
+}

api/server/server.go

@@ -119,6 +119,7 @@ func New(
 	namespace string,
 	registerer prometheus.Registerer,
 	httpConfig HTTPConfig,
+	allowedHosts []string,
 	wrappers ...Wrapper,
 ) (Server, error) {
 	m, err := newMetrics(namespace, registerer)
@@ -127,10 +128,11 @@ func New(
 	}
 
 	router := newRouter()
+	allowedHostsHandler := filterInvalidHosts(router, allowedHosts)
 	corsHandler := cors.New(cors.Options{
 		AllowedOrigins:   allowedOrigins,
 		AllowCredentials: true,
-	}).Handler(router)
+	}).Handler(allowedHostsHandler)
 	gzipHandler := gziphandler.GzipHandler(corsHandler)
 	var handler http.Handler = http.HandlerFunc(
 		func(w http.ResponseWriter, r *http.Request) {

codec/manager.go

@@ -23,14 +23,13 @@ const (
 )
 
 var (
-	ErrUnknownVersion = errors.New("unknown codec version")
-
-	errMarshalNil        = errors.New("can't marshal nil pointer or interface")
-	errUnmarshalNil      = errors.New("can't unmarshal nil")
-	errUnmarshalTooBig   = errors.New("byte array exceeds maximum length")
-	errCantPackVersion   = errors.New("couldn't pack codec version")
-	errCantUnpackVersion = errors.New("couldn't unpack codec version")
-	errDuplicatedVersion = errors.New("duplicated codec version")
+	ErrUnknownVersion    = errors.New("unknown codec version")
+	ErrMarshalNil        = errors.New("can't marshal nil pointer or interface")
+	ErrUnmarshalNil      = errors.New("can't unmarshal nil")
+	ErrUnmarshalTooBig   = errors.New("byte array exceeds maximum length")
+	ErrCantPackVersion   = errors.New("couldn't pack codec version")
+	ErrCantUnpackVersion = errors.New("couldn't unpack codec version")
+	ErrDuplicatedVersion = errors.New("duplicated codec version")
 )
 
 var _ Manager = (*manager)(nil)
@@ -43,7 +42,7 @@ type Manager interface {
 	// Size returns the size, in bytes, of [value] when it's marshaled
 	// using the codec with the given version.
 	// RegisterCodec must have been called with that version.
-	// If [value] is nil, returns [errMarshalNil]
+	// If [value] is nil, returns [ErrMarshalNil]
 	Size(version uint16, value interface{}) (int, error)
 
 	// Marshal the given value using the codec with the given version.
@@ -82,15 +81,15 @@ func (m *manager) RegisterCodec(version uint16, codec Codec) error {
 	defer m.lock.Unlock()
 
 	if _, exists := m.codecs[version]; exists {
-		return errDuplicatedVersion
+		return ErrDuplicatedVersion
 	}
 	m.codecs[version] = codec
 	return nil
 }
 
 func (m *manager) Size(version uint16, value interface{}) (int, error) {
 	if value == nil {
-		return 0, errMarshalNil // can't marshal nil
+		return 0, ErrMarshalNil // can't marshal nil
 	}
 
 	m.lock.RLock()
@@ -110,7 +109,7 @@ func (m *manager) Size(version uint16, value interface{}) (int, error) {
 // To marshal an interface, [value] must be a pointer to the interface.
 func (m *manager) Marshal(version uint16, value interface{}) ([]byte, error) {
 	if value == nil {
-		return nil, errMarshalNil // can't marshal nil
+		return nil, ErrMarshalNil // can't marshal nil
 	}
 
 	m.lock.RLock()
@@ -126,7 +125,7 @@ func (m *manager) Marshal(version uint16, value interface{}) ([]byte, error) {
 	}
 	p.PackShort(version)
 	if p.Errored() {
-		return nil, errCantPackVersion // Should never happen
+		return nil, ErrCantPackVersion // Should never happen
 	}
 	return p.Bytes, c.MarshalInto(value, &p)
 }
@@ -135,19 +134,19 @@ func (m *manager) Marshal(version uint16, value interface{}) ([]byte, error) {
 // interface.
 func (m *manager) Unmarshal(bytes []byte, dest interface{}) (uint16, error) {
 	if dest == nil {
-		return 0, errUnmarshalNil
+		return 0, ErrUnmarshalNil
 	}
 
 	if byteLen := len(bytes); byteLen > m.maxSize {
-		return 0, fmt.Errorf("%w: %d > %d", errUnmarshalTooBig, byteLen, m.maxSize)
+		return 0, fmt.Errorf("%w: %d > %d", ErrUnmarshalTooBig, byteLen, m.maxSize)
 	}
 
 	p := wrappers.Packer{
 		Bytes: bytes,
 	}
 	version := p.UnpackShort()
 	if p.Errored() { // Make sure the codec version is correct
-		return 0, errCantUnpackVersion
+		return 0, ErrCantUnpackVersion
 	}
 
 	m.lock.RLock()

codec/test_codec.go

@@ -746,7 +746,7 @@ func TestTooLargeUnmarshal(codec GeneralCodec, t testing.TB) {
 
 	s := inner{}
 	_, err := manager.Unmarshal(bytes, &s)
-	require.ErrorIs(err, errUnmarshalTooBig)
+	require.ErrorIs(err, ErrUnmarshalTooBig)
 }
 
 type outerInterface interface {

config/config.go

@@ -108,14 +108,12 @@ func getConsensusConfig(v *viper.Viper) snowball.Parameters {
 		//
 		// TODO: After the X-chain linearization use the
 		// SnowVirtuousCommitThresholdKey as before.
-		BetaVirtuous:            v.GetInt(SnowRogueCommitThresholdKey),
-		BetaRogue:               v.GetInt(SnowRogueCommitThresholdKey),
-		ConcurrentRepolls:       v.GetInt(SnowConcurrentRepollsKey),
-		OptimalProcessing:       v.GetInt(SnowOptimalProcessingKey),
-		MaxOutstandingItems:     v.GetInt(SnowMaxProcessingKey),
-		MaxItemProcessingTime:   v.GetDuration(SnowMaxTimeProcessingKey),
-		MixedQueryNumPushVdr:    int(v.GetUint(SnowMixedQueryNumPushVdrKey)),
-		MixedQueryNumPushNonVdr: int(v.GetUint(SnowMixedQueryNumPushNonVdrKey)),
+		BetaVirtuous:          v.GetInt(SnowRogueCommitThresholdKey),
+		BetaRogue:             v.GetInt(SnowRogueCommitThresholdKey),
+		ConcurrentRepolls:     v.GetInt(SnowConcurrentRepollsKey),
+		OptimalProcessing:     v.GetInt(SnowOptimalProcessingKey),
+		MaxOutstandingItems:   v.GetInt(SnowMaxProcessingKey),
+		MaxItemProcessingTime: v.GetDuration(SnowMaxTimeProcessingKey),
 	}
 }
 
@@ -238,14 +236,15 @@ func getHTTPConfig(v *viper.Viper) (node.HTTPConfig, error) {
 			MetricsAPIEnabled:  v.GetBool(MetricsAPIEnabledKey),
 			HealthAPIEnabled:   v.GetBool(HealthAPIEnabledKey),
 		},
-		HTTPHost:          v.GetString(HTTPHostKey),
-		HTTPPort:          uint16(v.GetUint(HTTPPortKey)),
-		HTTPSEnabled:      v.GetBool(HTTPSEnabledKey),
-		HTTPSKey:          httpsKey,
-		HTTPSCert:         httpsCert,
-		APIAllowedOrigins: v.GetStringSlice(HTTPAllowedOrigins),
-		ShutdownTimeout:   v.GetDuration(HTTPShutdownTimeoutKey),
-		ShutdownWait:      v.GetDuration(HTTPShutdownWaitKey),
+		HTTPHost:           v.GetString(HTTPHostKey),
+		HTTPPort:           uint16(v.GetUint(HTTPPortKey)),
+		HTTPSEnabled:       v.GetBool(HTTPSEnabledKey),
+		HTTPSKey:           httpsKey,
+		HTTPSCert:          httpsCert,
+		HTTPAllowedOrigins: v.GetStringSlice(HTTPAllowedOrigins),
+		HTTPAllowedHosts:   v.GetStringSlice(HTTPAllowedHostsKey),
+		ShutdownTimeout:    v.GetDuration(HTTPShutdownTimeoutKey),
+		ShutdownWait:       v.GetDuration(HTTPShutdownWaitKey),
 	}
 
 	config.APIAuthConfig, err = getAPIAuthConfig(v)

config/flags.go

@@ -221,6 +221,7 @@ func addNodeFlags(fs *pflag.FlagSet) {
 	fs.String(HTTPSCertFileKey, "", fmt.Sprintf("TLS certificate file for the HTTPs server. Ignored if %s is specified", HTTPSCertContentKey))
 	fs.String(HTTPSCertContentKey, "", "Specifies base64 encoded TLS certificate for the HTTPs server")
 	fs.String(HTTPAllowedOrigins, "*", "Origins to allow on the HTTP port. Defaults to * which allows all origins. Example: https://*.avax.network https://*.avax-test.network")
+	fs.StringSlice(HTTPAllowedHostsKey, []string{"localhost"}, "List of acceptable host names in API requests. Provide the wildcard ('*') to accept requests from all hosts. API requests where the Host field is empty or an IP address will always be accepted. An API call whose HTTP Host field isn't acceptable will receive a 403 error code")
 	fs.Duration(HTTPShutdownWaitKey, 0, "Duration to wait after receiving SIGTERM or SIGINT before initiating shutdown. The /health endpoint will return unhealthy during this duration")
 	fs.Duration(HTTPShutdownTimeoutKey, 10*time.Second, "Maximum duration to wait for existing connections to complete during node shutdown")
 	fs.Duration(HTTPReadTimeoutKey, 30*time.Second, "Maximum duration for reading the entire request, including the body. A zero or negative value means there will be no timeout")
@@ -318,8 +319,6 @@ func addNodeFlags(fs *pflag.FlagSet) {
 	fs.Int(SnowOptimalProcessingKey, 10, "Optimal number of processing containers in consensus")
 	fs.Int(SnowMaxProcessingKey, 256, "Maximum number of processing items to be considered healthy")
 	fs.Duration(SnowMaxTimeProcessingKey, 30*time.Second, "Maximum amount of time an item should be processing and still be healthy")
-	fs.Uint(SnowMixedQueryNumPushVdrKey, 10, fmt.Sprintf("If this node is a validator, when a container is inserted into consensus, send a Push Query to %s validators and a Pull Query to the others. Must be <= k.", SnowMixedQueryNumPushVdrKey))
-	fs.Uint(SnowMixedQueryNumPushNonVdrKey, 0, fmt.Sprintf("If this node is not a validator, when a container is inserted into consensus, send a Push Query to %s validators and a Pull Query to the others. Must be <= k.", SnowMixedQueryNumPushNonVdrKey))
 
 	// ProposerVM
 	fs.Bool(ProposerVMUseCurrentHeightKey, false, "Have the ProposerVM always report the last accepted P-chain block height")

config/keys.go

@@ -54,6 +54,7 @@ const (
 	HTTPSCertFileKey                                   = "http-tls-cert-file"
 	HTTPSCertContentKey                                = "http-tls-cert-file-content"
 	HTTPAllowedOrigins                                 = "http-allowed-origins"
+	HTTPAllowedHostsKey                                = "http-allowed-hosts"
 	HTTPShutdownTimeoutKey                             = "http-shutdown-timeout"
 	HTTPShutdownWaitKey                                = "http-shutdown-wait"
 	HTTPReadTimeoutKey                                 = "http-read-timeout"
@@ -136,8 +137,6 @@ const (
 	SnowOptimalProcessingKey                           = "snow-optimal-processing"
 	SnowMaxProcessingKey                               = "snow-max-processing"
 	SnowMaxTimeProcessingKey                           = "snow-max-time-processing"
-	SnowMixedQueryNumPushVdrKey                        = "snow-mixed-query-num-push-vdr"
-	SnowMixedQueryNumPushNonVdrKey                     = "snow-mixed-query-num-push-non-vdr"
 	TrackSubnetsKey                                    = "track-subnets"
 	AdminAPIEnabledKey                                 = "api-admin-enabled"
 	InfoAPIEnabledKey                                  = "api-info-enabled"