Update geoip databases to include data about documentation ranges

go.mod

@@ -28,12 +28,14 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/magefile/mage v1.15.0
+	github.com/maxmind/mmdbwriter v1.0.0
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/spf13/cobra v1.9.1
 	github.com/stretchr/testify v1.10.0
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
 	golang.org/x/tools v0.31.0
 	gopkg.in/dnaeon/go-vcr.v3 v3.2.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -124,6 +126,7 @@ require (
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/oschwald/maxminddb-golang v1.12.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -234,6 +234,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
+github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -276,6 +278,8 @@ github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM
 github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
 github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
+github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
 github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8=
 github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I=
 github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs=
@@ -369,6 +373,8 @@ go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7
 go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

internal/fields/validate.go

@@ -1262,10 +1262,10 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti
 	return nil
 }
 
-// IsDocumentation reports whether ip is a reserved address for documentation,
+// isDocumentation reports whether ip is a reserved address for documentation,
 // according to RFC 5737 (IPv4 Address Blocks Reserved for Documentation) and
 // RFC 3849 (IPv6 Address Prefix Reserved for Documentation).
-func IsDocumentation(ip net.IP) bool {
+func isDocumentation(ip net.IP) bool {
 	if ip4 := ip.To4(); ip4 != nil {
 		// Following RFC 5737, Section 3. Documentation Address Blocks which says:
 		//   The blocks 192.0.2.0/24 (TEST-NET-1), 198.51.100.0/24 (TEST-NET-2),
@@ -1285,6 +1285,7 @@ func IsDocumentation(ip net.IP) bool {
 // The set of allowed IPs are:
 // - private IPs as described in RFC 1918 & RFC 4193
 // - public IPs allowed by MaxMind for testing
+// - Reserved IPs for documentation RFC 5737 and RFC 3849
 // - 0.0.0.0 and 255.255.255.255 for IPv4
 // - 0:0:0:0:0:0:0:0 and ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff for IPv6
 func (v *Validator) isAllowedIPValue(s string) bool {
@@ -1301,7 +1302,7 @@ func (v *Validator) isAllowedIPValue(s string) bool {
 
 	if ip.IsUnspecified() ||
 		ip.IsPrivate() ||
-		IsDocumentation(ip) ||
+		isDocumentation(ip) ||
 		ip.IsLoopback() ||
 		ip.IsLinkLocalUnicast() ||
 		ip.IsLinkLocalMulticast() ||

internal/fields/validate_test.go

@@ -6,6 +6,7 @@ package fields
 
 import (
 	"encoding/json"
+	"net"
 	"os"
 	"path/filepath"
 	"sort"
@@ -1184,3 +1185,87 @@ func readSampleEvent(t *testing.T, path string) json.RawMessage {
 	require.NoError(t, err)
 	return c
 }
+
+func Test_IsAllowedIPValue(t *testing.T) {
+	cases := []struct {
+		title      string
+		ip         string
+		allowedIps []string
+		expected   bool
+	}{
+		{
+			title:    "private ipv4",
+			ip:       "192.168.1.2",
+			expected: true,
+		},
+		{
+			title:    "private ipv4 other range",
+			ip:       "10.2.2.2",
+			expected: true,
+		},
+		{
+			title:    "documentation IPv4",
+			ip:       "192.0.2.10",
+			expected: true,
+		},
+		{
+			title:    "documentation IPv6",
+			ip:       "2001:0DB8:1000:1000:1000:1000:1000:1000",
+			expected: true,
+		},
+		{
+			title:    "unspecified ipv4",
+			ip:       "0.0.0.0",
+			expected: true,
+		},
+		{
+			title:    "unspecified ipv6",
+			ip:       "0:0:0:0:0:0:0:0",
+			expected: true,
+		},
+		{
+			title:    "ip allowed CIDR",
+			ip:       "89.160.20.115",
+			expected: true,
+			allowedIps: []string{
+				"89.160.20.112/28",
+			},
+		},
+		{
+			title:    "not valid ipv4",
+			ip:       "216.160.83.57",
+			expected: false,
+			allowedIps: []string{
+				"89.160.20.112/28",
+			},
+		},
+		{
+			title:    "not valid ipv6",
+			ip:       "2002:2002:1000:1000:1000:1000:1000:1000",
+			expected: false,
+			allowedIps: []string{
+				"89.160.20.112/28",
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			allowedCIDRs := []*net.IPNet{}
+			for _, cidr := range c.allowedIps {
+				_, cidr, err := net.ParseCIDR(cidr)
+				require.NoError(t, err)
+				allowedCIDRs = append(allowedCIDRs, cidr)
+			}
+			v := Validator{
+				disabledDependencyManagement: true,
+				enabledAllowedIPCheck:        true,
+				allowedCIDRs:                 allowedCIDRs,
+			}
+
+			allowed := v.isAllowedIPValue(c.ip)
+			assert.Equal(t, c.expected, allowed)
+		})
+	}
+
+}