From 575fe0d0b5083e86226f7d9c8c9def1845363533 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 10 May 2015 11:15:59 +0300 Subject: [PATCH 0001/1746] pool: gracefully close pool by giving users time to free connection. --- pool.go | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pool.go b/pool.go index 5194bc80ca..27125b414c 100644 --- a/pool.go +++ b/pool.go @@ -66,6 +66,7 @@ func (l *connList) Add(cn *conn) { l.mx.Unlock() } +// Remove closes connection and removes it from the list. func (l *connList) Remove(cn *conn) error { defer l.mx.Unlock() l.mx.Lock() @@ -172,8 +173,8 @@ func (p *connPool) First() *conn { } // wait waits for free non-idle connection. It returns nil on timeout. -func (p *connPool) wait(timeout time.Duration) *conn { - deadline := time.After(timeout) +func (p *connPool) wait() *conn { + deadline := time.After(p.opt.PoolTimeout) for { select { case cn := <-p.freeConns: @@ -208,14 +209,13 @@ func (p *connPool) new() (*conn, error) { return cn, nil } -// Get returns existed connection from the pool or creates a new one -// if needed. +// Get returns existed connection from the pool or creates a new one. func (p *connPool) Get() (*conn, error) { if p.closed() { return nil, errClosed } - // Fetch first non-idle connection, if available + // Fetch first non-idle connection, if available. if cn := p.First(); cn != nil { return cn, nil } @@ -231,8 +231,8 @@ func (p *connPool) Get() (*conn, error) { return cn, nil } - // Otherwise, wait for the available connection - if cn := p.wait(p.opt.PoolTimeout); cn != nil { + // Otherwise, wait for the available connection. + if cn := p.wait(); cn != nil { return cn, nil } @@ -277,11 +277,25 @@ func (p *connPool) FreeLen() int { return len(p.freeConns) } -func (p *connPool) Close() error { +func (p *connPool) Close() (retErr error) { if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { return errClosed } - return p.conns.Close() + // First close free connections. + for p.Len() > 0 { + cn := p.wait() + if cn == nil { + break + } + if err := p.conns.Remove(cn); err != nil { + retErr = err + } + } + // Then close the rest. + if err := p.conns.Close(); err != nil { + retErr = err + } + return retErr } func (p *connPool) reaper() { From 7f1eb05ba827fde075d2b0d139e79e78955064e6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 10 May 2015 16:01:38 +0300 Subject: [PATCH 0002/1746] cluster: don't reset command when there no attempts left. --- cluster.go | 9 +++++++-- cluster_test.go | 26 ++++++++++++++++++++------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/cluster.go b/cluster.go index 0ed0954d63..20ecbe9e65 100644 --- a/cluster.go +++ b/cluster.go @@ -130,6 +130,10 @@ func (c *ClusterClient) process(cmd Cmder) { } for attempt := 0; attempt <= c.opt.getMaxRedirects(); attempt++ { + if attempt > 0 { + cmd.reset() + } + if ask { pipe := client.Pipeline() pipe.Process(NewCmd("ASKING")) @@ -152,7 +156,6 @@ func (c *ClusterClient) process(cmd Cmder) { if err != nil { return } - cmd.reset() continue } @@ -167,7 +170,6 @@ func (c *ClusterClient) process(cmd Cmder) { if err != nil { return } - cmd.reset() continue } @@ -282,6 +284,9 @@ type ClusterOptions struct { } func (opt *ClusterOptions) getMaxRedirects() int { + if opt.MaxRedirects == -1 { + return 0 + } if opt.MaxRedirects == 0 { return 16 } diff --git a/cluster_test.go b/cluster_test.go index 6b066870e8..2ba8812ec2 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -40,14 +40,16 @@ func (s *clusterScenario) slaves() []*redis.Client { return result } -func (s *clusterScenario) clusterClient() *redis.ClusterClient { +func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.ClusterClient { addrs := make([]string, len(s.ports)) for i, port := range s.ports { addrs[i] = net.JoinHostPort("127.0.0.1", port) } - return redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: addrs, - }) + if opt == nil { + opt = &redis.ClusterOptions{} + } + opt.Addrs = addrs + return redis.NewClusterClient(opt) } func startCluster(scenario *clusterScenario) error { @@ -228,7 +230,7 @@ var _ = Describe("Cluster", func() { var client *redis.ClusterClient BeforeEach(func() { - client = scenario.clusterClient() + client = scenario.clusterClient(nil) }) AfterEach(func() { @@ -301,6 +303,18 @@ var _ = Describe("Cluster", func() { Expect(cmds[27].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) }) + It("should return error when there are no attempts left", func() { + client = scenario.clusterClient(&redis.ClusterOptions{ + MaxRedirects: -1, + }) + + slot := redis.HashSlot("A") + Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + + err := client.Get("A").Err() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("MOVED")) + }) }) }) @@ -317,7 +331,7 @@ func BenchmarkRedisClusterPing(b *testing.B) { b.Fatal(err) } defer stopCluster(scenario) - client := scenario.clusterClient() + client := scenario.clusterClient(nil) defer client.Close() b.ResetTimer() From 8c67e00efc41fac0745675b2ec90c0ee24a2dbc0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 10 May 2015 15:33:04 +0300 Subject: [PATCH 0003/1746] Add auto-retry and MaxRetries option. Fixes #84. --- command.go | 6 +++++ conn.go | 16 ++++++++----- error.go | 10 +++++++- export_test.go | 6 +++++ pipeline.go | 42 +++++++++++++++++++--------------- pool.go | 4 ---- pubsub.go | 3 ++- redis.go | 62 ++++++++++++++++++++++++++++++++++---------------- redis_test.go | 35 ++++++++++++++++++++++++++++ 9 files changed, 134 insertions(+), 50 deletions(-) diff --git a/command.go b/command.go index 977a313892..fcea708cb8 100644 --- a/command.go +++ b/command.go @@ -47,6 +47,12 @@ func setCmdsErr(cmds []Cmder, e error) { } } +func resetCmds(cmds []Cmder) { + for _, cmd := range cmds { + cmd.reset() + } +} + func cmdString(cmd Cmder, val interface{}) string { s := strings.Join(cmd.args(), " ") if err := cmd.Err(); err != nil { diff --git a/conn.go b/conn.go index 6ce558059a..751bb54da6 100644 --- a/conn.go +++ b/conn.go @@ -7,14 +7,18 @@ import ( "gopkg.in/bufio.v1" ) +var ( + zeroTime = time.Time{} +) + type conn struct { netcn net.Conn rd *bufio.Reader buf []byte usedAt time.Time - readTimeout time.Duration - writeTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration } func newConnDialer(opt *options) func() (*conn, error) { @@ -70,8 +74,8 @@ func (cn *conn) writeCmds(cmds ...Cmder) error { } func (cn *conn) Read(b []byte) (int, error) { - if cn.readTimeout != 0 { - cn.netcn.SetReadDeadline(time.Now().Add(cn.readTimeout)) + if cn.ReadTimeout != 0 { + cn.netcn.SetReadDeadline(time.Now().Add(cn.ReadTimeout)) } else { cn.netcn.SetReadDeadline(zeroTime) } @@ -79,8 +83,8 @@ func (cn *conn) Read(b []byte) (int, error) { } func (cn *conn) Write(b []byte) (int, error) { - if cn.writeTimeout != 0 { - cn.netcn.SetWriteDeadline(time.Now().Add(cn.writeTimeout)) + if cn.WriteTimeout != 0 { + cn.netcn.SetWriteDeadline(time.Now().Add(cn.WriteTimeout)) } else { cn.netcn.SetWriteDeadline(zeroTime) } diff --git a/error.go b/error.go index 0e031f3184..7709c510cd 100644 --- a/error.go +++ b/error.go @@ -26,7 +26,7 @@ func (err redisError) Error() string { } func isNetworkError(err error) bool { - if _, ok := err.(*net.OpError); ok || err == io.EOF { + if _, ok := err.(net.Error); ok || err == io.EOF { return true } return false @@ -53,3 +53,11 @@ func isMovedError(err error) (moved bool, ask bool, addr string) { return } + +// shouldRetry reports whether failed command should be retried. +func shouldRetry(err error) bool { + if err == nil { + return false + } + return isNetworkError(err) +} diff --git a/export_test.go b/export_test.go index 53519d5a9e..f4687296a8 100644 --- a/export_test.go +++ b/export_test.go @@ -1,9 +1,15 @@ package redis +import "net" + func (c *baseClient) Pool() pool { return c.connPool } +func (cn *conn) SetNetConn(netcn net.Conn) { + cn.netcn = netcn +} + func HashSlot(key string) int { return hashSlot(key) } diff --git a/pipeline.go b/pipeline.go index 8cfd5a11e2..62cf7fd171 100644 --- a/pipeline.go +++ b/pipeline.go @@ -50,26 +50,38 @@ func (c *Pipeline) Discard() error { // Exec always returns list of commands and error of the first failed // command if any. -func (c *Pipeline) Exec() ([]Cmder, error) { +func (c *Pipeline) Exec() (cmds []Cmder, retErr error) { if c.closed { return nil, errClosed } if len(c.cmds) == 0 { - return []Cmder{}, nil + return c.cmds, nil } - cmds := c.cmds + cmds = c.cmds c.cmds = make([]Cmder, 0, 0) - cn, err := c.client.conn() - if err != nil { - setCmdsErr(cmds, err) - return cmds, err + for i := 0; i <= c.client.opt.MaxRetries; i++ { + if i > 0 { + resetCmds(cmds) + } + + cn, err := c.client.conn() + if err != nil { + setCmdsErr(cmds, err) + return cmds, err + } + + retErr = c.execCmds(cn, cmds) + c.client.putConn(cn, err) + if shouldRetry(err) { + continue + } + + break } - err = c.execCmds(cn, cmds) - c.client.putConn(cn, err) - return cmds, err + return cmds, retErr } func (c *Pipeline) execCmds(cn *conn, cmds []Cmder) error { @@ -79,17 +91,11 @@ func (c *Pipeline) execCmds(cn *conn, cmds []Cmder) error { } var firstCmdErr error - for i, cmd := range cmds { + for _, cmd := range cmds { err := cmd.parseReply(cn.rd) - if err == nil { - continue - } - if firstCmdErr == nil { + if err != nil && firstCmdErr == nil { firstCmdErr = err } - if isNetworkError(err) { - setCmdsErr(cmds[i:], err) - } } return firstCmdErr diff --git a/pool.go b/pool.go index 5194bc80ca..8af50e56d6 100644 --- a/pool.go +++ b/pool.go @@ -16,10 +16,6 @@ var ( errPoolTimeout = errors.New("redis: connection pool timeout") ) -var ( - zeroTime = time.Time{} -) - type pool interface { First() *conn Get() (*conn, error) diff --git a/pubsub.go b/pubsub.go index 1f4e91179f..5a32220be2 100644 --- a/pubsub.go +++ b/pubsub.go @@ -63,7 +63,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { if err != nil { return nil, err } - cn.readTimeout = timeout + cn.ReadTimeout = timeout cmd := NewSliceCmd() if err := cmd.parseReply(cn.rd); err != nil { @@ -92,6 +92,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { Payload: reply[3].(string), }, nil } + return nil, fmt.Errorf("redis: unsupported message name: %q", msgName) } diff --git a/redis.go b/redis.go index a2fe519307..6d774e435c 100644 --- a/redis.go +++ b/redis.go @@ -32,32 +32,46 @@ func (c *baseClient) putConn(cn *conn, ei error) { } func (c *baseClient) process(cmd Cmder) { - cn, err := c.conn() - if err != nil { - cmd.setErr(err) - return - } + for i := 0; i <= c.opt.MaxRetries; i++ { + if i > 0 { + cmd.reset() + } - if timeout := cmd.writeTimeout(); timeout != nil { - cn.writeTimeout = *timeout - } else { - cn.writeTimeout = c.opt.WriteTimeout - } + cn, err := c.conn() + if err != nil { + cmd.setErr(err) + return + } - if timeout := cmd.readTimeout(); timeout != nil { - cn.readTimeout = *timeout - } else { - cn.readTimeout = c.opt.ReadTimeout - } + if timeout := cmd.writeTimeout(); timeout != nil { + cn.WriteTimeout = *timeout + } else { + cn.WriteTimeout = c.opt.WriteTimeout + } - if err := cn.writeCmds(cmd); err != nil { + if timeout := cmd.readTimeout(); timeout != nil { + cn.ReadTimeout = *timeout + } else { + cn.ReadTimeout = c.opt.ReadTimeout + } + + if err := cn.writeCmds(cmd); err != nil { + c.putConn(cn, err) + cmd.setErr(err) + if shouldRetry(err) { + continue + } + return + } + + err = cmd.parseReply(cn.rd) c.putConn(cn, err) - cmd.setErr(err) + if shouldRetry(err) { + continue + } + return } - - err = cmd.parseReply(cn.rd) - c.putConn(cn, err) } // Close closes the client, releasing any open resources. @@ -105,6 +119,10 @@ type Options struct { // than specified in this option. // Default: 0 = no eviction IdleTimeout time.Duration + + // MaxRetries specifies maximum number of times client will retry + // failed command. Default is to not retry failed command. + MaxRetries int } func (opt *Options) getDialer() func() (net.Conn, error) { @@ -157,6 +175,8 @@ func (opt *Options) options() *options { DialTimeout: opt.getDialTimeout(), ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, + + MaxRetries: opt.MaxRetries, } } @@ -172,6 +192,8 @@ type options struct { DialTimeout time.Duration ReadTimeout time.Duration WriteTimeout time.Duration + + MaxRetries int } func (opt *options) connPoolOptions() *connPoolOptions { diff --git a/redis_test.go b/redis_test.go index 58676bc490..0198aba43a 100644 --- a/redis_test.go +++ b/redis_test.go @@ -124,6 +124,23 @@ var _ = Describe("Client", func() { Expect(db1.FlushDb().Err()).NotTo(HaveOccurred()) }) + It("should retry command on network error", func() { + Expect(client.Close()).NotTo(HaveOccurred()) + + client = redis.NewClient(&redis.Options{ + Addr: redisAddr, + MaxRetries: 1, + }) + + // Put bad connection in the pool. + cn, err := client.Pool().Get() + Expect(err).NotTo(HaveOccurred()) + cn.SetNetConn(newBadNetConn()) + Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) + + err = client.Ping().Err() + Expect(err).NotTo(HaveOccurred()) + }) }) //------------------------------------------------------------------------------ @@ -266,6 +283,24 @@ func BenchmarkPipeline(b *testing.B) { //------------------------------------------------------------------------------ +type badNetConn struct { + net.TCPConn +} + +var _ net.Conn = &badNetConn{} + +func newBadNetConn() net.Conn { + return &badNetConn{} +} + +func (badNetConn) Read([]byte) (int, error) { + return 0, net.UnknownNetworkError("badNetConn") +} + +func (badNetConn) Write([]byte) (int, error) { + return 0, net.UnknownNetworkError("badNetConn") +} + // Replaces ginkgo's Eventually. func waitForSubstring(fn func() string, substr string, timeout time.Duration) error { var s string From 84cd16e214b89c1d50cd57b99ada642119a82479 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 13 May 2015 12:38:34 +0300 Subject: [PATCH 0004/1746] cluster: user ClusterInfo instead of Ping to find live node. --- cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 20ecbe9e65..d09298ee03 100644 --- a/cluster.go +++ b/cluster.go @@ -97,7 +97,7 @@ func (c *ClusterClient) slotAddrs(slot int) []string { return addrs } -// randomClient returns a Client for the first pingable node. +// randomClient returns a Client for the first live node. func (c *ClusterClient) randomClient() (client *Client, err error) { for i := 0; i < 10; i++ { n := rand.Intn(len(c.addrs)) @@ -105,7 +105,7 @@ func (c *ClusterClient) randomClient() (client *Client, err error) { if err != nil { continue } - err = client.Ping().Err() + err = client.ClusterInfo().Err() if err == nil { return client, nil } From f24d3ff013afa1833cbd6f8f5a0872bcfcd22350 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 14 May 2015 14:54:38 +0300 Subject: [PATCH 0005/1746] Increase test timeout. --- cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_test.go b/cluster_test.go index 2ba8812ec2..6ec8552fb2 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -271,7 +271,7 @@ var _ = Describe("Cluster", func() { Eventually(func() []string { return client.SlotAddrs(slot) - }).Should(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) + }, "5s").Should(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) }) It("should perform multi-pipelines", func() { From 1078a303ea52f2d577115ef18cc300678c766487 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 14 May 2015 16:13:45 +0300 Subject: [PATCH 0006/1746] sentine: don't pass DB and Password to Sentinel client. --- cluster_test.go | 35 +++----- commands_test.go | 5 +- main_test.go | 223 +++++++++++++++++++++++++++++++++++++++++++++++ redis_test.go | 167 +---------------------------------- sentinel.go | 3 - sentinel_test.go | 59 ++++++------- 6 files changed, 265 insertions(+), 227 deletions(-) create mode 100644 main_test.go diff --git a/cluster_test.go b/cluster_test.go index 6ec8552fb2..d892c1060f 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -144,21 +144,6 @@ func stopCluster(scenario *clusterScenario) error { //------------------------------------------------------------------------------ var _ = Describe("Cluster", func() { - scenario := &clusterScenario{ - ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, - nodeIds: make([]string, 6), - processes: make(map[string]*redisProcess, 6), - clients: make(map[string]*redis.Client, 6), - } - - BeforeSuite(func() { - Expect(startCluster(scenario)).NotTo(HaveOccurred()) - }) - - AfterSuite(func() { - Expect(stopCluster(scenario)).NotTo(HaveOccurred()) - }) - Describe("HashSlot", func() { It("should calculate hash slots", func() { @@ -202,7 +187,7 @@ var _ = Describe("Cluster", func() { Describe("Commands", func() { It("should CLUSTER SLOTS", func() { - res, err := scenario.primary().ClusterSlots().Result() + res, err := cluster.primary().ClusterSlots().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(3)) Expect(res).To(ConsistOf([]redis.ClusterSlotInfo{ @@ -213,13 +198,13 @@ var _ = Describe("Cluster", func() { }) It("should CLUSTER NODES", func() { - res, err := scenario.primary().ClusterNodes().Result() + res, err := cluster.primary().ClusterNodes().Result() Expect(err).NotTo(HaveOccurred()) Expect(len(res)).To(BeNumerically(">", 400)) }) It("should CLUSTER INFO", func() { - res, err := scenario.primary().ClusterInfo().Result() + res, err := cluster.primary().ClusterInfo().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(ContainSubstring("cluster_known_nodes:6")) }) @@ -230,11 +215,11 @@ var _ = Describe("Cluster", func() { var client *redis.ClusterClient BeforeEach(func() { - client = scenario.clusterClient(nil) + client = cluster.clusterClient(nil) }) AfterEach(func() { - for _, client := range scenario.masters() { + for _, client := range cluster.masters() { Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) } Expect(client.Close()).NotTo(HaveOccurred()) @@ -304,7 +289,7 @@ var _ = Describe("Cluster", func() { }) It("should return error when there are no attempts left", func() { - client = scenario.clusterClient(&redis.ClusterOptions{ + client = cluster.clusterClient(&redis.ClusterOptions{ MaxRedirects: -1, }) @@ -321,17 +306,17 @@ var _ = Describe("Cluster", func() { //------------------------------------------------------------------------------ func BenchmarkRedisClusterPing(b *testing.B) { - scenario := &clusterScenario{ + cluster := &clusterScenario{ ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, nodeIds: make([]string, 6), processes: make(map[string]*redisProcess, 6), clients: make(map[string]*redis.Client, 6), } - if err := startCluster(scenario); err != nil { + if err := startCluster(cluster); err != nil { b.Fatal(err) } - defer stopCluster(scenario) - client := scenario.clusterClient(nil) + defer stopCluster(cluster) + client := cluster.clusterClient(nil) defer client.Close() b.ResetTimer() diff --git a/commands_test.go b/commands_test.go index b451c521ca..7a2add4438 100644 --- a/commands_test.go +++ b/commands_test.go @@ -307,9 +307,8 @@ var _ = Describe("Commands", func() { Expect(refCount.Err()).NotTo(HaveOccurred()) Expect(refCount.Val()).To(Equal(int64(1))) - enc := client.ObjectEncoding("key") - Expect(enc.Err()).NotTo(HaveOccurred()) - Expect(enc.Val()).To(Equal("raw")) + err := client.ObjectEncoding("key").Err() + Expect(err).NotTo(HaveOccurred()) idleTime := client.ObjectIdleTime("key") Expect(idleTime.Err()).NotTo(HaveOccurred()) diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000000..2a0f83685e --- /dev/null +++ b/main_test.go @@ -0,0 +1,223 @@ +package redis_test + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v2" +) + +const redisAddr = ":6379" + +const ( + sentinelName = "mymaster" + sentinelMasterPort = "8123" + sentinelSlave1Port = "8124" + sentinelSlave2Port = "8125" + sentinelPort = "8126" +) + +var sentinelMaster, sentinelSlave1, sentinelSlave2, sentinel *redisProcess + +var cluster = &clusterScenario{ + ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, + nodeIds: make([]string, 6), + processes: make(map[string]*redisProcess, 6), + clients: make(map[string]*redis.Client, 6), +} + +var _ = BeforeSuite(func() { + var err error + + sentinelMaster, err = startRedis(sentinelMasterPort) + Expect(err).NotTo(HaveOccurred()) + + sentinel, err = startSentinel(sentinelPort, sentinelName, sentinelMasterPort) + Expect(err).NotTo(HaveOccurred()) + + sentinelSlave1, err = startRedis( + sentinelSlave1Port, "--slaveof", "127.0.0.1", sentinelMasterPort) + Expect(err).NotTo(HaveOccurred()) + + sentinelSlave2, err = startRedis( + sentinelSlave2Port, "--slaveof", "127.0.0.1", sentinelMasterPort) + Expect(err).NotTo(HaveOccurred()) + + Expect(startCluster(cluster)).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + Expect(sentinel.Close()).NotTo(HaveOccurred()) + Expect(sentinelSlave1.Close()).NotTo(HaveOccurred()) + Expect(sentinelSlave2.Close()).NotTo(HaveOccurred()) + Expect(sentinelMaster.Close()).NotTo(HaveOccurred()) + + Expect(stopCluster(cluster)).NotTo(HaveOccurred()) +}) + +func TestGinkgoSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "gopkg.in/redis.v2") +} + +//------------------------------------------------------------------------------ + +// Replaces ginkgo's Eventually. +func waitForSubstring(fn func() string, substr string, timeout time.Duration) error { + var s string + + found := make(chan struct{}) + var exit int32 + go func() { + for atomic.LoadInt32(&exit) == 0 { + s = fn() + if strings.Contains(s, substr) { + found <- struct{}{} + return + } + time.Sleep(timeout / 100) + } + }() + + select { + case <-found: + return nil + case <-time.After(timeout): + atomic.StoreInt32(&exit, 1) + } + return fmt.Errorf("%q does not contain %q", s, substr) +} + +func execCmd(name string, args ...string) (*os.Process, error) { + cmd := exec.Command(name, args...) + if testing.Verbose() { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + return cmd.Process, cmd.Start() +} + +func connectTo(port string) (client *redis.Client, err error) { + client = redis.NewClient(&redis.Options{ + Addr: ":" + port, + }) + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if err = client.Ping().Err(); err == nil { + return client, nil + } + time.Sleep(250 * time.Millisecond) + } + + return nil, err +} + +type redisProcess struct { + *os.Process + *redis.Client +} + +func (p *redisProcess) Close() error { + p.Client.Close() + return p.Kill() +} + +var ( + redisServerBin, _ = filepath.Abs(filepath.Join(".test", "redis", "src", "redis-server")) + redisServerConf, _ = filepath.Abs(filepath.Join(".test", "redis.conf")) +) + +func redisDir(port string) (string, error) { + dir, err := filepath.Abs(filepath.Join(".test", "instances", port)) + if err != nil { + return "", err + } else if err = os.RemoveAll(dir); err != nil { + return "", err + } else if err = os.MkdirAll(dir, 0775); err != nil { + return "", err + } + return dir, nil +} + +func startRedis(port string, args ...string) (*redisProcess, error) { + dir, err := redisDir(port) + if err != nil { + return nil, err + } + if err = exec.Command("cp", "-f", redisServerConf, dir).Run(); err != nil { + return nil, err + } + + baseArgs := []string{filepath.Join(dir, "redis.conf"), "--port", port, "--dir", dir} + process, err := execCmd(redisServerBin, append(baseArgs, args...)...) + if err != nil { + return nil, err + } + + client, err := connectTo(port) + if err != nil { + process.Kill() + return nil, err + } + return &redisProcess{process, client}, err +} + +func startSentinel(port, masterName, masterPort string) (*redisProcess, error) { + dir, err := redisDir(port) + if err != nil { + return nil, err + } + process, err := execCmd(redisServerBin, os.DevNull, "--sentinel", "--port", port, "--dir", dir) + if err != nil { + return nil, err + } + client, err := connectTo(port) + if err != nil { + process.Kill() + return nil, err + } + for _, cmd := range []*redis.StatusCmd{ + redis.NewStatusCmd("SENTINEL", "MONITOR", masterName, "127.0.0.1", masterPort, "1"), + redis.NewStatusCmd("SENTINEL", "SET", masterName, "down-after-milliseconds", "500"), + redis.NewStatusCmd("SENTINEL", "SET", masterName, "failover-timeout", "1000"), + redis.NewStatusCmd("SENTINEL", "SET", masterName, "parallel-syncs", "1"), + } { + client.Process(cmd) + if err := cmd.Err(); err != nil { + process.Kill() + return nil, err + } + } + return &redisProcess{process, client}, nil +} + +//------------------------------------------------------------------------------ + +type badNetConn struct { + net.TCPConn +} + +var _ net.Conn = &badNetConn{} + +func newBadNetConn() net.Conn { + return &badNetConn{} +} + +func (badNetConn) Read([]byte) (int, error) { + return 0, net.UnknownNetworkError("badNetConn") +} + +func (badNetConn) Write([]byte) (int, error) { + return 0, net.UnknownNetworkError("badNetConn") +} diff --git a/redis_test.go b/redis_test.go index 0198aba43a..1f089d74cf 100644 --- a/redis_test.go +++ b/redis_test.go @@ -1,28 +1,15 @@ package redis_test import ( - "fmt" "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync/atomic" "testing" "time" - "gopkg.in/redis.v2" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" -) - -const redisAddr = ":6379" -func TestGinkgoSuite(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "gopkg.in/redis.v2") -} + "gopkg.in/redis.v2" +) var _ = Describe("Client", func() { var client *redis.Client @@ -280,153 +267,3 @@ func BenchmarkPipeline(b *testing.B) { } }) } - -//------------------------------------------------------------------------------ - -type badNetConn struct { - net.TCPConn -} - -var _ net.Conn = &badNetConn{} - -func newBadNetConn() net.Conn { - return &badNetConn{} -} - -func (badNetConn) Read([]byte) (int, error) { - return 0, net.UnknownNetworkError("badNetConn") -} - -func (badNetConn) Write([]byte) (int, error) { - return 0, net.UnknownNetworkError("badNetConn") -} - -// Replaces ginkgo's Eventually. -func waitForSubstring(fn func() string, substr string, timeout time.Duration) error { - var s string - - found := make(chan struct{}) - var exit int32 - go func() { - for atomic.LoadInt32(&exit) == 0 { - s = fn() - if strings.Contains(s, substr) { - found <- struct{}{} - return - } - time.Sleep(timeout / 100) - } - }() - - select { - case <-found: - return nil - case <-time.After(timeout): - atomic.StoreInt32(&exit, 1) - } - return fmt.Errorf("%q does not contain %q", s, substr) -} - -func execCmd(name string, args ...string) (*os.Process, error) { - cmd := exec.Command(name, args...) - if testing.Verbose() { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - return cmd.Process, cmd.Start() -} - -func connectTo(port string) (client *redis.Client, err error) { - client = redis.NewClient(&redis.Options{ - Addr: ":" + port, - }) - - deadline := time.Now().Add(3 * time.Second) - for time.Now().Before(deadline) { - if err = client.Ping().Err(); err == nil { - return client, nil - } - time.Sleep(250 * time.Millisecond) - } - - return nil, err -} - -type redisProcess struct { - *os.Process - *redis.Client -} - -func (p *redisProcess) Close() error { - p.Client.Close() - return p.Kill() -} - -var ( - redisServerBin, _ = filepath.Abs(filepath.Join(".test", "redis", "src", "redis-server")) - redisServerConf, _ = filepath.Abs(filepath.Join(".test", "redis.conf")) -) - -func redisDir(port string) (string, error) { - dir, err := filepath.Abs(filepath.Join(".test", "instances", port)) - if err != nil { - return "", err - } else if err = os.RemoveAll(dir); err != nil { - return "", err - } else if err = os.MkdirAll(dir, 0775); err != nil { - return "", err - } - return dir, nil -} - -func startRedis(port string, args ...string) (*redisProcess, error) { - dir, err := redisDir(port) - if err != nil { - return nil, err - } - if err = exec.Command("cp", "-f", redisServerConf, dir).Run(); err != nil { - return nil, err - } - - baseArgs := []string{filepath.Join(dir, "redis.conf"), "--port", port, "--dir", dir} - process, err := execCmd(redisServerBin, append(baseArgs, args...)...) - if err != nil { - return nil, err - } - - client, err := connectTo(port) - if err != nil { - process.Kill() - return nil, err - } - return &redisProcess{process, client}, err -} - -func startSentinel(port, masterName, masterPort string) (*redisProcess, error) { - dir, err := redisDir(port) - if err != nil { - return nil, err - } - process, err := execCmd(redisServerBin, os.DevNull, "--sentinel", "--port", port, "--dir", dir) - if err != nil { - return nil, err - } - client, err := connectTo(port) - if err != nil { - process.Kill() - return nil, err - } - for _, cmd := range []*redis.StatusCmd{ - redis.NewStatusCmd("SENTINEL", "MONITOR", masterName, "127.0.0.1", masterPort, "1"), - redis.NewStatusCmd("SENTINEL", "SET", masterName, "down-after-milliseconds", "500"), - redis.NewStatusCmd("SENTINEL", "SET", masterName, "failover-timeout", "1000"), - redis.NewStatusCmd("SENTINEL", "SET", masterName, "parallel-syncs", "1"), - } { - client.Process(cmd) - if err := cmd.Err(); err != nil { - process.Kill() - return nil, err - } - } - return &redisProcess{process, client}, nil -} diff --git a/sentinel.go b/sentinel.go index 717f412287..26ab485577 100644 --- a/sentinel.go +++ b/sentinel.go @@ -185,9 +185,6 @@ func (d *sentinelFailover) MasterAddr() (string, error) { sentinel := newSentinel(&Options{ Addr: sentinelAddr, - DB: d.opt.DB, - Password: d.opt.Password, - DialTimeout: d.opt.DialTimeout, ReadTimeout: d.opt.ReadTimeout, WriteTimeout: d.opt.WriteTimeout, diff --git a/sentinel_test.go b/sentinel_test.go index 6fc2f4f4ff..42d509e65a 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -7,48 +7,34 @@ import ( ) var _ = Describe("Sentinel", func() { + var client *redis.Client - const masterName = "mymaster" - const masterPort = "8123" - const sentinelPort = "8124" - - It("should facilitate failover", func() { - master, err := startRedis(masterPort) - Expect(err).NotTo(HaveOccurred()) - defer master.Close() - - sentinel, err := startSentinel(sentinelPort, masterName, masterPort) - Expect(err).NotTo(HaveOccurred()) - defer sentinel.Close() - - slave1, err := startRedis("8125", "--slaveof", "127.0.0.1", masterPort) - Expect(err).NotTo(HaveOccurred()) - defer slave1.Close() - - slave2, err := startRedis("8126", "--slaveof", "127.0.0.1", masterPort) - Expect(err).NotTo(HaveOccurred()) - defer slave2.Close() - - client := redis.NewFailoverClient(&redis.FailoverOptions{ - MasterName: masterName, + BeforeEach(func() { + client = redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: sentinelName, SentinelAddrs: []string{":" + sentinelPort}, }) - defer client.Close() + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + It("should facilitate failover", func() { // Set value on master, verify - err = client.Set("foo", "master", 0).Err() + err := client.Set("foo", "master", 0).Err() Expect(err).NotTo(HaveOccurred()) - val, err := master.Get("foo").Result() + val, err := sentinelMaster.Get("foo").Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("master")) // Wait until replicated Eventually(func() string { - return slave1.Get("foo").Val() + return sentinelSlave1.Get("foo").Val() }, "1s", "100ms").Should(Equal("master")) Eventually(func() string { - return slave2.Get("foo").Val() + return sentinelSlave2.Get("foo").Val() }, "1s", "100ms").Should(Equal("master")) // Wait until slaves are picked up by sentinel. @@ -57,14 +43,14 @@ var _ = Describe("Sentinel", func() { }, "10s", "100ms").Should(ContainSubstring("slaves=2")) // Kill master. - master.Shutdown() + sentinelMaster.Shutdown() Eventually(func() error { - return master.Ping().Err() + return sentinelMaster.Ping().Err() }, "5s", "100ms").Should(HaveOccurred()) // Wait for Redis sentinel to elect new master. Eventually(func() string { - return slave1.Info().Val() + slave2.Info().Val() + return sentinelSlave1.Info().Val() + sentinelSlave2.Info().Val() }, "30s", "1s").Should(ContainSubstring("role:master")) // Check that client picked up new master. @@ -73,4 +59,15 @@ var _ = Describe("Sentinel", func() { }, "5s", "100ms").ShouldNot(HaveOccurred()) }) + It("supports DB selection", func() { + Expect(client.Close()).NotTo(HaveOccurred()) + + client = redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: sentinelName, + SentinelAddrs: []string{":" + sentinelPort}, + DB: 1, + }) + err := client.Ping().Err() + Expect(err).NotTo(HaveOccurred()) + }) }) From 716ecc313b810fd684028957ab3bcbdf818381c2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 14 May 2015 16:14:25 +0300 Subject: [PATCH 0007/1746] travis: don't test on tip (it is broken). --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1c57ddea88..9eca14708b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ services: go: - 1.3 - 1.4 - - tip install: - go get gopkg.in/bufio.v1 From 18ea75d2ad12abf24f242b637d10e91d79af5dbe Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 14 May 2015 15:19:29 +0300 Subject: [PATCH 0008/1746] Release redis.v3 beta. --- .travis.yml | 4 ++-- cluster_test.go | 2 +- command_test.go | 3 ++- commands_test.go | 3 ++- example_test.go | 2 +- main_test.go | 4 ++-- multi_test.go | 4 ++-- pipeline_test.go | 4 ++-- pool_test.go | 2 +- pubsub_test.go | 4 ++-- redis.go | 2 +- redis_test.go | 2 +- sentinel_test.go | 3 ++- 13 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9eca14708b..169ccd0ada 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,5 +13,5 @@ install: - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega - mkdir -p $HOME/gopath/src/gopkg.in - - mv $HOME/gopath/src/github.com/go-redis/redis $HOME/gopath/src/gopkg.in/redis.v2 - - cd $HOME/gopath/src/gopkg.in/redis.v2 + - mv $HOME/gopath/src/github.com/go-redis/redis $HOME/gopath/src/gopkg.in/redis.v3 + - cd $HOME/gopath/src/gopkg.in/redis.v3 diff --git a/cluster_test.go b/cluster_test.go index d892c1060f..e8e0802ab4 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -10,7 +10,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + "gopkg.in/redis.v3" ) type clusterScenario struct { diff --git a/command_test.go b/command_test.go index d07bbca339..c1c968e94e 100644 --- a/command_test.go +++ b/command_test.go @@ -8,7 +8,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + + "gopkg.in/redis.v3" ) var _ = Describe("Command", func() { diff --git a/commands_test.go b/commands_test.go index 7a2add4438..a34e9d1776 100644 --- a/commands_test.go +++ b/commands_test.go @@ -9,7 +9,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + + "gopkg.in/redis.v3" ) var _ = Describe("Commands", func() { diff --git a/example_test.go b/example_test.go index 230a558548..859b6458e5 100644 --- a/example_test.go +++ b/example_test.go @@ -4,7 +4,7 @@ import ( "fmt" "strconv" - "gopkg.in/redis.v2" + "gopkg.in/redis.v3" ) var client *redis.Client diff --git a/main_test.go b/main_test.go index 2a0f83685e..59032738e9 100644 --- a/main_test.go +++ b/main_test.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + "gopkg.in/redis.v3" ) const redisAddr = ":6379" @@ -67,7 +67,7 @@ var _ = AfterSuite(func() { func TestGinkgoSuite(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "gopkg.in/redis.v2") + RunSpecs(t, "gopkg.in/redis.v3") } //------------------------------------------------------------------------------ diff --git a/multi_test.go b/multi_test.go index 0a8eac94f1..b481a521ce 100644 --- a/multi_test.go +++ b/multi_test.go @@ -1,10 +1,10 @@ package redis_test import ( - "gopkg.in/redis.v2" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "gopkg.in/redis.v3" ) var _ = Describe("Multi", func() { diff --git a/pipeline_test.go b/pipeline_test.go index 8a6c09d3c4..ddf7480b3d 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -4,10 +4,10 @@ import ( "strconv" "sync" - "gopkg.in/redis.v2" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "gopkg.in/redis.v3" ) var _ = Describe("Pipelining", func() { diff --git a/pool_test.go b/pool_test.go index fae6120dc7..3b0d00afe8 100644 --- a/pool_test.go +++ b/pool_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + "gopkg.in/redis.v3" ) var _ = Describe("Pool", func() { diff --git a/pubsub_test.go b/pubsub_test.go index 408a42cc84..82c0ca49c7 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -4,10 +4,10 @@ import ( "net" "time" - "gopkg.in/redis.v2" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "gopkg.in/redis.v3" ) var _ = Describe("PubSub", func() { diff --git a/redis.go b/redis.go index 6d774e435c..3cc5841d81 100644 --- a/redis.go +++ b/redis.go @@ -1,4 +1,4 @@ -package redis // import "gopkg.in/redis.v2" +package redis // import "gopkg.in/redis.v3" import ( "log" diff --git a/redis_test.go b/redis_test.go index 1f089d74cf..945791855b 100644 --- a/redis_test.go +++ b/redis_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + "gopkg.in/redis.v3" ) var _ = Describe("Client", func() { diff --git a/sentinel_test.go b/sentinel_test.go index 42d509e65a..14dcf834ba 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -3,7 +3,8 @@ package redis_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v2" + + "gopkg.in/redis.v3" ) var _ = Describe("Sentinel", func() { From 40bad36dfb23f1fe0dbda4e26435610b17a2e41e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 5 May 2015 12:48:49 +0300 Subject: [PATCH 0009/1746] cluster: don't reload slots if address already changed. --- cluster.go | 16 ++++++++++------ cluster_pipeline.go | 8 +------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cluster.go b/cluster.go index d09298ee03..02a2ed5bee 100644 --- a/cluster.go +++ b/cluster.go @@ -97,6 +97,14 @@ func (c *ClusterClient) slotAddrs(slot int) []string { return addrs } +func (c *ClusterClient) slotMasterAddr(slot int) string { + addrs := c.slotAddrs(slot) + if len(addrs) > 0 { + return addrs[0] + } + return "" +} + // randomClient returns a Client for the first live node. func (c *ClusterClient) randomClient() (client *Client, err error) { for i := 0; i < 10; i++ { @@ -118,11 +126,7 @@ func (c *ClusterClient) process(cmd Cmder) { slot := hashSlot(cmd.clusterKey()) - var addr string - if addrs := c.slotAddrs(slot); len(addrs) > 0 { - addr = addrs[0] // First address is master. - } - + addr := c.slotMasterAddr(slot) client, err := c.getClient(addr) if err != nil { cmd.setErr(err) @@ -163,7 +167,7 @@ func (c *ClusterClient) process(cmd Cmder) { var addr string moved, ask, addr = isMovedError(err) if moved || ask { - if moved { + if moved && c.slotMasterAddr(slot) != addr { c.lazyReloadSlots() } client, err = c.getClient(addr) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 2ddc064d4c..466be7d4e8 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -53,13 +53,7 @@ func (c *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { slot := hashSlot(cmd.clusterKey()) - addrs := c.cluster.slotAddrs(slot) - - var addr string - if len(addrs) > 0 { - addr = addrs[0] // First address is master. - } - + addr := c.cluster.slotMasterAddr(slot) cmdsMap[addr] = append(cmdsMap[addr], cmd) } From 0b87c16b615f29334bf1ae51829dfe2f98657133 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 14 May 2015 16:49:47 +0300 Subject: [PATCH 0010/1746] Remove PSetEx and SetEx. Set should be used instead. --- commands.go | 12 ------------ commands_test.go | 26 -------------------------- 2 files changed, 38 deletions(-) diff --git a/commands.go b/commands.go index 513f52ba20..b6df30cfec 100644 --- a/commands.go +++ b/commands.go @@ -470,12 +470,6 @@ func (c *commandable) MSetNX(pairs ...string) *BoolCmd { return cmd } -func (c *commandable) PSetEx(key string, expiration time.Duration, value string) *StatusCmd { - cmd := NewStatusCmd("PSETEX", key, formatMs(expiration), value) - c.Process(cmd) - return cmd -} - func (c *commandable) Set(key, value string, expiration time.Duration) *StatusCmd { args := []string{"SET", key, value} if expiration > 0 { @@ -501,12 +495,6 @@ func (c *commandable) SetBit(key string, offset int64, value int) *IntCmd { return cmd } -func (c *commandable) SetEx(key string, expiration time.Duration, value string) *StatusCmd { - cmd := NewStatusCmd("SETEX", key, formatSec(expiration), value) - c.Process(cmd) - return cmd -} - func (c *commandable) SetNX(key, value string, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if expiration == 0 { diff --git a/commands_test.go b/commands_test.go index 7a2add4438..6b275a66d8 100644 --- a/commands_test.go +++ b/commands_test.go @@ -872,22 +872,6 @@ var _ = Describe("Commands", func() { Expect(mSetNX.Val()).To(Equal(false)) }) - It("should PSetEx", func() { - expiration := 50 * time.Millisecond - psetex := client.PSetEx("key", expiration, "hello") - Expect(psetex.Err()).NotTo(HaveOccurred()) - Expect(psetex.Val()).To(Equal("OK")) - - pttl := client.PTTL("key") - Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val() <= expiration).To(Equal(true)) - Expect(pttl.Val() >= expiration-time.Millisecond).To(Equal(true)) - - get := client.Get("key") - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal("hello")) - }) - It("should Set with expiration", func() { err := client.Set("key", "hello", 100*time.Millisecond).Err() Expect(err).NotTo(HaveOccurred()) @@ -911,16 +895,6 @@ var _ = Describe("Commands", func() { Expect(get.Val()).To(Equal("hello")) }) - It("should SetEx", func() { - setEx := client.SetEx("key", 10*time.Second, "hello") - Expect(setEx.Err()).NotTo(HaveOccurred()) - Expect(setEx.Val()).To(Equal("OK")) - - ttl := client.TTL("key") - Expect(ttl.Err()).NotTo(HaveOccurred()) - Expect(ttl.Val()).To(Equal(10 * time.Second)) - }) - It("should SetNX", func() { setNX := client.SetNX("key", "hello", 0) Expect(setNX.Err()).NotTo(HaveOccurred()) From 5f85be3173895c1e57c00a9e0bb3c236ba912477 Mon Sep 17 00:00:00 2001 From: Adrien Bustany Date: Mon, 11 May 2015 15:41:42 +0200 Subject: [PATCH 0011/1746] commands: reduce memory allocations in ZAdd. --- commands.go | 9 ++++++--- redis_test.go | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index b6df30cfec..dc68bea1fd 100644 --- a/commands.go +++ b/commands.go @@ -858,9 +858,12 @@ type ZStore struct { } func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { - args := []string{"ZADD", key} - for _, m := range members { - args = append(args, formatFloat(m.Score), m.Member) + args := make([]string, 2+2*len(members)) + args[0] = "ZADD" + args[1] = key + for i, m := range members { + args[2+2*i] = formatFloat(m.Score) + args[2+2*i+1] = m.Member } cmd := NewIntCmd(args...) c.Process(cmd) diff --git a/redis_test.go b/redis_test.go index 945791855b..5fb6d65a2d 100644 --- a/redis_test.go +++ b/redis_test.go @@ -267,3 +267,20 @@ func BenchmarkPipeline(b *testing.B) { } }) } + +func BenchmarkZAdd(b *testing.B) { + client := redis.NewClient(&redis.Options{ + Addr: redisAddr, + }) + defer client.Close() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.ZAdd("key", redis.Z{float64(1), "hello"}).Err(); err != nil { + b.Fatal(err) + } + } + }) +} From 4df8b2bbbce4692e4ef5e8335da223804f4fc798 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 15 May 2015 15:11:22 +0300 Subject: [PATCH 0012/1746] Add ClientPause command. --- commands.go | 7 +++++++ commands_test.go | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index dc68bea1fd..7f4cbe0828 100644 --- a/commands.go +++ b/commands.go @@ -1114,6 +1114,13 @@ func (c *commandable) ClientList() *StringCmd { return cmd } +func (c *commandable) ClientPause(dur time.Duration) *BoolCmd { + cmd := NewBoolCmd("CLIENT", "PAUSE", formatMs(dur)) + cmd._clusterKeyPos = 0 + c.Process(cmd) + return cmd +} + func (c *commandable) ConfigGet(parameter string) *SliceCmd { cmd := NewSliceCmd("CONFIG", "GET", parameter) cmd._clusterKeyPos = 0 diff --git a/commands_test.go b/commands_test.go index c0ab161f82..a13ced6ce8 100644 --- a/commands_test.go +++ b/commands_test.go @@ -18,8 +18,10 @@ var _ = Describe("Commands", func() { BeforeEach(func() { client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - PoolTimeout: 30 * time.Second, + Addr: redisAddr, + ReadTimeout: 100 * time.Millisecond, + WriteTimeout: 100 * time.Millisecond, + PoolTimeout: 30 * time.Second, }) }) @@ -75,6 +77,19 @@ var _ = Describe("Commands", func() { Expect(r.Val()).To(Equal("")) }) + It("should ClientPause", func() { + err := client.ClientPause(time.Second).Err() + Expect(err).NotTo(HaveOccurred()) + + Consistently(func() error { + return client.Ping().Err() + }, "500ms").Should(HaveOccurred()) + + Eventually(func() error { + return client.Ping().Err() + }, "1s").ShouldNot(HaveOccurred()) + }) + It("should ConfigGet", func() { r := client.ConfigGet("*") Expect(r.Err()).NotTo(HaveOccurred()) From bca8659b542abde08f95c20ea679aae373a0fd75 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 18 May 2015 14:43:08 +0300 Subject: [PATCH 0013/1746] Run tests against latest Redis version. --- commands_test.go | 4 ++-- main_test.go | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/commands_test.go b/commands_test.go index a13ced6ce8..dd7c6d665a 100644 --- a/commands_test.go +++ b/commands_test.go @@ -273,7 +273,7 @@ var _ = Describe("Commands", func() { }) It("should Migrate", func() { - migrate := client.Migrate("localhost", "6380", "key", 0, 0) + migrate := client.Migrate("localhost", redisSecondaryPort, "key", 0, 0) Expect(migrate.Err()).NotTo(HaveOccurred()) Expect(migrate.Val()).To(Equal("NOKEY")) @@ -281,7 +281,7 @@ var _ = Describe("Commands", func() { Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) - migrate = client.Migrate("localhost", "6380", "key", 0, 0) + migrate = client.Migrate("localhost", redisSecondaryPort, "key", 0, 0) Expect(migrate.Err()).To(MatchError("IOERR error or timeout writing to target instance")) Expect(migrate.Val()).To(Equal("")) }) diff --git a/main_test.go b/main_test.go index 59032738e9..57eb493399 100644 --- a/main_test.go +++ b/main_test.go @@ -17,7 +17,11 @@ import ( "gopkg.in/redis.v3" ) -const redisAddr = ":6379" +const ( + redisPort = "6380" + redisAddr = ":" + redisPort + redisSecondaryPort = "6381" +) const ( sentinelName = "mymaster" @@ -27,7 +31,7 @@ const ( sentinelPort = "8126" ) -var sentinelMaster, sentinelSlave1, sentinelSlave2, sentinel *redisProcess +var redisMain, sentinelMaster, sentinelSlave1, sentinelSlave2, sentinel *redisProcess var cluster = &clusterScenario{ ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, @@ -39,6 +43,9 @@ var cluster = &clusterScenario{ var _ = BeforeSuite(func() { var err error + redisMain, err = startRedis(redisPort) + Expect(err).NotTo(HaveOccurred()) + sentinelMaster, err = startRedis(sentinelMasterPort) Expect(err).NotTo(HaveOccurred()) @@ -57,6 +64,8 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { + Expect(redisMain.Close()).NotTo(HaveOccurred()) + Expect(sentinel.Close()).NotTo(HaveOccurred()) Expect(sentinelSlave1.Close()).NotTo(HaveOccurred()) Expect(sentinelSlave2.Close()).NotTo(HaveOccurred()) From e72b69b964ed7d24381d52266d74436b07792b49 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 18 May 2015 14:52:46 +0300 Subject: [PATCH 0014/1746] Increase read/write timeout. --- commands_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commands_test.go b/commands_test.go index dd7c6d665a..7aca5891b1 100644 --- a/commands_test.go +++ b/commands_test.go @@ -19,8 +19,8 @@ var _ = Describe("Commands", func() { BeforeEach(func() { client = redis.NewClient(&redis.Options{ Addr: redisAddr, - ReadTimeout: 100 * time.Millisecond, - WriteTimeout: 100 * time.Millisecond, + ReadTimeout: 500 * time.Millisecond, + WriteTimeout: 500 * time.Millisecond, PoolTimeout: 30 * time.Second, }) }) @@ -83,7 +83,7 @@ var _ = Describe("Commands", func() { Consistently(func() error { return client.Ping().Err() - }, "500ms").Should(HaveOccurred()) + }, "900ms").Should(HaveOccurred()) Eventually(func() error { return client.Ping().Err() From 55f551a44742ece489c7830b26a11db8448adf36 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 18 May 2015 14:59:39 +0300 Subject: [PATCH 0015/1746] multi: remove confusing comment. --- multi.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multi.go b/multi.go index 161275aeb0..edd5ce3a2e 100644 --- a/multi.go +++ b/multi.go @@ -7,7 +7,8 @@ import ( var errDiscard = errors.New("redis: Discard can be used only inside Exec") -// Not thread-safe. +// Multi implements Redis transactions as described in +// http://redis.io/topics/transactions. type Multi struct { commandable From f531b3b493be57f112a4e29cd440f599b7392172 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 15 May 2015 15:21:28 +0300 Subject: [PATCH 0016/1746] Add Client.String method. --- cluster_client_test.go | 2 -- redis.go | 7 +++++++ redis_test.go | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cluster_client_test.go b/cluster_client_test.go index e9ea16c23c..c7f695d448 100644 --- a/cluster_client_test.go +++ b/cluster_client_test.go @@ -31,11 +31,9 @@ var _ = Describe("ClusterClient", func() { } BeforeEach(func() { - var err error subject = NewClusterClient(&ClusterOptions{ Addrs: []string{"127.0.0.1:6379", "127.0.0.1:7003", "127.0.0.1:7006"}, }) - Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/redis.go b/redis.go index 3cc5841d81..4186ffbb17 100644 --- a/redis.go +++ b/redis.go @@ -1,6 +1,7 @@ package redis // import "gopkg.in/redis.v3" import ( + "fmt" "log" "net" "time" @@ -11,6 +12,10 @@ type baseClient struct { opt *options } +func (c *baseClient) String() string { + return fmt.Sprintf("Redis<%s db:%d>", c.opt.Addr, c.opt.DB) +} + func (c *baseClient) conn() (*conn, error) { return c.connPool.Get() } @@ -164,6 +169,7 @@ func (opt *Options) getPoolTimeout() time.Duration { func (opt *Options) options() *options { return &options{ + Addr: opt.Addr, Dialer: opt.getDialer(), PoolSize: opt.getPoolSize(), PoolTimeout: opt.getPoolTimeout(), @@ -181,6 +187,7 @@ func (opt *Options) options() *options { } type options struct { + Addr string Dialer func() (net.Conn, error) PoolSize int PoolTimeout time.Duration diff --git a/redis_test.go b/redis_test.go index 5fb6d65a2d..4ad4486603 100644 --- a/redis_test.go +++ b/redis_test.go @@ -24,6 +24,10 @@ var _ = Describe("Client", func() { client.Close() }) + It("should Stringer", func() { + Expect(client.String()).To(Equal("Redis<:6380 db:0>")) + }) + It("should ping", func() { val, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) From f79308a137ef4bd4cd88d786184b3ae4ad48e1e4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 14:15:05 +0300 Subject: [PATCH 0017/1746] Rewrite PubSub example. --- example_test.go | 38 +++++++++++++++++++++++++++++--------- pubsub.go | 3 ++- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/example_test.go b/example_test.go index 859b6458e5..fb63ecb803 100644 --- a/example_test.go +++ b/example_test.go @@ -2,7 +2,9 @@ package redis_test import ( "fmt" + "net" "strconv" + "time" "gopkg.in/redis.v3" ) @@ -125,19 +127,37 @@ func ExamplePubSub() { defer pubsub.Close() err := pubsub.Subscribe("mychannel") - _ = err + if err != nil { + panic(err) + } - msg, err := pubsub.Receive() - fmt.Println(msg, err) + err = client.Publish("mychannel", "hello").Err() + if err != nil { + panic(err) + } - pub := client.Publish("mychannel", "hello") - _ = pub.Err() + for { + msgi, err := pubsub.ReceiveTimeout(100 * time.Millisecond) + if err != nil { + if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + // There are no more messages to process. Stop. + break + } + panic(err) + } - msg, err = pubsub.Receive() - fmt.Println(msg, err) + switch msg := msgi.(type) { + case *redis.Subscription: + fmt.Println(msg.Kind, msg.Channel) + case *redis.Message: + fmt.Println(msg.Channel, msg.Payload) + default: + panic(fmt.Sprintf("unknown message: %#v", msgi)) + } + } - // Output: subscribe: mychannel - // Message + // Output: subscribe mychannel + // mychannel hello } func ExampleScript() { diff --git a/pubsub.go b/pubsub.go index 5a32220be2..9d63e6be16 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,7 +5,8 @@ import ( "time" ) -// Not thread-safe. +// PubSub implements Pub/Sub commands as described in +// http://redis.io/topics/pubsub. type PubSub struct { *baseClient } From 387330857d58e289fdee812fe343bc482fdd6a81 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 14:36:29 +0300 Subject: [PATCH 0018/1746] Fix ClientPause test. --- commands_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands_test.go b/commands_test.go index 7aca5891b1..0d19b2e54c 100644 --- a/commands_test.go +++ b/commands_test.go @@ -83,7 +83,7 @@ var _ = Describe("Commands", func() { Consistently(func() error { return client.Ping().Err() - }, "900ms").Should(HaveOccurred()) + }, "400ms").Should(HaveOccurred()) // pause time - read timeout Eventually(func() error { return client.Ping().Err() From 1d02fa85a908bc244eb175259732b76d843c2622 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 14:43:34 +0300 Subject: [PATCH 0019/1746] sentinel: add comment. --- sentinel.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentinel.go b/sentinel.go index 26ab485577..872496968c 100644 --- a/sentinel.go +++ b/sentinel.go @@ -84,6 +84,8 @@ func (opt *FailoverOptions) options() *options { } } +// NewFailoverClient returns Redis client with automatic failover +// capabilities using Redis Sentinel. func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt := failoverOpt.options() failover := &sentinelFailover{ From 36fcbb1f9419e8ca6729b467fe9f9a0f42c4b28d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 14:33:33 +0300 Subject: [PATCH 0020/1746] Update readme for v3. --- README.md | 69 +++++++++++++++++++++++++++++++++++++++++-------- error.go | 2 +- example_test.go | 22 +++++++++++++--- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 041c3dc59f..4691a40c79 100644 --- a/README.md +++ b/README.md @@ -3,32 +3,79 @@ Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.pn Supports: -- Redis 2.8 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- Pub/sub. +- Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. +- Pub/Sub. - Transactions. -- Pipelining. -- Connection pool. -- TLS connections. -- Thread safety. +- Pipelines. +- Connection pool. Client can be safely used from multiple goroutines. - Timeouts. -- Redis Sentinel. -- Redis Cluster: https://github.com/bsm/redis-cluster. +- [Redis Sentinel](http://godoc.org/gopkg.in/redis.v3#NewFailoverClient). +- [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). -API docs: http://godoc.org/gopkg.in/redis.v2. -Examples: http://godoc.org/gopkg.in/redis.v2#pkg-examples. +API docs: http://godoc.org/gopkg.in/redis.v3. +Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. Installation ------------ Install: - go get gopkg.in/redis.v2 + go get gopkg.in/redis.v3 + +Quickstart +---------- + +```go +func ExampleNewClient() { + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", // no password set + DB: 0, // use default DB + }) + + pong, err := client.Ping().Result() + fmt.Println(pong, err) + // Output: PONG +} + +func ExampleClient() { + err := client.Set("key", "value", 0).Err() + if err != nil { + panic(err) + } + + val, err := client.Get("key").Result() + if err != nil { + panic(err) + } + fmt.Println("key", val) + + val2, err := client.Get("key2").Result() + if err == redis.Nil { + fmt.Println("key2 does not exists") + } else if err != nil { + panic(err) + } else { + fmt.Println("key2", val2) + } + // Output: key value + // key2 does not exists +} +``` + +Howto +----- + +Please go through [examples](http://godoc.org/gopkg.in/redis.v3#pkg-examples) to get an idea how to use this package. Look and feel ------------- Some corner cases: + SET key value EX 10 NX + set, err := client.SetNX("key", "value", 10*time.Second).Result() + SORT list LIMIT 0 2 ASC vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() diff --git a/error.go b/error.go index 7709c510cd..9e5d973ce1 100644 --- a/error.go +++ b/error.go @@ -7,7 +7,7 @@ import ( "strings" ) -// Redis nil reply. +// Redis nil reply, .e.g. when key does not exist. var Nil = errorf("redis: nil") // Redis transaction failed. diff --git a/example_test.go b/example_test.go index fb63ecb803..fc0534f785 100644 --- a/example_test.go +++ b/example_test.go @@ -38,13 +38,27 @@ func ExampleNewFailoverClient() { } func ExampleClient() { - if err := client.Set("foo", "bar", 0).Err(); err != nil { + err := client.Set("key", "value", 0).Err() + if err != nil { panic(err) } - v, err := client.Get("hello").Result() - fmt.Printf("%q %q %v", v, err, err == redis.Nil) - // Output: "" "redis: nil" true + val, err := client.Get("key").Result() + if err != nil { + panic(err) + } + fmt.Println("key", val) + + val2, err := client.Get("key2").Result() + if err == redis.Nil { + fmt.Println("key2 does not exists") + } else if err != nil { + panic(err) + } else { + fmt.Println("key2", val2) + } + // Output: key value + // key2 does not exists } func ExampleClient_Incr() { From f6ef0fd3424d5ce67dc851560203e16e6659ea35 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 16:35:30 +0300 Subject: [PATCH 0021/1746] Add basic example how to use Redis Cluster. Unify comments. --- cluster.go | 4 ++-- example_test.go | 14 +++++++++++++- sentinel.go | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index d09298ee03..0f42faa4be 100644 --- a/cluster.go +++ b/cluster.go @@ -26,8 +26,8 @@ type ClusterClient struct { reloading uint32 } -// NewClusterClient initializes a new cluster-aware client using given options. -// A list of seed addresses must be provided. +// NewClusterClient returns a new Redis Cluster client as described in +// http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { client := &ClusterClient{ addrs: opt.Addrs, diff --git a/example_test.go b/example_test.go index fc0534f785..f8b3954809 100644 --- a/example_test.go +++ b/example_test.go @@ -31,10 +31,22 @@ func ExampleNewClient() { } func ExampleNewFailoverClient() { - redis.NewFailoverClient(&redis.FailoverOptions{ + // See http://redis.io/topics/sentinel for instructions how to + // setup Redis Sentinel. + client := redis.NewFailoverClient(&redis.FailoverOptions{ MasterName: "master", SentinelAddrs: []string{":26379"}, }) + client.Ping() +} + +func ExampleNewClusterClient() { + // See http://redis.io/topics/cluster-tutorial for instructions + // how to setup Redis Cluster. + client := redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{":7000", ":7001", ":7002", ":7003", ":7004", ":7005"}, + }) + client.Ping() } func ExampleClient() { diff --git a/sentinel.go b/sentinel.go index 872496968c..1fb4abff88 100644 --- a/sentinel.go +++ b/sentinel.go @@ -84,7 +84,7 @@ func (opt *FailoverOptions) options() *options { } } -// NewFailoverClient returns Redis client with automatic failover +// NewFailoverClient returns a Redis client with automatic failover // capabilities using Redis Sentinel. func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt := failoverOpt.options() From 40b429df49ec89ab3886970ebd2409ed9c9d9c65 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 17:55:08 +0300 Subject: [PATCH 0022/1746] Reduce number of various options structs. Slightly reword comments. --- cluster.go | 24 ++++++++----- conn.go | 7 ++-- pool.go | 34 ++++++++---------- redis.go | 102 ++++++++++++++++------------------------------------ sentinel.go | 75 +++++++++++--------------------------- 5 files changed, 85 insertions(+), 157 deletions(-) diff --git a/cluster.go b/cluster.go index 0f42faa4be..90e5444d88 100644 --- a/cluster.go +++ b/cluster.go @@ -267,20 +267,28 @@ func (c *ClusterClient) reaper() { //------------------------------------------------------------------------------ +// ClusterOptions are used to configure a cluster client and should be +// passed to NewClusterClient. type ClusterOptions struct { - // A seed-list of host:port addresses of known cluster nodes + // A seed list of host:port addresses of cluster nodes. Addrs []string - // An optional password + // The maximum number of MOVED/ASK redirects to follow before + // giving up. + // Default is 16 + MaxRedirects int + + // Following options are copied from Options struct. + Password string - // The maximum number of MOVED/ASK redirects to follow, before - // giving up. Default: 16 - MaxRedirects int + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration - // Following options are copied from `redis.Options`. - PoolSize int - DialTimeout, ReadTimeout, WriteTimeout, PoolTimeout, IdleTimeout time.Duration + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration } func (opt *ClusterOptions) getMaxRedirects() int { diff --git a/conn.go b/conn.go index 751bb54da6..72f82b0fdc 100644 --- a/conn.go +++ b/conn.go @@ -21,9 +21,10 @@ type conn struct { WriteTimeout time.Duration } -func newConnDialer(opt *options) func() (*conn, error) { +func newConnDialer(opt *Options) func() (*conn, error) { + dialer := opt.getDialer() return func() (*conn, error) { - netcn, err := opt.Dialer() + netcn, err := dialer() if err != nil { return nil, err } @@ -36,7 +37,7 @@ func newConnDialer(opt *options) func() (*conn, error) { } } -func (cn *conn) init(opt *options) error { +func (cn *conn) init(opt *Options) error { if opt.Password == "" && opt.DB == 0 { return nil } diff --git a/pool.go b/pool.go index 6eeae1fe89..16efc598d5 100644 --- a/pool.go +++ b/pool.go @@ -110,17 +110,11 @@ func (l *connList) Close() (retErr error) { return retErr } -type connPoolOptions struct { - Dialer func() (*conn, error) - PoolSize int - PoolTimeout time.Duration - IdleTimeout time.Duration - IdleCheckFrequency time.Duration -} - type connPool struct { + dialer func() (*conn, error) + rl *ratelimit.RateLimiter - opt *connPoolOptions + opt *Options conns *connList freeConns chan *conn @@ -129,14 +123,16 @@ type connPool struct { lastDialErr error } -func newConnPool(opt *connPoolOptions) *connPool { +func newConnPool(opt *Options) *connPool { p := &connPool{ - rl: ratelimit.New(2*opt.PoolSize, time.Second), + dialer: newConnDialer(opt), + + rl: ratelimit.New(2*opt.getPoolSize(), time.Second), opt: opt, - conns: newConnList(opt.PoolSize), - freeConns: make(chan *conn, opt.PoolSize), + conns: newConnList(opt.getPoolSize()), + freeConns: make(chan *conn, opt.getPoolSize()), } - if p.opt.IdleTimeout > 0 && p.opt.IdleCheckFrequency > 0 { + if p.opt.getIdleTimeout() > 0 { go p.reaper() } return p @@ -147,7 +143,7 @@ func (p *connPool) closed() bool { } func (p *connPool) isIdle(cn *conn) bool { - return p.opt.IdleTimeout > 0 && time.Since(cn.usedAt) > p.opt.IdleTimeout + return p.opt.getIdleTimeout() > 0 && time.Since(cn.usedAt) > p.opt.getIdleTimeout() } // First returns first non-idle connection from the pool or nil if @@ -170,7 +166,7 @@ func (p *connPool) First() *conn { // wait waits for free non-idle connection. It returns nil on timeout. func (p *connPool) wait() *conn { - deadline := time.After(p.opt.PoolTimeout) + deadline := time.After(p.opt.getPoolTimeout()) for { select { case cn := <-p.freeConns: @@ -196,7 +192,7 @@ func (p *connPool) new() (*conn, error) { return nil, err } - cn, err := p.opt.Dialer() + cn, err := p.dialer() if err != nil { p.lastDialErr = err return nil, err @@ -241,7 +237,7 @@ func (p *connPool) Put(cn *conn) error { log.Printf("redis: connection has unread data: %q", b) return p.Remove(cn) } - if p.opt.IdleTimeout > 0 { + if p.opt.getIdleTimeout() > 0 { cn.usedAt = time.Now() } p.freeConns <- cn @@ -295,7 +291,7 @@ func (p *connPool) Close() (retErr error) { } func (p *connPool) reaper() { - ticker := time.NewTicker(p.opt.IdleCheckFrequency) + ticker := time.NewTicker(time.Minute) defer ticker.Stop() for _ = range ticker.C { diff --git a/redis.go b/redis.go index 4186ffbb17..f77c663aac 100644 --- a/redis.go +++ b/redis.go @@ -9,7 +9,7 @@ import ( type baseClient struct { connPool pool - opt *options + opt *Options } func (c *baseClient) String() string { @@ -87,10 +87,10 @@ func (c *baseClient) Close() error { //------------------------------------------------------------------------------ type Options struct { - // The network type, either "tcp" or "unix". - // Default: "tcp" + // The network type, either tcp or unix. + // Default is tcp. Network string - // The network address. + // host:port address. Addr string // Dialer creates new network connection and has priority over @@ -98,14 +98,17 @@ type Options struct { Dialer func() (net.Conn, error) // An optional password. Must match the password specified in the - // `requirepass` server configuration option. + // requirepass server configuration option. Password string - // Select a database. - // Default: 0 + // A database to be selected after connecting to server. DB int64 + // The maximum number of retries before giving up. + // Default is to not retry failed commands. + MaxRetries int + // Sets the deadline for establishing new connections. If reached, - // deal attepts will fail with a timeout. + // dial will fail with a timeout. DialTimeout time.Duration // Sets the deadline for socket reads. If reached, commands will // fail with a timeout instead of blocking. @@ -115,37 +118,34 @@ type Options struct { WriteTimeout time.Duration // The maximum number of socket connections. - // Default: 10 + // Default is 10 connections. PoolSize int - // PoolTimeout specifies amount of time client waits for a free - // connection in the pool. Default timeout is 1s. + // Specifies amount of time client waits for connection if all + // connections are busy before returning an error. + // Default is 5 seconds. PoolTimeout time.Duration - // Evict connections from the pool after they have been idle for longer - // than specified in this option. - // Default: 0 = no eviction + // Specifies amount of time after which client closes idle + // connections. Should be less than server's timeout. + // Default is to not close idle connections. IdleTimeout time.Duration +} - // MaxRetries specifies maximum number of times client will retry - // failed command. Default is to not retry failed command. - MaxRetries int +func (opt *Options) getNetwork() string { + if opt.Network == "" { + return "tcp" + } + return opt.Network } func (opt *Options) getDialer() func() (net.Conn, error) { if opt.Dialer == nil { - return func() (net.Conn, error) { + opt.Dialer = func() (net.Conn, error) { return net.DialTimeout(opt.getNetwork(), opt.Addr, opt.getDialTimeout()) } } return opt.Dialer } -func (opt *Options) getNetwork() string { - if opt.Network == "" { - return "tcp" - } - return opt.Network -} - func (opt *Options) getPoolSize() int { if opt.PoolSize == 0 { return 10 @@ -167,49 +167,8 @@ func (opt *Options) getPoolTimeout() time.Duration { return opt.PoolTimeout } -func (opt *Options) options() *options { - return &options{ - Addr: opt.Addr, - Dialer: opt.getDialer(), - PoolSize: opt.getPoolSize(), - PoolTimeout: opt.getPoolTimeout(), - IdleTimeout: opt.IdleTimeout, - - DB: opt.DB, - Password: opt.Password, - - DialTimeout: opt.getDialTimeout(), - ReadTimeout: opt.ReadTimeout, - WriteTimeout: opt.WriteTimeout, - - MaxRetries: opt.MaxRetries, - } -} - -type options struct { - Addr string - Dialer func() (net.Conn, error) - PoolSize int - PoolTimeout time.Duration - IdleTimeout time.Duration - - Password string - DB int64 - - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - - MaxRetries int -} - -func (opt *options) connPoolOptions() *connPoolOptions { - return &connPoolOptions{ - Dialer: newConnDialer(opt), - PoolSize: opt.PoolSize, - PoolTimeout: opt.PoolTimeout, - IdleTimeout: opt.IdleTimeout, - } +func (opt *Options) getIdleTimeout() time.Duration { + return opt.IdleTimeout } //------------------------------------------------------------------------------ @@ -219,7 +178,7 @@ type Client struct { commandable } -func newClient(opt *options, pool pool) *Client { +func newClient(opt *Options, pool pool) *Client { base := &baseClient{opt: opt, connPool: pool} return &Client{ baseClient: base, @@ -227,8 +186,7 @@ func newClient(opt *options, pool pool) *Client { } } -func NewClient(clOpt *Options) *Client { - opt := clOpt.options() - pool := newConnPool(opt.connPoolOptions()) +func NewClient(opt *Options) *Client { + pool := newConnPool(opt) return newClient(opt, pool) } diff --git a/sentinel.go b/sentinel.go index 1fb4abff88..82d9bc9188 100644 --- a/sentinel.go +++ b/sentinel.go @@ -11,75 +11,41 @@ import ( //------------------------------------------------------------------------------ +// FailoverOptions are used to configure a failover client and should +// be passed to NewFailoverClient. type FailoverOptions struct { // The master name. MasterName string - // Seed addresses of sentinel nodes. + // A seed list of host:port addresses of sentinel nodes. SentinelAddrs []string - // An optional password. Must match the password specified in the - // `requirepass` server configuration option. + // Following options are copied from Options struct. + Password string - // Select a database. - // Default: 0 - DB int64 - - // Sets the deadline for establishing new connections. If reached, - // deal attepts will fail with a timeout. - DialTimeout time.Duration - // Sets the deadline for socket reads. If reached, commands will - // fail with a timeout instead of blocking. - ReadTimeout time.Duration - // Sets the deadline for socket writes. If reached, commands will - // fail with a timeout instead of blocking. + DB int64 + + DialTimeout time.Duration + ReadTimeout time.Duration WriteTimeout time.Duration - // The maximum number of socket connections. - // Default: 10 - PoolSize int - // If all socket connections is the pool are busy, the pool will wait - // this amount of time for a conection to become available, before - // returning an error. - // Default: 5s + PoolSize int PoolTimeout time.Duration - // Evict connections from the pool after they have been idle for longer - // than specified in this option. - // Default: 0 = no eviction IdleTimeout time.Duration } -func (opt *FailoverOptions) getPoolSize() int { - if opt.PoolSize == 0 { - return 10 - } - return opt.PoolSize -} - -func (opt *FailoverOptions) getPoolTimeout() time.Duration { - if opt.PoolTimeout == 0 { - return 5 * time.Second - } - return opt.PoolTimeout -} - -func (opt *FailoverOptions) getDialTimeout() time.Duration { - if opt.DialTimeout == 0 { - return 5 * time.Second - } - return opt.DialTimeout -} +func (opt *FailoverOptions) options() *Options { + return &Options{ + Addr: "FailoverClient", -func (opt *FailoverOptions) options() *options { - return &options{ DB: opt.DB, Password: opt.Password, - DialTimeout: opt.getDialTimeout(), + DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, - PoolSize: opt.getPoolSize(), - PoolTimeout: opt.getPoolTimeout(), + PoolSize: opt.PoolSize, + PoolTimeout: opt.PoolTimeout, IdleTimeout: opt.IdleTimeout, } } @@ -104,11 +70,10 @@ type sentinelClient struct { *baseClient } -func newSentinel(clOpt *Options) *sentinelClient { - opt := clOpt.options() +func newSentinel(opt *Options) *sentinelClient { base := &baseClient{ opt: opt, - connPool: newConnPool(opt.connPoolOptions()), + connPool: newConnPool(opt), } return &sentinelClient{ baseClient: base, @@ -141,7 +106,7 @@ type sentinelFailover struct { masterName string sentinelAddrs []string - opt *options + opt *Options pool pool poolOnce sync.Once @@ -161,7 +126,7 @@ func (d *sentinelFailover) dial() (net.Conn, error) { func (d *sentinelFailover) Pool() pool { d.poolOnce.Do(func() { d.opt.Dialer = d.dial - d.pool = newConnPool(d.opt.connPoolOptions()) + d.pool = newConnPool(d.opt) }) return d.pool } From b1946ee532efdd01f4651214002abbf7e740bd99 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 23 May 2015 18:17:45 +0300 Subject: [PATCH 0023/1746] More comments. --- pubsub.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pubsub.go b/pubsub.go index 9d63e6be16..86e4bf6c0e 100644 --- a/pubsub.go +++ b/pubsub.go @@ -26,6 +26,7 @@ func (c *Client) Publish(channel, message string) *IntCmd { return req } +// Message received as result of a PUBLISH command issued by another client. type Message struct { Channel string Payload string @@ -35,6 +36,8 @@ func (m *Message) String() string { return fmt.Sprintf("Message<%s: %s>", m.Channel, m.Payload) } +// Message matching a pattern-matching subscription received as result +// of a PUBLISH command issued by another client. type PMessage struct { Channel string Pattern string @@ -45,10 +48,14 @@ func (m *PMessage) String() string { return fmt.Sprintf("PMessage<%s: %s>", m.Channel, m.Payload) } +// Message received after a successful subscription to channel. type Subscription struct { - Kind string + // Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe". + Kind string + // Channel name we have subscribed to. Channel string - Count int + // Number of channels we are currently subscribed to. + Count int } func (m *Subscription) String() string { From 9d6c73eb9d9ca5f3d2dc31177122bd4932420b72 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 3 Jun 2015 14:18:15 +0300 Subject: [PATCH 0024/1746] Don't panic if list is closed and conn can't be found. --- pool.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pool.go b/pool.go index 16efc598d5..090270b5d3 100644 --- a/pool.go +++ b/pool.go @@ -80,6 +80,9 @@ func (l *connList) Remove(cn *conn) error { } } + if l.closed() { + return nil + } panic("conn not found in the list") } @@ -94,6 +97,9 @@ func (l *connList) Replace(cn, newcn *conn) error { } } + if l.closed() { + return newcn.Close() + } panic("conn not found in the list") } @@ -110,6 +116,10 @@ func (l *connList) Close() (retErr error) { return retErr } +func (l *connList) closed() bool { + return l.cns == nil +} + type connPool struct { dialer func() (*conn, error) @@ -245,11 +255,6 @@ func (p *connPool) Put(cn *conn) error { } func (p *connPool) Remove(cn *conn) error { - if p.closed() { - // Close already closed all connections. - return nil - } - // Replace existing connection with new one and unblock waiter. newcn, err := p.new() if err != nil { @@ -372,12 +377,6 @@ func (p *singleConnPool) Put(cn *conn) error { return nil } -func (p *singleConnPool) put() error { - err := p.pool.Put(p.cn) - p.cn = nil - return err -} - func (p *singleConnPool) Remove(cn *conn) error { defer p.cnMtx.Unlock() p.cnMtx.Lock() @@ -427,7 +426,8 @@ func (p *singleConnPool) Close() error { var err error if p.cn != nil { if p.reusable { - err = p.put() + err = p.pool.Put(p.cn) + p.cn = nil } else { err = p.remove() } From 46f49a17a54456682c79b3f4e06c08800a2defb8 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 25 May 2015 16:22:27 +0300 Subject: [PATCH 0025/1746] Add Redis Ring. --- .travis.yml | 1 + cluster.go | 12 ++- main_test.go | 19 ++++- ring.go | 237 +++++++++++++++++++++++++++++++++++++++++++++++++++ ring_test.go | 84 ++++++++++++++++++ 5 files changed, 348 insertions(+), 5 deletions(-) create mode 100644 ring.go create mode 100644 ring_test.go diff --git a/.travis.yml b/.travis.yml index 169ccd0ada..3af4dd982e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ go: install: - go get gopkg.in/bufio.v1 - go get gopkg.in/bsm/ratelimit.v1 + - go get github.com/golang/groupcache/consistenthash - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega - mkdir -p $HOME/gopath/src/gopkg.in diff --git a/cluster.go b/cluster.go index 2d46ca737d..99d3eebff9 100644 --- a/cluster.go +++ b/cluster.go @@ -307,7 +307,6 @@ func (opt *ClusterOptions) getMaxRedirects() int { func (opt *ClusterOptions) clientOptions() *Options { return &Options{ - DB: 0, Password: opt.Password, DialTimeout: opt.DialTimeout, @@ -324,14 +323,19 @@ func (opt *ClusterOptions) clientOptions() *Options { const hashSlots = 16384 -// hashSlot returns a consistent slot number between 0 and 16383 -// for any given string key. -func hashSlot(key string) int { +func hashKey(key string) string { if s := strings.IndexByte(key, '{'); s > -1 { if e := strings.IndexByte(key[s+1:], '}'); e > 0 { key = key[s+1 : s+e+1] } } + return key +} + +// hashSlot returns a consistent slot number between 0 and 16383 +// for any given string key. +func hashSlot(key string) int { + key = hashKey(key) if key == "" { return rand.Intn(hashSlots) } diff --git a/main_test.go b/main_test.go index 57eb493399..ed2f7de9a7 100644 --- a/main_test.go +++ b/main_test.go @@ -23,6 +23,11 @@ const ( redisSecondaryPort = "6381" ) +const ( + ringShard1Port = "6390" + ringShard2Port = "6391" +) + const ( sentinelName = "mymaster" sentinelMasterPort = "8123" @@ -31,7 +36,11 @@ const ( sentinelPort = "8126" ) -var redisMain, sentinelMaster, sentinelSlave1, sentinelSlave2, sentinel *redisProcess +var ( + redisMain *redisProcess + ringShard1, ringShard2 *redisProcess + sentinelMaster, sentinelSlave1, sentinelSlave2, sentinel *redisProcess +) var cluster = &clusterScenario{ ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, @@ -46,6 +55,12 @@ var _ = BeforeSuite(func() { redisMain, err = startRedis(redisPort) Expect(err).NotTo(HaveOccurred()) + ringShard1, err = startRedis(ringShard1Port) + Expect(err).NotTo(HaveOccurred()) + + ringShard2, err = startRedis(ringShard2Port) + Expect(err).NotTo(HaveOccurred()) + sentinelMaster, err = startRedis(sentinelMasterPort) Expect(err).NotTo(HaveOccurred()) @@ -65,6 +80,8 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { Expect(redisMain.Close()).NotTo(HaveOccurred()) + Expect(ringShard1.Close()).NotTo(HaveOccurred()) + Expect(ringShard2.Close()).NotTo(HaveOccurred()) Expect(sentinel.Close()).NotTo(HaveOccurred()) Expect(sentinelSlave1.Close()).NotTo(HaveOccurred()) diff --git a/ring.go b/ring.go new file mode 100644 index 0000000000..0735b16b24 --- /dev/null +++ b/ring.go @@ -0,0 +1,237 @@ +package redis + +import ( + "errors" + "fmt" + "log" + "sync" + "time" + + "github.com/golang/groupcache/consistenthash" +) + +var ( + errRingShardsDown = errors.New("redis: all ring shards are down") +) + +// RingOptions are used to configure a ring client and should be +// passed to NewRing. +type RingOptions struct { + // A map of name => host:port addresses of ring shards. + Addrs map[string]string + + // Following options are copied from Options struct. + + DB int64 + Password string + + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration +} + +func (opt *RingOptions) clientOptions() *Options { + return &Options{ + DB: opt.DB, + Password: opt.Password, + + DialTimeout: opt.DialTimeout, + ReadTimeout: opt.ReadTimeout, + WriteTimeout: opt.WriteTimeout, + + PoolSize: opt.PoolSize, + PoolTimeout: opt.PoolTimeout, + IdleTimeout: opt.IdleTimeout, + } +} + +type ringShard struct { + Client *Client + down int +} + +func (shard *ringShard) String() string { + var state string + if shard.IsUp() { + state = "up" + } else { + state = "down" + } + return fmt.Sprintf("%s is %s", shard.Client, state) +} + +func (shard *ringShard) IsDown() bool { + const threshold = 5 + return shard.down >= threshold +} + +func (shard *ringShard) IsUp() bool { + return !shard.IsDown() +} + +// Vote votes to set shard state and returns true if state was changed. +func (shard *ringShard) Vote(up bool) bool { + if up { + changed := shard.IsDown() + shard.down = 0 + return changed + } + + if shard.IsDown() { + return false + } + + shard.down++ + return shard.IsDown() +} + +// Ring is a Redis client that uses constistent hashing to distribute +// keys across multiple Redis servers (shards). +// +// It monitors the state of each shard and removes dead shards from +// the ring. When shard comes online it is added back to the ring. This +// gives you maximum availability and partition tolerance, but no +// consistency between different shards or even clients. Each client +// uses shards that are available to the client and does not do any +// coordination when shard state is changed. +// +// Ring should be used when you use multiple Redis servers for caching +// and can tolerate losing data when one of the servers dies. +// Otherwise you should use Redis Cluster. +type Ring struct { + commandable + + nreplicas int + + mx sync.RWMutex + hash *consistenthash.Map + shards map[string]*ringShard + + closed bool +} + +func NewRing(opt *RingOptions) *Ring { + const nreplicas = 100 + ring := &Ring{ + nreplicas: nreplicas, + hash: consistenthash.New(nreplicas, nil), + shards: make(map[string]*ringShard), + } + ring.commandable.process = ring.process + for name, addr := range opt.Addrs { + clopt := opt.clientOptions() + clopt.Addr = addr + ring.addClient(name, NewClient(clopt)) + } + go ring.heartbeat() + return ring +} + +func (ring *Ring) addClient(name string, cl *Client) { + ring.mx.Lock() + ring.hash.Add(name) + ring.shards[name] = &ringShard{Client: cl} + ring.mx.Unlock() +} + +func (ring *Ring) getClient(key string) (*Client, error) { + ring.mx.RLock() + + if ring.closed { + return nil, errClosed + } + + name := ring.hash.Get(key) + if name == "" { + ring.mx.RUnlock() + return nil, errRingShardsDown + } + + if shard, ok := ring.shards[name]; ok { + ring.mx.RUnlock() + return shard.Client, nil + } + + ring.mx.RUnlock() + return nil, errRingShardsDown +} + +func (ring *Ring) process(cmd Cmder) { + cl, err := ring.getClient(hashKey(cmd.clusterKey())) + if err != nil { + cmd.setErr(err) + return + } + cl.baseClient.process(cmd) +} + +// rebalance removes dead shards from the ring. +func (ring *Ring) rebalance() { + defer ring.mx.Unlock() + ring.mx.Lock() + + ring.hash = consistenthash.New(ring.nreplicas, nil) + for name, shard := range ring.shards { + if shard.IsUp() { + ring.hash.Add(name) + } + } +} + +// heartbeat monitors state of each shard in the ring. +func (ring *Ring) heartbeat() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for _ = range ticker.C { + var rebalance bool + + ring.mx.RLock() + + if ring.closed { + ring.mx.RUnlock() + break + } + + for _, shard := range ring.shards { + err := shard.Client.Ping().Err() + if shard.Vote(err == nil) { + log.Printf("redis: ring shard state changed: %s", shard) + rebalance = true + } + } + + ring.mx.RUnlock() + + if rebalance { + ring.rebalance() + } + } +} + +// Close closes the ring client, releasing any open resources. +// +// It is rare to Close a Client, as the Client is meant to be +// long-lived and shared between many goroutines. +func (ring *Ring) Close() (retErr error) { + defer ring.mx.Unlock() + ring.mx.Lock() + + if ring.closed { + return nil + } + ring.closed = true + + for _, shard := range ring.shards { + if err := shard.Client.Close(); err != nil { + retErr = err + } + } + ring.hash = nil + ring.shards = nil + + return retErr +} diff --git a/ring_test.go b/ring_test.go new file mode 100644 index 0000000000..35212c3f9a --- /dev/null +++ b/ring_test.go @@ -0,0 +1,84 @@ +package redis_test + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v3" +) + +var _ = Describe("Redis ring", func() { + var ring *redis.Ring + + setRingKeys := func() { + for i := 0; i < 100; i++ { + err := ring.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + } + + BeforeEach(func() { + ring = redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "ringShard1": ":" + ringShard1Port, + "ringShard2": ":" + ringShard2Port, + }, + }) + + // Shards should not have any keys. + Expect(ringShard1.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(ringShard1.Info().Val()).NotTo(ContainSubstring("keys=")) + + Expect(ringShard2.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(ringShard2.Info().Val()).NotTo(ContainSubstring("keys=")) + }) + + AfterEach(func() { + Expect(ring.Close()).NotTo(HaveOccurred()) + }) + + It("uses both shards", func() { + setRingKeys() + + // Both shards should have some keys now. + Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + }) + + It("uses one shard when other shard is down", func() { + // Stop ringShard2. + Expect(ringShard2.Close()).NotTo(HaveOccurred()) + + // Ring needs 5 * heartbeat time to detect that node is down. + // Give it more to be sure. + heartbeat := 100 * time.Millisecond + time.Sleep(5*heartbeat + heartbeat) + + setRingKeys() + + // RingShard1 should have all keys. + Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=100")) + + // Start ringShard2. + var err error + ringShard2, err = startRedis(ringShard2Port) + Expect(err).NotTo(HaveOccurred()) + + // Wait for ringShard2 to come up. + Eventually(func() error { + return ringShard2.Ping().Err() + }, "1s").ShouldNot(HaveOccurred()) + + // Ring needs heartbeat time to detect that node is up. + // Give it more to be sure. + time.Sleep(heartbeat + heartbeat) + + setRingKeys() + + // RingShard2 should have its keys. + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + }) +}) From 2103d88732e47fbe86aa70ba4e270bd96f2b5007 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 2 Jun 2015 17:19:51 +0300 Subject: [PATCH 0026/1746] Embed consistenthash package. --- .travis.yml | 1 - internal/consistenthash/consistenthash.go | 81 +++++++++++++ .../consistenthash/consistenthash_test.go | 110 ++++++++++++++++++ ring.go | 2 +- 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 internal/consistenthash/consistenthash.go create mode 100644 internal/consistenthash/consistenthash_test.go diff --git a/.travis.yml b/.travis.yml index 3af4dd982e..169ccd0ada 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ go: install: - go get gopkg.in/bufio.v1 - go get gopkg.in/bsm/ratelimit.v1 - - go get github.com/golang/groupcache/consistenthash - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega - mkdir -p $HOME/gopath/src/gopkg.in diff --git a/internal/consistenthash/consistenthash.go b/internal/consistenthash/consistenthash.go new file mode 100644 index 0000000000..a9c56f0762 --- /dev/null +++ b/internal/consistenthash/consistenthash.go @@ -0,0 +1,81 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package consistenthash provides an implementation of a ring hash. +package consistenthash + +import ( + "hash/crc32" + "sort" + "strconv" +) + +type Hash func(data []byte) uint32 + +type Map struct { + hash Hash + replicas int + keys []int // Sorted + hashMap map[int]string +} + +func New(replicas int, fn Hash) *Map { + m := &Map{ + replicas: replicas, + hash: fn, + hashMap: make(map[int]string), + } + if m.hash == nil { + m.hash = crc32.ChecksumIEEE + } + return m +} + +// Returns true if there are no items available. +func (m *Map) IsEmpty() bool { + return len(m.keys) == 0 +} + +// Adds some keys to the hash. +func (m *Map) Add(keys ...string) { + for _, key := range keys { + for i := 0; i < m.replicas; i++ { + hash := int(m.hash([]byte(strconv.Itoa(i) + key))) + m.keys = append(m.keys, hash) + m.hashMap[hash] = key + } + } + sort.Ints(m.keys) +} + +// Gets the closest item in the hash to the provided key. +func (m *Map) Get(key string) string { + if m.IsEmpty() { + return "" + } + + hash := int(m.hash([]byte(key))) + + // Binary search for appropriate replica. + idx := sort.Search(len(m.keys), func(i int) bool { return m.keys[i] >= hash }) + + // Means we have cycled back to the first replica. + if idx == len(m.keys) { + idx = 0 + } + + return m.hashMap[m.keys[idx]] +} diff --git a/internal/consistenthash/consistenthash_test.go b/internal/consistenthash/consistenthash_test.go new file mode 100644 index 0000000000..1a37fd7ff4 --- /dev/null +++ b/internal/consistenthash/consistenthash_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package consistenthash + +import ( + "fmt" + "strconv" + "testing" +) + +func TestHashing(t *testing.T) { + + // Override the hash function to return easier to reason about values. Assumes + // the keys can be converted to an integer. + hash := New(3, func(key []byte) uint32 { + i, err := strconv.Atoi(string(key)) + if err != nil { + panic(err) + } + return uint32(i) + }) + + // Given the above hash function, this will give replicas with "hashes": + // 2, 4, 6, 12, 14, 16, 22, 24, 26 + hash.Add("6", "4", "2") + + testCases := map[string]string{ + "2": "2", + "11": "2", + "23": "4", + "27": "2", + } + + for k, v := range testCases { + if hash.Get(k) != v { + t.Errorf("Asking for %s, should have yielded %s", k, v) + } + } + + // Adds 8, 18, 28 + hash.Add("8") + + // 27 should now map to 8. + testCases["27"] = "8" + + for k, v := range testCases { + if hash.Get(k) != v { + t.Errorf("Asking for %s, should have yielded %s", k, v) + } + } + +} + +func TestConsistency(t *testing.T) { + hash1 := New(1, nil) + hash2 := New(1, nil) + + hash1.Add("Bill", "Bob", "Bonny") + hash2.Add("Bob", "Bonny", "Bill") + + if hash1.Get("Ben") != hash2.Get("Ben") { + t.Errorf("Fetching 'Ben' from both hashes should be the same") + } + + hash2.Add("Becky", "Ben", "Bobby") + + if hash1.Get("Ben") != hash2.Get("Ben") || + hash1.Get("Bob") != hash2.Get("Bob") || + hash1.Get("Bonny") != hash2.Get("Bonny") { + t.Errorf("Direct matches should always return the same entry") + } + +} + +func BenchmarkGet8(b *testing.B) { benchmarkGet(b, 8) } +func BenchmarkGet32(b *testing.B) { benchmarkGet(b, 32) } +func BenchmarkGet128(b *testing.B) { benchmarkGet(b, 128) } +func BenchmarkGet512(b *testing.B) { benchmarkGet(b, 512) } + +func benchmarkGet(b *testing.B, shards int) { + + hash := New(50, nil) + + var buckets []string + for i := 0; i < shards; i++ { + buckets = append(buckets, fmt.Sprintf("shard-%d", i)) + } + + hash.Add(buckets...) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + hash.Get(buckets[i&(shards-1)]) + } +} diff --git a/ring.go b/ring.go index 0735b16b24..4772581f6e 100644 --- a/ring.go +++ b/ring.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/golang/groupcache/consistenthash" + "gopkg.in/redis.v3/internal/consistenthash" ) var ( From 90aaae2ba2f19f5e2f68b28becae53c56fc1edba Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 2 Jun 2015 18:24:14 +0300 Subject: [PATCH 0027/1746] makefile: remove ginkgo flags. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 33dc97330b..d3763d6172 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ all: testdeps - go test ./... -v 1 -ginkgo.slowSpecThreshold=10 -cpu=1,2,4 - go test ./... -ginkgo.slowSpecThreshold=10 -short -race + go test ./... -v=1 -cpu=1,2,4 + go test ./... -short -race test: testdeps - go test ./... -v 1 -ginkgo.slowSpecThreshold=10 + go test ./... -v=1 testdeps: .test/redis/src/redis-server From fc04a09033c98e5c3eff38c214c4c34dd40e980e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 3 Jun 2015 14:36:48 +0300 Subject: [PATCH 0028/1746] Fix flaky test. --- pool_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pool_test.go b/pool_test.go index 3b0d00afe8..a4fa76e256 100644 --- a/pool_test.go +++ b/pool_test.go @@ -101,8 +101,9 @@ var _ = Describe("Pool", func() { }) pool := client.Pool() - Expect(pool.Len()).To(Equal(10)) - Expect(pool.FreeLen()).To(Equal(10)) + Expect(pool.Len()).To(BeNumerically("<=", 10)) + Expect(pool.FreeLen()).To(BeNumerically("<=", 10)) + Expect(pool.Len()).To(Equal(pool.FreeLen())) }) It("should remove broken connections", func() { From 7d886330f114b726d3464c59a62e82b2cdfa7adc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 3 Jun 2015 14:50:43 +0300 Subject: [PATCH 0029/1746] pool: close all connections at once. --- pool.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pool.go b/pool.go index 090270b5d3..301b9e0057 100644 --- a/pool.go +++ b/pool.go @@ -278,17 +278,13 @@ func (p *connPool) Close() (retErr error) { if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { return errClosed } - // First close free connections. - for p.Len() > 0 { - cn := p.wait() - if cn == nil { + // Wait for app to free connections, but don't close them immediately. + for i := 0; i < p.Len(); i++ { + if cn := p.wait(); cn == nil { break } - if err := p.conns.Remove(cn); err != nil { - retErr = err - } } - // Then close the rest. + // Close all connections. if err := p.conns.Close(); err != nil { retErr = err } From 3fc16811b5b8a77c9c5b541c3c1d6525bb38f2d9 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 3 Jun 2015 15:09:56 +0300 Subject: [PATCH 0030/1746] Fix flaky tests by using better matcher. --- commands_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/commands_test.go b/commands_test.go index 0d19b2e54c..ef593e1eb7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -369,8 +369,7 @@ var _ = Describe("Commands", func() { pttl := client.PTTL("key") Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val() <= expiration).To(Equal(true)) - Expect(pttl.Val() >= expiration-time.Millisecond).To(Equal(true)) + Expect(pttl.Val()).To(BeNumerically("~", expiration, 10*time.Millisecond)) }) It("should PExpireAt", func() { @@ -389,8 +388,7 @@ var _ = Describe("Commands", func() { pttl := client.PTTL("key") Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val() <= expiration).To(Equal(true)) - Expect(pttl.Val() >= expiration-time.Millisecond).To(Equal(true)) + Expect(pttl.Val()).To(BeNumerically("~", expiration, 10*time.Millisecond)) }) It("should PTTL", func() { @@ -405,8 +403,7 @@ var _ = Describe("Commands", func() { pttl := client.PTTL("key") Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val() <= expiration).To(Equal(true)) - Expect(pttl.Val() >= expiration-time.Millisecond).To(Equal(true)) + Expect(pttl.Val()).To(BeNumerically("~", expiration, 10*time.Millisecond)) }) It("should RandomKey", func() { From a8fe55571b1d68ed68d1e7f19e2fe8ed0570412b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 3 Jun 2015 16:45:46 +0300 Subject: [PATCH 0031/1746] pool: put connection to the list before returning it. --- main_test.go | 1 + multi.go | 8 ++++++-- pool.go | 56 +++++++++++++++++++++++++++------------------------ redis_test.go | 18 +++++++++++++++++ 4 files changed, 55 insertions(+), 28 deletions(-) diff --git a/main_test.go b/main_test.go index ed2f7de9a7..c4b5a5972b 100644 --- a/main_test.go +++ b/main_test.go @@ -80,6 +80,7 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { Expect(redisMain.Close()).NotTo(HaveOccurred()) + Expect(ringShard1.Close()).NotTo(HaveOccurred()) Expect(ringShard2.Close()).NotTo(HaveOccurred()) diff --git a/multi.go b/multi.go index edd5ce3a2e..9d87de9a54 100644 --- a/multi.go +++ b/multi.go @@ -3,6 +3,7 @@ package redis import ( "errors" "fmt" + "log" ) var errDiscard = errors.New("redis: Discard can be used only inside Exec") @@ -18,7 +19,10 @@ type Multi struct { func (c *Client) Multi() *Multi { multi := &Multi{ - base: &baseClient{opt: c.opt, connPool: newSingleConnPool(c.connPool, true)}, + base: &baseClient{ + opt: c.opt, + connPool: newSingleConnPool(c.connPool, true), + }, } multi.commandable.process = multi.process return multi @@ -34,7 +38,7 @@ func (c *Multi) process(cmd Cmder) { func (c *Multi) Close() error { if err := c.Unwatch().Err(); err != nil { - return err + log.Printf("redis: Unwatch failed: %s", err) } return c.base.Close() } diff --git a/pool.go b/pool.go index 301b9e0057..714cbe5bc5 100644 --- a/pool.go +++ b/pool.go @@ -258,10 +258,12 @@ func (p *connPool) Remove(cn *conn) error { // Replace existing connection with new one and unblock waiter. newcn, err := p.new() if err != nil { + log.Printf("redis: new failed: %s", err) return p.conns.Remove(cn) } + err = p.conns.Replace(cn, newcn) p.freeConns <- newcn - return p.conns.Replace(cn, newcn) + return err } // Len returns total number of connections. @@ -312,14 +314,12 @@ func (p *connPool) reaper() { //------------------------------------------------------------------------------ type singleConnPool struct { - pool pool - - cnMtx sync.Mutex - cn *conn - + pool pool reusable bool + cn *conn closed bool + mx sync.Mutex } func newSingleConnPool(pool pool, reusable bool) *singleConnPool { @@ -330,20 +330,24 @@ func newSingleConnPool(pool pool, reusable bool) *singleConnPool { } func (p *singleConnPool) SetConn(cn *conn) { - p.cnMtx.Lock() + p.mx.Lock() + if p.cn != nil { + panic("p.cn != nil") + } p.cn = cn - p.cnMtx.Unlock() + p.mx.Unlock() } func (p *singleConnPool) First() *conn { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() - return p.cn + p.mx.Lock() + cn := p.cn + p.mx.Unlock() + return cn } func (p *singleConnPool) Get() (*conn, error) { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() + defer p.mx.Unlock() + p.mx.Lock() if p.closed { return nil, errClosed @@ -362,8 +366,8 @@ func (p *singleConnPool) Get() (*conn, error) { } func (p *singleConnPool) Put(cn *conn) error { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() + defer p.mx.Unlock() + p.mx.Lock() if p.cn != cn { panic("p.cn != cn") } @@ -374,8 +378,8 @@ func (p *singleConnPool) Put(cn *conn) error { } func (p *singleConnPool) Remove(cn *conn) error { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() + defer p.mx.Unlock() + p.mx.Lock() if p.cn == nil { panic("p.cn == nil") } @@ -395,8 +399,8 @@ func (p *singleConnPool) remove() error { } func (p *singleConnPool) Len() int { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() + defer p.mx.Unlock() + p.mx.Lock() if p.cn == nil { return 0 } @@ -404,19 +408,19 @@ func (p *singleConnPool) Len() int { } func (p *singleConnPool) FreeLen() int { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() + defer p.mx.Unlock() + p.mx.Lock() if p.cn == nil { - return 0 + return 1 } - return 1 + return 0 } func (p *singleConnPool) Close() error { - defer p.cnMtx.Unlock() - p.cnMtx.Lock() + defer p.mx.Unlock() + p.mx.Lock() if p.closed { - return nil + return errClosed } p.closed = true var err error diff --git a/redis_test.go b/redis_test.go index 4ad4486603..8a8663e113 100644 --- a/redis_test.go +++ b/redis_test.go @@ -88,6 +88,24 @@ var _ = Describe("Client", func() { Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) + It("should close pubsub when client is closed", func() { + pubsub := client.PubSub() + Expect(client.Close()).NotTo(HaveOccurred()) + Expect(pubsub.Close()).NotTo(HaveOccurred()) + }) + + It("should close multi when client is closed", func() { + multi := client.Multi() + Expect(client.Close()).NotTo(HaveOccurred()) + Expect(multi.Close()).NotTo(HaveOccurred()) + }) + + It("should close pipeline when client is closed", func() { + pipeline := client.Pipeline() + Expect(client.Close()).NotTo(HaveOccurred()) + Expect(pipeline.Close()).NotTo(HaveOccurred()) + }) + It("should support idle-timeouts", func() { idle := redis.NewClient(&redis.Options{ Addr: redisAddr, From c64b7819b9edd393bdde0a9dd206bc57b051d464 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 3 Jun 2015 17:08:27 +0300 Subject: [PATCH 0032/1746] Fix flaky test. --- cluster_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster_test.go b/cluster_test.go index e8e0802ab4..9173faea1d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -252,7 +252,6 @@ var _ = Describe("Cluster", func() { val, err := client.Get("A").Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("VALUE")) - Expect(client.SlotAddrs(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) Eventually(func() []string { return client.SlotAddrs(slot) From 0cf1b73698e0a507cb071a2a60a1a906f7e06f2d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 4 Jun 2015 11:50:24 +0300 Subject: [PATCH 0033/1746] Add RingPipeline. --- cluster_pipeline.go | 47 ++++++++--------- pipeline.go | 95 +++++++++++++++++++--------------- ring.go | 121 +++++++++++++++++++++++++++++++++++++++++++- ring_test.go | 21 ++++++++ 4 files changed, 217 insertions(+), 67 deletions(-) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 466be7d4e8..2e1194064d 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -20,48 +20,42 @@ func (c *ClusterClient) Pipeline() *ClusterPipeline { return pipe } -func (c *ClusterPipeline) process(cmd Cmder) { - c.cmds = append(c.cmds, cmd) +func (pipe *ClusterPipeline) process(cmd Cmder) { + pipe.cmds = append(pipe.cmds, cmd) } -// Close marks the pipeline as closed -func (c *ClusterPipeline) Close() error { - c.closed = true - return nil -} - -// Discard resets the pipeline and discards queued commands -func (c *ClusterPipeline) Discard() error { - if c.closed { +// Discard resets the pipeline and discards queued commands. +func (pipe *ClusterPipeline) Discard() error { + if pipe.closed { return errClosed } - c.cmds = c.cmds[:0] + pipe.cmds = pipe.cmds[:0] return nil } -func (c *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { - if c.closed { +func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { + if pipe.closed { return nil, errClosed } - if len(c.cmds) == 0 { + if len(pipe.cmds) == 0 { return []Cmder{}, nil } - cmds = c.cmds - c.cmds = make([]Cmder, 0, 10) + cmds = pipe.cmds + pipe.cmds = make([]Cmder, 0, 10) cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { slot := hashSlot(cmd.clusterKey()) - addr := c.cluster.slotMasterAddr(slot) + addr := pipe.cluster.slotMasterAddr(slot) cmdsMap[addr] = append(cmdsMap[addr], cmd) } - for attempt := 0; attempt <= c.cluster.opt.getMaxRedirects(); attempt++ { + for attempt := 0; attempt <= pipe.cluster.opt.getMaxRedirects(); attempt++ { failedCmds := make(map[string][]Cmder) for addr, cmds := range cmdsMap { - client, err := c.cluster.getClient(addr) + client, err := pipe.cluster.getClient(addr) if err != nil { setCmdsErr(cmds, err) retErr = err @@ -75,7 +69,7 @@ func (c *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { continue } - failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) + failedCmds, err = pipe.execClusterCmds(cn, cmds, failedCmds) if err != nil { retErr = err } @@ -88,7 +82,14 @@ func (c *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { return cmds, retErr } -func (c *ClusterPipeline) execClusterCmds( +// Close marks the pipeline as closed +func (pipe *ClusterPipeline) Close() error { + pipe.Discard() + pipe.closed = true + return nil +} + +func (pipe *ClusterPipeline) execClusterCmds( cn *conn, cmds []Cmder, failedCmds map[string][]Cmder, ) (map[string][]Cmder, error) { if err := cn.writeCmds(cmds...); err != nil { @@ -107,7 +108,7 @@ func (c *ClusterPipeline) execClusterCmds( failedCmds[""] = append(failedCmds[""], cmds[i:]...) break } else if moved, ask, addr := isMovedError(err); moved { - c.cluster.lazyReloadSlots() + pipe.cluster.lazyReloadSlots() cmd.reset() failedCmds[addr] = append(failedCmds[addr], cmd) } else if ask { diff --git a/pipeline.go b/pipeline.go index 62cf7fd171..8981cb5043 100644 --- a/pipeline.go +++ b/pipeline.go @@ -1,102 +1,113 @@ package redis -// Not thread-safe. +// Pipeline implements pipelining as described in +// http://redis.io/topics/pipelining. +// +// Pipeline is not thread-safe. type Pipeline struct { commandable - cmds []Cmder client *baseClient + + cmds []Cmder closed bool } func (c *Client) Pipeline() *Pipeline { pipe := &Pipeline{ - client: &baseClient{ - opt: c.opt, - connPool: c.connPool, - }, - cmds: make([]Cmder, 0, 10), + client: c.baseClient, + cmds: make([]Cmder, 0, 10), } pipe.commandable.process = pipe.process return pipe } -func (c *Client) Pipelined(f func(*Pipeline) error) ([]Cmder, error) { - pc := c.Pipeline() - if err := f(pc); err != nil { +func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + pipe := c.Pipeline() + if err := fn(pipe); err != nil { return nil, err } - cmds, err := pc.Exec() - pc.Close() + cmds, err := pipe.Exec() + pipe.Close() return cmds, err } -func (c *Pipeline) process(cmd Cmder) { - c.cmds = append(c.cmds, cmd) +func (pipe *Pipeline) process(cmd Cmder) { + pipe.cmds = append(pipe.cmds, cmd) } -func (c *Pipeline) Close() error { - c.closed = true +func (pipe *Pipeline) Close() error { + pipe.Discard() + pipe.closed = true return nil } -func (c *Pipeline) Discard() error { - if c.closed { +// Discard resets the pipeline and discards queued commands. +func (pipe *Pipeline) Discard() error { + if pipe.closed { return errClosed } - c.cmds = c.cmds[:0] + pipe.cmds = pipe.cmds[:0] return nil } // Exec always returns list of commands and error of the first failed // command if any. -func (c *Pipeline) Exec() (cmds []Cmder, retErr error) { - if c.closed { +func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { + if pipe.closed { return nil, errClosed } - if len(c.cmds) == 0 { - return c.cmds, nil + if len(pipe.cmds) == 0 { + return pipe.cmds, nil } - cmds = c.cmds - c.cmds = make([]Cmder, 0, 0) - - for i := 0; i <= c.client.opt.MaxRetries; i++ { - if i > 0 { - resetCmds(cmds) - } + cmds = pipe.cmds + pipe.cmds = make([]Cmder, 0, 10) - cn, err := c.client.conn() + failedCmds := cmds + for i := 0; i <= pipe.client.opt.MaxRetries; i++ { + cn, err := pipe.client.conn() if err != nil { - setCmdsErr(cmds, err) + setCmdsErr(failedCmds, err) return cmds, err } - retErr = c.execCmds(cn, cmds) - c.client.putConn(cn, err) - if shouldRetry(err) { - continue + if i > 0 { + resetCmds(failedCmds) + } + failedCmds, err = execCmds(cn, failedCmds) + pipe.client.putConn(cn, err) + if err != nil && retErr == nil { + retErr = err + } + if len(failedCmds) == 0 { + break } - - break } return cmds, retErr } -func (c *Pipeline) execCmds(cn *conn, cmds []Cmder) error { +func execCmds(cn *conn, cmds []Cmder) ([]Cmder, error) { if err := cn.writeCmds(cmds...); err != nil { setCmdsErr(cmds, err) - return err + return cmds, err } var firstCmdErr error + var failedCmds []Cmder for _, cmd := range cmds { err := cmd.parseReply(cn.rd) - if err != nil && firstCmdErr == nil { + if err == nil { + continue + } + if firstCmdErr == nil { firstCmdErr = err } + if shouldRetry(err) { + failedCmds = append(failedCmds, cmd) + } } - return firstCmdErr + return failedCmds, firstCmdErr } diff --git a/ring.go b/ring.go index 4772581f6e..2124357c5f 100644 --- a/ring.go +++ b/ring.go @@ -25,6 +25,8 @@ type RingOptions struct { DB int64 Password string + MaxRetries int + DialTimeout time.Duration ReadTimeout time.Duration WriteTimeout time.Duration @@ -105,6 +107,7 @@ func (shard *ringShard) Vote(up bool) bool { type Ring struct { commandable + opt *RingOptions nreplicas int mx sync.RWMutex @@ -117,9 +120,11 @@ type Ring struct { func NewRing(opt *RingOptions) *Ring { const nreplicas = 100 ring := &Ring{ + opt: opt, nreplicas: nreplicas, - hash: consistenthash.New(nreplicas, nil), - shards: make(map[string]*ringShard), + + hash: consistenthash.New(nreplicas, nil), + shards: make(map[string]*ringShard), } ring.commandable.process = ring.process for name, addr := range opt.Addrs { @@ -235,3 +240,115 @@ func (ring *Ring) Close() (retErr error) { return retErr } + +// RingPipeline creates a new pipeline which is able to execute commands +// against multiple shards. +type RingPipeline struct { + commandable + + ring *Ring + + cmds []Cmder + closed bool +} + +func (ring *Ring) Pipeline() *RingPipeline { + pipe := &RingPipeline{ + ring: ring, + cmds: make([]Cmder, 0, 10), + } + pipe.commandable.process = pipe.process + return pipe +} + +func (ring *Ring) Pipelined(fn func(*RingPipeline) error) ([]Cmder, error) { + pipe := ring.Pipeline() + if err := fn(pipe); err != nil { + return nil, err + } + cmds, err := pipe.Exec() + pipe.Close() + return cmds, err +} + +func (pipe *RingPipeline) process(cmd Cmder) { + pipe.cmds = append(pipe.cmds, cmd) +} + +// Discard resets the pipeline and discards queued commands. +func (pipe *RingPipeline) Discard() error { + if pipe.closed { + return errClosed + } + pipe.cmds = pipe.cmds[:0] + return nil +} + +// Exec always returns list of commands and error of the first failed +// command if any. +func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { + if pipe.closed { + return nil, errClosed + } + if len(pipe.cmds) == 0 { + return pipe.cmds, nil + } + + cmds = pipe.cmds + pipe.cmds = make([]Cmder, 0, 10) + + cmdsMap := make(map[string][]Cmder) + for _, cmd := range cmds { + name := pipe.ring.hash.Get(cmd.clusterKey()) + cmdsMap[name] = append(cmdsMap[name], cmd) + } + + for i := 0; i <= pipe.ring.opt.MaxRetries; i++ { + failedCmdsMap := make(map[string][]Cmder) + + for name, cmds := range cmdsMap { + client, err := pipe.ring.getClient(name) + if err != nil { + setCmdsErr(cmds, err) + if retErr == nil { + retErr = err + } + continue + } + + cn, err := client.conn() + if err != nil { + setCmdsErr(cmds, err) + if retErr == nil { + retErr = err + } + continue + } + + if i > 0 { + resetCmds(cmds) + } + failedCmds, err := execCmds(cn, cmds) + client.putConn(cn, err) + if err != nil && retErr == nil { + retErr = err + } + if len(failedCmds) > 0 { + failedCmdsMap[name] = failedCmds + } + } + + if len(failedCmdsMap) == 0 { + break + } + cmdsMap = failedCmdsMap + } + + return cmds, retErr +} + +func (pipe *RingPipeline) Close() error { + pipe.Discard() + pipe.closed = true + return nil +} diff --git a/ring_test.go b/ring_test.go index 35212c3f9a..6234aeedd3 100644 --- a/ring_test.go +++ b/ring_test.go @@ -81,4 +81,25 @@ var _ = Describe("Redis ring", func() { // RingShard2 should have its keys. Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) }) + + It("supports pipelining", func() { + pipe := ring.Pipeline() + for i := 0; i < 100; i++ { + err := pipe.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(100)) + Expect(pipe.Close()).NotTo(HaveOccurred()) + + for _, cmd := range cmds { + Expect(cmd.Err()).NotTo(HaveOccurred()) + Expect(cmd.(*redis.StatusCmd).Val()).To(Equal("OK")) + } + + // Both shards should have some keys now. + Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + }) }) From faa7ed46bd55be18d606f9a2542d318fb1a47356 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 5 Jun 2015 15:02:57 +0300 Subject: [PATCH 0034/1746] Add Ring example. Improve existing examples. --- README.md | 10 ++--- example_test.go | 107 ++++++++++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 4691a40c79..3589878366 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.pn Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- Pub/Sub. -- Transactions. -- Pipelines. -- Connection pool. Client can be safely used from multiple goroutines. -- Timeouts. +- [Pub/Sub](http://godoc.org/gopkg.in/redis.v3#example-PubSub). +- [Transactions](http://godoc.org/gopkg.in/redis.v3#example-Multi). +- [Pipelining](http://godoc.org/gopkg.in/redis.v3#example-Client-Pipelined). +- [Timeouts](http://godoc.org/gopkg.in/redis.v3#Options). - [Redis Sentinel](http://godoc.org/gopkg.in/redis.v3#NewFailoverClient). - [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). +- [Ring](http://godoc.org/gopkg.in/redis.v3#NewRing). API docs: http://godoc.org/gopkg.in/redis.v3. Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. diff --git a/example_test.go b/example_test.go index f8b3954809..0da5573f38 100644 --- a/example_test.go +++ b/example_test.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "strconv" + "sync" "time" "gopkg.in/redis.v3" @@ -49,6 +50,17 @@ func ExampleNewClusterClient() { client.Ping() } +func ExampleNewRing() { + client := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "shard1": ":7000", + "shard2": ":7001", + "shard3": ":7002", + }, + }) + client.Ping() +} + func ExampleClient() { err := client.Set("key", "value", 0).Err() if err != nil { @@ -84,68 +96,73 @@ func ExampleClient_Incr() { } func ExampleClient_Pipelined() { - cmds, err := client.Pipelined(func(c *redis.Pipeline) error { - c.Set("key1", "hello1", 0) - c.Get("key1") + var incr *redis.IntCmd + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + incr = pipe.Incr("counter1") + pipe.Expire("counter1", time.Hour) return nil }) - fmt.Println(err) - set := cmds[0].(*redis.StatusCmd) - fmt.Println(set) - get := cmds[1].(*redis.StringCmd) - fmt.Println(get) - // Output: - // SET key1 hello1: OK - // GET key1: hello1 + fmt.Println(incr.Val(), err) + // Output: 1 } func ExamplePipeline() { - pipeline := client.Pipeline() - set := pipeline.Set("key1", "hello1", 0) - get := pipeline.Get("key1") - cmds, err := pipeline.Exec() - fmt.Println(cmds, err) - fmt.Println(set) - fmt.Println(get) - // Output: [SET key1 hello1: OK GET key1: hello1] - // SET key1 hello1: OK - // GET key1: hello1 + pipe := client.Pipeline() + defer pipe.Close() + + incr := pipe.Incr("counter2") + pipe.Expire("counter2", time.Hour) + _, err := pipe.Exec() + fmt.Println(incr.Val(), err) + // Output: 1 } func ExampleMulti() { - incr := func(tx *redis.Multi) ([]redis.Cmder, error) { - s, err := tx.Get("key").Result() + // Transactionally increments key using GET and SET commands. + incr := func(tx *redis.Multi, key string) error { + err := tx.Watch(key).Err() + if err != nil { + return err + } + + n, err := tx.Get(key).Int64() if err != nil && err != redis.Nil { - return nil, err + return err } - n, _ := strconv.ParseInt(s, 10, 64) - return tx.Exec(func() error { - tx.Set("key", strconv.FormatInt(n+1, 10), 0) + _, err = tx.Exec(func() error { + tx.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) + return err } - client.Del("key") - - tx := client.Multi() - defer tx.Close() - - watch := tx.Watch("key") - _ = watch.Err() - - for { - cmds, err := incr(tx) - if err == redis.TxFailedErr { - continue - } else if err != nil { - panic(err) - } - fmt.Println(cmds, err) - break + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + tx := client.Multi() + defer tx.Close() + + for { + err := incr(tx, "counter3") + if err == redis.TxFailedErr { + // Retry. + continue + } else if err != nil { + panic(err) + } + break + } + }() } + wg.Wait() - // Output: [SET key 1: OK] + n, err := client.Get("counter3").Int64() + fmt.Println(n, err) + // Output: 10 } func ExamplePubSub() { From 74c3ac1ac7ef937d1942372be5bd89878b6c59ae Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 5 Jun 2015 15:05:49 +0300 Subject: [PATCH 0035/1746] Fix links. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3589878366..ea3ffd540f 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.pn Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- [Pub/Sub](http://godoc.org/gopkg.in/redis.v3#example-PubSub). -- [Transactions](http://godoc.org/gopkg.in/redis.v3#example-Multi). -- [Pipelining](http://godoc.org/gopkg.in/redis.v3#example-Client-Pipelined). +- [Pub/Sub](http://godoc.org/gopkg.in/redis.v3#PubSub). +- [Transactions](http://godoc.org/gopkg.in/redis.v3#Multi). +- [Pipelining](http://godoc.org/gopkg.in/redis.v3#Client.Pipeline). - [Timeouts](http://godoc.org/gopkg.in/redis.v3#Options). - [Redis Sentinel](http://godoc.org/gopkg.in/redis.v3#NewFailoverClient). - [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). From 9b31a45f9e714acc97e6a57002b95733461b71f3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 6 Jun 2015 10:19:51 +0300 Subject: [PATCH 0036/1746] Add BenchmarkRedisSetBytes. --- redis_test.go | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/redis_test.go b/redis_test.go index 8a8663e113..d8c4f7aee6 100644 --- a/redis_test.go +++ b/redis_test.go @@ -1,6 +1,7 @@ package redis_test import ( + "bytes" "net" "testing" "time" @@ -154,9 +155,11 @@ var _ = Describe("Client", func() { //------------------------------------------------------------------------------ +const benchRedisAddr = ":6379" + func BenchmarkRedisPing(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() @@ -173,15 +176,34 @@ func BenchmarkRedisPing(b *testing.B) { func BenchmarkRedisSet(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() + value := string(bytes.Repeat([]byte{'1'}, 10000)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Set("key", "hello", 0).Err(); err != nil { + if err := client.Set("key", value, 0).Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkRedisSetBytes(b *testing.B) { + client := redis.NewClient(&redis.Options{ + Addr: benchRedisAddr, + }) + defer client.Close() + value := bytes.Repeat([]byte{'1'}, 10000) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Set("key", string(value), 0).Err(); err != nil { b.Fatal(err) } } @@ -190,7 +212,7 @@ func BenchmarkRedisSet(b *testing.B) { func BenchmarkRedisGetNil(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() if err := client.FlushDb().Err(); err != nil { @@ -210,7 +232,7 @@ func BenchmarkRedisGetNil(b *testing.B) { func BenchmarkRedisGet(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() if err := client.Set("key", "hello", 0).Err(); err != nil { @@ -230,7 +252,7 @@ func BenchmarkRedisGet(b *testing.B) { func BenchmarkRedisMGet(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() if err := client.MSet("key1", "hello1", "key2", "hello2").Err(); err != nil { @@ -250,7 +272,7 @@ func BenchmarkRedisMGet(b *testing.B) { func BenchmarkSetExpire(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() @@ -270,7 +292,7 @@ func BenchmarkSetExpire(b *testing.B) { func BenchmarkPipeline(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() @@ -292,7 +314,7 @@ func BenchmarkPipeline(b *testing.B) { func BenchmarkZAdd(b *testing.B) { client := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: benchRedisAddr, }) defer client.Close() From 771ae5f899d89949f53ffae6ac246d81a2164aca Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 11 Jun 2015 17:17:06 +0300 Subject: [PATCH 0037/1746] script: fix small API inconsistency (backwards compatible). --- script.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script.go b/script.go index 96c35f5149..3f22f46959 100644 --- a/script.go +++ b/script.go @@ -43,7 +43,7 @@ func (s *Script) EvalSha(c scripter, keys []string, args []string) *Cmd { return c.EvalSha(s.hash, keys, args) } -func (s *Script) Run(c *Client, keys []string, args []string) *Cmd { +func (s *Script) Run(c scripter, keys []string, args []string) *Cmd { r := s.EvalSha(c, keys, args) if err := r.Err(); err != nil && strings.HasPrefix(err.Error(), "NOSCRIPT ") { return s.Eval(c, keys, args) From c9fb737f06e6a9ec2f109b30c9f52368815a4c4e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 16 Jun 2015 10:31:21 +0300 Subject: [PATCH 0038/1746] Better Script example. --- README.md | 1 + example_test.go | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ea3ffd540f..d55458274b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Supports: - [Pub/Sub](http://godoc.org/gopkg.in/redis.v3#PubSub). - [Transactions](http://godoc.org/gopkg.in/redis.v3#Multi). - [Pipelining](http://godoc.org/gopkg.in/redis.v3#Client.Pipeline). +- [Scripting](http://godoc.org/gopkg.in/redis.v3#Script). - [Timeouts](http://godoc.org/gopkg.in/redis.v3#Options). - [Redis Sentinel](http://godoc.org/gopkg.in/redis.v3#NewFailoverClient). - [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). diff --git a/example_test.go b/example_test.go index 0da5573f38..e948410461 100644 --- a/example_test.go +++ b/example_test.go @@ -204,26 +204,26 @@ func ExamplePubSub() { } func ExampleScript() { - setnx := redis.NewScript(` - if redis.call("get", KEYS[1]) == false then - redis.call("set", KEYS[1], ARGV[1]) - return 1 - end - return 0 - `) - - v1, err := setnx.Run(client, []string{"keynx"}, []string{"foo"}).Result() - fmt.Println(v1.(int64), err) + IncrByXX := redis.NewScript(` + if redis.call("GET", KEYS[1]) ~= false then + return redis.call("INCRBY", KEYS[1], ARGV[1]) + end + return false + `) + + n, err := IncrByXX.Run(client, []string{"xx_counter"}, []string{"2"}).Result() + fmt.Println(n, err) - v2, err := setnx.Run(client, []string{"keynx"}, []string{"bar"}).Result() - fmt.Println(v2.(int64), err) + err = client.Set("xx_counter", "40", 0).Err() + if err != nil { + panic(err) + } - get := client.Get("keynx") - fmt.Println(get) + n, err = IncrByXX.Run(client, []string{"xx_counter"}, []string{"2"}).Result() + fmt.Println(n, err) - // Output: 1 - // 0 - // GET keynx: foo + // Output: redis: nil + // 42 } func Example_customCommand() { From 74f9f4f7a0b559fa6b3da4912f32a36a8358c3d0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 16 Jun 2015 10:46:46 +0300 Subject: [PATCH 0039/1746] ring: ignore pool timeout when pinging shards. --- ring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ring.go b/ring.go index 2124357c5f..4b620da675 100644 --- a/ring.go +++ b/ring.go @@ -203,7 +203,7 @@ func (ring *Ring) heartbeat() { for _, shard := range ring.shards { err := shard.Client.Ping().Err() - if shard.Vote(err == nil) { + if shard.Vote(err == nil || err == errPoolTimeout) { log.Printf("redis: ring shard state changed: %s", shard) rebalance = true } From fee949ecbf7f4b3a69068d0be25985c689f7530d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 24 Jun 2015 13:41:50 +0300 Subject: [PATCH 0040/1746] ring: improve pipelining tests. --- ring_test.go | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/ring_test.go b/ring_test.go index 6234aeedd3..b77e39fef3 100644 --- a/ring_test.go +++ b/ring_test.go @@ -82,24 +82,38 @@ var _ = Describe("Redis ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) }) - It("supports pipelining", func() { - pipe := ring.Pipeline() - for i := 0; i < 100; i++ { - err := pipe.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + Describe("pipelining", func() { + It("uses both shards", func() { + pipe := ring.Pipeline() + for i := 0; i < 100; i++ { + err := pipe.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + cmds, err := pipe.Exec() Expect(err).NotTo(HaveOccurred()) - } - cmds, err := pipe.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(100)) - Expect(pipe.Close()).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(100)) + Expect(pipe.Close()).NotTo(HaveOccurred()) - for _, cmd := range cmds { - Expect(cmd.Err()).NotTo(HaveOccurred()) - Expect(cmd.(*redis.StatusCmd).Val()).To(Equal("OK")) - } + for _, cmd := range cmds { + Expect(cmd.Err()).NotTo(HaveOccurred()) + Expect(cmd.(*redis.StatusCmd).Val()).To(Equal("OK")) + } - // Both shards should have some keys now. - Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) - Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + // Both shards should have some keys now. + Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + }) + + It("is consistent", func() { + _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { + pipe.Set("mykey", "pipeline", 0) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + val, err := ring.Get("mykey").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("pipeline")) + }) }) }) From 1608a33e551385996c40013451d22c70d6d732d5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 24 Jun 2015 15:37:41 +0300 Subject: [PATCH 0041/1746] ring: fix key hashing in Ring pipeline. --- cluster.go | 2 +- ring.go | 28 ++++++++++------------------ ring_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/cluster.go b/cluster.go index 99d3eebff9..cbf00b20fe 100644 --- a/cluster.go +++ b/cluster.go @@ -326,7 +326,7 @@ const hashSlots = 16384 func hashKey(key string) string { if s := strings.IndexByte(key, '{'); s > -1 { if e := strings.IndexByte(key[s+1:], '}'); e > 0 { - key = key[s+1 : s+e+1] + return key[s+1 : s+e+1] } } return key diff --git a/ring.go b/ring.go index 4b620da675..26b06b6256 100644 --- a/ring.go +++ b/ring.go @@ -150,23 +150,19 @@ func (ring *Ring) getClient(key string) (*Client, error) { return nil, errClosed } - name := ring.hash.Get(key) + name := ring.hash.Get(hashKey(key)) if name == "" { ring.mx.RUnlock() return nil, errRingShardsDown } - if shard, ok := ring.shards[name]; ok { - ring.mx.RUnlock() - return shard.Client, nil - } - + cl := ring.shards[name].Client ring.mx.RUnlock() - return nil, errRingShardsDown + return cl, nil } func (ring *Ring) process(cmd Cmder) { - cl, err := ring.getClient(hashKey(cmd.clusterKey())) + cl, err := ring.getClient(cmd.clusterKey()) if err != nil { cmd.setErr(err) return @@ -299,7 +295,11 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - name := pipe.ring.hash.Get(cmd.clusterKey()) + name := pipe.ring.hash.Get(hashKey(cmd.clusterKey())) + if name == "" { + cmd.setErr(errRingShardsDown) + continue + } cmdsMap[name] = append(cmdsMap[name], cmd) } @@ -307,15 +307,7 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { failedCmdsMap := make(map[string][]Cmder) for name, cmds := range cmdsMap { - client, err := pipe.ring.getClient(name) - if err != nil { - setCmdsErr(cmds, err) - if retErr == nil { - retErr = err - } - continue - } - + client := pipe.ring.shards[name].Client cn, err := client.conn() if err != nil { setCmdsErr(cmds, err) diff --git a/ring_test.go b/ring_test.go index b77e39fef3..117a9731c6 100644 --- a/ring_test.go +++ b/ring_test.go @@ -1,6 +1,7 @@ package redis_test import ( + "crypto/rand" "fmt" "time" @@ -23,8 +24,8 @@ var _ = Describe("Redis ring", func() { BeforeEach(func() { ring = redis.NewRing(&redis.RingOptions{ Addrs: map[string]string{ - "ringShard1": ":" + ringShard1Port, - "ringShard2": ":" + ringShard2Port, + "ringShardOne": ":" + ringShard1Port, + "ringShardTwo": ":" + ringShard2Port, }, }) @@ -82,6 +83,16 @@ var _ = Describe("Redis ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) }) + It("supports hash tags", func() { + for i := 0; i < 100; i++ { + err := ring.Set(fmt.Sprintf("key%d{tag}", i), "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + + Expect(ringShard1.Info().Val()).ToNot(ContainSubstring("keys=")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=100")) + }) + Describe("pipelining", func() { It("uses both shards", func() { pipe := ring.Pipeline() @@ -104,16 +115,41 @@ var _ = Describe("Redis ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) }) - It("is consistent", func() { + It("is consistent with ring", func() { + var keys []string + for i := 0; i < 100; i++ { + key := make([]byte, 64) + _, err := rand.Read(key) + Expect(err).NotTo(HaveOccurred()) + keys = append(keys, string(key)) + } + _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { - pipe.Set("mykey", "pipeline", 0) + for _, key := range keys { + pipe.Set(key, "value", 0).Err() + } return nil }) Expect(err).NotTo(HaveOccurred()) - val, err := ring.Get("mykey").Result() + for _, key := range keys { + val, err := ring.Get(key).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("value")) + } + }) + + It("supports hash tags", func() { + _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { + for i := 0; i < 100; i++ { + pipe.Set(fmt.Sprintf("key%d{tag}", i), "value", 0).Err() + } + return nil + }) Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal("pipeline")) + + Expect(ringShard1.Info().Val()).ToNot(ContainSubstring("keys=")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=100")) }) }) }) From 3c1f2bd45a62c38c604b39ce4111ca0b1514641c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 28 May 2015 15:51:19 +0300 Subject: [PATCH 0042/1746] Allow setting and scaning interface{} values. --- command.go | 98 +++++++---- command_test.go | 9 +- commands.go | 411 ++++++++++++++++++++++++++++++++--------------- commands_test.go | 93 +++++++++++ conn.go | 6 +- multi.go | 12 +- parser.go | 239 ++++++++++++++++++++++++--- parser_test.go | 2 +- pubsub.go | 29 ++-- redis_test.go | 37 +++-- 10 files changed, 716 insertions(+), 220 deletions(-) diff --git a/command.go b/command.go index fcea708cb8..f32569749a 100644 --- a/command.go +++ b/command.go @@ -1,6 +1,7 @@ package redis import ( + "bytes" "fmt" "strconv" "strings" @@ -28,7 +29,7 @@ var ( ) type Cmder interface { - args() []string + args() []interface{} parseReply(*bufio.Reader) error setErr(error) reset() @@ -38,7 +39,7 @@ type Cmder interface { clusterKey() string Err() error - String() string + fmt.Stringer } func setCmdsErr(cmds []Cmder, e error) { @@ -54,12 +55,21 @@ func resetCmds(cmds []Cmder) { } func cmdString(cmd Cmder, val interface{}) string { - s := strings.Join(cmd.args(), " ") + var ss []string + for _, arg := range cmd.args() { + ss = append(ss, fmt.Sprint(arg)) + } + s := strings.Join(ss, " ") if err := cmd.Err(); err != nil { return s + ": " + err.Error() } if val != nil { - return s + ": " + fmt.Sprint(val) + switch vv := val.(type) { + case []byte: + return s + ": " + string(vv) + default: + return s + ": " + fmt.Sprint(val) + } } return s @@ -68,7 +78,7 @@ func cmdString(cmd Cmder, val interface{}) string { //------------------------------------------------------------------------------ type baseCmd struct { - _args []string + _args []interface{} err error @@ -84,7 +94,7 @@ func (cmd *baseCmd) Err() error { return nil } -func (cmd *baseCmd) args() []string { +func (cmd *baseCmd) args() []interface{} { return cmd._args } @@ -102,7 +112,7 @@ func (cmd *baseCmd) writeTimeout() *time.Duration { func (cmd *baseCmd) clusterKey() string { if cmd._clusterKeyPos > 0 && cmd._clusterKeyPos < len(cmd._args) { - return cmd._args[cmd._clusterKeyPos] + return fmt.Sprint(cmd._args[cmd._clusterKeyPos]) } return "" } @@ -123,7 +133,7 @@ type Cmd struct { val interface{} } -func NewCmd(args ...string) *Cmd { +func NewCmd(args ...interface{}) *Cmd { return &Cmd{baseCmd: baseCmd{_args: args}} } @@ -146,6 +156,11 @@ func (cmd *Cmd) String() string { func (cmd *Cmd) parseReply(rd *bufio.Reader) error { cmd.val, cmd.err = parseReply(rd, parseSlice) + // Convert to string to preserve old behaviour. + // TODO: remove in v4 + if v, ok := cmd.val.([]byte); ok { + cmd.val = string(v) + } return cmd.err } @@ -157,7 +172,7 @@ type SliceCmd struct { val []interface{} } -func NewSliceCmd(args ...string) *SliceCmd { +func NewSliceCmd(args ...interface{}) *SliceCmd { return &SliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -196,11 +211,11 @@ type StatusCmd struct { val string } -func NewStatusCmd(args ...string) *StatusCmd { +func NewStatusCmd(args ...interface{}) *StatusCmd { return &StatusCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } -func newKeylessStatusCmd(args ...string) *StatusCmd { +func newKeylessStatusCmd(args ...interface{}) *StatusCmd { return &StatusCmd{baseCmd: baseCmd{_args: args}} } @@ -227,7 +242,7 @@ func (cmd *StatusCmd) parseReply(rd *bufio.Reader) error { cmd.err = err return err } - cmd.val = v.(string) + cmd.val = string(v.([]byte)) return nil } @@ -239,7 +254,7 @@ type IntCmd struct { val int64 } -func NewIntCmd(args ...string) *IntCmd { +func NewIntCmd(args ...interface{}) *IntCmd { return &IntCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -279,7 +294,7 @@ type DurationCmd struct { precision time.Duration } -func NewDurationCmd(precision time.Duration, args ...string) *DurationCmd { +func NewDurationCmd(precision time.Duration, args ...interface{}) *DurationCmd { return &DurationCmd{ precision: precision, baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}, @@ -321,7 +336,7 @@ type BoolCmd struct { val bool } -func NewBoolCmd(args ...string) *BoolCmd { +func NewBoolCmd(args ...interface{}) *BoolCmd { return &BoolCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -342,6 +357,8 @@ func (cmd *BoolCmd) String() string { return cmdString(cmd, cmd.val) } +var ok = []byte("OK") + func (cmd *BoolCmd) parseReply(rd *bufio.Reader) error { v, err := parseReply(rd, nil) // `SET key value NX` returns nil when key already exists. @@ -357,8 +374,8 @@ func (cmd *BoolCmd) parseReply(rd *bufio.Reader) error { case int64: cmd.val = vv == 1 return nil - case string: - cmd.val = vv == "OK" + case []byte: + cmd.val = bytes.Equal(vv, ok) return nil default: return fmt.Errorf("got %T, wanted int64 or string") @@ -370,23 +387,27 @@ func (cmd *BoolCmd) parseReply(rd *bufio.Reader) error { type StringCmd struct { baseCmd - val string + val []byte } -func NewStringCmd(args ...string) *StringCmd { +func NewStringCmd(args ...interface{}) *StringCmd { return &StringCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } func (cmd *StringCmd) reset() { - cmd.val = "" + cmd.val = nil cmd.err = nil } func (cmd *StringCmd) Val() string { - return cmd.val + return string(cmd.val) } func (cmd *StringCmd) Result() (string, error) { + return cmd.Val(), cmd.err +} + +func (cmd *StringCmd) Bytes() ([]byte, error) { return cmd.val, cmd.err } @@ -394,21 +415,28 @@ func (cmd *StringCmd) Int64() (int64, error) { if cmd.err != nil { return 0, cmd.err } - return strconv.ParseInt(cmd.val, 10, 64) + return strconv.ParseInt(cmd.Val(), 10, 64) } func (cmd *StringCmd) Uint64() (uint64, error) { if cmd.err != nil { return 0, cmd.err } - return strconv.ParseUint(cmd.val, 10, 64) + return strconv.ParseUint(cmd.Val(), 10, 64) } func (cmd *StringCmd) Float64() (float64, error) { if cmd.err != nil { return 0, cmd.err } - return strconv.ParseFloat(cmd.val, 64) + return strconv.ParseFloat(cmd.Val(), 64) +} + +func (cmd *StringCmd) Scan(val interface{}) error { + if cmd.err != nil { + return cmd.err + } + return scan(cmd.val, val) } func (cmd *StringCmd) String() string { @@ -421,7 +449,9 @@ func (cmd *StringCmd) parseReply(rd *bufio.Reader) error { cmd.err = err return err } - cmd.val = v.(string) + b := v.([]byte) + cmd.val = make([]byte, len(b)) + copy(cmd.val, b) return nil } @@ -433,7 +463,7 @@ type FloatCmd struct { val float64 } -func NewFloatCmd(args ...string) *FloatCmd { +func NewFloatCmd(args ...interface{}) *FloatCmd { return &FloatCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -456,7 +486,7 @@ func (cmd *FloatCmd) parseReply(rd *bufio.Reader) error { cmd.err = err return err } - cmd.val, cmd.err = strconv.ParseFloat(v.(string), 64) + cmd.val, cmd.err = strconv.ParseFloat(string(v.([]byte)), 64) return cmd.err } @@ -468,7 +498,7 @@ type StringSliceCmd struct { val []string } -func NewStringSliceCmd(args ...string) *StringSliceCmd { +func NewStringSliceCmd(args ...interface{}) *StringSliceCmd { return &StringSliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -507,7 +537,7 @@ type BoolSliceCmd struct { val []bool } -func NewBoolSliceCmd(args ...string) *BoolSliceCmd { +func NewBoolSliceCmd(args ...interface{}) *BoolSliceCmd { return &BoolSliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -546,7 +576,7 @@ type StringStringMapCmd struct { val map[string]string } -func NewStringStringMapCmd(args ...string) *StringStringMapCmd { +func NewStringStringMapCmd(args ...interface{}) *StringStringMapCmd { return &StringStringMapCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -585,7 +615,7 @@ type StringIntMapCmd struct { val map[string]int64 } -func NewStringIntMapCmd(args ...string) *StringIntMapCmd { +func NewStringIntMapCmd(args ...interface{}) *StringIntMapCmd { return &StringIntMapCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -624,7 +654,7 @@ type ZSliceCmd struct { val []Z } -func NewZSliceCmd(args ...string) *ZSliceCmd { +func NewZSliceCmd(args ...interface{}) *ZSliceCmd { return &ZSliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -664,7 +694,7 @@ type ScanCmd struct { keys []string } -func NewScanCmd(args ...string) *ScanCmd { +func NewScanCmd(args ...interface{}) *ScanCmd { return &ScanCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } @@ -720,7 +750,7 @@ type ClusterSlotCmd struct { val []ClusterSlotInfo } -func NewClusterSlotCmd(args ...string) *ClusterSlotCmd { +func NewClusterSlotCmd(args ...interface{}) *ClusterSlotCmd { return &ClusterSlotCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } diff --git a/command_test.go b/command_test.go index c1c968e94e..1218724e4e 100644 --- a/command_test.go +++ b/command_test.go @@ -26,7 +26,7 @@ var _ = Describe("Command", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("should have a plain string result", func() { + It("should implement Stringer", func() { set := client.Set("foo", "bar", 0) Expect(set.String()).To(Equal("SET foo bar: OK")) @@ -117,6 +117,13 @@ var _ = Describe("Command", func() { Expect(f).To(Equal(float64(10))) }) + It("Cmd should return string", func() { + cmd := redis.NewCmd("PING") + client.Process(cmd) + Expect(cmd.Err()).NotTo(HaveOccurred()) + Expect(cmd.Val()).To(Equal("PONG")) + }) + Describe("races", func() { var C, N = 10, 1000 if testing.Short() { diff --git a/commands.go b/commands.go index 7f4cbe0828..56167d508c 100644 --- a/commands.go +++ b/commands.go @@ -7,14 +7,18 @@ import ( "time" ) -func formatFloat(f float64) string { - return strconv.FormatFloat(f, 'f', -1, 64) -} - func formatInt(i int64) string { return strconv.FormatInt(i, 10) } +func formatUint(i uint64) string { + return strconv.FormatUint(i, 10) +} + +func formatFloat(f float64) string { + return strconv.FormatFloat(f, 'f', -1, 64) +} + func readTimeout(timeout time.Duration) time.Duration { if timeout == 0 { return 0 @@ -22,14 +26,6 @@ func readTimeout(timeout time.Duration) time.Duration { return timeout + time.Second } -type commandable struct { - process func(cmd Cmder) -} - -func (c *commandable) Process(cmd Cmder) { - c.process(cmd) -} - func usePrecise(dur time.Duration) bool { return dur < time.Second || dur%time.Second != 0 } @@ -41,7 +37,7 @@ func formatMs(dur time.Duration) string { dur, time.Millisecond, ) } - return strconv.FormatInt(int64(dur/time.Millisecond), 10) + return formatInt(int64(dur / time.Millisecond)) } func formatSec(dur time.Duration) string { @@ -51,7 +47,15 @@ func formatSec(dur time.Duration) string { dur, time.Second, ) } - return strconv.FormatInt(int64(dur/time.Second), 10) + return formatInt(int64(dur / time.Second)) +} + +type commandable struct { + process func(cmd Cmder) +} + +func (c *commandable) Process(cmd Cmder) { + c.process(cmd) } //------------------------------------------------------------------------------ @@ -80,7 +84,7 @@ func (c *commandable) Quit() *StatusCmd { } func (c *commandable) Select(index int64) *StatusCmd { - cmd := newKeylessStatusCmd("SELECT", strconv.FormatInt(index, 10)) + cmd := newKeylessStatusCmd("SELECT", formatInt(index)) c.Process(cmd) return cmd } @@ -88,7 +92,11 @@ func (c *commandable) Select(index int64) *StatusCmd { //------------------------------------------------------------------------------ func (c *commandable) Del(keys ...string) *IntCmd { - args := append([]string{"DEL"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "DEL" + for i, key := range keys { + args[1+i] = key + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -113,7 +121,7 @@ func (c *commandable) Expire(key string, expiration time.Duration) *BoolCmd { } func (c *commandable) ExpireAt(key string, tm time.Time) *BoolCmd { - cmd := NewBoolCmd("EXPIREAT", key, strconv.FormatInt(tm.Unix(), 10)) + cmd := NewBoolCmd("EXPIREAT", key, formatInt(tm.Unix())) c.Process(cmd) return cmd } @@ -130,7 +138,7 @@ func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Dur host, port, key, - strconv.FormatInt(db, 10), + formatInt(db), formatMs(timeout), ) cmd._clusterKeyPos = 3 @@ -140,13 +148,18 @@ func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Dur } func (c *commandable) Move(key string, db int64) *BoolCmd { - cmd := NewBoolCmd("MOVE", key, strconv.FormatInt(db, 10)) + cmd := NewBoolCmd("MOVE", key, formatInt(db)) c.Process(cmd) return cmd } func (c *commandable) ObjectRefCount(keys ...string) *IntCmd { - args := append([]string{"OBJECT", "REFCOUNT"}, keys...) + args := make([]interface{}, 2+len(keys)) + args[0] = "OBJECT" + args[1] = "REFCOUNT" + for i, key := range keys { + args[2+i] = key + } cmd := NewIntCmd(args...) cmd._clusterKeyPos = 2 c.Process(cmd) @@ -154,7 +167,12 @@ func (c *commandable) ObjectRefCount(keys ...string) *IntCmd { } func (c *commandable) ObjectEncoding(keys ...string) *StringCmd { - args := append([]string{"OBJECT", "ENCODING"}, keys...) + args := make([]interface{}, 2+len(keys)) + args[0] = "OBJECT" + args[1] = "ENCODING" + for i, key := range keys { + args[2+i] = key + } cmd := NewStringCmd(args...) cmd._clusterKeyPos = 2 c.Process(cmd) @@ -162,7 +180,12 @@ func (c *commandable) ObjectEncoding(keys ...string) *StringCmd { } func (c *commandable) ObjectIdleTime(keys ...string) *DurationCmd { - args := append([]string{"OBJECT", "IDLETIME"}, keys...) + args := make([]interface{}, 2+len(keys)) + args[0] = "OBJECT" + args[1] = "IDLETIME" + for i, key := range keys { + args[2+i] = key + } cmd := NewDurationCmd(time.Second, args...) cmd._clusterKeyPos = 2 c.Process(cmd) @@ -185,7 +208,7 @@ func (c *commandable) PExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd( "PEXPIREAT", key, - strconv.FormatInt(tm.UnixNano()/int64(time.Millisecond), 10), + formatInt(tm.UnixNano()/int64(time.Millisecond)), ) c.Process(cmd) return cmd @@ -219,7 +242,7 @@ func (c *commandable) Restore(key string, ttl int64, value string) *StatusCmd { cmd := NewStatusCmd( "RESTORE", key, - strconv.FormatInt(ttl, 10), + formatInt(ttl), value, ) c.Process(cmd) @@ -236,7 +259,7 @@ type Sort struct { } func (c *commandable) Sort(key string, sort Sort) *StringSliceCmd { - args := []string{"SORT", key} + args := []interface{}{"SORT", key} if sort.By != "" { args = append(args, "BY", sort.By) } @@ -273,12 +296,12 @@ func (c *commandable) Type(key string) *StatusCmd { } func (c *commandable) Scan(cursor int64, match string, count int64) *ScanCmd { - args := []string{"SCAN", strconv.FormatInt(cursor, 10)} + args := []interface{}{"SCAN", formatInt(cursor)} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", strconv.FormatInt(count, 10)) + args = append(args, "COUNT", formatInt(count)) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -286,12 +309,12 @@ func (c *commandable) Scan(cursor int64, match string, count int64) *ScanCmd { } func (c *commandable) SScan(key string, cursor int64, match string, count int64) *ScanCmd { - args := []string{"SSCAN", key, strconv.FormatInt(cursor, 10)} + args := []interface{}{"SSCAN", key, formatInt(cursor)} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", strconv.FormatInt(count, 10)) + args = append(args, "COUNT", formatInt(count)) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -299,12 +322,12 @@ func (c *commandable) SScan(key string, cursor int64, match string, count int64) } func (c *commandable) HScan(key string, cursor int64, match string, count int64) *ScanCmd { - args := []string{"HSCAN", key, strconv.FormatInt(cursor, 10)} + args := []interface{}{"HSCAN", key, formatInt(cursor)} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", strconv.FormatInt(count, 10)) + args = append(args, "COUNT", formatInt(count)) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -312,12 +335,12 @@ func (c *commandable) HScan(key string, cursor int64, match string, count int64) } func (c *commandable) ZScan(key string, cursor int64, match string, count int64) *ScanCmd { - args := []string{"ZSCAN", key, strconv.FormatInt(cursor, 10)} + args := []interface{}{"ZSCAN", key, formatInt(cursor)} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", strconv.FormatInt(count, 10)) + args = append(args, "COUNT", formatInt(count)) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -337,12 +360,12 @@ type BitCount struct { } func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { - args := []string{"BITCOUNT", key} + args := []interface{}{"BITCOUNT", key} if bitCount != nil { args = append( args, - strconv.FormatInt(bitCount.Start, 10), - strconv.FormatInt(bitCount.End, 10), + formatInt(bitCount.Start), + formatInt(bitCount.End), ) } cmd := NewIntCmd(args...) @@ -351,8 +374,13 @@ func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { } func (c *commandable) bitOp(op, destKey string, keys ...string) *IntCmd { - args := []string{"BITOP", op, destKey} - args = append(args, keys...) + args := make([]interface{}, 3+len(keys)) + args[0] = "BITOP" + args[1] = op + args[2] = destKey + for i, key := range keys { + args[3+i] = key + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -375,13 +403,17 @@ func (c *commandable) BitOpNot(destKey string, key string) *IntCmd { } func (c *commandable) BitPos(key string, bit int64, pos ...int64) *IntCmd { - args := []string{"BITPOS", key, formatInt(bit)} + args := make([]interface{}, 3+len(pos)) + args[0] = "BITPOS" + args[1] = key + args[2] = formatInt(bit) switch len(pos) { case 0: case 1: - args = append(args, formatInt(pos[0])) + args[3] = formatInt(pos[0]) case 2: - args = append(args, formatInt(pos[0]), formatInt(pos[1])) + args[3] = formatInt(pos[0]) + args[4] = formatInt(pos[1]) default: panic("too many arguments") } @@ -397,7 +429,7 @@ func (c *commandable) Decr(key string) *IntCmd { } func (c *commandable) DecrBy(key string, decrement int64) *IntCmd { - cmd := NewIntCmd("DECRBY", key, strconv.FormatInt(decrement, 10)) + cmd := NewIntCmd("DECRBY", key, formatInt(decrement)) c.Process(cmd) return cmd } @@ -409,7 +441,7 @@ func (c *commandable) Get(key string) *StringCmd { } func (c *commandable) GetBit(key string, offset int64) *IntCmd { - cmd := NewIntCmd("GETBIT", key, strconv.FormatInt(offset, 10)) + cmd := NewIntCmd("GETBIT", key, formatInt(offset)) c.Process(cmd) return cmd } @@ -418,14 +450,14 @@ func (c *commandable) GetRange(key string, start, end int64) *StringCmd { cmd := NewStringCmd( "GETRANGE", key, - strconv.FormatInt(start, 10), - strconv.FormatInt(end, 10), + formatInt(start), + formatInt(end), ) c.Process(cmd) return cmd } -func (c *commandable) GetSet(key, value string) *StringCmd { +func (c *commandable) GetSet(key string, value interface{}) *StringCmd { cmd := NewStringCmd("GETSET", key, value) c.Process(cmd) return cmd @@ -438,7 +470,7 @@ func (c *commandable) Incr(key string) *IntCmd { } func (c *commandable) IncrBy(key string, value int64) *IntCmd { - cmd := NewIntCmd("INCRBY", key, strconv.FormatInt(value, 10)) + cmd := NewIntCmd("INCRBY", key, formatInt(value)) c.Process(cmd) return cmd } @@ -450,28 +482,43 @@ func (c *commandable) IncrByFloat(key string, value float64) *FloatCmd { } func (c *commandable) MGet(keys ...string) *SliceCmd { - args := append([]string{"MGET"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "MGET" + for i, key := range keys { + args[1+i] = key + } cmd := NewSliceCmd(args...) c.Process(cmd) return cmd } func (c *commandable) MSet(pairs ...string) *StatusCmd { - args := append([]string{"MSET"}, pairs...) + args := make([]interface{}, 1+len(pairs)) + args[0] = "MSET" + for i, pair := range pairs { + args[1+i] = pair + } cmd := NewStatusCmd(args...) c.Process(cmd) return cmd } func (c *commandable) MSetNX(pairs ...string) *BoolCmd { - args := append([]string{"MSETNX"}, pairs...) + args := make([]interface{}, 1+len(pairs)) + args[0] = "MSETNX" + for i, pair := range pairs { + args[1+i] = pair + } cmd := NewBoolCmd(args...) c.Process(cmd) return cmd } -func (c *commandable) Set(key, value string, expiration time.Duration) *StatusCmd { - args := []string{"SET", key, value} +func (c *commandable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { + args := make([]interface{}, 3, 5) + args[0] = "SET" + args[1] = key + args[2] = value if expiration > 0 { if usePrecise(expiration) { args = append(args, "PX", formatMs(expiration)) @@ -488,14 +535,14 @@ func (c *commandable) SetBit(key string, offset int64, value int) *IntCmd { cmd := NewIntCmd( "SETBIT", key, - strconv.FormatInt(offset, 10), - strconv.FormatInt(int64(value), 10), + formatInt(offset), + formatInt(int64(value)), ) c.Process(cmd) return cmd } -func (c *commandable) SetNX(key, value string, expiration time.Duration) *BoolCmd { +func (c *commandable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if expiration == 0 { // Use old `SETNX` to support old Redis versions. @@ -511,7 +558,7 @@ func (c *commandable) SetNX(key, value string, expiration time.Duration) *BoolCm return cmd } -func (c *Client) SetXX(key, value string, expiration time.Duration) *BoolCmd { +func (c *Client) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if usePrecise(expiration) { cmd = NewBoolCmd("SET", key, value, "PX", formatMs(expiration), "XX") @@ -523,7 +570,7 @@ func (c *Client) SetXX(key, value string, expiration time.Duration) *BoolCmd { } func (c *commandable) SetRange(key string, offset int64, value string) *IntCmd { - cmd := NewIntCmd("SETRANGE", key, strconv.FormatInt(offset, 10), value) + cmd := NewIntCmd("SETRANGE", key, formatInt(offset), value) c.Process(cmd) return cmd } @@ -537,7 +584,12 @@ func (c *commandable) StrLen(key string) *IntCmd { //------------------------------------------------------------------------------ func (c *commandable) HDel(key string, fields ...string) *IntCmd { - args := append([]string{"HDEL", key}, fields...) + args := make([]interface{}, 2+len(fields)) + args[0] = "HDEL" + args[1] = key + for i, field := range fields { + args[2+i] = field + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -568,7 +620,7 @@ func (c *commandable) HGetAllMap(key string) *StringStringMapCmd { } func (c *commandable) HIncrBy(key, field string, incr int64) *IntCmd { - cmd := NewIntCmd("HINCRBY", key, field, strconv.FormatInt(incr, 10)) + cmd := NewIntCmd("HINCRBY", key, field, formatInt(incr)) c.Process(cmd) return cmd } @@ -592,14 +644,26 @@ func (c *commandable) HLen(key string) *IntCmd { } func (c *commandable) HMGet(key string, fields ...string) *SliceCmd { - args := append([]string{"HMGET", key}, fields...) + args := make([]interface{}, 2+len(fields)) + args[0] = "HMGET" + args[1] = key + for i, field := range fields { + args[2+i] = field + } cmd := NewSliceCmd(args...) c.Process(cmd) return cmd } func (c *commandable) HMSet(key, field, value string, pairs ...string) *StatusCmd { - args := append([]string{"HMSET", key, field, value}, pairs...) + args := make([]interface{}, 4+len(pairs)) + args[0] = "HMSET" + args[1] = key + args[2] = field + args[3] = value + for i, pair := range pairs { + args[4+i] = pair + } cmd := NewStatusCmd(args...) c.Process(cmd) return cmd @@ -626,8 +690,12 @@ func (c *commandable) HVals(key string) *StringSliceCmd { //------------------------------------------------------------------------------ func (c *commandable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { - args := append([]string{"BLPOP"}, keys...) - args = append(args, formatSec(timeout)) + args := make([]interface{}, 2+len(keys)) + args[0] = "BLPOP" + for i, key := range keys { + args[1+i] = key + } + args[len(args)-1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) cmd.setReadTimeout(readTimeout(timeout)) c.Process(cmd) @@ -635,8 +703,12 @@ func (c *commandable) BLPop(timeout time.Duration, keys ...string) *StringSliceC } func (c *commandable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { - args := append([]string{"BRPOP"}, keys...) - args = append(args, formatSec(timeout)) + args := make([]interface{}, 2+len(keys)) + args[0] = "BRPOP" + for i, key := range keys { + args[1+i] = key + } + args[len(args)-1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) cmd.setReadTimeout(readTimeout(timeout)) c.Process(cmd) @@ -656,7 +728,7 @@ func (c *commandable) BRPopLPush(source, destination string, timeout time.Durati } func (c *commandable) LIndex(key string, index int64) *StringCmd { - cmd := NewStringCmd("LINDEX", key, strconv.FormatInt(index, 10)) + cmd := NewStringCmd("LINDEX", key, formatInt(index)) c.Process(cmd) return cmd } @@ -680,7 +752,12 @@ func (c *commandable) LPop(key string) *StringCmd { } func (c *commandable) LPush(key string, values ...string) *IntCmd { - args := append([]string{"LPUSH", key}, values...) + args := make([]interface{}, 2+len(values)) + args[0] = "LPUSH" + args[1] = key + for i, value := range values { + args[2+i] = value + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -696,21 +773,21 @@ func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd( "LRANGE", key, - strconv.FormatInt(start, 10), - strconv.FormatInt(stop, 10), + formatInt(start), + formatInt(stop), ) c.Process(cmd) return cmd } func (c *commandable) LRem(key string, count int64, value string) *IntCmd { - cmd := NewIntCmd("LREM", key, strconv.FormatInt(count, 10), value) + cmd := NewIntCmd("LREM", key, formatInt(count), value) c.Process(cmd) return cmd } func (c *commandable) LSet(key string, index int64, value string) *StatusCmd { - cmd := NewStatusCmd("LSET", key, strconv.FormatInt(index, 10), value) + cmd := NewStatusCmd("LSET", key, formatInt(index), value) c.Process(cmd) return cmd } @@ -719,8 +796,8 @@ func (c *commandable) LTrim(key string, start, stop int64) *StatusCmd { cmd := NewStatusCmd( "LTRIM", key, - strconv.FormatInt(start, 10), - strconv.FormatInt(stop, 10), + formatInt(start), + formatInt(stop), ) c.Process(cmd) return cmd @@ -739,7 +816,12 @@ func (c *commandable) RPopLPush(source, destination string) *StringCmd { } func (c *commandable) RPush(key string, values ...string) *IntCmd { - args := append([]string{"RPUSH", key}, values...) + args := make([]interface{}, 2+len(values)) + args[0] = "RPUSH" + args[1] = key + for i, value := range values { + args[2+i] = value + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -754,7 +836,12 @@ func (c *commandable) RPushX(key string, value string) *IntCmd { //------------------------------------------------------------------------------ func (c *commandable) SAdd(key string, members ...string) *IntCmd { - args := append([]string{"SADD", key}, members...) + args := make([]interface{}, 2+len(members)) + args[0] = "SADD" + args[1] = key + for i, member := range members { + args[2+i] = member + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -767,28 +854,46 @@ func (c *commandable) SCard(key string) *IntCmd { } func (c *commandable) SDiff(keys ...string) *StringSliceCmd { - args := append([]string{"SDIFF"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "SDIFF" + for i, key := range keys { + args[1+i] = key + } cmd := NewStringSliceCmd(args...) c.Process(cmd) return cmd } func (c *commandable) SDiffStore(destination string, keys ...string) *IntCmd { - args := append([]string{"SDIFFSTORE", destination}, keys...) + args := make([]interface{}, 2+len(keys)) + args[0] = "SDIFFSTORE" + args[1] = destination + for i, key := range keys { + args[2+i] = key + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd } func (c *commandable) SInter(keys ...string) *StringSliceCmd { - args := append([]string{"SINTER"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "SINTER" + for i, key := range keys { + args[1+i] = key + } cmd := NewStringSliceCmd(args...) c.Process(cmd) return cmd } func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { - args := append([]string{"SINTERSTORE", destination}, keys...) + args := make([]interface{}, 2+len(keys)) + args[0] = "SINTERSTORE" + args[1] = destination + for i, key := range keys { + args[2+i] = key + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -825,21 +930,35 @@ func (c *commandable) SRandMember(key string) *StringCmd { } func (c *commandable) SRem(key string, members ...string) *IntCmd { - args := append([]string{"SREM", key}, members...) + args := make([]interface{}, 2+len(members)) + args[0] = "SREM" + args[1] = key + for i, member := range members { + args[2+i] = member + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd } func (c *commandable) SUnion(keys ...string) *StringSliceCmd { - args := append([]string{"SUNION"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "SUNION" + for i, key := range keys { + args[1+i] = key + } cmd := NewStringSliceCmd(args...) c.Process(cmd) return cmd } func (c *commandable) SUnionStore(destination string, keys ...string) *IntCmd { - args := append([]string{"SUNIONSTORE", destination}, keys...) + args := make([]interface{}, 2+len(keys)) + args[0] = "SUNIONSTORE" + args[1] = destination + for i, key := range keys { + args[2+i] = key + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -858,7 +977,7 @@ type ZStore struct { } func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { - args := make([]string, 2+2*len(members)) + args := make([]interface{}, 2+2*len(members)) args[0] = "ZADD" args[1] = key for i, m := range members { @@ -893,12 +1012,17 @@ func (c *commandable) ZInterStore( store ZStore, keys ...string, ) *IntCmd { - args := []string{"ZINTERSTORE", destination, strconv.FormatInt(int64(len(keys)), 10)} - args = append(args, keys...) + args := make([]interface{}, 3+len(keys)) + args[0] = "ZINTERSTORE" + args[1] = destination + args[2] = strconv.Itoa(len(keys)) + for i, key := range keys { + args[3+i] = key + } if len(store.Weights) > 0 { args = append(args, "WEIGHTS") for _, weight := range store.Weights { - args = append(args, strconv.FormatInt(weight, 10)) + args = append(args, formatInt(weight)) } } if store.Aggregate != "" { @@ -910,11 +1034,11 @@ func (c *commandable) ZInterStore( } func (c *commandable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { - args := []string{ + args := []interface{}{ "ZRANGE", key, - strconv.FormatInt(start, 10), - strconv.FormatInt(stop, 10), + formatInt(start), + formatInt(stop), } if withScores { args = append(args, "WITHSCORES") @@ -929,11 +1053,11 @@ func (c *commandable) ZRange(key string, start, stop int64) *StringSliceCmd { } func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { - args := []string{ + args := []interface{}{ "ZRANGE", key, - strconv.FormatInt(start, 10), - strconv.FormatInt(stop, 10), + formatInt(start), + formatInt(stop), "WITHSCORES", } cmd := NewZSliceCmd(args...) @@ -947,7 +1071,7 @@ type ZRangeByScore struct { } func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores bool) *StringSliceCmd { - args := []string{"ZRANGEBYSCORE", key, opt.Min, opt.Max} + args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max} if withScores { args = append(args, "WITHSCORES") } @@ -955,8 +1079,8 @@ func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores bo args = append( args, "LIMIT", - strconv.FormatInt(opt.Offset, 10), - strconv.FormatInt(opt.Count, 10), + formatInt(opt.Offset), + formatInt(opt.Count), ) } cmd := NewStringSliceCmd(args...) @@ -969,13 +1093,13 @@ func (c *commandable) ZRangeByScore(key string, opt ZRangeByScore) *StringSliceC } func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { - args := []string{"ZRANGEBYSCORE", key, opt.Min, opt.Max, "WITHSCORES"} + args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( args, "LIMIT", - strconv.FormatInt(opt.Offset, 10), - strconv.FormatInt(opt.Count, 10), + formatInt(opt.Offset), + formatInt(opt.Count), ) } cmd := NewZSliceCmd(args...) @@ -990,7 +1114,12 @@ func (c *commandable) ZRank(key, member string) *IntCmd { } func (c *commandable) ZRem(key string, members ...string) *IntCmd { - args := append([]string{"ZREM", key}, members...) + args := make([]interface{}, 2+len(members)) + args[0] = "ZREM" + args[1] = key + for i, member := range members { + args[2+i] = member + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -1000,8 +1129,8 @@ func (c *commandable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { cmd := NewIntCmd( "ZREMRANGEBYRANK", key, - strconv.FormatInt(start, 10), - strconv.FormatInt(stop, 10), + formatInt(start), + formatInt(stop), ) c.Process(cmd) return cmd @@ -1026,13 +1155,13 @@ func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSlice } func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { - args := []string{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min} + args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min} if opt.Offset != 0 || opt.Count != 0 { args = append( args, "LIMIT", - strconv.FormatInt(opt.Offset, 10), - strconv.FormatInt(opt.Count, 10), + formatInt(opt.Offset), + formatInt(opt.Count), ) } cmd := NewStringSliceCmd(args...) @@ -1041,13 +1170,13 @@ func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSli } func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { - args := []string{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min, "WITHSCORES"} + args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( args, "LIMIT", - strconv.FormatInt(opt.Offset, 10), - strconv.FormatInt(opt.Count, 10), + formatInt(opt.Offset), + formatInt(opt.Count), ) } cmd := NewZSliceCmd(args...) @@ -1068,12 +1197,17 @@ func (c *commandable) ZScore(key, member string) *FloatCmd { } func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { - args := []string{"ZUNIONSTORE", dest, strconv.FormatInt(int64(len(keys)), 10)} - args = append(args, keys...) + args := make([]interface{}, 3+len(keys)) + args[0] = "ZUNIONSTORE" + args[1] = dest + args[2] = strconv.Itoa(len(keys)) + for i, key := range keys { + args[3+i] = key + } if len(store.Weights) > 0 { args = append(args, "WEIGHTS") for _, weight := range store.Weights { - args = append(args, strconv.FormatInt(weight, 10)) + args = append(args, formatInt(weight)) } } if store.Aggregate != "" { @@ -1182,11 +1316,11 @@ func (c *commandable) Save() *StatusCmd { } func (c *commandable) shutdown(modifier string) *StatusCmd { - var args []string + var args []interface{} if modifier == "" { - args = []string{"SHUTDOWN"} + args = []interface{}{"SHUTDOWN"} } else { - args = []string{"SHUTDOWN", modifier} + args = []interface{}{"SHUTDOWN", modifier} } cmd := newKeylessStatusCmd(args...) c.Process(cmd) @@ -1239,9 +1373,17 @@ func (c *commandable) Time() *StringSliceCmd { //------------------------------------------------------------------------------ func (c *commandable) Eval(script string, keys []string, args []string) *Cmd { - cmdArgs := []string{"EVAL", script, strconv.FormatInt(int64(len(keys)), 10)} - cmdArgs = append(cmdArgs, keys...) - cmdArgs = append(cmdArgs, args...) + cmdArgs := make([]interface{}, 3+len(keys)+len(args)) + cmdArgs[0] = "EVAL" + cmdArgs[1] = script + cmdArgs[2] = strconv.Itoa(len(keys)) + for i, key := range keys { + cmdArgs[3+i] = key + } + pos := 3 + len(keys) + for i, arg := range args { + cmdArgs[pos+i] = arg + } cmd := NewCmd(cmdArgs...) if len(keys) > 0 { cmd._clusterKeyPos = 3 @@ -1251,9 +1393,17 @@ func (c *commandable) Eval(script string, keys []string, args []string) *Cmd { } func (c *commandable) EvalSha(sha1 string, keys []string, args []string) *Cmd { - cmdArgs := []string{"EVALSHA", sha1, strconv.FormatInt(int64(len(keys)), 10)} - cmdArgs = append(cmdArgs, keys...) - cmdArgs = append(cmdArgs, args...) + cmdArgs := make([]interface{}, 3+len(keys)+len(args)) + cmdArgs[0] = "EVALSHA" + cmdArgs[1] = sha1 + cmdArgs[2] = strconv.Itoa(len(keys)) + for i, key := range keys { + cmdArgs[3+i] = key + } + pos := 3 + len(keys) + for i, arg := range args { + cmdArgs[pos+i] = arg + } cmd := NewCmd(cmdArgs...) if len(keys) > 0 { cmd._clusterKeyPos = 3 @@ -1263,7 +1413,12 @@ func (c *commandable) EvalSha(sha1 string, keys []string, args []string) *Cmd { } func (c *commandable) ScriptExists(scripts ...string) *BoolSliceCmd { - args := append([]string{"SCRIPT", "EXISTS"}, scripts...) + args := make([]interface{}, 2+len(scripts)) + args[0] = "SCRIPT" + args[1] = "EXISTS" + for i, script := range scripts { + args[2+i] = script + } cmd := NewBoolSliceCmd(args...) cmd._clusterKeyPos = 0 c.Process(cmd) @@ -1301,7 +1456,7 @@ func (c *commandable) DebugObject(key string) *StringCmd { //------------------------------------------------------------------------------ func (c *commandable) PubSubChannels(pattern string) *StringSliceCmd { - args := []string{"PUBSUB", "CHANNELS"} + args := []interface{}{"PUBSUB", "CHANNELS"} if pattern != "*" { args = append(args, pattern) } @@ -1312,8 +1467,12 @@ func (c *commandable) PubSubChannels(pattern string) *StringSliceCmd { } func (c *commandable) PubSubNumSub(channels ...string) *StringIntMapCmd { - args := []string{"PUBSUB", "NUMSUB"} - args = append(args, channels...) + args := make([]interface{}, 2+len(channels)) + args[0] = "PUBSUB" + args[1] = "NUMSUB" + for i, channel := range channels { + args[2+i] = channel + } cmd := NewStringIntMapCmd(args...) cmd._clusterKeyPos = 0 c.Process(cmd) @@ -1369,11 +1528,11 @@ func (c *commandable) ClusterFailover() *StatusCmd { } func (c *commandable) ClusterAddSlots(slots ...int) *StatusCmd { - args := make([]string, len(slots)+2) + args := make([]interface{}, 2+len(slots)) args[0] = "CLUSTER" - args[1] = "addslots" + args[1] = "ADDSLOTS" for i, num := range slots { - args[i+2] = strconv.Itoa(num) + args[2+i] = strconv.Itoa(num) } cmd := newKeylessStatusCmd(args...) c.Process(cmd) diff --git a/commands_test.go b/commands_test.go index ef593e1eb7..0e0405eddb 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1,7 +1,9 @@ package redis_test import ( + "encoding/json" "fmt" + "reflect" "strconv" "sync" "testing" @@ -2286,4 +2288,95 @@ var _ = Describe("Commands", func() { }) + Describe("marshaling/unmarshaling", func() { + + type convTest struct { + value interface{} + wanted string + dest interface{} + } + + convTests := []convTest{ + {nil, "", nil}, + {"hello", "hello", new(string)}, + {[]byte("hello"), "hello", new([]byte)}, + {int(1), "1", new(int)}, + {int8(1), "1", new(int8)}, + {int16(1), "1", new(int16)}, + {int32(1), "1", new(int32)}, + {int64(1), "1", new(int64)}, + {uint(1), "1", new(uint)}, + {uint8(1), "1", new(uint8)}, + {uint16(1), "1", new(uint16)}, + {uint32(1), "1", new(uint32)}, + {uint64(1), "1", new(uint64)}, + {float32(1.0), "1", new(float32)}, + {float64(1.0), "1", new(float64)}, + {true, "1", new(bool)}, + {false, "0", new(bool)}, + } + + It("should convert to string", func() { + for _, test := range convTests { + err := client.Set("key", test.value, 0).Err() + Expect(err).NotTo(HaveOccurred()) + + s, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(s).To(Equal(test.wanted)) + + if test.dest == nil { + continue + } + + err = client.Get("key").Scan(test.dest) + Expect(err).NotTo(HaveOccurred()) + Expect(deref(test.dest)).To(Equal(test.value)) + } + }) + + }) + + Describe("json marshaling/unmarshaling", func() { + BeforeEach(func() { + value := &numberStruct{Number: 42} + err := client.Set("key", value, 0).Err() + Expect(err).NotTo(HaveOccurred()) + }) + + It("should marshal custom values using json", func() { + s, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(s).To(Equal(`{"Number":42}`)) + }) + + It("should scan custom values using json", func() { + value := &numberStruct{} + err := client.Get("key").Scan(value) + Expect(err).To(BeNil()) + Expect(value.Number).To(Equal(42)) + }) + + }) + }) + +type numberStruct struct { + Number int +} + +func (s *numberStruct) MarshalBinary() ([]byte, error) { + return json.Marshal(s) +} + +func (s *numberStruct) UnmarshalBinary(b []byte) error { + return json.Unmarshal(b, s) +} + +func deref(viface interface{}) interface{} { + v := reflect.ValueOf(viface) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + return v.Interface() +} diff --git a/conn.go b/conn.go index 72f82b0fdc..bf104d9dd2 100644 --- a/conn.go +++ b/conn.go @@ -67,7 +67,11 @@ func (cn *conn) init(opt *Options) error { func (cn *conn) writeCmds(cmds ...Cmder) error { buf := cn.buf[:0] for _, cmd := range cmds { - buf = appendArgs(buf, cmd.args()) + var err error + buf, err = appendArgs(buf, cmd.args()) + if err != nil { + return err + } } _, err := cn.Write(buf) diff --git a/multi.go b/multi.go index 9d87de9a54..63ecdd5896 100644 --- a/multi.go +++ b/multi.go @@ -44,14 +44,22 @@ func (c *Multi) Close() error { } func (c *Multi) Watch(keys ...string) *StatusCmd { - args := append([]string{"WATCH"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "WATCH" + for i, key := range keys { + args[1+i] = key + } cmd := NewStatusCmd(args...) c.Process(cmd) return cmd } func (c *Multi) Unwatch(keys ...string) *StatusCmd { - args := append([]string{"UNWATCH"}, keys...) + args := make([]interface{}, 1+len(keys)) + args[0] = "UNWATCH" + for i, key := range keys { + args[1+i] = key + } cmd := NewStatusCmd(args...) c.Process(cmd) return cmd diff --git a/parser.go b/parser.go index 5a9c79f447..48fe7696e6 100644 --- a/parser.go +++ b/parser.go @@ -17,18 +17,201 @@ var ( //------------------------------------------------------------------------------ -func appendArgs(buf []byte, args []string) []byte { - buf = append(buf, '*') - buf = strconv.AppendUint(buf, uint64(len(args)), 10) - buf = append(buf, '\r', '\n') +// Copy of encoding.BinaryMarshaler. +type binaryMarshaler interface { + MarshalBinary() (data []byte, err error) +} + +// Copy of encoding.BinaryUnmarshaler. +type binaryUnmarshaler interface { + UnmarshalBinary(data []byte) error +} + +func appendString(b []byte, s string) []byte { + b = append(b, '$') + b = strconv.AppendUint(b, uint64(len(s)), 10) + b = append(b, '\r', '\n') + b = append(b, s...) + b = append(b, '\r', '\n') + return b +} + +func appendBytes(b, bb []byte) []byte { + b = append(b, '$') + b = strconv.AppendUint(b, uint64(len(bb)), 10) + b = append(b, '\r', '\n') + b = append(b, bb...) + b = append(b, '\r', '\n') + return b +} + +func appendArg(b []byte, val interface{}) ([]byte, error) { + switch v := val.(type) { + case nil: + b = appendString(b, "") + case string: + b = appendString(b, v) + case []byte: + b = appendBytes(b, v) + case int: + b = appendString(b, formatInt(int64(v))) + case int8: + b = appendString(b, formatInt(int64(v))) + case int16: + b = appendString(b, formatInt(int64(v))) + case int32: + b = appendString(b, formatInt(int64(v))) + case int64: + b = appendString(b, formatInt(v)) + case uint: + b = appendString(b, formatUint(uint64(v))) + case uint8: + b = appendString(b, formatUint(uint64(v))) + case uint16: + b = appendString(b, formatUint(uint64(v))) + case uint32: + b = appendString(b, formatUint(uint64(v))) + case uint64: + b = appendString(b, formatUint(v)) + case float32: + b = appendString(b, formatFloat(float64(v))) + case float64: + b = appendString(b, formatFloat(v)) + case bool: + if v { + b = appendString(b, "1") + } else { + b = appendString(b, "0") + } + default: + if bm, ok := val.(binaryMarshaler); ok { + bb, err := bm.MarshalBinary() + if err != nil { + return nil, err + } + b = appendBytes(b, bb) + } else { + err := fmt.Errorf( + "redis: can't marshal %T (consider implementing BinaryMarshaler)", val) + return nil, err + } + } + return b, nil +} + +func appendArgs(b []byte, args []interface{}) ([]byte, error) { + b = append(b, '*') + b = strconv.AppendUint(b, uint64(len(args)), 10) + b = append(b, '\r', '\n') for _, arg := range args { - buf = append(buf, '$') - buf = strconv.AppendUint(buf, uint64(len(arg)), 10) - buf = append(buf, '\r', '\n') - buf = append(buf, arg...) - buf = append(buf, '\r', '\n') + var err error + b, err = appendArg(b, arg) + if err != nil { + return nil, err + } + } + return b, nil +} + +func scan(b []byte, val interface{}) error { + switch v := val.(type) { + case nil: + return errorf("redis: Scan(nil)") + case *string: + *v = string(b) + return nil + case *[]byte: + *v = b + return nil + case *int: + var err error + *v, err = strconv.Atoi(string(b)) + return err + case *int8: + n, err := strconv.ParseInt(string(b), 10, 8) + if err != nil { + return err + } + *v = int8(n) + return nil + case *int16: + n, err := strconv.ParseInt(string(b), 10, 16) + if err != nil { + return err + } + *v = int16(n) + return nil + case *int32: + n, err := strconv.ParseInt(string(b), 10, 16) + if err != nil { + return err + } + *v = int32(n) + return nil + case *int64: + n, err := strconv.ParseInt(string(b), 10, 64) + if err != nil { + return err + } + *v = n + return nil + case *uint: + n, err := strconv.ParseUint(string(b), 10, 64) + if err != nil { + return err + } + *v = uint(n) + return nil + case *uint8: + n, err := strconv.ParseUint(string(b), 10, 8) + if err != nil { + return err + } + *v = uint8(n) + return nil + case *uint16: + n, err := strconv.ParseUint(string(b), 10, 16) + if err != nil { + return err + } + *v = uint16(n) + return nil + case *uint32: + n, err := strconv.ParseUint(string(b), 10, 32) + if err != nil { + return err + } + *v = uint32(n) + return nil + case *uint64: + n, err := strconv.ParseUint(string(b), 10, 64) + if err != nil { + return err + } + *v = n + return nil + case *float32: + n, err := strconv.ParseFloat(string(b), 32) + if err != nil { + return err + } + *v = float32(n) + return err + case *float64: + var err error + *v, err = strconv.ParseFloat(string(b), 64) + return err + case *bool: + *v = len(b) == 1 && b[0] == '1' + return nil + default: + if bu, ok := val.(binaryUnmarshaler); ok { + return bu.UnmarshalBinary(b) + } + err := fmt.Errorf( + "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", val) + return err } - return buf } //------------------------------------------------------------------------------ @@ -120,7 +303,7 @@ func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { case '-': return nil, errorf(string(line[1:])) case '+': - return string(line[1:]), nil + return line[1:], nil case ':': v, err := strconv.ParseInt(string(line[1:]), 10, 64) if err != nil { @@ -141,7 +324,7 @@ func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { if err != nil { return nil, err } - return string(b[:replyLen]), nil + return b[:replyLen], nil case '*': if len(line) == 3 && line[1] == '-' && line[2] == '1' { return nil, Nil @@ -166,7 +349,12 @@ func parseSlice(rd *bufio.Reader, n int64) (interface{}, error) { } else if err != nil { return nil, err } else { - vals = append(vals, v) + switch vv := v.(type) { + case []byte: + vals = append(vals, string(vv)) + default: + vals = append(vals, v) + } } } return vals, nil @@ -179,11 +367,11 @@ func parseStringSlice(rd *bufio.Reader, n int64) (interface{}, error) { if err != nil { return nil, err } - v, ok := viface.(string) + v, ok := viface.([]byte) if !ok { return nil, fmt.Errorf("got %T, expected string", viface) } - vals = append(vals, v) + vals = append(vals, string(v)) } return vals, nil } @@ -211,7 +399,7 @@ func parseStringStringMap(rd *bufio.Reader, n int64) (interface{}, error) { if err != nil { return nil, err } - key, ok := keyiface.(string) + key, ok := keyiface.([]byte) if !ok { return nil, fmt.Errorf("got %T, expected string", keyiface) } @@ -220,12 +408,12 @@ func parseStringStringMap(rd *bufio.Reader, n int64) (interface{}, error) { if err != nil { return nil, err } - value, ok := valueiface.(string) + value, ok := valueiface.([]byte) if !ok { return nil, fmt.Errorf("got %T, expected string", valueiface) } - m[key] = value + m[string(key)] = string(value) } return m, nil } @@ -237,7 +425,7 @@ func parseStringIntMap(rd *bufio.Reader, n int64) (interface{}, error) { if err != nil { return nil, err } - key, ok := keyiface.(string) + key, ok := keyiface.([]byte) if !ok { return nil, fmt.Errorf("got %T, expected string", keyiface) } @@ -248,15 +436,14 @@ func parseStringIntMap(rd *bufio.Reader, n int64) (interface{}, error) { } switch value := valueiface.(type) { case int64: - m[key] = value + m[string(key)] = value case string: - m[key], err = strconv.ParseInt(value, 10, 64) + m[string(key)], err = strconv.ParseInt(value, 10, 64) if err != nil { return nil, fmt.Errorf("got %v, expected number", value) } default: return nil, fmt.Errorf("got %T, expected number or string", valueiface) - } } return m, nil @@ -271,21 +458,21 @@ func parseZSlice(rd *bufio.Reader, n int64) (interface{}, error) { if err != nil { return nil, err } - member, ok := memberiface.(string) + member, ok := memberiface.([]byte) if !ok { return nil, fmt.Errorf("got %T, expected string", memberiface) } - z.Member = member + z.Member = string(member) scoreiface, err := parseReply(rd, nil) if err != nil { return nil, err } - scorestr, ok := scoreiface.(string) + scoreb, ok := scoreiface.([]byte) if !ok { return nil, fmt.Errorf("got %T, expected string", scoreiface) } - score, err := strconv.ParseFloat(scorestr, 64) + score, err := strconv.ParseFloat(string(scoreb), 64) if err != nil { return nil, err } diff --git a/parser_test.go b/parser_test.go index 1b9e15810a..b71305a7ad 100644 --- a/parser_test.go +++ b/parser_test.go @@ -47,7 +47,7 @@ func benchmarkParseReply(b *testing.B, reply string, p multiBulkParser, wanterr func BenchmarkAppendArgs(b *testing.B) { buf := make([]byte, 0, 64) - args := []string{"hello", "world", "foo", "bar"} + args := []interface{}{"hello", "world", "foo", "bar"} for i := 0; i < b.N; i++ { appendArgs(buf, args) } diff --git a/pubsub.go b/pubsub.go index 86e4bf6c0e..26fa85289e 100644 --- a/pubsub.go +++ b/pubsub.go @@ -80,11 +80,11 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { reply := cmd.Val() - msgName := reply[0].(string) - switch msgName { + kind := reply[0].(string) + switch kind { case "subscribe", "unsubscribe", "psubscribe", "punsubscribe": return &Subscription{ - Kind: msgName, + Kind: kind, Channel: reply[1].(string), Count: int(reply[2].(int64)), }, nil @@ -101,7 +101,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { }, nil } - return nil, fmt.Errorf("redis: unsupported message name: %q", msgName) + return nil, fmt.Errorf("redis: unsupported pubsub notification: %q", kind) } func (c *PubSub) subscribe(cmd string, channels ...string) error { @@ -110,7 +110,11 @@ func (c *PubSub) subscribe(cmd string, channels ...string) error { return err } - args := append([]string{cmd}, channels...) + args := make([]interface{}, 1+len(channels)) + args[0] = cmd + for i, channel := range channels { + args[1+i] = channel + } req := NewSliceCmd(args...) return cn.writeCmds(req) } @@ -123,21 +127,10 @@ func (c *PubSub) PSubscribe(patterns ...string) error { return c.subscribe("PSUBSCRIBE", patterns...) } -func (c *PubSub) unsubscribe(cmd string, channels ...string) error { - cn, err := c.conn() - if err != nil { - return err - } - - args := append([]string{cmd}, channels...) - req := NewSliceCmd(args...) - return cn.writeCmds(req) -} - func (c *PubSub) Unsubscribe(channels ...string) error { - return c.unsubscribe("UNSUBSCRIBE", channels...) + return c.subscribe("UNSUBSCRIBE", channels...) } func (c *PubSub) PUnsubscribe(patterns ...string) error { - return c.unsubscribe("PUNSUBSCRIBE", patterns...) + return c.subscribe("PUNSUBSCRIBE", patterns...) } diff --git a/redis_test.go b/redis_test.go index d8c4f7aee6..ac5ee7c99a 100644 --- a/redis_test.go +++ b/redis_test.go @@ -192,30 +192,34 @@ func BenchmarkRedisSet(b *testing.B) { }) } -func BenchmarkRedisSetBytes(b *testing.B) { +func BenchmarkRedisGetNil(b *testing.B) { client := redis.NewClient(&redis.Options{ Addr: benchRedisAddr, }) defer client.Close() - value := bytes.Repeat([]byte{'1'}, 10000) + if err := client.FlushDb().Err(); err != nil { + b.Fatal(err) + } b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Set("key", string(value), 0).Err(); err != nil { + if err := client.Get("key").Err(); err != redis.Nil { b.Fatal(err) } } }) } -func BenchmarkRedisGetNil(b *testing.B) { +func BenchmarkRedisGet(b *testing.B) { client := redis.NewClient(&redis.Options{ Addr: benchRedisAddr, }) defer client.Close() - if err := client.FlushDb().Err(); err != nil { + + value := bytes.Repeat([]byte{'1'}, 10000) + if err := client.Set("key", value, 0).Err(); err != nil { b.Fatal(err) } @@ -223,29 +227,40 @@ func BenchmarkRedisGetNil(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Get("key").Err(); err != redis.Nil { + s, err := client.Get("key").Result() + if err != nil { b.Fatal(err) } + if len(s) != 10000 { + panic("len(s) != 10000") + } } }) } -func BenchmarkRedisGet(b *testing.B) { +func BenchmarkRedisGetSetBytes(b *testing.B) { client := redis.NewClient(&redis.Options{ Addr: benchRedisAddr, }) defer client.Close() - if err := client.Set("key", "hello", 0).Err(); err != nil { - b.Fatal(err) - } + + src := bytes.Repeat([]byte{'1'}, 10000) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Get("key").Err(); err != nil { + if err := client.Set("key", src, 0).Err(); err != nil { b.Fatal(err) } + + dst, err := client.Get("key").Bytes() + if err != nil { + b.Fatal(err) + } + if !bytes.Equal(dst, src) { + panic("len(dst) != 10000") + } } }) } From fa185a564fd76981830e4ad8ecdbd1dd20e302aa Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 11 Jul 2015 12:10:41 +0300 Subject: [PATCH 0043/1746] Use time.Duration in Restore. --- commands.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index 56167d508c..05de2f0a51 100644 --- a/commands.go +++ b/commands.go @@ -238,11 +238,11 @@ func (c *commandable) RenameNX(key, newkey string) *BoolCmd { return cmd } -func (c *commandable) Restore(key string, ttl int64, value string) *StatusCmd { +func (c *commandable) Restore(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( "RESTORE", key, - formatInt(ttl), + formatMs(ttl), value, ) c.Process(cmd) From 412baf447b1b47109bc4c6b5e50ea7d53447ce8f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 11 Jul 2015 12:23:04 +0300 Subject: [PATCH 0044/1746] Add RestoreReplace. --- commands.go | 12 ++++++++++++ commands_test.go | 47 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/commands.go b/commands.go index 56167d508c..c9a3a35138 100644 --- a/commands.go +++ b/commands.go @@ -249,6 +249,18 @@ func (c *commandable) Restore(key string, ttl int64, value string) *StatusCmd { return cmd } +func (c *commandable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { + cmd := NewStatusCmd( + "RESTORE", + key, + formatMs(ttl), + value, + "REPLACE", + ) + c.Process(cmd) + return cmd +} + type Sort struct { By string Offset, Count float64 diff --git a/commands_test.go b/commands_test.go index 0e0405eddb..ae0535eee0 100644 --- a/commands_test.go +++ b/commands_test.go @@ -451,27 +451,46 @@ var _ = Describe("Commands", func() { }) It("should Restore", func() { - set := client.Set("key", "hello", 0) - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) + err := client.Set("key", "hello", 0).Err() + Expect(err).NotTo(HaveOccurred()) dump := client.Dump("key") Expect(dump.Err()).NotTo(HaveOccurred()) - del := client.Del("key") - Expect(del.Err()).NotTo(HaveOccurred()) + err = client.Del("key").Err() + Expect(err).NotTo(HaveOccurred()) - restore := client.Restore("key", 0, dump.Val()) - Expect(restore.Err()).NotTo(HaveOccurred()) - Expect(restore.Val()).To(Equal("OK")) + restore, err := client.Restore("key", 0, dump.Val()).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(restore).To(Equal("OK")) - type_ := client.Type("key") - Expect(type_.Err()).NotTo(HaveOccurred()) - Expect(type_.Val()).To(Equal("string")) + type_, err := client.Type("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(type_).To(Equal("string")) - lRange := client.Get("key") - Expect(lRange.Err()).NotTo(HaveOccurred()) - Expect(lRange.Val()).To(Equal("hello")) + val, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("hello")) + }) + + It("should RestoreReplace", func() { + err := client.Set("key", "hello", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + dump := client.Dump("key") + Expect(dump.Err()).NotTo(HaveOccurred()) + + restore, err := client.RestoreReplace("key", 0, dump.Val()).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(restore).To(Equal("OK")) + + type_, err := client.Type("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(type_).To(Equal("string")) + + val, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("hello")) }) It("should Sort", func() { From f901321d846ad0f37964210647bdecb191a9b02e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 11 Jul 2015 13:12:47 +0300 Subject: [PATCH 0045/1746] pubsub: add PING support. --- example_test.go | 14 ++++++----- pubsub.go | 64 ++++++++++++++++++++++++++++++++++++------------- pubsub_test.go | 53 +++++++++++++++++++++++++++------------- 3 files changed, 91 insertions(+), 40 deletions(-) diff --git a/example_test.go b/example_test.go index e948410461..b54e043a6a 100644 --- a/example_test.go +++ b/example_test.go @@ -2,7 +2,6 @@ package redis_test import ( "fmt" - "net" "strconv" "sync" "time" @@ -179,14 +178,14 @@ func ExamplePubSub() { panic(err) } - for { + for i := 0; i < 4; i++ { msgi, err := pubsub.ReceiveTimeout(100 * time.Millisecond) if err != nil { - if neterr, ok := err.(net.Error); ok && neterr.Timeout() { - // There are no more messages to process. Stop. - break + err := pubsub.Ping("") + if err != nil { + panic(err) } - panic(err) + continue } switch msg := msgi.(type) { @@ -194,6 +193,8 @@ func ExamplePubSub() { fmt.Println(msg.Kind, msg.Channel) case *redis.Message: fmt.Println(msg.Channel, msg.Payload) + case *redis.Pong: + fmt.Println(msg) default: panic(fmt.Sprintf("unknown message: %#v", msgi)) } @@ -201,6 +202,7 @@ func ExamplePubSub() { // Output: subscribe mychannel // mychannel hello + // Pong } func ExampleScript() { diff --git a/pubsub.go b/pubsub.go index 26fa85289e..e143448b11 100644 --- a/pubsub.go +++ b/pubsub.go @@ -26,6 +26,20 @@ func (c *Client) Publish(channel, message string) *IntCmd { return req } +func (c *PubSub) Ping(payload string) error { + cn, err := c.conn() + if err != nil { + return err + } + + args := []interface{}{"PING"} + if payload != "" { + args = append(args, payload) + } + cmd := NewCmd(args...) + return cn.writeCmds(cmd) +} + // Message received as result of a PUBLISH command issued by another client. type Message struct { Channel string @@ -48,6 +62,18 @@ func (m *PMessage) String() string { return fmt.Sprintf("PMessage<%s: %s>", m.Channel, m.Payload) } +// Pong received as result of a PING command issued by another client. +type Pong struct { + Payload string +} + +func (p *Pong) String() string { + if p.Payload != "" { + return fmt.Sprintf("Pong<%s>", p.Payload) + } + return "Pong" +} + // Message received after a successful subscription to channel. type Subscription struct { // Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe". @@ -66,22 +92,8 @@ func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } -func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { - cn, err := c.conn() - if err != nil { - return nil, err - } - cn.ReadTimeout = timeout - - cmd := NewSliceCmd() - if err := cmd.parseReply(cn.rd); err != nil { - return nil, err - } - - reply := cmd.Val() - - kind := reply[0].(string) - switch kind { +func newMessage(reply []interface{}) (interface{}, error) { + switch kind := reply[0].(string); kind { case "subscribe", "unsubscribe", "psubscribe", "punsubscribe": return &Subscription{ Kind: kind, @@ -99,9 +111,27 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { Channel: reply[2].(string), Payload: reply[3].(string), }, nil + case "pong": + return &Pong{ + Payload: reply[1].(string), + }, nil + default: + return nil, fmt.Errorf("redis: unsupported pubsub notification: %q", kind) + } +} + +func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { + cn, err := c.conn() + if err != nil { + return nil, err } + cn.ReadTimeout = timeout - return nil, fmt.Errorf("redis: unsupported pubsub notification: %q", kind) + cmd := NewSliceCmd() + if err := cmd.parseReply(cn.rd); err != nil { + return nil, err + } + return newMessage(cmd.Val()) } func (c *PubSub) subscribe(cmd string, channels ...string) error { diff --git a/pubsub_test.go b/pubsub_test.go index 82c0ca49c7..bf59a2cd2e 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -12,24 +12,22 @@ import ( var _ = Describe("PubSub", func() { var client *redis.Client + var pubsub *redis.PubSub BeforeEach(func() { client = redis.NewClient(&redis.Options{ Addr: redisAddr, }) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + pubsub = client.PubSub() }) AfterEach(func() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(pubsub.Close()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) It("should support pattern matching", func() { - pubsub := client.PubSub() - defer func() { - Expect(pubsub.Close()).NotTo(HaveOccurred()) - }() - Expect(pubsub.PSubscribe("mychannel*")).NotTo(HaveOccurred()) pub := client.Publish("mychannel1", "hello") @@ -77,8 +75,6 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(channels).To(BeEmpty()) - pubsub := client.PubSub() - defer pubsub.Close() Expect(pubsub.Subscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) channels, err = client.PubSubChannels("mychannel*").Result() @@ -95,8 +91,6 @@ var _ = Describe("PubSub", func() { }) It("should return the numbers of subscribers", func() { - pubsub := client.PubSub() - defer pubsub.Close() Expect(pubsub.Subscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) channels, err := client.PubSubNumSub("mychannel", "mychannel2", "mychannel3").Result() @@ -113,8 +107,6 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(num).To(Equal(int64(0))) - pubsub := client.PubSub() - defer pubsub.Close() Expect(pubsub.PSubscribe("*")).NotTo(HaveOccurred()) num, err = client.PubSubNumPat().Result() @@ -123,11 +115,6 @@ var _ = Describe("PubSub", func() { }) It("should pub/sub", func() { - pubsub := client.PubSub() - defer func() { - Expect(pubsub.Close()).NotTo(HaveOccurred()) - }() - Expect(pubsub.Subscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) pub := client.Publish("mychannel", "hello") @@ -199,4 +186,36 @@ var _ = Describe("PubSub", func() { } }) + It("should ping/pong", func() { + err := pubsub.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + + _, err = pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + + err = pubsub.Ping("") + Expect(err).NotTo(HaveOccurred()) + + msgi, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + pong := msgi.(*redis.Pong) + Expect(pong.Payload).To(Equal("")) + }) + + It("should ping/pong with payload", func() { + err := pubsub.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + + _, err = pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + + err = pubsub.Ping("hello") + Expect(err).NotTo(HaveOccurred()) + + msgi, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + pong := msgi.(*redis.Pong) + Expect(pong.Payload).To(Equal("hello")) + }) + }) From d7edae84cf2a63bd3e0c7951de2ec4b9b448e72a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 11 Jul 2015 13:42:44 +0300 Subject: [PATCH 0046/1746] pubsub: improve API and docs. --- example_test.go | 6 ++--- pubsub.go | 60 ++++++++++++++++++++++++++++++++++--------------- pubsub_test.go | 47 ++++++++++++++++++++++---------------- 3 files changed, 72 insertions(+), 41 deletions(-) diff --git a/example_test.go b/example_test.go index b54e043a6a..25869bea4a 100644 --- a/example_test.go +++ b/example_test.go @@ -165,13 +165,11 @@ func ExampleMulti() { } func ExamplePubSub() { - pubsub := client.PubSub() - defer pubsub.Close() - - err := pubsub.Subscribe("mychannel") + pubsub, err := client.Subscribe("mychannel") if err != nil { panic(err) } + defer pubsub.Close() err = client.Publish("mychannel", "hello").Err() if err != nil { diff --git a/pubsub.go b/pubsub.go index e143448b11..1f4f5b6362 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,12 +5,20 @@ import ( "time" ) +// Posts a message to the given channel. +func (c *Client) Publish(channel, message string) *IntCmd { + req := NewIntCmd("PUBLISH", channel, message) + c.Process(req) + return req +} + // PubSub implements Pub/Sub commands as described in // http://redis.io/topics/pubsub. type PubSub struct { *baseClient } +// Deprecated. Use Subscribe/PSubscribe instead. func (c *Client) PubSub() *PubSub { return &PubSub{ baseClient: &baseClient{ @@ -20,10 +28,16 @@ func (c *Client) PubSub() *PubSub { } } -func (c *Client) Publish(channel, message string) *IntCmd { - req := NewIntCmd("PUBLISH", channel, message) - c.Process(req) - return req +// Subscribes the client to the specified channels. +func (c *Client) Subscribe(channels ...string) (*PubSub, error) { + pubsub := c.PubSub() + return pubsub, pubsub.Subscribe(channels...) +} + +// Subscribes the client to the given patterns. +func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { + pubsub := c.PubSub() + return pubsub, pubsub.PSubscribe(channels...) } func (c *PubSub) Ping(payload string) error { @@ -40,6 +54,20 @@ func (c *PubSub) Ping(payload string) error { return cn.writeCmds(cmd) } +// Message received after a successful subscription to channel. +type Subscription struct { + // Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe". + Kind string + // Channel name we have subscribed to. + Channel string + // Number of channels we are currently subscribed to. + Count int +} + +func (m *Subscription) String() string { + return fmt.Sprintf("%s: %s", m.Kind, m.Channel) +} + // Message received as result of a PUBLISH command issued by another client. type Message struct { Channel string @@ -74,20 +102,8 @@ func (p *Pong) String() string { return "Pong" } -// Message received after a successful subscription to channel. -type Subscription struct { - // Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe". - Kind string - // Channel name we have subscribed to. - Channel string - // Number of channels we are currently subscribed to. - Count int -} - -func (m *Subscription) String() string { - return fmt.Sprintf("%s: %s", m.Kind, m.Channel) -} - +// Returns a message as a Subscription, Message, PMessage, Pong or +// error. See PubSub example for details. func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } @@ -120,6 +136,8 @@ func newMessage(reply []interface{}) (interface{}, error) { } } +// ReceiveTimeout acts like Receive but returns an error if message +// is not received in time. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { cn, err := c.conn() if err != nil { @@ -149,18 +167,24 @@ func (c *PubSub) subscribe(cmd string, channels ...string) error { return cn.writeCmds(req) } +// Subscribes the client to the specified channels. func (c *PubSub) Subscribe(channels ...string) error { return c.subscribe("SUBSCRIBE", channels...) } +// Subscribes the client to the given patterns. func (c *PubSub) PSubscribe(patterns ...string) error { return c.subscribe("PSUBSCRIBE", patterns...) } +// Unsubscribes the client from the given channels, or from all of +// them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { return c.subscribe("UNSUBSCRIBE", channels...) } +// Unsubscribes the client from the given patterns, or from all of +// them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { return c.subscribe("PUNSUBSCRIBE", patterns...) } diff --git a/pubsub_test.go b/pubsub_test.go index bf59a2cd2e..ac1d629b92 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -12,27 +12,26 @@ import ( var _ = Describe("PubSub", func() { var client *redis.Client - var pubsub *redis.PubSub BeforeEach(func() { client = redis.NewClient(&redis.Options{ Addr: redisAddr, }) Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - pubsub = client.PubSub() }) AfterEach(func() { - Expect(pubsub.Close()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) It("should support pattern matching", func() { - Expect(pubsub.PSubscribe("mychannel*")).NotTo(HaveOccurred()) + pubsub, err := client.PSubscribe("mychannel*") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() - pub := client.Publish("mychannel1", "hello") - Expect(pub.Err()).NotTo(HaveOccurred()) - Expect(pub.Val()).To(Equal(int64(1))) + n, err := client.Publish("mychannel1", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) Expect(pubsub.PUnsubscribe("mychannel*")).NotTo(HaveOccurred()) @@ -75,7 +74,9 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(channels).To(BeEmpty()) - Expect(pubsub.Subscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) + pubsub, err := client.Subscribe("mychannel", "mychannel2") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() channels, err = client.PubSubChannels("mychannel*").Result() Expect(err).NotTo(HaveOccurred()) @@ -91,7 +92,9 @@ var _ = Describe("PubSub", func() { }) It("should return the numbers of subscribers", func() { - Expect(pubsub.Subscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) + pubsub, err := client.Subscribe("mychannel", "mychannel2") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() channels, err := client.PubSubNumSub("mychannel", "mychannel2", "mychannel3").Result() Expect(err).NotTo(HaveOccurred()) @@ -107,7 +110,9 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(num).To(Equal(int64(0))) - Expect(pubsub.PSubscribe("*")).NotTo(HaveOccurred()) + pubsub, err := client.PSubscribe("*") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() num, err = client.PubSubNumPat().Result() Expect(err).NotTo(HaveOccurred()) @@ -115,15 +120,17 @@ var _ = Describe("PubSub", func() { }) It("should pub/sub", func() { - Expect(pubsub.Subscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) + pubsub, err := client.Subscribe("mychannel", "mychannel2") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() - pub := client.Publish("mychannel", "hello") - Expect(pub.Err()).NotTo(HaveOccurred()) - Expect(pub.Val()).To(Equal(int64(1))) + n, err := client.Publish("mychannel", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) - pub = client.Publish("mychannel2", "hello2") - Expect(pub.Err()).NotTo(HaveOccurred()) - Expect(pub.Val()).To(Equal(int64(1))) + n, err = client.Publish("mychannel2", "hello2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) Expect(pubsub.Unsubscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) @@ -187,8 +194,9 @@ var _ = Describe("PubSub", func() { }) It("should ping/pong", func() { - err := pubsub.Subscribe("mychannel") + pubsub, err := client.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() _, err = pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) @@ -203,8 +211,9 @@ var _ = Describe("PubSub", func() { }) It("should ping/pong with payload", func() { - err := pubsub.Subscribe("mychannel") + pubsub, err := client.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() _, err = pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) From 54dcf59909272af0781595a409c1895785529d0b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 13 Jul 2015 12:56:16 +0300 Subject: [PATCH 0047/1746] ring: return an error in pipeline when all shards are down. --- ring.go | 3 +++ ring_test.go | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/ring.go b/ring.go index 26b06b6256..4b20e7a90f 100644 --- a/ring.go +++ b/ring.go @@ -298,6 +298,9 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { name := pipe.ring.hash.Get(hashKey(cmd.clusterKey())) if name == "" { cmd.setErr(errRingShardsDown) + if retErr == nil { + retErr = errRingShardsDown + } continue } cmdsMap[name] = append(cmdsMap[name], cmd) diff --git a/ring_test.go b/ring_test.go index 117a9731c6..5b52b3204d 100644 --- a/ring_test.go +++ b/ring_test.go @@ -94,6 +94,15 @@ var _ = Describe("Redis ring", func() { }) Describe("pipelining", func() { + It("returns an error when all shards are down", func() { + ring := redis.NewRing(&redis.RingOptions{}) + _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { + pipe.Ping() + return nil + }) + Expect(err).To(MatchError("redis: all ring shards are down")) + }) + It("uses both shards", func() { pipe := ring.Pipeline() for i := 0; i < 100; i++ { From 029065eb68819816ef5da0343397d4cefbba7ad0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 13 Jul 2015 13:45:37 +0300 Subject: [PATCH 0048/1746] Fix nil pool on read timeout. Fixes #135. --- conn.go | 5 ++--- pool.go | 34 ++++++++++++++++++++-------------- redis_test.go | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/conn.go b/conn.go index bf104d9dd2..9dc2ede038 100644 --- a/conn.go +++ b/conn.go @@ -42,9 +42,8 @@ func (cn *conn) init(opt *Options) error { return nil } - // Use connection to connect to redis - pool := newSingleConnPool(nil, false) - pool.SetConn(cn) + // Use connection to connect to Redis. + pool := newSingleConnPoolConn(cn) // Client is not closed because we want to reuse underlying connection. client := newClient(opt, pool) diff --git a/pool.go b/pool.go index 714cbe5bc5..71ac456d5e 100644 --- a/pool.go +++ b/pool.go @@ -329,13 +329,10 @@ func newSingleConnPool(pool pool, reusable bool) *singleConnPool { } } -func (p *singleConnPool) SetConn(cn *conn) { - p.mx.Lock() - if p.cn != nil { - panic("p.cn != nil") +func newSingleConnPoolConn(cn *conn) *singleConnPool { + return &singleConnPool{ + cn: cn, } - p.cn = cn - p.mx.Unlock() } func (p *singleConnPool) First() *conn { @@ -365,6 +362,14 @@ func (p *singleConnPool) Get() (*conn, error) { return p.cn, nil } +func (p *singleConnPool) put() (err error) { + if p.pool != nil { + err = p.pool.Put(p.cn) + } + p.cn = nil + return err +} + func (p *singleConnPool) Put(cn *conn) error { defer p.mx.Unlock() p.mx.Lock() @@ -377,6 +382,14 @@ func (p *singleConnPool) Put(cn *conn) error { return nil } +func (p *singleConnPool) remove() (err error) { + if p.pool != nil { + err = p.pool.Remove(p.cn) + } + p.cn = nil + return err +} + func (p *singleConnPool) Remove(cn *conn) error { defer p.mx.Unlock() p.mx.Lock() @@ -392,12 +405,6 @@ func (p *singleConnPool) Remove(cn *conn) error { return p.remove() } -func (p *singleConnPool) remove() error { - err := p.pool.Remove(p.cn) - p.cn = nil - return err -} - func (p *singleConnPool) Len() int { defer p.mx.Unlock() p.mx.Lock() @@ -426,8 +433,7 @@ func (p *singleConnPool) Close() error { var err error if p.cn != nil { if p.reusable { - err = p.pool.Put(p.cn) - p.cn = nil + err = p.put() } else { err = p.remove() } diff --git a/redis_test.go b/redis_test.go index ac5ee7c99a..3c0d6358a0 100644 --- a/redis_test.go +++ b/redis_test.go @@ -134,6 +134,20 @@ var _ = Describe("Client", func() { Expect(db1.FlushDb().Err()).NotTo(HaveOccurred()) }) + It("should support DB selection with read timeout (issue #135)", func() { + for i := 0; i < 100; i++ { + db1 := redis.NewClient(&redis.Options{ + Addr: redisAddr, + DB: 1, + ReadTimeout: time.Nanosecond, + }) + + err := db1.Ping().Err() + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + } + }) + It("should retry command on network error", func() { Expect(client.Close()).NotTo(HaveOccurred()) From 379b44f44ade439344b0b35ae4b863f35487af4e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 16 Jul 2015 17:30:16 +0300 Subject: [PATCH 0049/1746] Add bytes support for sorted set. --- commands.go | 7 +++++-- commands_test.go | 46 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/commands.go b/commands.go index 2bc10d6323..7f10a2f079 100644 --- a/commands.go +++ b/commands.go @@ -978,13 +978,16 @@ func (c *commandable) SUnionStore(destination string, keys ...string) *IntCmd { //------------------------------------------------------------------------------ +// Sorted set member. type Z struct { Score float64 - Member string + Member interface{} } +// Sorted set store operation. type ZStore struct { - Weights []int64 + Weights []int64 + // Can be SUM, MIN or MAX. Aggregate string } diff --git a/commands_test.go b/commands_test.go index ae0535eee0..6cd9fc0639 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1822,21 +1822,43 @@ var _ = Describe("Commands", func() { Describe("sorted sets", func() { It("should ZAdd", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - Expect(zAdd.Val()).To(Equal(int64(1))) + added, err := client.ZAdd("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) - zAdd = client.ZAdd("zset", redis.Z{1, "uno"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - Expect(zAdd.Val()).To(Equal(int64(1))) + added, err = client.ZAdd("zset", redis.Z{1, "uno"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - Expect(zAdd.Val()).To(Equal(int64(1))) + added, err = client.ZAdd("zset", redis.Z{2, "two"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) - zAdd = client.ZAdd("zset", redis.Z{3, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - Expect(zAdd.Val()).To(Equal(int64(0))) + added, err = client.ZAdd("zset", redis.Z{3, "two"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(0))) + + val, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal([]redis.Z{{1, "one"}, {1, "uno"}, {3, "two"}})) + }) + + It("should ZAdd bytes", func() { + added, err := client.ZAdd("zset", redis.Z{1, []byte("one")}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + added, err = client.ZAdd("zset", redis.Z{1, []byte("uno")}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + added, err = client.ZAdd("zset", redis.Z{2, []byte("two")}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + added, err = client.ZAdd("zset", redis.Z{3, []byte("two")}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(0))) val, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) From 704068597fcb397e66625214ac57ec379b28dbd4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Jul 2015 10:39:03 +0300 Subject: [PATCH 0050/1746] Use unsafe for bytes->string conversion. --- command.go | 5 +++-- parser.go | 32 ++++++++++++++++---------------- unsafe.go | 12 ++++++++++++ 3 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 unsafe.go diff --git a/command.go b/command.go index f32569749a..dab9fc3c83 100644 --- a/command.go +++ b/command.go @@ -400,7 +400,7 @@ func (cmd *StringCmd) reset() { } func (cmd *StringCmd) Val() string { - return string(cmd.val) + return bytesToString(cmd.val) } func (cmd *StringCmd) Result() (string, error) { @@ -486,7 +486,8 @@ func (cmd *FloatCmd) parseReply(rd *bufio.Reader) error { cmd.err = err return err } - cmd.val, cmd.err = strconv.ParseFloat(string(v.([]byte)), 64) + b := v.([]byte) + cmd.val, cmd.err = strconv.ParseFloat(bytesToString(b), 64) return cmd.err } diff --git a/parser.go b/parser.go index 48fe7696e6..32646ff847 100644 --- a/parser.go +++ b/parser.go @@ -118,80 +118,80 @@ func scan(b []byte, val interface{}) error { case nil: return errorf("redis: Scan(nil)") case *string: - *v = string(b) + *v = bytesToString(b) return nil case *[]byte: *v = b return nil case *int: var err error - *v, err = strconv.Atoi(string(b)) + *v, err = strconv.Atoi(bytesToString(b)) return err case *int8: - n, err := strconv.ParseInt(string(b), 10, 8) + n, err := strconv.ParseInt(bytesToString(b), 10, 8) if err != nil { return err } *v = int8(n) return nil case *int16: - n, err := strconv.ParseInt(string(b), 10, 16) + n, err := strconv.ParseInt(bytesToString(b), 10, 16) if err != nil { return err } *v = int16(n) return nil case *int32: - n, err := strconv.ParseInt(string(b), 10, 16) + n, err := strconv.ParseInt(bytesToString(b), 10, 16) if err != nil { return err } *v = int32(n) return nil case *int64: - n, err := strconv.ParseInt(string(b), 10, 64) + n, err := strconv.ParseInt(bytesToString(b), 10, 64) if err != nil { return err } *v = n return nil case *uint: - n, err := strconv.ParseUint(string(b), 10, 64) + n, err := strconv.ParseUint(bytesToString(b), 10, 64) if err != nil { return err } *v = uint(n) return nil case *uint8: - n, err := strconv.ParseUint(string(b), 10, 8) + n, err := strconv.ParseUint(bytesToString(b), 10, 8) if err != nil { return err } *v = uint8(n) return nil case *uint16: - n, err := strconv.ParseUint(string(b), 10, 16) + n, err := strconv.ParseUint(bytesToString(b), 10, 16) if err != nil { return err } *v = uint16(n) return nil case *uint32: - n, err := strconv.ParseUint(string(b), 10, 32) + n, err := strconv.ParseUint(bytesToString(b), 10, 32) if err != nil { return err } *v = uint32(n) return nil case *uint64: - n, err := strconv.ParseUint(string(b), 10, 64) + n, err := strconv.ParseUint(bytesToString(b), 10, 64) if err != nil { return err } *v = n return nil case *float32: - n, err := strconv.ParseFloat(string(b), 32) + n, err := strconv.ParseFloat(bytesToString(b), 32) if err != nil { return err } @@ -199,7 +199,7 @@ func scan(b []byte, val interface{}) error { return err case *float64: var err error - *v, err = strconv.ParseFloat(string(b), 64) + *v, err = strconv.ParseFloat(bytesToString(b), 64) return err case *bool: *v = len(b) == 1 && b[0] == '1' @@ -305,7 +305,7 @@ func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { case '+': return line[1:], nil case ':': - v, err := strconv.ParseInt(string(line[1:]), 10, 64) + v, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) if err != nil { return nil, err } @@ -330,7 +330,7 @@ func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { return nil, Nil } - repliesNum, err := strconv.ParseInt(string(line[1:]), 10, 64) + repliesNum, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) if err != nil { return nil, err } @@ -472,7 +472,7 @@ func parseZSlice(rd *bufio.Reader, n int64) (interface{}, error) { if !ok { return nil, fmt.Errorf("got %T, expected string", scoreiface) } - score, err := strconv.ParseFloat(string(scoreb), 64) + score, err := strconv.ParseFloat(bytesToString(scoreb), 64) if err != nil { return nil, err } diff --git a/unsafe.go b/unsafe.go new file mode 100644 index 0000000000..1c4d55fa24 --- /dev/null +++ b/unsafe.go @@ -0,0 +1,12 @@ +package redis + +import ( + "reflect" + "unsafe" +) + +func bytesToString(b []byte) string { + bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + strHeader := reflect.StringHeader{bytesHeader.Data, bytesHeader.Len} + return *(*string)(unsafe.Pointer(&strHeader)) +} From ba4682c2a34e5667e2096ab897f5692794201fe9 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Jul 2015 11:22:42 +0300 Subject: [PATCH 0051/1746] Fix benchmarks to use Redis on default port and FLUSHDB before run. --- cluster_test.go | 4 ++++ pool_test.go | 6 ++---- redis_test.go | 51 +++++++++++++++++++------------------------------ 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 9173faea1d..136340ccb6 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -305,6 +305,10 @@ var _ = Describe("Cluster", func() { //------------------------------------------------------------------------------ func BenchmarkRedisClusterPing(b *testing.B) { + if testing.Short() { + b.Skip("skipping in short mode") + } + cluster := &clusterScenario{ ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, nodeIds: make([]string, 6), diff --git a/pool_test.go b/pool_test.go index a4fa76e256..bff892cc7f 100644 --- a/pool_test.go +++ b/pool_test.go @@ -182,11 +182,9 @@ var _ = Describe("Pool", func() { }) func BenchmarkPool(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: redisAddr, - IdleTimeout: 100 * time.Millisecond, - }) + client := benchRedisClient() defer client.Close() + pool := client.Pool() b.ResetTimer() diff --git a/redis_test.go b/redis_test.go index 3c0d6358a0..1122df9a75 100644 --- a/redis_test.go +++ b/redis_test.go @@ -169,12 +169,18 @@ var _ = Describe("Client", func() { //------------------------------------------------------------------------------ -const benchRedisAddr = ":6379" - -func BenchmarkRedisPing(b *testing.B) { +func benchRedisClient() *redis.Client { client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, + Addr: ":6379", }) + if err := client.FlushDb().Err(); err != nil { + panic(err) + } + return client +} + +func BenchmarkRedisPing(b *testing.B) { + client := benchRedisClient() defer client.Close() b.ResetTimer() @@ -189,10 +195,9 @@ func BenchmarkRedisPing(b *testing.B) { } func BenchmarkRedisSet(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() + value := string(bytes.Repeat([]byte{'1'}, 10000)) b.ResetTimer() @@ -207,13 +212,8 @@ func BenchmarkRedisSet(b *testing.B) { } func BenchmarkRedisGetNil(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() - if err := client.FlushDb().Err(); err != nil { - b.Fatal(err) - } b.ResetTimer() @@ -227,9 +227,7 @@ func BenchmarkRedisGetNil(b *testing.B) { } func BenchmarkRedisGet(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() value := bytes.Repeat([]byte{'1'}, 10000) @@ -253,9 +251,7 @@ func BenchmarkRedisGet(b *testing.B) { } func BenchmarkRedisGetSetBytes(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() src := bytes.Repeat([]byte{'1'}, 10000) @@ -280,10 +276,9 @@ func BenchmarkRedisGetSetBytes(b *testing.B) { } func BenchmarkRedisMGet(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() + if err := client.MSet("key1", "hello1", "key2", "hello2").Err(); err != nil { b.Fatal(err) } @@ -300,9 +295,7 @@ func BenchmarkRedisMGet(b *testing.B) { } func BenchmarkSetExpire(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() b.ResetTimer() @@ -320,9 +313,7 @@ func BenchmarkSetExpire(b *testing.B) { } func BenchmarkPipeline(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() b.ResetTimer() @@ -342,9 +333,7 @@ func BenchmarkPipeline(b *testing.B) { } func BenchmarkZAdd(b *testing.B) { - client := redis.NewClient(&redis.Options{ - Addr: benchRedisAddr, - }) + client := benchRedisClient() defer client.Close() b.ResetTimer() From a78354cb126313fd3bfb9eaee977769e72486fbc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 20 Jul 2015 15:01:40 +0300 Subject: [PATCH 0052/1746] Benchmark set/get with bigger values. --- redis_test.go | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/redis_test.go b/redis_test.go index 1122df9a75..b1a25475cc 100644 --- a/redis_test.go +++ b/redis_test.go @@ -226,50 +226,67 @@ func BenchmarkRedisGetNil(b *testing.B) { }) } -func BenchmarkRedisGet(b *testing.B) { +func benchmarkRedisSetGet(b *testing.B, size int) { client := benchRedisClient() defer client.Close() - value := bytes.Repeat([]byte{'1'}, 10000) - if err := client.Set("key", value, 0).Err(); err != nil { - b.Fatal(err) - } + value := string(bytes.Repeat([]byte{'1'}, size)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - s, err := client.Get("key").Result() + if err := client.Set("key", value, 0).Err(); err != nil { + b.Fatal(err) + } + + got, err := client.Get("key").Result() if err != nil { b.Fatal(err) } - if len(s) != 10000 { - panic("len(s) != 10000") + if got != value { + b.Fatalf("got != value") } } }) } -func BenchmarkRedisGetSetBytes(b *testing.B) { +func BenchmarkRedisSetGet64Bytes(b *testing.B) { + benchmarkRedisSetGet(b, 64) +} + +func BenchmarkRedisSetGet1KB(b *testing.B) { + benchmarkRedisSetGet(b, 1024) +} + +func BenchmarkRedisSetGet10KB(b *testing.B) { + benchmarkRedisSetGet(b, 10*1024) +} + +func BenchmarkRedisSetGet1MB(b *testing.B) { + benchmarkRedisSetGet(b, 1024*1024) +} + +func BenchmarkRedisSetGetBytes(b *testing.B) { client := benchRedisClient() defer client.Close() - src := bytes.Repeat([]byte{'1'}, 10000) + value := bytes.Repeat([]byte{'1'}, 10000) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Set("key", src, 0).Err(); err != nil { + if err := client.Set("key", value, 0).Err(); err != nil { b.Fatal(err) } - dst, err := client.Get("key").Bytes() + got, err := client.Get("key").Bytes() if err != nil { b.Fatal(err) } - if !bytes.Equal(dst, src) { - panic("len(dst) != 10000") + if !bytes.Equal(got, value) { + b.Fatalf("got != value") } } }) From 0e6443b1ac44a67023a14449e3079f4ecb4aaa20 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 3 Aug 2015 13:22:12 +0300 Subject: [PATCH 0053/1746] readme: add link to Redis cache. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d55458274b..42e0686a17 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Supports: - [Redis Sentinel](http://godoc.org/gopkg.in/redis.v3#NewFailoverClient). - [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). - [Ring](http://godoc.org/gopkg.in/redis.v3#NewRing). +- [Cache friendly](https://github.com/go-redis/cache). API docs: http://godoc.org/gopkg.in/redis.v3. Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. From c0b0c99ec5ebf3d86f93db4e4ab929dadf74667d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 7 Aug 2015 17:02:17 +0300 Subject: [PATCH 0054/1746] Document zero expiration. Fixes #143. --- commands.go | 9 +++++++++ example_test.go | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/commands.go b/commands.go index 7f10a2f079..887fd7377b 100644 --- a/commands.go +++ b/commands.go @@ -526,6 +526,9 @@ func (c *commandable) MSetNX(pairs ...string) *BoolCmd { return cmd } +// Redis `SET key value [expiration]` command. +// +// Zero expiration means the key has no expiration time. func (c *commandable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { args := make([]interface{}, 3, 5) args[0] = "SET" @@ -554,6 +557,9 @@ func (c *commandable) SetBit(key string, offset int64, value int) *IntCmd { return cmd } +// Redis `SET key value [expiration] NX` command. +// +// Zero expiration means the key has no expiration time. func (c *commandable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if expiration == 0 { @@ -570,6 +576,9 @@ func (c *commandable) SetNX(key string, value interface{}, expiration time.Durat return cmd } +// Redis `SET key value [expiration] XX` command. +// +// Zero expiration means the key has no expiration time. func (c *Client) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if usePrecise(expiration) { diff --git a/example_test.go b/example_test.go index 25869bea4a..de9f8da99a 100644 --- a/example_test.go +++ b/example_test.go @@ -84,6 +84,21 @@ func ExampleClient() { // key2 does not exists } +func ExampleClient_Set() { + // Last argument is expiration. Zero means the key has no + // expiration time. + err := client.Set("key", "value", 0).Err() + if err != nil { + panic(err) + } + + // key2 will expire in an hour. + err = client.Set("key2", "value", time.Hour).Err() + if err != nil { + panic(err) + } +} + func ExampleClient_Incr() { if err := client.Incr("counter").Err(); err != nil { panic(err) From d522cd5ae5e5f936ba35efc880435c26daf762d8 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 7 Aug 2015 18:07:27 +0300 Subject: [PATCH 0055/1746] travis: use Redis 3.0.3 to fix the build. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d3763d6172..1107e5fe51 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ testdeps: .test/redis/src/redis-server .test/redis: mkdir -p $@ - wget -qO- https://github.com/antirez/redis/archive/3.0.tar.gz | tar xvz --strip-components=1 -C $@ + wget -qO- https://github.com/antirez/redis/archive/3.0.3.tar.gz | tar xvz --strip-components=1 -C $@ .test/redis/src/redis-server: .test/redis cd $< && make all From bb8a57d39576085c35164261c58b471737045904 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 8 Aug 2015 13:49:27 +0300 Subject: [PATCH 0056/1746] travis: run tests on Go tip. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 169ccd0ada..8a951ffd6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ services: go: - 1.3 - 1.4 + - tip install: - go get gopkg.in/bufio.v1 From 37782aca57a983ce94dd1d70fdca32af956120a7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 19 Aug 2015 11:44:46 +0300 Subject: [PATCH 0057/1746] Add Scan example. Fixes #149. --- example_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/example_test.go b/example_test.go index de9f8da99a..58b7dfea79 100644 --- a/example_test.go +++ b/example_test.go @@ -109,6 +109,34 @@ func ExampleClient_Incr() { // Output: 1 } +func ExampleClient_Scan() { + client.FlushDb() + for i := 0; i < 33; i++ { + err := client.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + if err != nil { + panic(err) + } + } + + var cursor int64 + var n int + for { + var keys []string + var err error + cursor, keys, err = client.Scan(cursor, "", 10).Result() + if err != nil { + panic(err) + } + n += len(keys) + if cursor == 0 { + break + } + } + + fmt.Printf("found %d keys\n", n) + // Output: found 33 keys +} + func ExampleClient_Pipelined() { var incr *redis.IntCmd _, err := client.Pipelined(func(pipe *redis.Pipeline) error { From b7e0e5aa0d1796710c80dc4c135308e6be95533b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 21 Aug 2015 11:23:52 +0300 Subject: [PATCH 0058/1746] Don't use unsafe on appengine. Fixes #151. --- safe.go | 7 +++++++ unsafe.go | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 safe.go diff --git a/safe.go b/safe.go new file mode 100644 index 0000000000..d66dc56a09 --- /dev/null +++ b/safe.go @@ -0,0 +1,7 @@ +// +build appengine + +package redis + +func bytesToString(b []byte) string { + return string(b) +} diff --git a/unsafe.go b/unsafe.go index 1c4d55fa24..3cd8d1c182 100644 --- a/unsafe.go +++ b/unsafe.go @@ -1,3 +1,5 @@ +// +build !appengine + package redis import ( From 2de07f24930aaf7f4f0d6c34656fcf5229675bbd Mon Sep 17 00:00:00 2001 From: Jeff Pierce Date: Sat, 22 Aug 2015 20:38:37 -0700 Subject: [PATCH 0059/1746] Implemented ZRangeByLex with tests. --- commands.go | 16 +++++++++++++--- commands_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 887fd7377b..b07a8046fc 100644 --- a/commands.go +++ b/commands.go @@ -1094,8 +1094,14 @@ type ZRangeByScore struct { Offset, Count int64 } -func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores bool) *StringSliceCmd { - args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max} +func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores, isLex bool) *StringSliceCmd { + var zcmd string + if isLex { + zcmd = "ZRANGEBYLEX" + } else { + zcmd = "ZRANGEBYSCORE" + } + args := []interface{}{zcmd, key, opt.Min, opt.Max} if withScores { args = append(args, "WITHSCORES") } @@ -1113,7 +1119,11 @@ func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores bo } func (c *commandable) ZRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { - return c.zRangeByScore(key, opt, false) + return c.zRangeByScore(key, opt, false, false) +} + +func (c *commandable) ZRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { + return c.zRangeByScore(key, opt, false, true) } func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { diff --git a/commands_test.go b/commands_test.go index 6cd9fc0639..d0dbe43c7d 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2010,6 +2010,42 @@ var _ = Describe("Commands", func() { Expect(zRangeByScore.Val()).To(Equal([]string{})) }) + It("should ZRangeByLex", func() { + zAdd := client.ZAdd("zset", redis.Z{0, "a"}) + Expect(zAdd.Err()).NotTo(HaveOccurred()) + zAdd = client.ZAdd("zset", redis.Z{0, "b"}) + Expect(zAdd.Err()).NotTo(HaveOccurred()) + zAdd = client.ZAdd("zset", redis.Z{0, "c"}) + + zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeByScore{ + Min: "-", + Max: "+", + }) + Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) + Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b", "c"})) + + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + Min: "[a", + Max: "[b", + }) + Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) + Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b"})) + + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + Min: "(a", + Max: "[b", + }) + Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) + Expect(zRangeByLex.Val()).To(Equal([]string{"b"})) + + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + Min: "(a", + Max: "(b", + }) + Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) + Expect(zRangeByLex.Val()).To(Equal([]string{})) + }) + It("should ZRangeByScoreWithScoresMap", func() { zAdd := client.ZAdd("zset", redis.Z{1, "one"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) From 8b26a9f71e01aa2034c140e108a8733e643b4ac1 Mon Sep 17 00:00:00 2001 From: arnaud briche Date: Mon, 24 Aug 2015 10:46:48 +0200 Subject: [PATCH 0060/1746] Add MaxRetries to FailoverOptions --- sentinel.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sentinel.go b/sentinel.go index 82d9bc9188..255416ecb3 100644 --- a/sentinel.go +++ b/sentinel.go @@ -31,6 +31,8 @@ type FailoverOptions struct { PoolSize int PoolTimeout time.Duration IdleTimeout time.Duration + + MaxRetries int } func (opt *FailoverOptions) options() *Options { @@ -47,6 +49,8 @@ func (opt *FailoverOptions) options() *Options { PoolSize: opt.PoolSize, PoolTimeout: opt.PoolTimeout, IdleTimeout: opt.IdleTimeout, + + MaxRetries: opt.MaxRetries, } } From 2abf5c5f146e7ce1740eb1ed942596459b71ce1e Mon Sep 17 00:00:00 2001 From: Jeff Pierce Date: Mon, 24 Aug 2015 14:52:00 -0700 Subject: [PATCH 0061/1746] Refactored zRangeByScore into zRangeBy. --- commands.go | 26 ++++++++++---------------- commands_test.go | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/commands.go b/commands.go index b07a8046fc..a289cd32e3 100644 --- a/commands.go +++ b/commands.go @@ -1089,19 +1089,13 @@ func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd return cmd } -type ZRangeByScore struct { +type ZRangeBy struct { Min, Max string Offset, Count int64 } -func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores, isLex bool) *StringSliceCmd { - var zcmd string - if isLex { - zcmd = "ZRANGEBYLEX" - } else { - zcmd = "ZRANGEBYSCORE" - } - args := []interface{}{zcmd, key, opt.Min, opt.Max} +func (c *commandable) zRangeBy(zRangeType, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { + args := []interface{}{zRangeType, key, opt.Min, opt.Max} if withScores { args = append(args, "WITHSCORES") } @@ -1118,15 +1112,15 @@ func (c *commandable) zRangeByScore(key string, opt ZRangeByScore, withScores, i return cmd } -func (c *commandable) ZRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { - return c.zRangeByScore(key, opt, false, false) +func (c *commandable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { + return c.zRangeBy("ZRANGEBYSCORE", key, opt, false) } -func (c *commandable) ZRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { - return c.zRangeByScore(key, opt, false, true) +func (c *commandable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { + return c.zRangeBy("ZRANGEBYLEX", key, opt, false) } -func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { +func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1188,7 +1182,7 @@ func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSlice return cmd } -func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1203,7 +1197,7 @@ func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSli return cmd } -func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { +func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( diff --git a/commands_test.go b/commands_test.go index d0dbe43c7d..4ef419e8d6 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1981,28 +1981,28 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{3, "three"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) - zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "-inf", Max: "+inf", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"one", "two", "three"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "1", Max: "2", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"one", "two"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "(1", Max: "2", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"two"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore = client.ZRangeByScore("ZRANGEBYSCORE", "zset", redis.ZRangeBy{ Min: "(1", Max: "(2", }) @@ -2017,28 +2017,28 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) zAdd = client.ZAdd("zset", redis.Z{0, "c"}) - zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "-", Max: "+", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b", "c"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "[a", Max: "[b", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "(a", Max: "[b", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"b"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "(a", Max: "(b", }) @@ -2054,28 +2054,28 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{3, "three"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) - val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "-inf", Max: "+inf", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}, {3, "three"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "(1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{2, "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "(1", Max: "(2", }).Result() From 15c887f70091e468b44c76af8e6a323b70195896 Mon Sep 17 00:00:00 2001 From: Jeff Pierce Date: Mon, 24 Aug 2015 15:01:32 -0700 Subject: [PATCH 0062/1746] Fixed tests. --- commands_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/commands_test.go b/commands_test.go index 4ef419e8d6..78a9fbc140 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2002,7 +2002,7 @@ var _ = Describe("Commands", func() { Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"two"})) - zRangeByScore = client.ZRangeByScore("ZRANGEBYSCORE", "zset", redis.ZRangeBy{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "(1", Max: "(2", }) @@ -2202,17 +2202,17 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScore( - "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"three", "two", "one"})) vals, err = client.ZRevRangeByScore( - "zset", redis.ZRangeByScore{Max: "2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"two"})) vals, err = client.ZRevRangeByScore( - "zset", redis.ZRangeByScore{Max: "(2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{})) }) @@ -2226,7 +2226,7 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) }) @@ -2240,17 +2240,17 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) val, err := client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) val, err = client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{2, "two"}})) val, err = client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "(2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{})) }) From 9ead45f9e902067a5690758f6138234b08a0db13 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 25 Aug 2015 14:02:16 +0300 Subject: [PATCH 0063/1746] Add SRandMemberN. Fixes #155. --- commands.go | 8 ++++++++ commands_test.go | 30 +++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/commands.go b/commands.go index 887fd7377b..769133083f 100644 --- a/commands.go +++ b/commands.go @@ -944,12 +944,20 @@ func (c *commandable) SPop(key string) *StringCmd { return cmd } +// Redis `SRANDMEMBER key` command. func (c *commandable) SRandMember(key string) *StringCmd { cmd := NewStringCmd("SRANDMEMBER", key) c.Process(cmd) return cmd } +// Redis `SRANDMEMBER key count` command. +func (c *commandable) SRandMemberN(key string, count int64) *StringSliceCmd { + cmd := NewStringSliceCmd("SRANDMEMBER", key, formatInt(count)) + c.Process(cmd) + return cmd +} + func (c *commandable) SRem(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "SREM" diff --git a/commands_test.go b/commands_test.go index 6cd9fc0639..5fb52907f4 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1733,21 +1733,25 @@ var _ = Describe("Commands", func() { Expect(sMembers.Val()).To(HaveLen(2)) }) - It("should SRandMember", func() { - sAdd := client.SAdd("set", "one") - Expect(sAdd.Err()).NotTo(HaveOccurred()) - sAdd = client.SAdd("set", "two") - Expect(sAdd.Err()).NotTo(HaveOccurred()) - sAdd = client.SAdd("set", "three") - Expect(sAdd.Err()).NotTo(HaveOccurred()) + It("should SRandMember and SRandMemberN", func() { + err := client.SAdd("set", "one").Err() + Expect(err).NotTo(HaveOccurred()) + err = client.SAdd("set", "two").Err() + Expect(err).NotTo(HaveOccurred()) + err = client.SAdd("set", "three").Err() + Expect(err).NotTo(HaveOccurred()) - sRandMember := client.SRandMember("set") - Expect(sRandMember.Err()).NotTo(HaveOccurred()) - Expect(sRandMember.Val()).NotTo(Equal("")) + members, err := client.SMembers("set").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(members).To(HaveLen(3)) - sMembers := client.SMembers("set") - Expect(sMembers.Err()).NotTo(HaveOccurred()) - Expect(sMembers.Val()).To(HaveLen(3)) + member, err := client.SRandMember("set").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(member).NotTo(Equal("")) + + members, err = client.SRandMemberN("set", 2).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(members).To(HaveLen(2)) }) It("should SRem", func() { From 5498ba400da826ef70e8829563fb2c8585d35cd2 Mon Sep 17 00:00:00 2001 From: Jeff Pierce Date: Tue, 25 Aug 2015 12:15:01 -0700 Subject: [PATCH 0064/1746] Reverted change to struct ZRangeByScore, implemented ZRevRangeByLex. --- commands.go | 27 ++++++++++++++------- commands_test.go | 62 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/commands.go b/commands.go index a289cd32e3..a601645ddd 100644 --- a/commands.go +++ b/commands.go @@ -1089,13 +1089,14 @@ func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd return cmd } -type ZRangeBy struct { +// TODO: Rename to something more generic in v4 +type ZRangeByScore struct { Min, Max string Offset, Count int64 } -func (c *commandable) zRangeBy(zRangeType, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { - args := []interface{}{zRangeType, key, opt.Min, opt.Max} +func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeByScore, withScores bool) *StringSliceCmd { + args := []interface{}{zcmd, key, opt.Min, opt.Max} if withScores { args = append(args, "WITHSCORES") } @@ -1112,15 +1113,15 @@ func (c *commandable) zRangeBy(zRangeType, key string, opt ZRangeBy, withScores return cmd } -func (c *commandable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { +func (c *commandable) ZRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { return c.zRangeBy("ZRANGEBYSCORE", key, opt, false) } -func (c *commandable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { +func (c *commandable) ZRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { return c.zRangeBy("ZRANGEBYLEX", key, opt, false) } -func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { +func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1182,8 +1183,8 @@ func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSlice return cmd } -func (c *commandable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { - args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min} +func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeByScore) *StringSliceCmd { + args := []interface{}{zcmd, key, opt.Max, opt.Min} if opt.Offset != 0 || opt.Count != 0 { args = append( args, @@ -1197,7 +1198,15 @@ func (c *commandable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd return cmd } -func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { +func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { + return c.zRevRangeBy("ZREVRANGEBYSCORE", key, opt) +} + +func (c commandable) ZRevRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { + return c.zRevRangeBy("ZREVRANGEBYLEX", key, opt) +} + +func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( diff --git a/commands_test.go b/commands_test.go index 78a9fbc140..61b06852f2 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1981,28 +1981,28 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{3, "three"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) - zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeBy{ + zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeByScore{ Min: "-inf", Max: "+inf", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"one", "two", "three"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ Min: "1", Max: "2", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"one", "two"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ Min: "(1", Max: "2", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"two"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ Min: "(1", Max: "(2", }) @@ -2017,28 +2017,28 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) zAdd = client.ZAdd("zset", redis.Z{0, "c"}) - zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeBy{ + zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeByScore{ Min: "-", Max: "+", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b", "c"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ Min: "[a", Max: "[b", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ Min: "(a", Max: "[b", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"b"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ Min: "(a", Max: "(b", }) @@ -2054,28 +2054,28 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{3, "three"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) - val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ Min: "-inf", Max: "+inf", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}, {3, "three"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ Min: "1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ Min: "(1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{2, "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ Min: "(1", Max: "(2", }).Result() @@ -2202,17 +2202,41 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScore( - "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"three", "two", "one"})) vals, err = client.ZRevRangeByScore( - "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() + "zset", redis.ZRangeByScore{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"two"})) vals, err = client.ZRevRangeByScore( - "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() + "zset", redis.ZRangeByScore{Max: "(2", Min: "(1"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]string{})) + }) + + It("should ZRevRangeByLex", func() { + zadd := client.ZAdd("zset", redis.Z{0, "a"}) + Expect(zadd.Err()).NotTo(HaveOccurred()) + zadd = client.ZAdd("zset", redis.Z{0, "b"}) + Expect(zadd.Err()).NotTo(HaveOccurred()) + zadd = client.ZAdd("zset", redis.Z{0, "c"}) + Expect(zadd.Err()).NotTo(HaveOccurred()) + + vals, err := client.ZRevRangeByLex( + "zset", redis.ZRangeByScore{Max: "+", Min: "-"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]string{"c", "b", "a"})) + + vals, err := client.ZRevRangeByLex( + "zset", redis.ZRangeByScore{Max: "[b", Min: "(a"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]string{"b"})) + + vals, err := client.ZRevRangeByLex( + "zset", redis.ZRangeByScore{Max: "(b", Min: "(a"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{})) }) @@ -2226,7 +2250,7 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) }) @@ -2240,17 +2264,17 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) val, err := client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) val, err = client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() + "zset", redis.ZRangeByScore{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{2, "two"}})) val, err = client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() + "zset", redis.ZRangeByScore{Max: "(2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{})) }) From 7cafa2f3a5aca129d684384809cb8ef847796d07 Mon Sep 17 00:00:00 2001 From: Jeff Pierce Date: Tue, 25 Aug 2015 12:18:17 -0700 Subject: [PATCH 0065/1746] Fixed syntax on ZRevRangeByLex test. --- commands_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands_test.go b/commands_test.go index 61b06852f2..2ca079fba7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2230,12 +2230,12 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"c", "b", "a"})) - vals, err := client.ZRevRangeByLex( + vals, err = client.ZRevRangeByLex( "zset", redis.ZRangeByScore{Max: "[b", Min: "(a"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"b"})) - vals, err := client.ZRevRangeByLex( + vals, err = client.ZRevRangeByLex( "zset", redis.ZRangeByScore{Max: "(b", Min: "(a"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{})) From 03faf5daedea79a9eb391ccc23be57c22fba2f68 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 26 Aug 2015 09:38:44 +0300 Subject: [PATCH 0066/1746] Disable unsafe. --- unsafe.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/unsafe.go b/unsafe.go index 3cd8d1c182..75465d43dc 100644 --- a/unsafe.go +++ b/unsafe.go @@ -2,13 +2,6 @@ package redis -import ( - "reflect" - "unsafe" -) - func bytesToString(b []byte) string { - bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - strHeader := reflect.StringHeader{bytesHeader.Data, bytesHeader.Len} - return *(*string)(unsafe.Pointer(&strHeader)) + return string(b) } From d2689feb56094702cd10f0cb4c132fad09659a4c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 29 Aug 2015 13:08:27 +0300 Subject: [PATCH 0067/1746] Add ZADD modifiers. --- command.go | 8 ++- commands.go | 91 +++++++++++++++++++++++-- commands_test.go | 172 +++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 242 insertions(+), 29 deletions(-) diff --git a/command.go b/command.go index dab9fc3c83..8dceb32505 100644 --- a/command.go +++ b/command.go @@ -361,7 +361,9 @@ var ok = []byte("OK") func (cmd *BoolCmd) parseReply(rd *bufio.Reader) error { v, err := parseReply(rd, nil) - // `SET key value NX` returns nil when key already exists. + // `SET key value NX` returns nil when key already exists, which + // is inconsistent with `SETNX key value`. + // TODO: is this okay? if err == Nil { cmd.val = false return nil @@ -476,6 +478,10 @@ func (cmd *FloatCmd) Val() float64 { return cmd.val } +func (cmd *FloatCmd) Result() (float64, error) { + return cmd.Val(), cmd.Err() +} + func (cmd *FloatCmd) String() string { return cmdString(cmd, cmd.val) } diff --git a/commands.go b/commands.go index c273e1088f..44a77d0d91 100644 --- a/commands.go +++ b/commands.go @@ -1008,19 +1008,98 @@ type ZStore struct { Aggregate string } +func (c *commandable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { + for i, m := range members { + a[n+2*i] = formatFloat(m.Score) + a[n+2*i+1] = m.Member + } + cmd := NewIntCmd(a...) + c.Process(cmd) + return cmd +} + +// Redis `ZADD key score member [score member ...]` command. func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { - args := make([]interface{}, 2+2*len(members)) - args[0] = "ZADD" - args[1] = key + const n = 2 + a := make([]interface{}, n+2*len(members)) + a[0], a[1] = "ZADD", key + return c.zAdd(a, n, members...) +} + +// Redis `ZADD key NX score member [score member ...]` command. +func (c *commandable) ZAddNX(key string, members ...Z) *IntCmd { + const n = 3 + a := make([]interface{}, n+2*len(members)) + a[0], a[1], a[2] = "ZADD", key, "NX" + return c.zAdd(a, n, members...) +} + +// Redis `ZADD key XX score member [score member ...]` command. +func (c *commandable) ZAddXX(key string, members ...Z) *IntCmd { + const n = 3 + a := make([]interface{}, n+2*len(members)) + a[0], a[1], a[2] = "ZADD", key, "XX" + return c.zAdd(a, n, members...) +} + +// Redis `ZADD key CH score member [score member ...]` command. +func (c *commandable) ZAddCh(key string, members ...Z) *IntCmd { + const n = 3 + a := make([]interface{}, n+2*len(members)) + a[0], a[1], a[2] = "ZADD", key, "CH" + return c.zAdd(a, n, members...) +} + +// Redis `ZADD key NX CH score member [score member ...]` command. +func (c *commandable) ZAddNXCh(key string, members ...Z) *IntCmd { + const n = 4 + a := make([]interface{}, n+2*len(members)) + a[0], a[1], a[2], a[3] = "ZADD", key, "NX", "CH" + return c.zAdd(a, n, members...) +} + +// Redis `ZADD key XX CH score member [score member ...]` command. +func (c *commandable) ZAddXXCh(key string, members ...Z) *IntCmd { + const n = 4 + a := make([]interface{}, n+2*len(members)) + a[0], a[1], a[2], a[3] = "ZADD", key, "XX", "CH" + return c.zAdd(a, n, members...) +} + +func (c *commandable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { for i, m := range members { - args[2+2*i] = formatFloat(m.Score) - args[2+2*i+1] = m.Member + a[n+2*i] = formatFloat(m.Score) + a[n+2*i+1] = m.Member } - cmd := NewIntCmd(args...) + cmd := NewFloatCmd(a...) c.Process(cmd) return cmd } +// Redis `ZADD key INCR score member` command. +func (c *commandable) ZIncr(key string, member Z) *FloatCmd { + const n = 3 + a := make([]interface{}, n+2) + a[0], a[1], a[2] = "ZADD", key, "INCR" + return c.zIncr(a, n, member) +} + +// Redis `ZADD key NX INCR score member` command. +func (c *commandable) ZIncrNX(key string, member Z) *FloatCmd { + const n = 4 + a := make([]interface{}, n+2) + a[0], a[1], a[2], a[3] = "ZADD", key, "INCR", "NX" + return c.zIncr(a, n, member) +} + +// Redis `ZADD key XX INCR score member` command. +func (c *commandable) ZIncrXX(key string, member Z) *FloatCmd { + const n = 4 + a := make([]interface{}, n+2) + a[0], a[1], a[2], a[3] = "ZADD", key, "INCR", "XX" + return c.zIncr(a, n, member) +} + func (c *commandable) ZCard(key string) *IntCmd { cmd := NewIntCmd("ZCARD", key) c.Process(cmd) diff --git a/commands_test.go b/commands_test.go index b14f5663a1..448e0423f9 100644 --- a/commands_test.go +++ b/commands_test.go @@ -32,8 +32,6 @@ var _ = Describe("Commands", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - //------------------------------------------------------------------------------ - Describe("server", func() { It("should Auth", func() { @@ -158,8 +156,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("debugging", func() { It("should DebugObject", func() { @@ -175,8 +171,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("keys", func() { It("should Del", func() { @@ -539,8 +533,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("scanning", func() { It("should Scan", func() { @@ -593,8 +585,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("strings", func() { It("should Append", func() { @@ -1004,8 +994,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("hashes", func() { It("should HDel", func() { @@ -1192,8 +1180,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("lists", func() { It("should BLPop", func() { @@ -1546,8 +1532,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("sets", func() { It("should SAdd", func() { @@ -1821,8 +1805,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("sorted sets", func() { It("should ZAdd", func() { @@ -1842,9 +1824,9 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(0))) - val, err := client.ZRangeWithScores("zset", 0, -1).Result() + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}, {1, "uno"}, {3, "two"}})) + Expect(vals).To(Equal([]redis.Z{{1, "one"}, {1, "uno"}, {3, "two"}})) }) It("should ZAdd bytes", func() { @@ -1869,6 +1851,154 @@ var _ = Describe("Commands", func() { Expect(val).To(Equal([]redis.Z{{1, "one"}, {1, "uno"}, {3, "two"}})) }) + It("should ZAddNX", func() { + added, err := client.ZAddNX("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + + added, err = client.ZAddNX("zset", redis.Z{2, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(0))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + }) + + It("should ZAddXX", func() { + added, err := client.ZAddXX("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(0))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(BeEmpty()) + + added, err = client.ZAdd("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + added, err = client.ZAddXX("zset", redis.Z{2, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(0))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + }) + + It("should ZAddCh", func() { + changed, err := client.ZAddCh("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(Equal(int64(1))) + + changed, err = client.ZAddCh("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(Equal(int64(0))) + }) + + It("should ZAddNXCh", func() { + changed, err := client.ZAddNXCh("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(Equal(int64(1))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + + changed, err = client.ZAddNXCh("zset", redis.Z{2, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(Equal(int64(0))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + }) + + It("should ZAddXXCh", func() { + changed, err := client.ZAddXXCh("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(Equal(int64(0))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(BeEmpty()) + + added, err := client.ZAdd("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + changed, err = client.ZAddXXCh("zset", redis.Z{2, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(Equal(int64(1))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + }) + + It("should ZIncr", func() { + score, err := client.ZIncr("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(score).To(Equal(float64(1))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + + score, err = client.ZIncr("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(score).To(Equal(float64(2))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + }) + + It("should ZIncrNX", func() { + score, err := client.ZIncrNX("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(score).To(Equal(float64(1))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + + score, err = client.ZIncrNX("zset", redis.Z{1, "one"}).Result() + Expect(err).To(Equal(redis.Nil)) + Expect(score).To(Equal(float64(0))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + }) + + It("should ZIncrXX", func() { + score, err := client.ZIncrXX("zset", redis.Z{1, "one"}).Result() + Expect(err).To(Equal(redis.Nil)) + Expect(score).To(Equal(float64(0))) + + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(BeEmpty()) + + added, err := client.ZAdd("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(added).To(Equal(int64(1))) + + score, err = client.ZIncrXX("zset", redis.Z{1, "one"}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(score).To(Equal(float64(2))) + + vals, err = client.ZRangeWithScores("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + }) + It("should ZCard", func() { zAdd := client.ZAdd("zset", redis.Z{1, "one"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) @@ -2334,8 +2464,6 @@ var _ = Describe("Commands", func() { }) - //------------------------------------------------------------------------------ - Describe("watch/unwatch", func() { It("should WatchUnwatch", func() { From 5a976d17b89d72cd3ed3f8316f615b2e0b345b91 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 1 Sep 2015 13:27:19 +0800 Subject: [PATCH 0068/1746] Fix missing argument. --- command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command.go b/command.go index dab9fc3c83..559a11f518 100644 --- a/command.go +++ b/command.go @@ -378,7 +378,7 @@ func (cmd *BoolCmd) parseReply(rd *bufio.Reader) error { cmd.val = bytes.Equal(vv, ok) return nil default: - return fmt.Errorf("got %T, wanted int64 or string") + return fmt.Errorf("got %T, wanted int64 or string", v) } } From 58cb170ac08953bc38c89464e3e78a7903041675 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 3 Sep 2015 17:55:31 +0300 Subject: [PATCH 0069/1746] Get rid of custom bufio package. --- .travis.yml | 1 - cluster_pipeline.go | 2 +- command.go | 68 +++++++++++----------- conn.go | 20 ++++++- multi.go | 6 +- parser.go | 133 +++++++++++++------------------------------- parser_test.go | 19 ++++--- pipeline.go | 2 +- pool.go | 2 +- pubsub.go | 2 +- redis.go | 2 +- 11 files changed, 107 insertions(+), 150 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a951ffd6a..1dea73b099 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ go: - tip install: - - go get gopkg.in/bufio.v1 - go get gopkg.in/bsm/ratelimit.v1 - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 2e1194064d..01f06e7d1b 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -99,7 +99,7 @@ func (pipe *ClusterPipeline) execClusterCmds( var firstCmdErr error for i, cmd := range cmds { - err := cmd.parseReply(cn.rd) + err := cmd.parseReply(cn) if err == nil { continue } diff --git a/command.go b/command.go index 298c68d737..6c809060db 100644 --- a/command.go +++ b/command.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" "time" - - "gopkg.in/bufio.v1" ) var ( @@ -30,7 +28,7 @@ var ( type Cmder interface { args() []interface{} - parseReply(*bufio.Reader) error + parseReply(*conn) error setErr(error) reset() @@ -154,8 +152,8 @@ func (cmd *Cmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *Cmd) parseReply(rd *bufio.Reader) error { - cmd.val, cmd.err = parseReply(rd, parseSlice) +func (cmd *Cmd) parseReply(cn *conn) error { + cmd.val, cmd.err = parseReply(cn, parseSlice) // Convert to string to preserve old behaviour. // TODO: remove in v4 if v, ok := cmd.val.([]byte); ok { @@ -193,8 +191,8 @@ func (cmd *SliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *SliceCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseSlice) +func (cmd *SliceCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseSlice) if err != nil { cmd.err = err return err @@ -236,8 +234,8 @@ func (cmd *StatusCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StatusCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, nil) +func (cmd *StatusCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, nil) if err != nil { cmd.err = err return err @@ -275,8 +273,8 @@ func (cmd *IntCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *IntCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, nil) +func (cmd *IntCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, nil) if err != nil { cmd.err = err return err @@ -318,8 +316,8 @@ func (cmd *DurationCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *DurationCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, nil) +func (cmd *DurationCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, nil) if err != nil { cmd.err = err return err @@ -359,8 +357,8 @@ func (cmd *BoolCmd) String() string { var ok = []byte("OK") -func (cmd *BoolCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, nil) +func (cmd *BoolCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, nil) // `SET key value NX` returns nil when key already exists, which // is inconsistent with `SETNX key value`. // TODO: is this okay? @@ -445,15 +443,13 @@ func (cmd *StringCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, nil) +func (cmd *StringCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, nil) if err != nil { cmd.err = err return err } - b := v.([]byte) - cmd.val = make([]byte, len(b)) - copy(cmd.val, b) + cmd.val = cn.copyBuf(v.([]byte)) return nil } @@ -486,8 +482,8 @@ func (cmd *FloatCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *FloatCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, nil) +func (cmd *FloatCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, nil) if err != nil { cmd.err = err return err @@ -526,8 +522,8 @@ func (cmd *StringSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringSliceCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseStringSlice) +func (cmd *StringSliceCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseStringSlice) if err != nil { cmd.err = err return err @@ -565,8 +561,8 @@ func (cmd *BoolSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *BoolSliceCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseBoolSlice) +func (cmd *BoolSliceCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseBoolSlice) if err != nil { cmd.err = err return err @@ -604,8 +600,8 @@ func (cmd *StringStringMapCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringStringMapCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseStringStringMap) +func (cmd *StringStringMapCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseStringStringMap) if err != nil { cmd.err = err return err @@ -643,8 +639,8 @@ func (cmd *StringIntMapCmd) reset() { cmd.err = nil } -func (cmd *StringIntMapCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseStringIntMap) +func (cmd *StringIntMapCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseStringIntMap) if err != nil { cmd.err = err return err @@ -682,8 +678,8 @@ func (cmd *ZSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *ZSliceCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseZSlice) +func (cmd *ZSliceCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseZSlice) if err != nil { cmd.err = err return err @@ -723,8 +719,8 @@ func (cmd *ScanCmd) String() string { return cmdString(cmd, cmd.keys) } -func (cmd *ScanCmd) parseReply(rd *bufio.Reader) error { - vi, err := parseReply(rd, parseSlice) +func (cmd *ScanCmd) parseReply(cn *conn) error { + vi, err := parseReply(cn, parseSlice) if err != nil { cmd.err = err return cmd.err @@ -778,8 +774,8 @@ func (cmd *ClusterSlotCmd) reset() { cmd.err = nil } -func (cmd *ClusterSlotCmd) parseReply(rd *bufio.Reader) error { - v, err := parseReply(rd, parseClusterSlotInfoSlice) +func (cmd *ClusterSlotCmd) parseReply(cn *conn) error { + v, err := parseReply(cn, parseClusterSlotInfoSlice) if err != nil { cmd.err = err return err diff --git a/conn.go b/conn.go index 9dc2ede038..36ba99adbb 100644 --- a/conn.go +++ b/conn.go @@ -1,12 +1,13 @@ package redis import ( + "bufio" "net" "time" - - "gopkg.in/bufio.v1" ) +const defaultBufSize = 4096 + var ( zeroTime = time.Time{} ) @@ -30,7 +31,7 @@ func newConnDialer(opt *Options) func() (*conn, error) { } cn := &conn{ netcn: netcn, - buf: make([]byte, 0, 64), + buf: make([]byte, defaultBufSize), } cn.rd = bufio.NewReader(cn) return cn, cn.init(opt) @@ -102,3 +103,16 @@ func (cn *conn) RemoteAddr() net.Addr { func (cn *conn) Close() error { return cn.netcn.Close() } + +func isSameSlice(s1, s2 []byte) bool { + return len(s1) > 0 && len(s2) > 0 && &s1[0] == &s2[0] +} + +func (cn *conn) copyBuf(b []byte) []byte { + if isSameSlice(b, cn.buf) { + new := make([]byte, len(b)) + copy(new, b) + return new + } + return b +} diff --git a/multi.go b/multi.go index 63ecdd5896..1cc419c0f7 100644 --- a/multi.go +++ b/multi.go @@ -115,14 +115,14 @@ func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { // Parse queued replies. for i := 0; i < cmdsLen; i++ { - if err := statusCmd.parseReply(cn.rd); err != nil { + if err := statusCmd.parseReply(cn); err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) return err } } // Parse number of replies. - line, err := readLine(cn.rd) + line, err := readLine(cn) if err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) return err @@ -143,7 +143,7 @@ func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { // Loop starts from 1 to omit MULTI cmd. for i := 1; i < cmdsLen; i++ { cmd := cmds[i] - if err := cmd.parseReply(cn.rd); err != nil { + if err := cmd.parseReply(cn); err != nil { if firstCmdErr == nil { firstCmdErr = err } diff --git a/parser.go b/parser.go index 32646ff847..5b6e073335 100644 --- a/parser.go +++ b/parser.go @@ -3,13 +3,12 @@ package redis import ( "errors" "fmt" + "io" "net" "strconv" - - "gopkg.in/bufio.v1" ) -type multiBulkParser func(rd *bufio.Reader, n int64) (interface{}, error) +type multiBulkParser func(cn *conn, n int64) (interface{}, error) var ( errReaderTooSmall = errors.New("redis: reader is too small") @@ -216,8 +215,8 @@ func scan(b []byte, val interface{}) error { //------------------------------------------------------------------------------ -func readLine(rd *bufio.Reader) ([]byte, error) { - line, isPrefix, err := rd.ReadLine() +func readLine(cn *conn) ([]byte, error) { + line, isPrefix, err := cn.rd.ReadLine() if err != nil { return line, err } @@ -227,74 +226,21 @@ func readLine(rd *bufio.Reader) ([]byte, error) { return line, nil } -func readN(rd *bufio.Reader, n int) ([]byte, error) { - b, err := rd.ReadN(n) - if err == bufio.ErrBufferFull { - tmp := make([]byte, n) - r := copy(tmp, b) - b = tmp - - for { - nn, err := rd.Read(b[r:]) - r += nn - if r >= n { - // Ignore error if we read enough. - break - } - if err != nil { - return nil, err - } - } - } else if err != nil { - return nil, err - } - return b, nil -} - -//------------------------------------------------------------------------------ - -func parseReq(rd *bufio.Reader) ([]string, error) { - line, err := readLine(rd) - if err != nil { - return nil, err - } - - if line[0] != '*' { - return []string{string(line)}, nil - } - numReplies, err := strconv.ParseInt(string(line[1:]), 10, 64) - if err != nil { - return nil, err - } - - args := make([]string, 0, numReplies) - for i := int64(0); i < numReplies; i++ { - line, err = readLine(rd) - if err != nil { - return nil, err - } - if line[0] != '$' { - return nil, fmt.Errorf("redis: expected '$', but got %q", line) - } - - argLen, err := strconv.ParseInt(string(line[1:]), 10, 32) - if err != nil { - return nil, err - } - - arg, err := readN(rd, int(argLen)+2) - if err != nil { - return nil, err - } - args = append(args, string(arg[:argLen])) +func readN(cn *conn, n int) ([]byte, error) { + var b []byte + if cap(cn.buf) < n { + b = make([]byte, n) + } else { + b = cn.buf[:n] } - return args, nil + _, err := io.ReadFull(cn.rd, b) + return b, err } //------------------------------------------------------------------------------ -func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { - line, err := readLine(rd) +func parseReply(cn *conn, p multiBulkParser) (interface{}, error) { + line, err := readLine(cn) if err != nil { return nil, err } @@ -315,12 +261,12 @@ func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { return nil, Nil } - replyLen, err := strconv.Atoi(string(line[1:])) + replyLen, err := strconv.Atoi(bytesToString(line[1:])) if err != nil { return nil, err } - b, err := readN(rd, replyLen+2) + b, err := readN(cn, replyLen+2) if err != nil { return nil, err } @@ -335,15 +281,15 @@ func parseReply(rd *bufio.Reader, p multiBulkParser) (interface{}, error) { return nil, err } - return p(rd, repliesNum) + return p(cn, repliesNum) } return nil, fmt.Errorf("redis: can't parse %q", line) } -func parseSlice(rd *bufio.Reader, n int64) (interface{}, error) { +func parseSlice(cn *conn, n int64) (interface{}, error) { vals := make([]interface{}, 0, n) for i := int64(0); i < n; i++ { - v, err := parseReply(rd, parseSlice) + v, err := parseReply(cn, parseSlice) if err == Nil { vals = append(vals, nil) } else if err != nil { @@ -360,10 +306,10 @@ func parseSlice(rd *bufio.Reader, n int64) (interface{}, error) { return vals, nil } -func parseStringSlice(rd *bufio.Reader, n int64) (interface{}, error) { +func parseStringSlice(cn *conn, n int64) (interface{}, error) { vals := make([]string, 0, n) for i := int64(0); i < n; i++ { - viface, err := parseReply(rd, nil) + viface, err := parseReply(cn, nil) if err != nil { return nil, err } @@ -376,10 +322,10 @@ func parseStringSlice(rd *bufio.Reader, n int64) (interface{}, error) { return vals, nil } -func parseBoolSlice(rd *bufio.Reader, n int64) (interface{}, error) { +func parseBoolSlice(cn *conn, n int64) (interface{}, error) { vals := make([]bool, 0, n) for i := int64(0); i < n; i++ { - viface, err := parseReply(rd, nil) + viface, err := parseReply(cn, nil) if err != nil { return nil, err } @@ -392,36 +338,37 @@ func parseBoolSlice(rd *bufio.Reader, n int64) (interface{}, error) { return vals, nil } -func parseStringStringMap(rd *bufio.Reader, n int64) (interface{}, error) { +func parseStringStringMap(cn *conn, n int64) (interface{}, error) { m := make(map[string]string, n/2) for i := int64(0); i < n; i += 2 { - keyiface, err := parseReply(rd, nil) + keyIface, err := parseReply(cn, nil) if err != nil { return nil, err } - key, ok := keyiface.([]byte) + keyBytes, ok := keyIface.([]byte) if !ok { - return nil, fmt.Errorf("got %T, expected string", keyiface) + return nil, fmt.Errorf("got %T, expected []byte", keyIface) } + key := string(keyBytes) - valueiface, err := parseReply(rd, nil) + valueIface, err := parseReply(cn, nil) if err != nil { return nil, err } - value, ok := valueiface.([]byte) + valueBytes, ok := valueIface.([]byte) if !ok { - return nil, fmt.Errorf("got %T, expected string", valueiface) + return nil, fmt.Errorf("got %T, expected []byte", valueIface) } - m[string(key)] = string(value) + m[key] = string(valueBytes) } return m, nil } -func parseStringIntMap(rd *bufio.Reader, n int64) (interface{}, error) { +func parseStringIntMap(cn *conn, n int64) (interface{}, error) { m := make(map[string]int64, n/2) for i := int64(0); i < n; i += 2 { - keyiface, err := parseReply(rd, nil) + keyiface, err := parseReply(cn, nil) if err != nil { return nil, err } @@ -430,7 +377,7 @@ func parseStringIntMap(rd *bufio.Reader, n int64) (interface{}, error) { return nil, fmt.Errorf("got %T, expected string", keyiface) } - valueiface, err := parseReply(rd, nil) + valueiface, err := parseReply(cn, nil) if err != nil { return nil, err } @@ -449,12 +396,12 @@ func parseStringIntMap(rd *bufio.Reader, n int64) (interface{}, error) { return m, nil } -func parseZSlice(rd *bufio.Reader, n int64) (interface{}, error) { +func parseZSlice(cn *conn, n int64) (interface{}, error) { zz := make([]Z, n/2) for i := int64(0); i < n; i += 2 { z := &zz[i/2] - memberiface, err := parseReply(rd, nil) + memberiface, err := parseReply(cn, nil) if err != nil { return nil, err } @@ -464,7 +411,7 @@ func parseZSlice(rd *bufio.Reader, n int64) (interface{}, error) { } z.Member = string(member) - scoreiface, err := parseReply(rd, nil) + scoreiface, err := parseReply(cn, nil) if err != nil { return nil, err } @@ -481,10 +428,10 @@ func parseZSlice(rd *bufio.Reader, n int64) (interface{}, error) { return zz, nil } -func parseClusterSlotInfoSlice(rd *bufio.Reader, n int64) (interface{}, error) { +func parseClusterSlotInfoSlice(cn *conn, n int64) (interface{}, error) { infos := make([]ClusterSlotInfo, 0, n) for i := int64(0); i < n; i++ { - viface, err := parseReply(rd, parseSlice) + viface, err := parseReply(cn, parseSlice) if err != nil { return nil, err } diff --git a/parser_test.go b/parser_test.go index b71305a7ad..10403f6242 100644 --- a/parser_test.go +++ b/parser_test.go @@ -1,9 +1,9 @@ package redis import ( + "bufio" + "bytes" "testing" - - "gopkg.in/bufio.v1" ) func BenchmarkParseReplyStatus(b *testing.B) { @@ -27,20 +27,21 @@ func BenchmarkParseReplySlice(b *testing.B) { } func benchmarkParseReply(b *testing.B, reply string, p multiBulkParser, wanterr bool) { - b.StopTimer() - - buf := &bufio.Buffer{} - rd := bufio.NewReader(buf) + buf := &bytes.Buffer{} for i := 0; i < b.N; i++ { buf.WriteString(reply) } + cn := &conn{ + rd: bufio.NewReader(buf), + buf: make([]byte, 0, defaultBufSize), + } - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := parseReply(rd, p) + _, err := parseReply(cn, p) if !wanterr && err != nil { - panic(err) + b.Fatal(err) } } } diff --git a/pipeline.go b/pipeline.go index 8981cb5043..6fb1db1a12 100644 --- a/pipeline.go +++ b/pipeline.go @@ -97,7 +97,7 @@ func execCmds(cn *conn, cmds []Cmder) ([]Cmder, error) { var firstCmdErr error var failedCmds []Cmder for _, cmd := range cmds { - err := cmd.parseReply(cn.rd) + err := cmd.parseReply(cn) if err == nil { continue } diff --git a/pool.go b/pool.go index 71ac456d5e..f52eb6f943 100644 --- a/pool.go +++ b/pool.go @@ -243,7 +243,7 @@ func (p *connPool) Get() (*conn, error) { func (p *connPool) Put(cn *conn) error { if cn.rd.Buffered() != 0 { - b, _ := cn.rd.ReadN(cn.rd.Buffered()) + b, _ := cn.rd.Peek(cn.rd.Buffered()) log.Printf("redis: connection has unread data: %q", b) return p.Remove(cn) } diff --git a/pubsub.go b/pubsub.go index 1f4f5b6362..be36caa189 100644 --- a/pubsub.go +++ b/pubsub.go @@ -146,7 +146,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { cn.ReadTimeout = timeout cmd := NewSliceCmd() - if err := cmd.parseReply(cn.rd); err != nil { + if err := cmd.parseReply(cn); err != nil { return nil, err } return newMessage(cmd.Val()) diff --git a/redis.go b/redis.go index f77c663aac..1504e6cc4c 100644 --- a/redis.go +++ b/redis.go @@ -69,7 +69,7 @@ func (c *baseClient) process(cmd Cmder) { return } - err = cmd.parseReply(cn.rd) + err = cmd.parseReply(cn) c.putConn(cn, err) if shouldRetry(err) { continue From a689c20777ed9ad661f3c9e4017ebdca23bbba3c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 4 Sep 2015 13:30:37 +0300 Subject: [PATCH 0070/1746] Enable unsafe. --- unsafe.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/unsafe.go b/unsafe.go index 75465d43dc..3cd8d1c182 100644 --- a/unsafe.go +++ b/unsafe.go @@ -2,6 +2,13 @@ package redis +import ( + "reflect" + "unsafe" +) + func bytesToString(b []byte) string { - return string(b) + bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + strHeader := reflect.StringHeader{bytesHeader.Data, bytesHeader.Len} + return *(*string)(unsafe.Pointer(&strHeader)) } From 9987f2abaa5c0ed52fec654f9ad2de45bb444270 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 6 Sep 2015 13:50:16 +0300 Subject: [PATCH 0071/1746] Add PubSub.ReceiveMessage. --- example_test.go | 32 ++++++--- main_test.go | 34 +++++++--- pool.go | 4 +- pubsub.go | 173 ++++++++++++++++++++++++++++++++++++++---------- pubsub_test.go | 51 +++++++++++++- redis_test.go | 3 +- 6 files changed, 239 insertions(+), 58 deletions(-) diff --git a/example_test.go b/example_test.go index 58b7dfea79..cb4b8f670a 100644 --- a/example_test.go +++ b/example_test.go @@ -219,14 +219,31 @@ func ExamplePubSub() { panic(err) } - for i := 0; i < 4; i++ { + msg, err := pubsub.ReceiveMessage() + if err != nil { + panic(err) + } + + fmt.Println(msg.Channel, msg.Payload) + // Output: mychannel hello +} + +func ExamplePubSub_Receive() { + pubsub, err := client.Subscribe("mychannel") + if err != nil { + panic(err) + } + defer pubsub.Close() + + err = client.Publish("mychannel", "hello").Err() + if err != nil { + panic(err) + } + + for i := 0; i < 2; i++ { msgi, err := pubsub.ReceiveTimeout(100 * time.Millisecond) if err != nil { - err := pubsub.Ping("") - if err != nil { - panic(err) - } - continue + panic(err) } switch msg := msgi.(type) { @@ -234,8 +251,6 @@ func ExamplePubSub() { fmt.Println(msg.Kind, msg.Channel) case *redis.Message: fmt.Println(msg.Channel, msg.Payload) - case *redis.Pong: - fmt.Println(msg) default: panic(fmt.Sprintf("unknown message: %#v", msgi)) } @@ -243,7 +258,6 @@ func ExamplePubSub() { // Output: subscribe mychannel // mychannel hello - // Pong } func ExampleScript() { diff --git a/main_test.go b/main_test.go index c4b5a5972b..d2f8d2adbf 100644 --- a/main_test.go +++ b/main_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "sync/atomic" + "syscall" "testing" "time" @@ -231,20 +232,33 @@ func startSentinel(port, masterName, masterPort string) (*redisProcess, error) { //------------------------------------------------------------------------------ -type badNetConn struct { - net.TCPConn -} +var errTimeout = syscall.ETIMEDOUT -var _ net.Conn = &badNetConn{} +type badConn struct { + net.TCPConn -func newBadNetConn() net.Conn { - return &badNetConn{} + readDelay, writeDelay time.Duration + readErr, writeErr error } -func (badNetConn) Read([]byte) (int, error) { - return 0, net.UnknownNetworkError("badNetConn") +var _ net.Conn = &badConn{} + +func (cn *badConn) Read([]byte) (int, error) { + if cn.readDelay != 0 { + time.Sleep(cn.readDelay) + } + if cn.readErr != nil { + return 0, cn.readErr + } + return 0, net.UnknownNetworkError("badConn") } -func (badNetConn) Write([]byte) (int, error) { - return 0, net.UnknownNetworkError("badNetConn") +func (cn *badConn) Write([]byte) (int, error) { + if cn.writeDelay != 0 { + time.Sleep(cn.writeDelay) + } + if cn.writeErr != nil { + return 0, cn.writeErr + } + return 0, net.UnknownNetworkError("badConn") } diff --git a/pool.go b/pool.go index f52eb6f943..bd494d85a6 100644 --- a/pool.go +++ b/pool.go @@ -396,8 +396,8 @@ func (p *singleConnPool) Remove(cn *conn) error { if p.cn == nil { panic("p.cn == nil") } - if p.cn != cn { - panic("p.cn != cn") + if cn != nil && cn != p.cn { + panic("cn != p.cn") } if p.closed { return errClosed diff --git a/pubsub.go b/pubsub.go index be36caa189..b85e475c45 100644 --- a/pubsub.go +++ b/pubsub.go @@ -2,6 +2,8 @@ package redis import ( "fmt" + "log" + "net" "time" ) @@ -16,6 +18,9 @@ func (c *Client) Publish(channel, message string) *IntCmd { // http://redis.io/topics/pubsub. type PubSub struct { *baseClient + + channels []string + patterns []string } // Deprecated. Use Subscribe/PSubscribe instead. @@ -40,6 +45,71 @@ func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { return pubsub, pubsub.PSubscribe(channels...) } +func (c *PubSub) subscribe(cmd string, channels ...string) error { + cn, err := c.conn() + if err != nil { + return err + } + + args := make([]interface{}, 1+len(channels)) + args[0] = cmd + for i, channel := range channels { + args[1+i] = channel + } + req := NewSliceCmd(args...) + return cn.writeCmds(req) +} + +// Subscribes the client to the specified channels. +func (c *PubSub) Subscribe(channels ...string) error { + err := c.subscribe("SUBSCRIBE", channels...) + if err == nil { + c.channels = append(c.channels, channels...) + } + return err +} + +// Subscribes the client to the given patterns. +func (c *PubSub) PSubscribe(patterns ...string) error { + err := c.subscribe("PSUBSCRIBE", patterns...) + if err == nil { + c.channels = append(c.channels, patterns...) + } + return err +} + +func remove(ss []string, es ...string) []string { + for _, e := range es { + for i, s := range ss { + if s == e { + ss = append(ss[:i], ss[i+1:]...) + break + } + } + } + return ss +} + +// Unsubscribes the client from the given channels, or from all of +// them if none is given. +func (c *PubSub) Unsubscribe(channels ...string) error { + err := c.subscribe("UNSUBSCRIBE", channels...) + if err == nil { + c.channels = remove(c.channels, channels...) + } + return err +} + +// Unsubscribes the client from the given patterns, or from all of +// them if none is given. +func (c *PubSub) PUnsubscribe(patterns ...string) error { + err := c.subscribe("PUNSUBSCRIBE", patterns...) + if err == nil { + c.patterns = remove(c.patterns, patterns...) + } + return err +} + func (c *PubSub) Ping(payload string) error { cn, err := c.conn() if err != nil { @@ -71,6 +141,7 @@ func (m *Subscription) String() string { // Message received as result of a PUBLISH command issued by another client. type Message struct { Channel string + Pattern string Payload string } @@ -78,6 +149,8 @@ func (m *Message) String() string { return fmt.Sprintf("Message<%s: %s>", m.Channel, m.Payload) } +// TODO: remove PMessage if favor of Message + // Message matching a pattern-matching subscription received as result // of a PUBLISH command issued by another client. type PMessage struct { @@ -102,12 +175,6 @@ func (p *Pong) String() string { return "Pong" } -// Returns a message as a Subscription, Message, PMessage, Pong or -// error. See PubSub example for details. -func (c *PubSub) Receive() (interface{}, error) { - return c.ReceiveTimeout(0) -} - func newMessage(reply []interface{}) (interface{}, error) { switch kind := reply[0].(string); kind { case "subscribe", "unsubscribe", "psubscribe", "punsubscribe": @@ -137,7 +204,8 @@ func newMessage(reply []interface{}) (interface{}, error) { } // ReceiveTimeout acts like Receive but returns an error if message -// is not received in time. +// is not received in time. This is low-level API and most clients +// should use ReceiveMessage. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { cn, err := c.conn() if err != nil { @@ -152,39 +220,74 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { return newMessage(cmd.Val()) } -func (c *PubSub) subscribe(cmd string, channels ...string) error { - cn, err := c.conn() - if err != nil { - return err - } +// Receive returns a message as a Subscription, Message, PMessage, +// Pong or error. See PubSub example for details. This is low-level +// API and most clients should use ReceiveMessage. +func (c *PubSub) Receive() (interface{}, error) { + return c.ReceiveTimeout(0) +} - args := make([]interface{}, 1+len(channels)) - args[0] = cmd - for i, channel := range channels { - args[1+i] = channel +func (c *PubSub) reconnect() { + c.connPool.Remove(nil) // close current connection + if len(c.channels) > 0 { + if err := c.Subscribe(c.channels...); err != nil { + log.Printf("redis: Subscribe failed: %s", err) + } + } + if len(c.patterns) > 0 { + if err := c.PSubscribe(c.patterns...); err != nil { + log.Printf("redis: Subscribe failed: %s", err) + } } - req := NewSliceCmd(args...) - return cn.writeCmds(req) } -// Subscribes the client to the specified channels. -func (c *PubSub) Subscribe(channels ...string) error { - return c.subscribe("SUBSCRIBE", channels...) -} +// ReceiveMessage returns a message or error. It automatically +// reconnects to Redis in case of network errors. +func (c *PubSub) ReceiveMessage() (*Message, error) { + var badConn bool + for { + msgi, err := c.ReceiveTimeout(5 * time.Second) + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if badConn { + c.reconnect() + badConn = false + continue + } -// Subscribes the client to the given patterns. -func (c *PubSub) PSubscribe(patterns ...string) error { - return c.subscribe("PSUBSCRIBE", patterns...) -} + err := c.Ping("") + if err != nil { + c.reconnect() + } else { + badConn = true + } + continue + } -// Unsubscribes the client from the given channels, or from all of -// them if none is given. -func (c *PubSub) Unsubscribe(channels ...string) error { - return c.subscribe("UNSUBSCRIBE", channels...) -} + if isNetworkError(err) { + c.reconnect() + continue + } -// Unsubscribes the client from the given patterns, or from all of -// them if none is given. -func (c *PubSub) PUnsubscribe(patterns ...string) error { - return c.subscribe("PUNSUBSCRIBE", patterns...) + return nil, err + } + + switch msg := msgi.(type) { + case *Subscription: + // Ignore. + case *Pong: + badConn = false + // Ignore. + case *Message: + return msg, nil + case *PMessage: + return &Message{ + Channel: msg.Channel, + Pattern: msg.Pattern, + Payload: msg.Payload, + }, nil + default: + return nil, fmt.Errorf("redis: unknown message: %T", msgi) + } + } } diff --git a/pubsub_test.go b/pubsub_test.go index ac1d629b92..5a7b0dad21 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -12,10 +12,12 @@ import ( var _ = Describe("PubSub", func() { var client *redis.Client + readTimeout := 3 * time.Second BeforeEach(func() { client = redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: redisAddr, + ReadTimeout: readTimeout, }) Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) @@ -227,4 +229,51 @@ var _ = Describe("PubSub", func() { Expect(pong.Payload).To(Equal("hello")) }) + It("should ReceiveMessage", func() { + pubsub, err := client.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + go func() { + defer GinkgoRecover() + + time.Sleep(readTimeout + 100*time.Millisecond) + n, err := client.Publish("mychannel", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + }() + + msg, err := pubsub.ReceiveMessage() + Expect(err).NotTo(HaveOccurred()) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("hello")) + }) + + It("should reconnect on ReceiveMessage error", func() { + pubsub, err := client.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + cn, err := pubsub.Pool().Get() + Expect(err).NotTo(HaveOccurred()) + cn.SetNetConn(&badConn{ + readErr: errTimeout, + writeErr: errTimeout, + }) + + go func() { + defer GinkgoRecover() + + time.Sleep(100 * time.Millisecond) + n, err := client.Publish("mychannel", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(2))) + }() + + msg, err := pubsub.ReceiveMessage() + Expect(err).NotTo(HaveOccurred()) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("hello")) + }) + }) diff --git a/redis_test.go b/redis_test.go index b1a25475cc..acc8ca1d47 100644 --- a/redis_test.go +++ b/redis_test.go @@ -159,7 +159,8 @@ var _ = Describe("Client", func() { // Put bad connection in the pool. cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(newBadNetConn()) + + cn.SetNetConn(&badConn{}) Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) err = client.Ping().Err() From c809246d8b39d8bbe9d03b67fee954081a863f32 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 12 Sep 2015 09:36:03 +0300 Subject: [PATCH 0072/1746] Clarify thread safety. Fixes #166. --- cluster.go | 9 ++++++--- cluster_pipeline.go | 5 +++-- multi.go | 3 ++- pipeline.go | 6 +++--- pubsub.go | 3 ++- redis.go | 7 +++++++ ring.go | 11 +++++++---- sentinel.go | 5 +++-- 8 files changed, 33 insertions(+), 16 deletions(-) diff --git a/cluster.go b/cluster.go index cbf00b20fe..463cae4f03 100644 --- a/cluster.go +++ b/cluster.go @@ -9,6 +9,9 @@ import ( "time" ) +// ClusterClient is a Redis Cluster client representing a pool of zero +// or more underlying connections. It's safe for concurrent use by +// multiple goroutines. type ClusterClient struct { commandable @@ -26,7 +29,7 @@ type ClusterClient struct { reloading uint32 } -// NewClusterClient returns a new Redis Cluster client as described in +// NewClusterClient returns a Redis Cluster client as described in // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { client := &ClusterClient{ @@ -43,8 +46,8 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { // Close closes the cluster client, releasing any open resources. // -// It is rare to Close a Client, as the Client is meant to be -// long-lived and shared between many goroutines. +// It is rare to Close a ClusterClient, as the ClusterClient is meant +// to be long-lived and shared between many goroutines. func (c *ClusterClient) Close() error { defer c.clientsMx.Unlock() c.clientsMx.Lock() diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 01f06e7d1b..619ad82f48 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -10,7 +10,8 @@ type ClusterPipeline struct { } // Pipeline creates a new pipeline which is able to execute commands -// against multiple shards. +// against multiple shards. It's NOT safe for concurrent use by +// multiple goroutines. func (c *ClusterClient) Pipeline() *ClusterPipeline { pipe := &ClusterPipeline{ cluster: c, @@ -82,7 +83,7 @@ func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { return cmds, retErr } -// Close marks the pipeline as closed +// Close closes the pipeline, releasing any open resources. func (pipe *ClusterPipeline) Close() error { pipe.Discard() pipe.closed = true diff --git a/multi.go b/multi.go index 1cc419c0f7..7b55c7b64f 100644 --- a/multi.go +++ b/multi.go @@ -9,7 +9,8 @@ import ( var errDiscard = errors.New("redis: Discard can be used only inside Exec") // Multi implements Redis transactions as described in -// http://redis.io/topics/transactions. +// http://redis.io/topics/transactions. It's NOT safe for concurrent +// use by multiple goroutines. type Multi struct { commandable diff --git a/pipeline.go b/pipeline.go index 6fb1db1a12..02ecbacc11 100644 --- a/pipeline.go +++ b/pipeline.go @@ -1,9 +1,8 @@ package redis // Pipeline implements pipelining as described in -// http://redis.io/topics/pipelining. -// -// Pipeline is not thread-safe. +// http://redis.io/topics/pipelining. It's NOT safe for concurrent use +// by multiple goroutines. type Pipeline struct { commandable @@ -36,6 +35,7 @@ func (pipe *Pipeline) process(cmd Cmder) { pipe.cmds = append(pipe.cmds, cmd) } +// Close closes the pipeline, releasing any open resources. func (pipe *Pipeline) Close() error { pipe.Discard() pipe.closed = true diff --git a/pubsub.go b/pubsub.go index b85e475c45..fa804eb479 100644 --- a/pubsub.go +++ b/pubsub.go @@ -15,7 +15,8 @@ func (c *Client) Publish(channel, message string) *IntCmd { } // PubSub implements Pub/Sub commands as described in -// http://redis.io/topics/pubsub. +// http://redis.io/topics/pubsub. It's NOT safe for concurrent use by +// multiple goroutines. type PubSub struct { *baseClient diff --git a/redis.go b/redis.go index 1504e6cc4c..aea53a2231 100644 --- a/redis.go +++ b/redis.go @@ -80,6 +80,9 @@ func (c *baseClient) process(cmd Cmder) { } // Close closes the client, releasing any open resources. +// +// It is rare to Close a Client, as the Client is meant to be +// long-lived and shared between many goroutines. func (c *baseClient) Close() error { return c.connPool.Close() } @@ -173,6 +176,9 @@ func (opt *Options) getIdleTimeout() time.Duration { //------------------------------------------------------------------------------ +// Client is a Redis client representing a pool of zero or more +// underlying connections. It's safe for concurrent use by multiple +// goroutines. type Client struct { *baseClient commandable @@ -186,6 +192,7 @@ func newClient(opt *Options, pool pool) *Client { } } +// NewClient returns a client to the Redis Server specified by Options. func NewClient(opt *Options) *Client { pool := newConnPool(opt) return newClient(opt, pool) diff --git a/ring.go b/ring.go index 4b20e7a90f..8005af97f0 100644 --- a/ring.go +++ b/ring.go @@ -92,7 +92,8 @@ func (shard *ringShard) Vote(up bool) bool { } // Ring is a Redis client that uses constistent hashing to distribute -// keys across multiple Redis servers (shards). +// keys across multiple Redis servers (shards). It's safe for +// concurrent use by multiple goroutines. // // It monitors the state of each shard and removes dead shards from // the ring. When shard comes online it is added back to the ring. This @@ -215,8 +216,8 @@ func (ring *Ring) heartbeat() { // Close closes the ring client, releasing any open resources. // -// It is rare to Close a Client, as the Client is meant to be -// long-lived and shared between many goroutines. +// It is rare to Close a Ring, as the Ring is meant to be long-lived +// and shared between many goroutines. func (ring *Ring) Close() (retErr error) { defer ring.mx.Unlock() ring.mx.Lock() @@ -238,7 +239,8 @@ func (ring *Ring) Close() (retErr error) { } // RingPipeline creates a new pipeline which is able to execute commands -// against multiple shards. +// against multiple shards. It's NOT safe for concurrent use by +// multiple goroutines. type RingPipeline struct { commandable @@ -342,6 +344,7 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { return cmds, retErr } +// Close closes the pipeline, releasing any open resources. func (pipe *RingPipeline) Close() error { pipe.Discard() pipe.closed = true diff --git a/sentinel.go b/sentinel.go index 255416ecb3..04a821bd6b 100644 --- a/sentinel.go +++ b/sentinel.go @@ -54,8 +54,9 @@ func (opt *FailoverOptions) options() *Options { } } -// NewFailoverClient returns a Redis client with automatic failover -// capabilities using Redis Sentinel. +// NewFailoverClient returns a Redis client that uses Redis Sentinel +// for automatic failover. It's safe for concurrent use by multiple +// goroutines. func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt := failoverOpt.options() failover := &sentinelFailover{ From 8a05670e7a78ea5a2d9e30fe8e29dd5fb6f36e0f Mon Sep 17 00:00:00 2001 From: Ian Chan Date: Sun, 20 Sep 2015 14:13:13 +0100 Subject: [PATCH 0073/1746] Added binding for GEOADD and GEORADIUS. --- command.go | 99 ++++++++++++++++++++++++++++++++++++++++++++++++ commands.go | 72 +++++++++++++++++++++++++++++++++++ commands_test.go | 60 +++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+) diff --git a/command.go b/command.go index 6c809060db..8bae5f4b5a 100644 --- a/command.go +++ b/command.go @@ -783,3 +783,102 @@ func (cmd *ClusterSlotCmd) parseReply(cn *conn) error { cmd.val = v.([]ClusterSlotInfo) return nil } + +//------------------------------------------------------------------------------ + +// Location type for GEO operations in Redis +type GeoLocation struct { + Name string + Longitude, Latitude, Distance float64 + GeoHash int64 +} + +type GeoCmd struct { + baseCmd + + locations []GeoLocation +} + +// Query type for geo radius +type GeoRadiusQuery struct { + Key string + Longitude, Latitude, Radius float64 + // Unit default to km when nil + Unit string + WithCoordinates, WithDistance, WithGeoHash bool + // Count default to 0 and ignored limit. + Count int + // Sort default to unsorted, ASC or DESC otherwise + Sort string +} + +func NewGeoCmd(args ...interface{}) *GeoCmd { + return &GeoCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} +} + +func (cmd *GeoCmd) reset() { + cmd.locations = nil + cmd.err = nil +} + +func (cmd *GeoCmd) Val() ([]GeoLocation) { + return cmd.locations +} + +func (cmd *GeoCmd) Result() ([]GeoLocation, error) { + return cmd.locations, cmd.err +} + +func (cmd *GeoCmd) String() string { + return cmdString(cmd, cmd.locations) +} + +func (cmd *GeoCmd) parseReply(cn *conn) error { + vi, err := parseReply(cn, parseSlice) + if err != nil { + cmd.err = err + return cmd.err + } + + v := vi.([]interface{}) + + if len(v) == 0 { + return nil + } + + if _, ok := v[0].(string); ok { // Location names only (single level string array) + for _, keyi := range v { + cmd.locations = append(cmd.locations, GeoLocation{Name: keyi.(string)}) + } + } else { // Full location details (nested arrays) + for _, keyi := range v { + tmpLocation := GeoLocation{} + keyiface := keyi.([]interface{}) + for _, subKeyi := range keyiface { + if strVal, ok := subKeyi.(string); ok { + if len(tmpLocation.Name) == 0 { + tmpLocation.Name = strVal + } else { + tmpLocation.Distance, err = strconv.ParseFloat(strVal, 64) + if err != nil { + return err + } + } + } else if intVal, ok := subKeyi.(int64); ok { + tmpLocation.GeoHash = intVal + } else if ifcVal, ok := subKeyi.([]interface{}); ok { + tmpLocation.Longitude, err = strconv.ParseFloat(ifcVal[0].(string), 64) + if err != nil { + return err + } + tmpLocation.Latitude, err = strconv.ParseFloat(ifcVal[1].(string), 64) + if err != nil { + return err + } + } + } + cmd.locations = append(cmd.locations, tmpLocation) + } + } + return nil +} diff --git a/commands.go b/commands.go index 44a77d0d91..9e8abd94b5 100644 --- a/commands.go +++ b/commands.go @@ -1671,3 +1671,75 @@ func (c *commandable) ClusterAddSlotsRange(min, max int) *StatusCmd { } return c.ClusterAddSlots(slots...) } + +//------------------------------------------------------------------------------ + +func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { + args := make([]interface{}, 2+3*len(geoLocation)) + args[0] = "GEOADD" + args[1] = key + for i, eachLoc := range geoLocation { + args[2+3*i] = eachLoc.Longitude + args[2+3*i+1] = eachLoc.Latitude + args[2+3*i+2] = eachLoc.Name + } + cmd := NewIntCmd(args...) + c.Process(cmd) + return cmd +} + +func (c *commandable) GeoRadius(query *GeoRadiusQuery) *GeoCmd { + var options, optionsCtr int + if query.WithCoordinates { + options++ + } + if query.WithDistance { + options++ + } + if query.WithGeoHash { + options++ + } + if query.Count > 0 { + options += 2 + } + if query.Sort != "" { + options++ + } + + args := make([]interface{}, 6 + options) + args[0] = "GEORADIUS" + args[1] = query.Key + args[2] = query.Longitude + args[3] = query.Latitude + args[4] = query.Radius + if query.Unit != "" { + args[5] = query.Unit + } else { + args[5] = "km" + } + if query.WithCoordinates { + args[6+optionsCtr] = "WITHCOORD" + optionsCtr++ + } + if query.WithDistance { + args[6+optionsCtr] = "WITHDIST" + optionsCtr++ + } + if query.WithGeoHash { + args[6+optionsCtr] = "WITHHASH" + optionsCtr++ + } + if query.Count > 0 { + args[6+optionsCtr] = "COUNT" + optionsCtr++ + args[6+optionsCtr] = query.Count + optionsCtr++ + } + if query.Sort != "" { + args[6+optionsCtr] = query.Sort + } + + cmd := NewGeoCmd(args...) + c.Process(cmd) + return cmd +} diff --git a/commands_test.go b/commands_test.go index 448e0423f9..ecf27cd3a4 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2521,6 +2521,66 @@ var _ = Describe("Commands", func() { }) + Describe("Geo add and radius search", func() { + It("should add one geo location", func() { + geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(1))) + }) + + It("should add multiple geo locations", func() { + geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + }) + + It("should search geo radius", func() { + geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + + geoRadius := client.GeoRadius(&redis.GeoRadiusQuery{Key: "Sicily", Longitude: 15, Latitude: 37, Radius: 200}) + Expect(geoRadius.Err()).NotTo(HaveOccurred()) + Expect(geoRadius.Val()[0].Name).To(Equal("Palermo")) + Expect(geoRadius.Val()[1].Name).To(Equal("Catania")) + }) + + It("should search geo radius with options", func() { + locations := []*redis.GeoLocation{&redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}} + + geoAdd := client.GeoAdd("Sicily", locations...) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + + geoRadius := client.GeoRadius(&redis.GeoRadiusQuery{Key: "Sicily", Longitude: 15, Latitude: 37, Radius: 200, Unit: "km", WithGeoHash: true, WithCoordinates: true, WithDistance: true, Count: 2, Sort: "ASC"}) + Expect(geoRadius.Err()).NotTo(HaveOccurred()) + Expect(geoRadius.Val()[1].Name).To(Equal("Palermo")) + Expect(geoRadius.Val()[1].Distance).To(Equal(190.4424)) + Expect(geoRadius.Val()[1].GeoHash).To(Equal(int64(3479099956230698))) + Expect(geoRadius.Val()[1].Longitude).To(Equal(13.361389338970184)) + Expect(geoRadius.Val()[1].Latitude).To(Equal(38.115556395496299)) + Expect(geoRadius.Val()[0].Name).To(Equal("Catania")) + Expect(geoRadius.Val()[0].Distance).To(Equal(56.4413)) + Expect(geoRadius.Val()[0].GeoHash).To(Equal(int64(3479447370796909))) + Expect(geoRadius.Val()[0].Longitude).To(Equal(15.087267458438873)) + Expect(geoRadius.Val()[0].Latitude).To(Equal(37.50266842333162)) + }) + + It("should search geo radius with no results", func() { + geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + + geoRadius := client.GeoRadius(&redis.GeoRadiusQuery{Key: "Sicily", Longitude: 99, Latitude: 37, Radius: 200, Unit: "km", WithGeoHash: true, WithCoordinates: true, WithDistance: true}) + Expect(geoRadius.Err()).NotTo(HaveOccurred()) + Expect(len(geoRadius.Val())).To(Equal(0)) + }) + }) + Describe("marshaling/unmarshaling", func() { type convTest struct { From 2d374744745d59ccecf076e114e56610feec7318 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 28 Sep 2015 11:13:32 +0300 Subject: [PATCH 0074/1746] travis: fix build. --- .travis.yml | 1 - Makefile | 8 ++++---- commands_test.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1dea73b099..574bae484a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ services: go: - 1.3 - 1.4 - - tip install: - go get gopkg.in/bsm/ratelimit.v1 diff --git a/Makefile b/Makefile index 1107e5fe51..1b43765b4f 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ all: testdeps - go test ./... -v=1 -cpu=1,2,4 - go test ./... -short -race + go test ./... -test.v -test.cpu=1,2,4 + go test ./... -test.short -test.race test: testdeps - go test ./... -v=1 + go test ./... -test.v=1 testdeps: .test/redis/src/redis-server @@ -11,7 +11,7 @@ testdeps: .test/redis/src/redis-server .test/redis: mkdir -p $@ - wget -qO- https://github.com/antirez/redis/archive/3.0.3.tar.gz | tar xvz --strip-components=1 -C $@ + wget -qO- https://github.com/antirez/redis/archive/unstable.tar.gz | tar xvz --strip-components=1 -C $@ .test/redis/src/redis-server: .test/redis cd $< && make all diff --git a/commands_test.go b/commands_test.go index ecf27cd3a4..f10cfc3dda 100644 --- a/commands_test.go +++ b/commands_test.go @@ -193,7 +193,7 @@ var _ = Describe("Commands", func() { dump := client.Dump("key") Expect(dump.Err()).NotTo(HaveOccurred()) - Expect(dump.Val()).To(Equal("\x00\x05hello\x06\x00\xf5\x9f\xb7\xf6\x90a\x1c\x99")) + Expect(dump.Val()).NotTo(BeEmpty()) }) It("should Exists", func() { From b23b9b77279f0bcfc2a1bef121f639efc64aa3d6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Oct 2015 17:09:20 +0300 Subject: [PATCH 0075/1746] Refactor reply parser. --- cluster_pipeline.go | 2 +- command.go | 178 ++++++++----------- commands.go | 39 +---- multi.go | 4 +- parser.go | 409 ++++++++++++++++++++++++++++++-------------- parser_test.go | 4 +- pipeline.go | 2 +- pubsub.go | 2 +- redis.go | 2 +- 9 files changed, 375 insertions(+), 267 deletions(-) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 619ad82f48..90687a8b39 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -100,7 +100,7 @@ func (pipe *ClusterPipeline) execClusterCmds( var firstCmdErr error for i, cmd := range cmds { - err := cmd.parseReply(cn) + err := cmd.readReply(cn) if err == nil { continue } diff --git a/command.go b/command.go index 8bae5f4b5a..6b4465e840 100644 --- a/command.go +++ b/command.go @@ -28,7 +28,7 @@ var ( type Cmder interface { args() []interface{} - parseReply(*conn) error + readReply(*conn) error setErr(error) reset() @@ -152,14 +152,20 @@ func (cmd *Cmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *Cmd) parseReply(cn *conn) error { - cmd.val, cmd.err = parseReply(cn, parseSlice) - // Convert to string to preserve old behaviour. - // TODO: remove in v4 - if v, ok := cmd.val.([]byte); ok { +func (cmd *Cmd) readReply(cn *conn) error { + val, err := readReply(cn, sliceParser) + if err != nil { + cmd.err = err + return cmd.err + } + if v, ok := val.([]byte); ok { + // Convert to string to preserve old behaviour. + // TODO: remove in v4 cmd.val = string(v) + } else { + cmd.val = val } - return cmd.err + return nil } //------------------------------------------------------------------------------ @@ -191,8 +197,8 @@ func (cmd *SliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *SliceCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseSlice) +func (cmd *SliceCmd) readReply(cn *conn) error { + v, err := readReply(cn, sliceParser) if err != nil { cmd.err = err return err @@ -234,8 +240,8 @@ func (cmd *StatusCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StatusCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, nil) +func (cmd *StatusCmd) readReply(cn *conn) error { + v, err := readReply(cn, nil) if err != nil { cmd.err = err return err @@ -273,8 +279,8 @@ func (cmd *IntCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *IntCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, nil) +func (cmd *IntCmd) readReply(cn *conn) error { + v, err := readReply(cn, nil) if err != nil { cmd.err = err return err @@ -316,8 +322,8 @@ func (cmd *DurationCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *DurationCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, nil) +func (cmd *DurationCmd) readReply(cn *conn) error { + v, err := readReply(cn, nil) if err != nil { cmd.err = err return err @@ -357,8 +363,8 @@ func (cmd *BoolCmd) String() string { var ok = []byte("OK") -func (cmd *BoolCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, nil) +func (cmd *BoolCmd) readReply(cn *conn) error { + v, err := readReply(cn, nil) // `SET key value NX` returns nil when key already exists, which // is inconsistent with `SETNX key value`. // TODO: is this okay? @@ -443,8 +449,8 @@ func (cmd *StringCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, nil) +func (cmd *StringCmd) readReply(cn *conn) error { + v, err := readReply(cn, nil) if err != nil { cmd.err = err return err @@ -482,8 +488,8 @@ func (cmd *FloatCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *FloatCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, nil) +func (cmd *FloatCmd) readReply(cn *conn) error { + v, err := readReply(cn, nil) if err != nil { cmd.err = err return err @@ -522,8 +528,8 @@ func (cmd *StringSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringSliceCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseStringSlice) +func (cmd *StringSliceCmd) readReply(cn *conn) error { + v, err := readReply(cn, stringSliceParser) if err != nil { cmd.err = err return err @@ -561,8 +567,8 @@ func (cmd *BoolSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *BoolSliceCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseBoolSlice) +func (cmd *BoolSliceCmd) readReply(cn *conn) error { + v, err := readReply(cn, boolSliceParser) if err != nil { cmd.err = err return err @@ -600,8 +606,8 @@ func (cmd *StringStringMapCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringStringMapCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseStringStringMap) +func (cmd *StringStringMapCmd) readReply(cn *conn) error { + v, err := readReply(cn, stringStringMapParser) if err != nil { cmd.err = err return err @@ -639,8 +645,8 @@ func (cmd *StringIntMapCmd) reset() { cmd.err = nil } -func (cmd *StringIntMapCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseStringIntMap) +func (cmd *StringIntMapCmd) readReply(cn *conn) error { + v, err := readReply(cn, stringIntMapParser) if err != nil { cmd.err = err return err @@ -678,8 +684,8 @@ func (cmd *ZSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *ZSliceCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseZSlice) +func (cmd *ZSliceCmd) readReply(cn *conn) error { + v, err := readReply(cn, zSliceParser) if err != nil { cmd.err = err return err @@ -719,8 +725,8 @@ func (cmd *ScanCmd) String() string { return cmdString(cmd, cmd.keys) } -func (cmd *ScanCmd) parseReply(cn *conn) error { - vi, err := parseReply(cn, parseSlice) +func (cmd *ScanCmd) readReply(cn *conn) error { + vi, err := readReply(cn, sliceParser) if err != nil { cmd.err = err return cmd.err @@ -743,8 +749,9 @@ func (cmd *ScanCmd) parseReply(cn *conn) error { //------------------------------------------------------------------------------ type ClusterSlotInfo struct { - Start, End int - Addrs []string + Start int + End int + Addrs []string } type ClusterSlotCmd struct { @@ -774,8 +781,8 @@ func (cmd *ClusterSlotCmd) reset() { cmd.err = nil } -func (cmd *ClusterSlotCmd) parseReply(cn *conn) error { - v, err := parseReply(cn, parseClusterSlotInfoSlice) +func (cmd *ClusterSlotCmd) readReply(cn *conn) error { + v, err := readReply(cn, clusterSlotInfoSliceParser) if err != nil { cmd.err = err return err @@ -786,99 +793,62 @@ func (cmd *ClusterSlotCmd) parseReply(cn *conn) error { //------------------------------------------------------------------------------ -// Location type for GEO operations in Redis +// GeoLocation is used with GeoAdd to add geospatial location. type GeoLocation struct { - Name string + Name string Longitude, Latitude, Distance float64 - GeoHash int64 + GeoHash int64 } -type GeoCmd struct { +// GeoRadiusQuery is used with GeoRadius to query geospatial index. +type GeoRadiusQuery struct { + Key string + Longitude float64 + Latitude float64 + Radius float64 + // Can be m, km, ft, or mi. Default is km. + Unit string + WithCoordinates bool + WithDistance bool + WithGeoHash bool + Count int + // Can be ASC or DESC. Default is no sort order. + Sort string +} + +type GeoLocationCmd struct { baseCmd locations []GeoLocation } -// Query type for geo radius -type GeoRadiusQuery struct { - Key string - Longitude, Latitude, Radius float64 - // Unit default to km when nil - Unit string - WithCoordinates, WithDistance, WithGeoHash bool - // Count default to 0 and ignored limit. - Count int - // Sort default to unsorted, ASC or DESC otherwise - Sort string +func NewGeoLocationCmd(args ...interface{}) *GeoLocationCmd { + return &GeoLocationCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } -func NewGeoCmd(args ...interface{}) *GeoCmd { - return &GeoCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} -} - -func (cmd *GeoCmd) reset() { +func (cmd *GeoLocationCmd) reset() { cmd.locations = nil cmd.err = nil } -func (cmd *GeoCmd) Val() ([]GeoLocation) { +func (cmd *GeoLocationCmd) Val() []GeoLocation { return cmd.locations } -func (cmd *GeoCmd) Result() ([]GeoLocation, error) { +func (cmd *GeoLocationCmd) Result() ([]GeoLocation, error) { return cmd.locations, cmd.err } -func (cmd *GeoCmd) String() string { +func (cmd *GeoLocationCmd) String() string { return cmdString(cmd, cmd.locations) } -func (cmd *GeoCmd) parseReply(cn *conn) error { - vi, err := parseReply(cn, parseSlice) +func (cmd *GeoLocationCmd) readReply(cn *conn) error { + reply, err := readReply(cn, geoLocationSliceParser) if err != nil { cmd.err = err - return cmd.err - } - - v := vi.([]interface{}) - - if len(v) == 0 { - return nil - } - - if _, ok := v[0].(string); ok { // Location names only (single level string array) - for _, keyi := range v { - cmd.locations = append(cmd.locations, GeoLocation{Name: keyi.(string)}) - } - } else { // Full location details (nested arrays) - for _, keyi := range v { - tmpLocation := GeoLocation{} - keyiface := keyi.([]interface{}) - for _, subKeyi := range keyiface { - if strVal, ok := subKeyi.(string); ok { - if len(tmpLocation.Name) == 0 { - tmpLocation.Name = strVal - } else { - tmpLocation.Distance, err = strconv.ParseFloat(strVal, 64) - if err != nil { - return err - } - } - } else if intVal, ok := subKeyi.(int64); ok { - tmpLocation.GeoHash = intVal - } else if ifcVal, ok := subKeyi.([]interface{}); ok { - tmpLocation.Longitude, err = strconv.ParseFloat(ifcVal[0].(string), 64) - if err != nil { - return err - } - tmpLocation.Latitude, err = strconv.ParseFloat(ifcVal[1].(string), 64) - if err != nil { - return err - } - } - } - cmd.locations = append(cmd.locations, tmpLocation) - } + return err } + cmd.locations = reply.([]GeoLocation) return nil } diff --git a/commands.go b/commands.go index 9e8abd94b5..1916aecfb5 100644 --- a/commands.go +++ b/commands.go @@ -1688,25 +1688,8 @@ func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { return cmd } -func (c *commandable) GeoRadius(query *GeoRadiusQuery) *GeoCmd { - var options, optionsCtr int - if query.WithCoordinates { - options++ - } - if query.WithDistance { - options++ - } - if query.WithGeoHash { - options++ - } - if query.Count > 0 { - options += 2 - } - if query.Sort != "" { - options++ - } - - args := make([]interface{}, 6 + options) +func (c *commandable) GeoRadius(query *GeoRadiusQuery) *GeoLocationCmd { + args := make([]interface{}, 6) args[0] = "GEORADIUS" args[1] = query.Key args[2] = query.Longitude @@ -1718,28 +1701,22 @@ func (c *commandable) GeoRadius(query *GeoRadiusQuery) *GeoCmd { args[5] = "km" } if query.WithCoordinates { - args[6+optionsCtr] = "WITHCOORD" - optionsCtr++ + args = append(args, "WITHCOORD") } if query.WithDistance { - args[6+optionsCtr] = "WITHDIST" - optionsCtr++ + args = append(args, "WITHDIST") } if query.WithGeoHash { - args[6+optionsCtr] = "WITHHASH" - optionsCtr++ + args = append(args, "WITHHASH") } if query.Count > 0 { - args[6+optionsCtr] = "COUNT" - optionsCtr++ - args[6+optionsCtr] = query.Count - optionsCtr++ + args = append(args, "COUNT", query.Count) } if query.Sort != "" { - args[6+optionsCtr] = query.Sort + args = append(args, query.Sort) } - cmd := NewGeoCmd(args...) + cmd := NewGeoLocationCmd(args...) c.Process(cmd) return cmd } diff --git a/multi.go b/multi.go index 7b55c7b64f..e3d628fd4c 100644 --- a/multi.go +++ b/multi.go @@ -116,7 +116,7 @@ func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { // Parse queued replies. for i := 0; i < cmdsLen; i++ { - if err := statusCmd.parseReply(cn); err != nil { + if err := statusCmd.readReply(cn); err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) return err } @@ -144,7 +144,7 @@ func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { // Loop starts from 1 to omit MULTI cmd. for i := 1; i < cmdsLen; i++ { cmd := cmds[i] - if err := cmd.parseReply(cn); err != nil { + if err := cmd.readReply(cn); err != nil { if firstCmdErr == nil { firstCmdErr = err } diff --git a/parser.go b/parser.go index 5b6e073335..9acb0f1746 100644 --- a/parser.go +++ b/parser.go @@ -8,6 +8,14 @@ import ( "strconv" ) +const ( + errorReply = '-' + statusReply = '+' + intReply = ':' + stringReply = '$' + arrayReply = '*' +) + type multiBulkParser func(cn *conn, n int64) (interface{}, error) var ( @@ -239,57 +247,157 @@ func readN(cn *conn, n int) ([]byte, error) { //------------------------------------------------------------------------------ -func parseReply(cn *conn, p multiBulkParser) (interface{}, error) { +func parseErrorReply(cn *conn, line []byte) error { + return errorf(string(line[1:])) +} + +func parseIntReply(cn *conn, line []byte) (int64, error) { + n, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) + if err != nil { + return 0, err + } + return n, nil +} + +func readIntReply(cn *conn) (int64, error) { line, err := readLine(cn) + if err != nil { + return 0, err + } + switch line[0] { + case errorReply: + return 0, parseErrorReply(cn, line) + case intReply: + return parseIntReply(cn, line) + default: + return 0, fmt.Errorf("readIntReply: can't parse %.100q", line) + } +} + +func parseBytesReply(cn *conn, line []byte) ([]byte, error) { + if len(line) == 3 && line[1] == '-' && line[2] == '1' { + return nil, Nil + } + + replyLen, err := strconv.Atoi(bytesToString(line[1:])) + if err != nil { + return nil, err + } + + b, err := readN(cn, replyLen+2) if err != nil { return nil, err } + return b[:replyLen], nil +} + +func readBytesReply(cn *conn) ([]byte, error) { + line, err := readLine(cn) + if err != nil { + return nil, err + } switch line[0] { - case '-': - return nil, errorf(string(line[1:])) - case '+': - return line[1:], nil - case ':': - v, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) - if err != nil { - return nil, err - } - return v, nil - case '$': - if len(line) == 3 && line[1] == '-' && line[2] == '1' { - return nil, Nil - } + case errorReply: + return nil, parseErrorReply(cn, line) + case stringReply: + return parseBytesReply(cn, line) + default: + return nil, fmt.Errorf("readBytesReply: can't parse %.100q", line) + } +} - replyLen, err := strconv.Atoi(bytesToString(line[1:])) - if err != nil { - return nil, err - } +func readStringReply(cn *conn) (string, error) { + b, err := readBytesReply(cn) + if err != nil { + return "", err + } + return string(b), nil +} - b, err := readN(cn, replyLen+2) - if err != nil { - return nil, err - } - return b[:replyLen], nil - case '*': - if len(line) == 3 && line[1] == '-' && line[2] == '1' { - return nil, Nil - } +func readFloatReply(cn *conn) (float64, error) { + b, err := readBytesReply(cn) + if err != nil { + return 0, err + } + return strconv.ParseFloat(bytesToString(b), 64) +} - repliesNum, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) - if err != nil { - return nil, err - } +func parseArrayHeader(cn *conn, line []byte) (int64, error) { + if len(line) == 3 && line[1] == '-' && line[2] == '1' { + return 0, Nil + } - return p(cn, repliesNum) + n, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) + if err != nil { + return 0, err + } + return n, nil +} + +func parseArrayReply(cn *conn, p multiBulkParser, line []byte) (interface{}, error) { + n, err := parseArrayHeader(cn, line) + if err != nil { + return nil, err + } + return p(cn, n) +} + +func readArrayHeader(cn *conn) (int64, error) { + line, err := readLine(cn) + if err != nil { + return 0, err + } + switch line[0] { + case errorReply: + return 0, parseErrorReply(cn, line) + case arrayReply: + return parseArrayHeader(cn, line) + default: + return 0, fmt.Errorf("readArrayReply: can't parse %.100q", line) + } +} + +func readArrayReply(cn *conn, p multiBulkParser) (interface{}, error) { + line, err := readLine(cn) + if err != nil { + return nil, err + } + switch line[0] { + case errorReply: + return nil, parseErrorReply(cn, line) + case arrayReply: + return parseArrayReply(cn, p, line) + default: + return nil, fmt.Errorf("readArrayReply: can't parse %.100q", line) + } +} + +func readReply(cn *conn, p multiBulkParser) (interface{}, error) { + line, err := readLine(cn) + if err != nil { + return nil, err + } + + switch line[0] { + case errorReply: + return nil, parseErrorReply(cn, line) + case statusReply: + return line[1:], nil + case intReply: + return parseIntReply(cn, line) + case stringReply: + return parseBytesReply(cn, line) + case arrayReply: + return parseArrayReply(cn, p, line) } return nil, fmt.Errorf("redis: can't parse %q", line) } -func parseSlice(cn *conn, n int64) (interface{}, error) { +func sliceParser(cn *conn, n int64) (interface{}, error) { vals := make([]interface{}, 0, n) for i := int64(0); i < n; i++ { - v, err := parseReply(cn, parseSlice) + v, err := readReply(cn, sliceParser) if err == Nil { vals = append(vals, nil) } else if err != nil { @@ -306,171 +414,224 @@ func parseSlice(cn *conn, n int64) (interface{}, error) { return vals, nil } -func parseStringSlice(cn *conn, n int64) (interface{}, error) { - vals := make([]string, 0, n) +func intSliceParser(cn *conn, n int64) (interface{}, error) { + ints := make([]int64, 0, n) for i := int64(0); i < n; i++ { - viface, err := parseReply(cn, nil) + n, err := readIntReply(cn) if err != nil { return nil, err } - v, ok := viface.([]byte) - if !ok { - return nil, fmt.Errorf("got %T, expected string", viface) + ints = append(ints, n) + } + return ints, nil +} + +func boolSliceParser(cn *conn, n int64) (interface{}, error) { + bools := make([]bool, 0, n) + for i := int64(0); i < n; i++ { + n, err := readIntReply(cn) + if err != nil { + return nil, err } - vals = append(vals, string(v)) + bools = append(bools, n == 1) } - return vals, nil + return bools, nil } -func parseBoolSlice(cn *conn, n int64) (interface{}, error) { - vals := make([]bool, 0, n) +func stringSliceParser(cn *conn, n int64) (interface{}, error) { + ss := make([]string, 0, n) for i := int64(0); i < n; i++ { - viface, err := parseReply(cn, nil) + s, err := readStringReply(cn) if err != nil { return nil, err } - v, ok := viface.(int64) - if !ok { - return nil, fmt.Errorf("got %T, expected int64", viface) + ss = append(ss, s) + } + return ss, nil +} + +func floatSliceParser(cn *conn, n int64) (interface{}, error) { + nn := make([]float64, 0, n) + for i := int64(0); i < n; i++ { + n, err := readFloatReply(cn) + if err != nil { + return nil, err } - vals = append(vals, v == 1) + nn = append(nn, n) } - return vals, nil + return nn, nil } -func parseStringStringMap(cn *conn, n int64) (interface{}, error) { +func stringStringMapParser(cn *conn, n int64) (interface{}, error) { m := make(map[string]string, n/2) for i := int64(0); i < n; i += 2 { - keyIface, err := parseReply(cn, nil) + key, err := readStringReply(cn) if err != nil { return nil, err } - keyBytes, ok := keyIface.([]byte) - if !ok { - return nil, fmt.Errorf("got %T, expected []byte", keyIface) - } - key := string(keyBytes) - valueIface, err := parseReply(cn, nil) + value, err := readStringReply(cn) if err != nil { return nil, err } - valueBytes, ok := valueIface.([]byte) - if !ok { - return nil, fmt.Errorf("got %T, expected []byte", valueIface) - } - m[key] = string(valueBytes) + m[key] = value } return m, nil } -func parseStringIntMap(cn *conn, n int64) (interface{}, error) { +func stringIntMapParser(cn *conn, n int64) (interface{}, error) { m := make(map[string]int64, n/2) for i := int64(0); i < n; i += 2 { - keyiface, err := parseReply(cn, nil) + key, err := readStringReply(cn) if err != nil { return nil, err } - key, ok := keyiface.([]byte) - if !ok { - return nil, fmt.Errorf("got %T, expected string", keyiface) - } - valueiface, err := parseReply(cn, nil) + n, err := readIntReply(cn) if err != nil { return nil, err } - switch value := valueiface.(type) { - case int64: - m[string(key)] = value - case string: - m[string(key)], err = strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, fmt.Errorf("got %v, expected number", value) - } - default: - return nil, fmt.Errorf("got %T, expected number or string", valueiface) - } + + m[key] = n } return m, nil } -func parseZSlice(cn *conn, n int64) (interface{}, error) { +func zSliceParser(cn *conn, n int64) (interface{}, error) { zz := make([]Z, n/2) for i := int64(0); i < n; i += 2 { + var err error + z := &zz[i/2] - memberiface, err := parseReply(cn, nil) + z.Member, err = readStringReply(cn) if err != nil { return nil, err } - member, ok := memberiface.([]byte) - if !ok { - return nil, fmt.Errorf("got %T, expected string", memberiface) - } - z.Member = string(member) - scoreiface, err := parseReply(cn, nil) - if err != nil { - return nil, err - } - scoreb, ok := scoreiface.([]byte) - if !ok { - return nil, fmt.Errorf("got %T, expected string", scoreiface) - } - score, err := strconv.ParseFloat(bytesToString(scoreb), 64) + z.Score, err = readFloatReply(cn) if err != nil { return nil, err } - z.Score = score } return zz, nil } -func parseClusterSlotInfoSlice(cn *conn, n int64) (interface{}, error) { +func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { infos := make([]ClusterSlotInfo, 0, n) for i := int64(0); i < n; i++ { - viface, err := parseReply(cn, parseSlice) + n, err := readArrayHeader(cn) if err != nil { return nil, err } + if n < 2 { + return nil, fmt.Errorf("got %d elements in cluster info, expected at least 2", n) + } - item, ok := viface.([]interface{}) - if !ok { - return nil, fmt.Errorf("got %T, expected []interface{}", viface) - } else if len(item) < 3 { - return nil, fmt.Errorf("got %v, expected {int64, int64, string...}", item) + start, err := readIntReply(cn) + if err != nil { + return nil, err } - start, ok := item[0].(int64) - if !ok || start < 0 || start > hashSlots { - return nil, fmt.Errorf("got %v, expected {int64, int64, string...}", item) + end, err := readIntReply(cn) + if err != nil { + return nil, err } - end, ok := item[1].(int64) - if !ok || end < 0 || end > hashSlots { - return nil, fmt.Errorf("got %v, expected {int64, int64, string...}", item) + + addrsn := n - 2 + info := ClusterSlotInfo{ + Start: int(start), + End: int(end), + Addrs: make([]string, addrsn), } - info := ClusterSlotInfo{int(start), int(end), make([]string, len(item)-2)} - for n, ipair := range item[2:] { - pair, ok := ipair.([]interface{}) - if !ok || len(pair) != 2 { - return nil, fmt.Errorf("got %v, expected []interface{host, port}", viface) + for i := int64(0); i < addrsn; i++ { + n, err := readArrayHeader(cn) + if err != nil { + return nil, err + } + if n != 2 { + return nil, fmt.Errorf("got %d elements in cluster info address, expected 2", n) } - ip, ok := pair[0].(string) - if !ok || len(ip) < 1 { - return nil, fmt.Errorf("got %v, expected IP PORT pair", pair) + ip, err := readStringReply(cn) + if err != nil { + return nil, err } - port, ok := pair[1].(int64) - if !ok || port < 1 { - return nil, fmt.Errorf("got %v, expected IP PORT pair", pair) + + port, err := readIntReply(cn) + if err != nil { + return nil, err } - info.Addrs[n] = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) + info.Addrs[i] = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) } + infos = append(infos, info) } return infos, nil } + +func geoLocationParser(cn *conn, n int64) (interface{}, error) { + loc := &GeoLocation{} + + var err error + loc.Name, err = readStringReply(cn) + if err != nil { + return nil, err + } + if n >= 2 { + loc.Distance, err = readFloatReply(cn) + if err != nil { + return nil, err + } + } + if n >= 3 { + loc.GeoHash, err = readIntReply(cn) + if err != nil { + return nil, err + } + } + if n >= 4 { + n, err := readArrayHeader(cn) + if err != nil { + return nil, err + } + if n != 2 { + return nil, fmt.Errorf("got %d coordinates, expected 2", n) + } + + loc.Longitude, err = readFloatReply(cn) + if err != nil { + return nil, err + } + loc.Latitude, err = readFloatReply(cn) + if err != nil { + return nil, err + } + } + + return loc, nil +} + +func geoLocationSliceParser(cn *conn, n int64) (interface{}, error) { + locs := make([]GeoLocation, 0, n) + for i := int64(0); i < n; i++ { + v, err := readReply(cn, geoLocationParser) + if err != nil { + return nil, err + } + switch vv := v.(type) { + case []byte: + locs = append(locs, GeoLocation{ + Name: string(vv), + }) + case *GeoLocation: + locs = append(locs, *vv) + default: + return nil, fmt.Errorf("got %T, expected string or *GeoLocation", v) + } + } + return locs, nil +} diff --git a/parser_test.go b/parser_test.go index 10403f6242..b1c7434439 100644 --- a/parser_test.go +++ b/parser_test.go @@ -23,7 +23,7 @@ func BenchmarkParseReplyString(b *testing.B) { } func BenchmarkParseReplySlice(b *testing.B) { - benchmarkParseReply(b, "*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n", parseSlice, false) + benchmarkParseReply(b, "*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n", sliceParser, false) } func benchmarkParseReply(b *testing.B, reply string, p multiBulkParser, wanterr bool) { @@ -39,7 +39,7 @@ func benchmarkParseReply(b *testing.B, reply string, p multiBulkParser, wanterr b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := parseReply(cn, p) + _, err := readReply(cn, p) if !wanterr && err != nil { b.Fatal(err) } diff --git a/pipeline.go b/pipeline.go index 02ecbacc11..d7d130425b 100644 --- a/pipeline.go +++ b/pipeline.go @@ -97,7 +97,7 @@ func execCmds(cn *conn, cmds []Cmder) ([]Cmder, error) { var firstCmdErr error var failedCmds []Cmder for _, cmd := range cmds { - err := cmd.parseReply(cn) + err := cmd.readReply(cn) if err == nil { continue } diff --git a/pubsub.go b/pubsub.go index fa804eb479..ba053e4794 100644 --- a/pubsub.go +++ b/pubsub.go @@ -215,7 +215,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { cn.ReadTimeout = timeout cmd := NewSliceCmd() - if err := cmd.parseReply(cn); err != nil { + if err := cmd.readReply(cn); err != nil { return nil, err } return newMessage(cmd.Val()) diff --git a/redis.go b/redis.go index aea53a2231..2d1076d2d4 100644 --- a/redis.go +++ b/redis.go @@ -69,7 +69,7 @@ func (c *baseClient) process(cmd Cmder) { return } - err = cmd.parseReply(cn) + err = cmd.readReply(cn) c.putConn(cn, err) if shouldRetry(err) { continue From e89ca81306a39a9bb21288ad13f7c9f32c4c2d3f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Oct 2015 17:10:09 +0300 Subject: [PATCH 0076/1746] travis: test on Go 1.5. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 574bae484a..6ef52f7206 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ services: go: - 1.3 - 1.4 + - 1.5 install: - go get gopkg.in/bsm/ratelimit.v1 From 25cb844f82b09545ceb4b137bb1326f1f33686d6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Oct 2015 17:56:49 +0300 Subject: [PATCH 0077/1746] Add readScanReply. --- command.go | 73 +++++++++++++++++++----------------------------------- parser.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 89 insertions(+), 56 deletions(-) diff --git a/command.go b/command.go index 6b4465e840..850d1eb4a4 100644 --- a/command.go +++ b/command.go @@ -198,7 +198,7 @@ func (cmd *SliceCmd) String() string { } func (cmd *SliceCmd) readReply(cn *conn) error { - v, err := readReply(cn, sliceParser) + v, err := readArrayReply(cn, sliceParser) if err != nil { cmd.err = err return err @@ -241,13 +241,8 @@ func (cmd *StatusCmd) String() string { } func (cmd *StatusCmd) readReply(cn *conn) error { - v, err := readReply(cn, nil) - if err != nil { - cmd.err = err - return err - } - cmd.val = string(v.([]byte)) - return nil + cmd.val, cmd.err = readStringReply(cn) + return cmd.err } //------------------------------------------------------------------------------ @@ -280,13 +275,8 @@ func (cmd *IntCmd) String() string { } func (cmd *IntCmd) readReply(cn *conn) error { - v, err := readReply(cn, nil) - if err != nil { - cmd.err = err - return err - } - cmd.val = v.(int64) - return nil + cmd.val, cmd.err = readIntReply(cn) + return cmd.err } //------------------------------------------------------------------------------ @@ -323,12 +313,12 @@ func (cmd *DurationCmd) String() string { } func (cmd *DurationCmd) readReply(cn *conn) error { - v, err := readReply(cn, nil) + n, err := readIntReply(cn) if err != nil { cmd.err = err return err } - cmd.val = time.Duration(v.(int64)) * cmd.precision + cmd.val = time.Duration(n) * cmd.precision return nil } @@ -365,8 +355,8 @@ var ok = []byte("OK") func (cmd *BoolCmd) readReply(cn *conn) error { v, err := readReply(cn, nil) - // `SET key value NX` returns nil when key already exists, which - // is inconsistent with `SETNX key value`. + // `SET key value NX` returns nil when key already exists. But + // `SETNX key value` returns bool (0/1). So convert nil to bool. // TODO: is this okay? if err == Nil { cmd.val = false @@ -450,12 +440,12 @@ func (cmd *StringCmd) String() string { } func (cmd *StringCmd) readReply(cn *conn) error { - v, err := readReply(cn, nil) + b, err := readBytesReply(cn) if err != nil { cmd.err = err return err } - cmd.val = cn.copyBuf(v.([]byte)) + cmd.val = cn.copyBuf(b) return nil } @@ -489,13 +479,7 @@ func (cmd *FloatCmd) String() string { } func (cmd *FloatCmd) readReply(cn *conn) error { - v, err := readReply(cn, nil) - if err != nil { - cmd.err = err - return err - } - b := v.([]byte) - cmd.val, cmd.err = strconv.ParseFloat(bytesToString(b), 64) + cmd.val, cmd.err = readFloatReply(cn) return cmd.err } @@ -529,7 +513,7 @@ func (cmd *StringSliceCmd) String() string { } func (cmd *StringSliceCmd) readReply(cn *conn) error { - v, err := readReply(cn, stringSliceParser) + v, err := readArrayReply(cn, stringSliceParser) if err != nil { cmd.err = err return err @@ -568,7 +552,7 @@ func (cmd *BoolSliceCmd) String() string { } func (cmd *BoolSliceCmd) readReply(cn *conn) error { - v, err := readReply(cn, boolSliceParser) + v, err := readArrayReply(cn, boolSliceParser) if err != nil { cmd.err = err return err @@ -607,7 +591,7 @@ func (cmd *StringStringMapCmd) String() string { } func (cmd *StringStringMapCmd) readReply(cn *conn) error { - v, err := readReply(cn, stringStringMapParser) + v, err := readArrayReply(cn, stringStringMapParser) if err != nil { cmd.err = err return err @@ -646,7 +630,7 @@ func (cmd *StringIntMapCmd) reset() { } func (cmd *StringIntMapCmd) readReply(cn *conn) error { - v, err := readReply(cn, stringIntMapParser) + v, err := readArrayReply(cn, stringIntMapParser) if err != nil { cmd.err = err return err @@ -685,7 +669,7 @@ func (cmd *ZSliceCmd) String() string { } func (cmd *ZSliceCmd) readReply(cn *conn) error { - v, err := readReply(cn, zSliceParser) + v, err := readArrayReply(cn, zSliceParser) if err != nil { cmd.err = err return err @@ -713,6 +697,9 @@ func (cmd *ScanCmd) reset() { cmd.err = nil } +// TODO: cursor should be string to match redis type +// TODO: swap return values + func (cmd *ScanCmd) Val() (int64, []string) { return cmd.cursor, cmd.keys } @@ -726,23 +713,13 @@ func (cmd *ScanCmd) String() string { } func (cmd *ScanCmd) readReply(cn *conn) error { - vi, err := readReply(cn, sliceParser) + keys, cursor, err := readScanReply(cn) if err != nil { cmd.err = err return cmd.err } - v := vi.([]interface{}) - - cmd.cursor, cmd.err = strconv.ParseInt(v[0].(string), 10, 64) - if cmd.err != nil { - return cmd.err - } - - keys := v[1].([]interface{}) - for _, keyi := range keys { - cmd.keys = append(cmd.keys, keyi.(string)) - } - + cmd.keys = keys + cmd.cursor = cursor return nil } @@ -782,7 +759,7 @@ func (cmd *ClusterSlotCmd) reset() { } func (cmd *ClusterSlotCmd) readReply(cn *conn) error { - v, err := readReply(cn, clusterSlotInfoSliceParser) + v, err := readArrayReply(cn, clusterSlotInfoSliceParser) if err != nil { cmd.err = err return err @@ -844,7 +821,7 @@ func (cmd *GeoLocationCmd) String() string { } func (cmd *GeoLocationCmd) readReply(cn *conn) error { - reply, err := readReply(cn, geoLocationSliceParser) + reply, err := readArrayReply(cn, geoLocationSliceParser) if err != nil { cmd.err = err return err diff --git a/parser.go b/parser.go index 9acb0f1746..dd242ca1a1 100644 --- a/parser.go +++ b/parser.go @@ -251,6 +251,21 @@ func parseErrorReply(cn *conn, line []byte) error { return errorf(string(line[1:])) } +func isNilReply(b []byte) bool { + return len(b) == 3 && b[1] == '-' && b[2] == '1' +} + +func parseNilReply(cn *conn, line []byte) error { + if isNilReply(line) { + return Nil + } + return fmt.Errorf("redis: can't parse nil reply: %.100", line) +} + +func parseStatusReply(cn *conn, line []byte) ([]byte, error) { + return line[1:], nil +} + func parseIntReply(cn *conn, line []byte) (int64, error) { n, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) if err != nil { @@ -267,15 +282,17 @@ func readIntReply(cn *conn) (int64, error) { switch line[0] { case errorReply: return 0, parseErrorReply(cn, line) + case stringReply: + return 0, parseNilReply(cn, line) case intReply: return parseIntReply(cn, line) default: - return 0, fmt.Errorf("readIntReply: can't parse %.100q", line) + return 0, fmt.Errorf("redis: can't parse int reply: %.100q", line) } } func parseBytesReply(cn *conn, line []byte) ([]byte, error) { - if len(line) == 3 && line[1] == '-' && line[2] == '1' { + if isNilReply(line) { return nil, Nil } @@ -302,8 +319,10 @@ func readBytesReply(cn *conn) ([]byte, error) { return nil, parseErrorReply(cn, line) case stringReply: return parseBytesReply(cn, line) + case statusReply: + return parseStatusReply(cn, line) default: - return nil, fmt.Errorf("readBytesReply: can't parse %.100q", line) + return nil, fmt.Errorf("redis: can't parse string reply: %.100q", line) } } @@ -354,7 +373,7 @@ func readArrayHeader(cn *conn) (int64, error) { case arrayReply: return parseArrayHeader(cn, line) default: - return 0, fmt.Errorf("readArrayReply: can't parse %.100q", line) + return 0, fmt.Errorf("redis: can't parse array reply: %.100q", line) } } @@ -369,7 +388,7 @@ func readArrayReply(cn *conn, p multiBulkParser) (interface{}, error) { case arrayReply: return parseArrayReply(cn, p, line) default: - return nil, fmt.Errorf("readArrayReply: can't parse %.100q", line) + return nil, fmt.Errorf("redis: can't parse array reply: %.100q", line) } } @@ -383,7 +402,7 @@ func readReply(cn *conn, p multiBulkParser) (interface{}, error) { case errorReply: return nil, parseErrorReply(cn, line) case statusReply: - return line[1:], nil + return parseStatusReply(cn, line) case intReply: return parseIntReply(cn, line) case stringReply: @@ -391,7 +410,43 @@ func readReply(cn *conn, p multiBulkParser) (interface{}, error) { case arrayReply: return parseArrayReply(cn, p, line) } - return nil, fmt.Errorf("redis: can't parse %q", line) + return nil, fmt.Errorf("redis: can't parse %.100q", line) +} + +func readScanReply(cn *conn) ([]string, int64, error) { + n, err := readArrayHeader(cn) + if err != nil { + return nil, 0, err + } + if n != 2 { + return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2") + } + + b, err := readBytesReply(cn) + if err != nil { + return nil, 0, err + } + + cursor, err := strconv.ParseInt(bytesToString(b), 10, 64) + if err != nil { + return nil, 0, err + } + + n, err = readArrayHeader(cn) + if err != nil { + return nil, 0, err + } + + keys := make([]string, n) + for i := int64(0); i < n; i++ { + key, err := readStringReply(cn) + if err != nil { + return nil, 0, err + } + keys[i] = key + } + + return keys, cursor, err } func sliceParser(cn *conn, n int64) (interface{}, error) { @@ -526,7 +581,8 @@ func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { return nil, err } if n < 2 { - return nil, fmt.Errorf("got %d elements in cluster info, expected at least 2", n) + err := fmt.Errorf("redis: got %d elements in cluster info, expected at least 2", n) + return nil, err } start, err := readIntReply(cn) From 0944d0167b5886c195a151a5553b9fbdff0815d7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Oct 2015 18:21:18 +0300 Subject: [PATCH 0078/1746] Remove unneeded formatting. --- commands.go | 128 ++++++++++++++++++++++++---------------------------- 1 file changed, 58 insertions(+), 70 deletions(-) diff --git a/commands.go b/commands.go index 1916aecfb5..6572c81eb6 100644 --- a/commands.go +++ b/commands.go @@ -84,7 +84,7 @@ func (c *commandable) Quit() *StatusCmd { } func (c *commandable) Select(index int64) *StatusCmd { - cmd := newKeylessStatusCmd("SELECT", formatInt(index)) + cmd := newKeylessStatusCmd("SELECT", index) c.Process(cmd) return cmd } @@ -121,7 +121,7 @@ func (c *commandable) Expire(key string, expiration time.Duration) *BoolCmd { } func (c *commandable) ExpireAt(key string, tm time.Time) *BoolCmd { - cmd := NewBoolCmd("EXPIREAT", key, formatInt(tm.Unix())) + cmd := NewBoolCmd("EXPIREAT", key, tm.Unix()) c.Process(cmd) return cmd } @@ -138,7 +138,7 @@ func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Dur host, port, key, - formatInt(db), + db, formatMs(timeout), ) cmd._clusterKeyPos = 3 @@ -148,7 +148,7 @@ func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Dur } func (c *commandable) Move(key string, db int64) *BoolCmd { - cmd := NewBoolCmd("MOVE", key, formatInt(db)) + cmd := NewBoolCmd("MOVE", key, db) c.Process(cmd) return cmd } @@ -208,7 +208,7 @@ func (c *commandable) PExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd( "PEXPIREAT", key, - formatInt(tm.UnixNano()/int64(time.Millisecond)), + tm.UnixNano()/int64(time.Millisecond), ) c.Process(cmd) return cmd @@ -276,7 +276,7 @@ func (c *commandable) Sort(key string, sort Sort) *StringSliceCmd { args = append(args, "BY", sort.By) } if sort.Offset != 0 || sort.Count != 0 { - args = append(args, "LIMIT", formatFloat(sort.Offset), formatFloat(sort.Count)) + args = append(args, "LIMIT", sort.Offset, sort.Count) } for _, get := range sort.Get { args = append(args, "GET", get) @@ -308,12 +308,12 @@ func (c *commandable) Type(key string) *StatusCmd { } func (c *commandable) Scan(cursor int64, match string, count int64) *ScanCmd { - args := []interface{}{"SCAN", formatInt(cursor)} + args := []interface{}{"SCAN", cursor} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", formatInt(count)) + args = append(args, "COUNT", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -321,12 +321,12 @@ func (c *commandable) Scan(cursor int64, match string, count int64) *ScanCmd { } func (c *commandable) SScan(key string, cursor int64, match string, count int64) *ScanCmd { - args := []interface{}{"SSCAN", key, formatInt(cursor)} + args := []interface{}{"SSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", formatInt(count)) + args = append(args, "COUNT", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -334,12 +334,12 @@ func (c *commandable) SScan(key string, cursor int64, match string, count int64) } func (c *commandable) HScan(key string, cursor int64, match string, count int64) *ScanCmd { - args := []interface{}{"HSCAN", key, formatInt(cursor)} + args := []interface{}{"HSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", formatInt(count)) + args = append(args, "COUNT", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -347,12 +347,12 @@ func (c *commandable) HScan(key string, cursor int64, match string, count int64) } func (c *commandable) ZScan(key string, cursor int64, match string, count int64) *ScanCmd { - args := []interface{}{"ZSCAN", key, formatInt(cursor)} + args := []interface{}{"ZSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) } if count > 0 { - args = append(args, "COUNT", formatInt(count)) + args = append(args, "COUNT", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -376,8 +376,8 @@ func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { if bitCount != nil { args = append( args, - formatInt(bitCount.Start), - formatInt(bitCount.End), + bitCount.Start, + bitCount.End, ) } cmd := NewIntCmd(args...) @@ -418,14 +418,14 @@ func (c *commandable) BitPos(key string, bit int64, pos ...int64) *IntCmd { args := make([]interface{}, 3+len(pos)) args[0] = "BITPOS" args[1] = key - args[2] = formatInt(bit) + args[2] = bit switch len(pos) { case 0: case 1: - args[3] = formatInt(pos[0]) + args[3] = pos[0] case 2: - args[3] = formatInt(pos[0]) - args[4] = formatInt(pos[1]) + args[3] = pos[0] + args[4] = pos[1] default: panic("too many arguments") } @@ -441,7 +441,7 @@ func (c *commandable) Decr(key string) *IntCmd { } func (c *commandable) DecrBy(key string, decrement int64) *IntCmd { - cmd := NewIntCmd("DECRBY", key, formatInt(decrement)) + cmd := NewIntCmd("DECRBY", key, decrement) c.Process(cmd) return cmd } @@ -453,18 +453,13 @@ func (c *commandable) Get(key string) *StringCmd { } func (c *commandable) GetBit(key string, offset int64) *IntCmd { - cmd := NewIntCmd("GETBIT", key, formatInt(offset)) + cmd := NewIntCmd("GETBIT", key, offset) c.Process(cmd) return cmd } func (c *commandable) GetRange(key string, start, end int64) *StringCmd { - cmd := NewStringCmd( - "GETRANGE", - key, - formatInt(start), - formatInt(end), - ) + cmd := NewStringCmd("GETRANGE", key, start, end) c.Process(cmd) return cmd } @@ -482,13 +477,13 @@ func (c *commandable) Incr(key string) *IntCmd { } func (c *commandable) IncrBy(key string, value int64) *IntCmd { - cmd := NewIntCmd("INCRBY", key, formatInt(value)) + cmd := NewIntCmd("INCRBY", key, value) c.Process(cmd) return cmd } func (c *commandable) IncrByFloat(key string, value float64) *FloatCmd { - cmd := NewFloatCmd("INCRBYFLOAT", key, formatFloat(value)) + cmd := NewFloatCmd("INCRBYFLOAT", key, value) c.Process(cmd) return cmd } @@ -550,8 +545,8 @@ func (c *commandable) SetBit(key string, offset int64, value int) *IntCmd { cmd := NewIntCmd( "SETBIT", key, - formatInt(offset), - formatInt(int64(value)), + offset, + value, ) c.Process(cmd) return cmd @@ -591,7 +586,7 @@ func (c *Client) SetXX(key string, value interface{}, expiration time.Duration) } func (c *commandable) SetRange(key string, offset int64, value string) *IntCmd { - cmd := NewIntCmd("SETRANGE", key, formatInt(offset), value) + cmd := NewIntCmd("SETRANGE", key, offset, value) c.Process(cmd) return cmd } @@ -641,13 +636,13 @@ func (c *commandable) HGetAllMap(key string) *StringStringMapCmd { } func (c *commandable) HIncrBy(key, field string, incr int64) *IntCmd { - cmd := NewIntCmd("HINCRBY", key, field, formatInt(incr)) + cmd := NewIntCmd("HINCRBY", key, field, incr) c.Process(cmd) return cmd } func (c *commandable) HIncrByFloat(key, field string, incr float64) *FloatCmd { - cmd := NewFloatCmd("HINCRBYFLOAT", key, field, formatFloat(incr)) + cmd := NewFloatCmd("HINCRBYFLOAT", key, field, incr) c.Process(cmd) return cmd } @@ -749,7 +744,7 @@ func (c *commandable) BRPopLPush(source, destination string, timeout time.Durati } func (c *commandable) LIndex(key string, index int64) *StringCmd { - cmd := NewStringCmd("LINDEX", key, formatInt(index)) + cmd := NewStringCmd("LINDEX", key, index) c.Process(cmd) return cmd } @@ -794,21 +789,21 @@ func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd( "LRANGE", key, - formatInt(start), - formatInt(stop), + start, + stop, ) c.Process(cmd) return cmd } func (c *commandable) LRem(key string, count int64, value string) *IntCmd { - cmd := NewIntCmd("LREM", key, formatInt(count), value) + cmd := NewIntCmd("LREM", key, count, value) c.Process(cmd) return cmd } func (c *commandable) LSet(key string, index int64, value string) *StatusCmd { - cmd := NewStatusCmd("LSET", key, formatInt(index), value) + cmd := NewStatusCmd("LSET", key, index, value) c.Process(cmd) return cmd } @@ -817,8 +812,8 @@ func (c *commandable) LTrim(key string, start, stop int64) *StatusCmd { cmd := NewStatusCmd( "LTRIM", key, - formatInt(start), - formatInt(stop), + start, + stop, ) c.Process(cmd) return cmd @@ -953,7 +948,7 @@ func (c *commandable) SRandMember(key string) *StringCmd { // Redis `SRANDMEMBER key count` command. func (c *commandable) SRandMemberN(key string, count int64) *StringSliceCmd { - cmd := NewStringSliceCmd("SRANDMEMBER", key, formatInt(count)) + cmd := NewStringSliceCmd("SRANDMEMBER", key, count) c.Process(cmd) return cmd } @@ -1010,7 +1005,7 @@ type ZStore struct { func (c *commandable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { for i, m := range members { - a[n+2*i] = formatFloat(m.Score) + a[n+2*i] = m.Score a[n+2*i+1] = m.Member } cmd := NewIntCmd(a...) @@ -1068,7 +1063,7 @@ func (c *commandable) ZAddXXCh(key string, members ...Z) *IntCmd { func (c *commandable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { for i, m := range members { - a[n+2*i] = formatFloat(m.Score) + a[n+2*i] = m.Score a[n+2*i+1] = m.Member } cmd := NewFloatCmd(a...) @@ -1113,7 +1108,7 @@ func (c *commandable) ZCount(key, min, max string) *IntCmd { } func (c *commandable) ZIncrBy(key string, increment float64, member string) *FloatCmd { - cmd := NewFloatCmd("ZINCRBY", key, formatFloat(increment), member) + cmd := NewFloatCmd("ZINCRBY", key, increment, member) c.Process(cmd) return cmd } @@ -1133,7 +1128,7 @@ func (c *commandable) ZInterStore( if len(store.Weights) > 0 { args = append(args, "WEIGHTS") for _, weight := range store.Weights { - args = append(args, formatInt(weight)) + args = append(args, weight) } } if store.Aggregate != "" { @@ -1148,8 +1143,8 @@ func (c *commandable) zRange(key string, start, stop int64, withScores bool) *St args := []interface{}{ "ZRANGE", key, - formatInt(start), - formatInt(stop), + start, + stop, } if withScores { args = append(args, "WITHSCORES") @@ -1164,14 +1159,7 @@ func (c *commandable) ZRange(key string, start, stop int64) *StringSliceCmd { } func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { - args := []interface{}{ - "ZRANGE", - key, - formatInt(start), - formatInt(stop), - "WITHSCORES", - } - cmd := NewZSliceCmd(args...) + cmd := NewZSliceCmd("ZRANGE", key, start, stop, "WITHSCORES") c.Process(cmd) return cmd } @@ -1191,8 +1179,8 @@ func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeByScore, withScores b args = append( args, "LIMIT", - formatInt(opt.Offset), - formatInt(opt.Count), + opt.Offset, + opt.Count, ) } cmd := NewStringSliceCmd(args...) @@ -1214,8 +1202,8 @@ func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeByScore) *ZS args = append( args, "LIMIT", - formatInt(opt.Offset), - formatInt(opt.Count), + opt.Offset, + opt.Count, ) } cmd := NewZSliceCmd(args...) @@ -1245,8 +1233,8 @@ func (c *commandable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { cmd := NewIntCmd( "ZREMRANGEBYRANK", key, - formatInt(start), - formatInt(stop), + start, + stop, ) c.Process(cmd) return cmd @@ -1259,13 +1247,13 @@ func (c *commandable) ZRemRangeByScore(key, min, max string) *IntCmd { } func (c *commandable) ZRevRange(key string, start, stop int64) *StringSliceCmd { - cmd := NewStringSliceCmd("ZREVRANGE", key, formatInt(start), formatInt(stop)) + cmd := NewStringSliceCmd("ZREVRANGE", key, start, stop) c.Process(cmd) return cmd } func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { - cmd := NewZSliceCmd("ZREVRANGE", key, formatInt(start), formatInt(stop), "WITHSCORES") + cmd := NewZSliceCmd("ZREVRANGE", key, start, stop, "WITHSCORES") c.Process(cmd) return cmd } @@ -1276,8 +1264,8 @@ func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeByScore) *StringSl args = append( args, "LIMIT", - formatInt(opt.Offset), - formatInt(opt.Count), + opt.Offset, + opt.Count, ) } cmd := NewStringSliceCmd(args...) @@ -1299,8 +1287,8 @@ func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeByScore) args = append( args, "LIMIT", - formatInt(opt.Offset), - formatInt(opt.Count), + opt.Offset, + opt.Count, ) } cmd := NewZSliceCmd(args...) @@ -1331,7 +1319,7 @@ func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *In if len(store.Weights) > 0 { args = append(args, "WEIGHTS") for _, weight := range store.Weights { - args = append(args, formatInt(weight)) + args = append(args, weight) } } if store.Aggregate != "" { From 25164333ff3cf6e288ffe54cb05d03674befb934 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 13 Oct 2015 12:02:29 +0300 Subject: [PATCH 0079/1746] Fix pool panic on slow connection with MaxRetries > 0. --- conn.go | 7 ++-- conn_test.go | 25 ++++++++++++++ export_test.go | 2 ++ main_test.go | 14 ++++++-- multi.go | 2 +- pool.go | 92 ++++++++++++++++++++++++++++++++++++-------------- pool_test.go | 2 +- pubsub.go | 2 +- redis_test.go | 3 +- sentinel.go | 2 +- 10 files changed, 112 insertions(+), 39 deletions(-) create mode 100644 conn_test.go diff --git a/conn.go b/conn.go index 36ba99adbb..ec4abd1811 100644 --- a/conn.go +++ b/conn.go @@ -43,11 +43,8 @@ func (cn *conn) init(opt *Options) error { return nil } - // Use connection to connect to Redis. - pool := newSingleConnPoolConn(cn) - - // Client is not closed because we want to reuse underlying connection. - client := newClient(opt, pool) + // Temp client for Auth and Select. + client := newClient(opt, newSingleConnPool(cn)) if opt.Password != "" { if err := client.Auth(opt.Password).Err(); err != nil { diff --git a/conn_test.go b/conn_test.go new file mode 100644 index 0000000000..4f1eef6d73 --- /dev/null +++ b/conn_test.go @@ -0,0 +1,25 @@ +package redis_test + +import ( + "net" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v3" +) + +var _ = Describe("newConnDialer with bad connection", func() { + It("should return an error", func() { + dialer := redis.NewConnDialer(&redis.Options{ + Dialer: func() (net.Conn, error) { + return &badConn{}, nil + }, + MaxRetries: 3, + Password: "password", + DB: 1, + }) + _, err := dialer() + Expect(err).To(MatchError("bad connection")) + }) +}) diff --git a/export_test.go b/export_test.go index f4687296a8..66ccec25d0 100644 --- a/export_test.go +++ b/export_test.go @@ -6,6 +6,8 @@ func (c *baseClient) Pool() pool { return c.connPool } +var NewConnDialer = newConnDialer + func (cn *conn) SetNetConn(netcn net.Conn) { cn.netcn = netcn } diff --git a/main_test.go b/main_test.go index d2f8d2adbf..eafbeee5db 100644 --- a/main_test.go +++ b/main_test.go @@ -232,7 +232,15 @@ func startSentinel(port, masterName, masterPort string) (*redisProcess, error) { //------------------------------------------------------------------------------ -var errTimeout = syscall.ETIMEDOUT +var ( + errTimeout = syscall.ETIMEDOUT +) + +type badConnError string + +func (e badConnError) Error() string { return string(e) } +func (e badConnError) Timeout() bool { return false } +func (e badConnError) Temporary() bool { return false } type badConn struct { net.TCPConn @@ -250,7 +258,7 @@ func (cn *badConn) Read([]byte) (int, error) { if cn.readErr != nil { return 0, cn.readErr } - return 0, net.UnknownNetworkError("badConn") + return 0, badConnError("bad connection") } func (cn *badConn) Write([]byte) (int, error) { @@ -260,5 +268,5 @@ func (cn *badConn) Write([]byte) (int, error) { if cn.writeErr != nil { return 0, cn.writeErr } - return 0, net.UnknownNetworkError("badConn") + return 0, badConnError("bad connection") } diff --git a/multi.go b/multi.go index e3d628fd4c..00dc99913f 100644 --- a/multi.go +++ b/multi.go @@ -22,7 +22,7 @@ func (c *Client) Multi() *Multi { multi := &Multi{ base: &baseClient{ opt: c.opt, - connPool: newSingleConnPool(c.connPool, true), + connPool: newStickyConnPool(c.connPool, true), }, } multi.commandable.process = multi.process diff --git a/pool.go b/pool.go index bd494d85a6..d048468a42 100644 --- a/pool.go +++ b/pool.go @@ -314,6 +314,52 @@ func (p *connPool) reaper() { //------------------------------------------------------------------------------ type singleConnPool struct { + cn *conn +} + +func newSingleConnPool(cn *conn) *singleConnPool { + return &singleConnPool{ + cn: cn, + } +} + +func (p *singleConnPool) First() *conn { + return p.cn +} + +func (p *singleConnPool) Get() (*conn, error) { + return p.cn, nil +} + +func (p *singleConnPool) Put(cn *conn) error { + if p.cn != cn { + panic("p.cn != cn") + } + return nil +} + +func (p *singleConnPool) Remove(cn *conn) error { + if p.cn != cn { + panic("p.cn != cn") + } + return nil +} + +func (p *singleConnPool) Len() int { + return 1 +} + +func (p *singleConnPool) FreeLen() int { + return 0 +} + +func (p *singleConnPool) Close() error { + return nil +} + +//------------------------------------------------------------------------------ + +type stickyConnPool struct { pool pool reusable bool @@ -322,27 +368,21 @@ type singleConnPool struct { mx sync.Mutex } -func newSingleConnPool(pool pool, reusable bool) *singleConnPool { - return &singleConnPool{ +func newStickyConnPool(pool pool, reusable bool) *stickyConnPool { + return &stickyConnPool{ pool: pool, reusable: reusable, } } -func newSingleConnPoolConn(cn *conn) *singleConnPool { - return &singleConnPool{ - cn: cn, - } -} - -func (p *singleConnPool) First() *conn { +func (p *stickyConnPool) First() *conn { p.mx.Lock() cn := p.cn p.mx.Unlock() return cn } -func (p *singleConnPool) Get() (*conn, error) { +func (p *stickyConnPool) Get() (*conn, error) { defer p.mx.Unlock() p.mx.Lock() @@ -362,15 +402,13 @@ func (p *singleConnPool) Get() (*conn, error) { return p.cn, nil } -func (p *singleConnPool) put() (err error) { - if p.pool != nil { - err = p.pool.Put(p.cn) - } +func (p *stickyConnPool) put() (err error) { + err = p.pool.Put(p.cn) p.cn = nil return err } -func (p *singleConnPool) Put(cn *conn) error { +func (p *stickyConnPool) Put(cn *conn) error { defer p.mx.Unlock() p.mx.Lock() if p.cn != cn { @@ -382,30 +420,32 @@ func (p *singleConnPool) Put(cn *conn) error { return nil } -func (p *singleConnPool) remove() (err error) { - if p.pool != nil { - err = p.pool.Remove(p.cn) - } +func (p *stickyConnPool) remove() (err error) { + err = p.pool.Remove(p.cn) p.cn = nil return err } -func (p *singleConnPool) Remove(cn *conn) error { +func (p *stickyConnPool) Remove(cn *conn) error { defer p.mx.Unlock() p.mx.Lock() if p.cn == nil { panic("p.cn == nil") } - if cn != nil && cn != p.cn { - panic("cn != p.cn") + if cn != nil && p.cn != cn { + panic("p.cn != cn") } if p.closed { return errClosed } - return p.remove() + if cn == nil { + return p.remove() + } else { + return nil + } } -func (p *singleConnPool) Len() int { +func (p *stickyConnPool) Len() int { defer p.mx.Unlock() p.mx.Lock() if p.cn == nil { @@ -414,7 +454,7 @@ func (p *singleConnPool) Len() int { return 1 } -func (p *singleConnPool) FreeLen() int { +func (p *stickyConnPool) FreeLen() int { defer p.mx.Unlock() p.mx.Lock() if p.cn == nil { @@ -423,7 +463,7 @@ func (p *singleConnPool) FreeLen() int { return 0 } -func (p *singleConnPool) Close() error { +func (p *stickyConnPool) Close() error { defer p.mx.Unlock() p.mx.Lock() if p.closed { diff --git a/pool_test.go b/pool_test.go index bff892cc7f..d59c7d2d04 100644 --- a/pool_test.go +++ b/pool_test.go @@ -11,7 +11,7 @@ import ( "gopkg.in/redis.v3" ) -var _ = Describe("Pool", func() { +var _ = Describe("pool", func() { var client *redis.Client var perform = func(n int, cb func()) { diff --git a/pubsub.go b/pubsub.go index ba053e4794..8096c938d9 100644 --- a/pubsub.go +++ b/pubsub.go @@ -29,7 +29,7 @@ func (c *Client) PubSub() *PubSub { return &PubSub{ baseClient: &baseClient{ opt: c.opt, - connPool: newSingleConnPool(c.connPool, false), + connPool: newStickyConnPool(c.connPool, false), }, } } diff --git a/redis_test.go b/redis_test.go index acc8ca1d47..3ad4ae256b 100644 --- a/redis_test.go +++ b/redis_test.go @@ -161,7 +161,8 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) cn.SetNetConn(&badConn{}) - Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) + err = client.Pool().Put(cn) + Expect(err).NotTo(HaveOccurred()) err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) diff --git a/sentinel.go b/sentinel.go index 04a821bd6b..63c011d471 100644 --- a/sentinel.go +++ b/sentinel.go @@ -90,7 +90,7 @@ func (c *sentinelClient) PubSub() *PubSub { return &PubSub{ baseClient: &baseClient{ opt: c.opt, - connPool: newSingleConnPool(c.connPool, false), + connPool: newStickyConnPool(c.connPool, false), }, } } From 126513f1fb84749ec26e99a47369964fd7db8f74 Mon Sep 17 00:00:00 2001 From: Ian Chan Date: Wed, 14 Oct 2015 22:39:39 +0100 Subject: [PATCH 0080/1746] Added binding for GEORADIUSBYMEMBER, GEODIST AND GEOHASH. Change-Id: Ia6144617f42629af4c022e595c444ddc6d66f1a3 --- command.go | 15 ++--- commands.go | 49 +++++++++++---- commands_test.go | 161 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 176 insertions(+), 49 deletions(-) diff --git a/command.go b/command.go index 850d1eb4a4..1fbadc089f 100644 --- a/command.go +++ b/command.go @@ -779,16 +779,13 @@ type GeoLocation struct { // GeoRadiusQuery is used with GeoRadius to query geospatial index. type GeoRadiusQuery struct { - Key string - Longitude float64 - Latitude float64 - Radius float64 + Radius float64 // Can be m, km, ft, or mi. Default is km. - Unit string - WithCoordinates bool - WithDistance bool - WithGeoHash bool - Count int + Unit string + WithCoord bool + WithDist bool + WithGeoHash bool + Count int // Can be ASC or DESC. Default is no sort order. Sort string } diff --git a/commands.go b/commands.go index 6572c81eb6..6f9a7c7c03 100644 --- a/commands.go +++ b/commands.go @@ -1676,22 +1676,17 @@ func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { return cmd } -func (c *commandable) GeoRadius(query *GeoRadiusQuery) *GeoLocationCmd { - args := make([]interface{}, 6) - args[0] = "GEORADIUS" - args[1] = query.Key - args[2] = query.Longitude - args[3] = query.Latitude - args[4] = query.Radius +func (c *commandable) geoRadius(args []interface{}, query *GeoRadiusQuery) *GeoLocationCmd { + args = append(args, query.Radius) if query.Unit != "" { - args[5] = query.Unit + args = append(args, query.Unit) } else { - args[5] = "km" + args = append(args, "km") } - if query.WithCoordinates { + if query.WithCoord { args = append(args, "WITHCOORD") } - if query.WithDistance { + if query.WithDist { args = append(args, "WITHDIST") } if query.WithGeoHash { @@ -1703,8 +1698,38 @@ func (c *commandable) GeoRadius(query *GeoRadiusQuery) *GeoLocationCmd { if query.Sort != "" { args = append(args, query.Sort) } - cmd := NewGeoLocationCmd(args...) c.Process(cmd) return cmd } + +func (c *commandable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { + args := []interface{}{"GEORADIUS", key, longitude, latitude} + return c.geoRadius(args, query) +} + +func (c *commandable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { + args := []interface{}{"GEORADIUSBYMEMBER", key, member} + return c.geoRadius(args, query) +} + +func (c *commandable) GeoDist(key string, member1, member2, unit string) *FloatCmd { + if unit == "" { + unit = "km" + } + cmd := NewFloatCmd("GEODIST", key, member1, member2, unit) + c.Process(cmd) + return cmd +} + +func (c *commandable) GeoHash(key string, members ...string) *StringSliceCmd { + args := make([]interface{}, 2+len(members)) + args[0] = "GEOHASH" + args[1] = key + for i, member := range members { + args[2+i] = member + } + cmd := NewStringSliceCmd(args...) + c.Process(cmd) + return cmd +} diff --git a/commands_test.go b/commands_test.go index f10cfc3dda..35527969e2 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2522,63 +2522,168 @@ var _ = Describe("Commands", func() { }) Describe("Geo add and radius search", func() { + It("should add one geo location", func() { - geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}) + geoAdd := client.GeoAdd( + "Sicily", + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + ) Expect(geoAdd.Err()).NotTo(HaveOccurred()) Expect(geoAdd.Val()).To(Equal(int64(1))) }) It("should add multiple geo locations", func() { - geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}) + geoAdd := client.GeoAdd( + "Sicily", + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + ) Expect(geoAdd.Err()).NotTo(HaveOccurred()) Expect(geoAdd.Val()).To(Equal(int64(2))) }) It("should search geo radius", func() { - geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}) + geoAdd := client.GeoAdd( + "Sicily", + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + ) Expect(geoAdd.Err()).NotTo(HaveOccurred()) Expect(geoAdd.Val()).To(Equal(int64(2))) - geoRadius := client.GeoRadius(&redis.GeoRadiusQuery{Key: "Sicily", Longitude: 15, Latitude: 37, Radius: 200}) - Expect(geoRadius.Err()).NotTo(HaveOccurred()) - Expect(geoRadius.Val()[0].Name).To(Equal("Palermo")) - Expect(geoRadius.Val()[1].Name).To(Equal("Catania")) + res, err := client.GeoRadius("Sicily", 15, 37, &redis.GeoRadiusQuery{Radius: 200}).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(2)) + Expect(res[0].Name).To(Equal("Palermo")) + Expect(res[1].Name).To(Equal("Catania")) }) It("should search geo radius with options", func() { - locations := []*redis.GeoLocation{&redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}} + locations := []*redis.GeoLocation{ + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + } + geoAdd := client.GeoAdd("Sicily", locations...) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + + res, err := client.GeoRadius("Sicily", 15, 37, &redis.GeoRadiusQuery{ + Radius: 200, + Unit: "km", + WithGeoHash: true, + WithCoord: true, + WithDist: true, + Count: 2, + Sort: "ASC", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(2)) + Expect(res[1].Name).To(Equal("Palermo")) + Expect(res[1].Distance).To(Equal(190.4424)) + Expect(res[1].GeoHash).To(Equal(int64(3479099956230698))) + Expect(res[1].Longitude).To(Equal(13.361389338970184)) + Expect(res[1].Latitude).To(Equal(38.115556395496299)) + Expect(res[0].Name).To(Equal("Catania")) + Expect(res[0].Distance).To(Equal(56.4413)) + Expect(res[0].GeoHash).To(Equal(int64(3479447370796909))) + Expect(res[0].Longitude).To(Equal(15.087267458438873)) + Expect(res[0].Latitude).To(Equal(37.50266842333162)) + }) + + It("should search geo radius by member with options", func() { + locations := []*redis.GeoLocation{ + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + } geoAdd := client.GeoAdd("Sicily", locations...) Expect(geoAdd.Err()).NotTo(HaveOccurred()) Expect(geoAdd.Val()).To(Equal(int64(2))) - geoRadius := client.GeoRadius(&redis.GeoRadiusQuery{Key: "Sicily", Longitude: 15, Latitude: 37, Radius: 200, Unit: "km", WithGeoHash: true, WithCoordinates: true, WithDistance: true, Count: 2, Sort: "ASC"}) - Expect(geoRadius.Err()).NotTo(HaveOccurred()) - Expect(geoRadius.Val()[1].Name).To(Equal("Palermo")) - Expect(geoRadius.Val()[1].Distance).To(Equal(190.4424)) - Expect(geoRadius.Val()[1].GeoHash).To(Equal(int64(3479099956230698))) - Expect(geoRadius.Val()[1].Longitude).To(Equal(13.361389338970184)) - Expect(geoRadius.Val()[1].Latitude).To(Equal(38.115556395496299)) - Expect(geoRadius.Val()[0].Name).To(Equal("Catania")) - Expect(geoRadius.Val()[0].Distance).To(Equal(56.4413)) - Expect(geoRadius.Val()[0].GeoHash).To(Equal(int64(3479447370796909))) - Expect(geoRadius.Val()[0].Longitude).To(Equal(15.087267458438873)) - Expect(geoRadius.Val()[0].Latitude).To(Equal(37.50266842333162)) + res, err := client.GeoRadiusByMember("Sicily", "Catania", &redis.GeoRadiusQuery{ + Radius: 200, + Unit: "km", + WithGeoHash: true, + WithCoord: true, + WithDist: true, + Count: 2, + Sort: "ASC", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(2)) + Expect(res[0].Name).To(Equal("Catania")) + Expect(res[0].Distance).To(Equal(0.0)) + Expect(res[0].GeoHash).To(Equal(int64(3479447370796909))) + Expect(res[0].Longitude).To(Equal(15.087267458438873)) + Expect(res[0].Latitude).To(Equal(37.50266842333162)) + Expect(res[1].Name).To(Equal("Palermo")) + Expect(res[1].Distance).To(Equal(166.2742)) + Expect(res[1].GeoHash).To(Equal(int64(3479099956230698))) + Expect(res[1].Longitude).To(Equal(13.361389338970184)) + Expect(res[1].Latitude).To(Equal(38.115556395496299)) }) It("should search geo radius with no results", func() { - geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}) + geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{ + Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + ) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + + res, err := client.GeoRadius("Sicily", 99, 37, &redis.GeoRadiusQuery{ + Radius: 200, + Unit: "km", + WithGeoHash: true, + WithCoord: true, + WithDist: true, + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(0)) + }) + + It("should get geo distance with unit options", func() { + // From Redis CLI, note the difference in rounding in m vs + // km on Redis itself. + // + // GEOADD Sicily 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania" + // GEODIST Sicily Palermo Catania m + // "166274.15156960033" + // GEODIST Sicily Palermo Catania km + // "166.27415156960032" + locations := []*redis.GeoLocation{ + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + } + + geoAdd := client.GeoAdd("Sicily", locations...) Expect(geoAdd.Err()).NotTo(HaveOccurred()) Expect(geoAdd.Val()).To(Equal(int64(2))) - geoRadius := client.GeoRadius(&redis.GeoRadiusQuery{Key: "Sicily", Longitude: 99, Latitude: 37, Radius: 200, Unit: "km", WithGeoHash: true, WithCoordinates: true, WithDistance: true}) - Expect(geoRadius.Err()).NotTo(HaveOccurred()) - Expect(len(geoRadius.Val())).To(Equal(0)) + geoDist := client.GeoDist("Sicily", "Palermo", "Catania", "km") + Expect(geoDist.Err()).NotTo(HaveOccurred()) + Expect(geoDist.Val()).To(Equal(166.27415156960032)) + + geoDist = client.GeoDist("Sicily", "Palermo", "Catania", "m") + Expect(geoDist.Err()).NotTo(HaveOccurred()) + Expect(geoDist.Val()).To(Equal(166274.15156960033)) }) + + It("should get geo hash in string representation", func() { + locations := []*redis.GeoLocation{ + &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, + &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, + } + geoAdd := client.GeoAdd("Sicily", locations...) + Expect(geoAdd.Err()).NotTo(HaveOccurred()) + Expect(geoAdd.Val()).To(Equal(int64(2))) + + res, err := client.GeoHash("Sicily", "Palermo", "Catania").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res[0]).To(Equal("sqc8b49rny0")) + Expect(res[1]).To(Equal("sqdtr74hyu0")) + }) + }) Describe("marshaling/unmarshaling", func() { From ffeacb8b03b6e1677e82f5694142a42fefd63c67 Mon Sep 17 00:00:00 2001 From: Will Jessop Date: Tue, 20 Oct 2015 20:21:58 +0100 Subject: [PATCH 0081/1746] Implement SetName and GetName Allows setting and getting the client connection name. http://redis.io/commands/client-setname http://redis.io/commands/client-getname --- commands.go | 12 ++++++++++++ commands_test.go | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/commands.go b/commands.go index 6572c81eb6..ae6d576e73 100644 --- a/commands.go +++ b/commands.go @@ -1367,6 +1367,18 @@ func (c *commandable) ClientPause(dur time.Duration) *BoolCmd { return cmd } +func (c *commandable) SetName(name string) *StatusCmd { + cmd := NewStatusCmd("CLIENT", "SETNAME", name) + c.Process(cmd) + return cmd +} + +func (c *Client) GetName() *StringCmd { + cmd := NewStringCmd("CLIENT", "GETNAME") + c.Process(cmd) + return cmd +} + func (c *commandable) ConfigGet(parameter string) *SliceCmd { cmd := NewSliceCmd("CONFIG", "GET", parameter) cmd._clusterKeyPos = 0 diff --git a/commands_test.go b/commands_test.go index f10cfc3dda..40f76410a3 100644 --- a/commands_test.go +++ b/commands_test.go @@ -90,6 +90,16 @@ var _ = Describe("Commands", func() { }, "1s").ShouldNot(HaveOccurred()) }) + It("should SetName", func() { + isSet, err := client.SetName("theclientname").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(isSet).To(Equal("OK")) + + val, err := client.GetName().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("theclientname")) + }) + It("should ConfigGet", func() { r := client.ConfigGet("*") Expect(r.Err()).NotTo(HaveOccurred()) From 2bf0ea529db4da2f904fd1851a33ebfbd3415925 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 22 Oct 2015 14:15:47 +0300 Subject: [PATCH 0082/1746] Fix names and add comments. --- commands.go | 8 +++++--- commands_test.go | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/commands.go b/commands.go index ae6d576e73..79ae09827f 100644 --- a/commands.go +++ b/commands.go @@ -1367,13 +1367,15 @@ func (c *commandable) ClientPause(dur time.Duration) *BoolCmd { return cmd } -func (c *commandable) SetName(name string) *StatusCmd { - cmd := NewStatusCmd("CLIENT", "SETNAME", name) +// ClientSetName assigns a name to the one of many connections in the pool. +func (c *commandable) ClientSetName(name string) *BoolCmd { + cmd := NewBoolCmd("CLIENT", "SETNAME", name) c.Process(cmd) return cmd } -func (c *Client) GetName() *StringCmd { +// ClientGetName returns the name of the one of many connections in the pool. +func (c *Client) ClientGetName() *StringCmd { cmd := NewStringCmd("CLIENT", "GETNAME") c.Process(cmd) return cmd diff --git a/commands_test.go b/commands_test.go index 40f76410a3..d048473713 100644 --- a/commands_test.go +++ b/commands_test.go @@ -90,12 +90,12 @@ var _ = Describe("Commands", func() { }, "1s").ShouldNot(HaveOccurred()) }) - It("should SetName", func() { - isSet, err := client.SetName("theclientname").Result() + It("should ClientSetName and ClientGetName", func() { + isSet, err := client.ClientSetName("theclientname").Result() Expect(err).NotTo(HaveOccurred()) - Expect(isSet).To(Equal("OK")) + Expect(isSet).To(BeTrue()) - val, err := client.GetName().Result() + val, err := client.ClientGetName().Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("theclientname")) }) From 05394bee7cbd1c5771438f34eb02cc8eee832a66 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 25 Oct 2015 13:30:26 +0200 Subject: [PATCH 0083/1746] Poorman SEO. --- README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 42e0686a17..25b9c819f5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) -======================= +# Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) Supports: @@ -17,15 +16,13 @@ Supports: API docs: http://godoc.org/gopkg.in/redis.v3. Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. -Installation ------------- +## Installation Install: go get gopkg.in/redis.v3 -Quickstart ----------- +## Quickstart ```go func ExampleNewClient() { @@ -65,13 +62,11 @@ func ExampleClient() { } ``` -Howto ------ +## Howto Please go through [examples](http://godoc.org/gopkg.in/redis.v3#pkg-examples) to get an idea how to use this package. -Look and feel -------------- +## Look and feel Some corner cases: @@ -94,3 +89,7 @@ Some corner cases: EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, []string{"hello"}).Result() + +## Shameless plug + +Check my [PostgreSQL client for Go](https://github.com/go-pg/pg). From 43603e1ea47e7876901dce3a37e8e50f583170b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cosmin=20Lut=CC=A6a=CC=86?= Date: Wed, 4 Nov 2015 09:34:58 +0200 Subject: [PATCH 0084/1746] Implemented PFADD, PFCOUNT, PFMERGE --- commands.go | 32 ++++++++++++++++++++++++++++++++ commands_test.go | 21 +++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/commands.go b/commands.go index 9a65d6689e..694cf4e9e0 100644 --- a/commands.go +++ b/commands.go @@ -1332,6 +1332,38 @@ func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *In //------------------------------------------------------------------------------ +func (c *commandable) PFAdd(key string, fields ...string) *IntCmd { + args := make([]interface{}, 2+len(fields)) + args[0] = "PFADD" + args[1] = key + for i, field := range fields { + args[2+i] = field + } + cmd := NewIntCmd(args...) + c.Process(cmd) + return cmd +} + +func (c *commandable) PFCount(key string) *IntCmd { + cmd := NewIntCmd("PFCOUNT", key) + c.Process(cmd) + return cmd +} + +func (c *commandable) PFMerge(dest string, keys ...string) *StatusCmd { + args := make([]interface{}, 2+len(keys)) + args[0] = "PFMERGE" + args[1] = dest + for i, key := range keys { + args[2+i] = key + } + cmd := NewStatusCmd(args...) + c.Process(cmd) + return cmd +} + +//------------------------------------------------------------------------------ + func (c *commandable) BgRewriteAOF() *StatusCmd { cmd := NewStatusCmd("BGREWRITEAOF") cmd._clusterKeyPos = 0 diff --git a/commands_test.go b/commands_test.go index f7fd42e20c..3304a1ee57 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1190,6 +1190,27 @@ var _ = Describe("Commands", func() { }) + Describe("hyperloglog", func() { + It("should PFMerge", func() { + pfAdd := client.PFAdd("hll1", "1", "2", "3", "4", "5") + Expect(pfAdd.Err()).NotTo(HaveOccurred()) + + pfCount := client.PFCount("hll1") + Expect(pfCount.Err()).NotTo(HaveOccurred()) + Expect(pfCount.Val()).To(Equal(int64(5))) + + pfAdd = client.PFAdd("hll2", "a", "b", "c", "d", "e") + Expect(pfAdd.Err()).NotTo(HaveOccurred()) + + pfMerge := client.PFMerge("hllMerged", "hll1", "hll2") + Expect(pfMerge.Err()).NotTo(HaveOccurred()) + + pfCount = client.PFCount("hllMerged") + Expect(pfCount.Err()).NotTo(HaveOccurred()) + Expect(pfCount.Val()).To(Equal(int64(10))) + }) + }) + Describe("lists", func() { It("should BLPop", func() { From d0d3920e69467352e1ea1b3edf28e2aa2c6e9b4a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 4 Nov 2015 14:25:48 +0200 Subject: [PATCH 0085/1746] Make Pipeline thread-safe. Fixes #166. --- multi.go | 27 ++++++++++++++++++++++++--- pipeline.go | 36 +++++++++++++++++++++++++++++------- pipeline_test.go | 21 +++++++++++++++++++++ 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/multi.go b/multi.go index 00dc99913f..d0859e6142 100644 --- a/multi.go +++ b/multi.go @@ -9,13 +9,18 @@ import ( var errDiscard = errors.New("redis: Discard can be used only inside Exec") // Multi implements Redis transactions as described in -// http://redis.io/topics/transactions. It's NOT safe for concurrent -// use by multiple goroutines. +// http://redis.io/topics/transactions. It's NOT safe for concurrent use +// by multiple goroutines, because Exec resets connection state. +// If you don't need WATCH it is better to use Pipeline. +// +// TODO(vmihailenco): rename to Tx type Multi struct { commandable base *baseClient - cmds []Cmder + + cmds []Cmder + closed bool } func (c *Client) Multi() *Multi { @@ -37,13 +42,17 @@ func (c *Multi) process(cmd Cmder) { } } +// Close closes the client, releasing any open resources. func (c *Multi) Close() error { + c.closed = true if err := c.Unwatch().Err(); err != nil { log.Printf("redis: Unwatch failed: %s", err) } return c.base.Close() } +// Watch marks the keys to be watched for conditional execution +// of a transaction. func (c *Multi) Watch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) args[0] = "WATCH" @@ -55,6 +64,7 @@ func (c *Multi) Watch(keys ...string) *StatusCmd { return cmd } +// Unwatch flushes all the previously watched keys for a transaction. func (c *Multi) Unwatch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) args[0] = "UNWATCH" @@ -66,6 +76,7 @@ func (c *Multi) Unwatch(keys ...string) *StatusCmd { return cmd } +// Discard discards queued commands. func (c *Multi) Discard() error { if c.cmds == nil { return errDiscard @@ -74,10 +85,20 @@ func (c *Multi) Discard() error { return nil } +// Exec executes all previously queued commands in a transaction +// and restores the connection state to normal. +// +// When using WATCH, EXEC will execute commands only if the watched keys +// were not modified, allowing for a check-and-set mechanism. +// // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. func (c *Multi) Exec(f func() error) ([]Cmder, error) { + if c.closed { + return nil, errClosed + } + c.cmds = []Cmder{NewStatusCmd("MULTI")} if err := f(); err != nil { return nil, err diff --git a/pipeline.go b/pipeline.go index d7d130425b..27c9f8eff9 100644 --- a/pipeline.go +++ b/pipeline.go @@ -1,15 +1,22 @@ package redis +import ( + "sync" + "sync/atomic" +) + // Pipeline implements pipelining as described in -// http://redis.io/topics/pipelining. It's NOT safe for concurrent use +// http://redis.io/topics/pipelining. It's safe for concurrent use // by multiple goroutines. type Pipeline struct { commandable client *baseClient - cmds []Cmder - closed bool + mu sync.Mutex // protects cmds + cmds []Cmder + + closed int32 } func (c *Client) Pipeline() *Pipeline { @@ -27,36 +34,51 @@ func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { return nil, err } cmds, err := pipe.Exec() - pipe.Close() + _ = pipe.Close() return cmds, err } func (pipe *Pipeline) process(cmd Cmder) { + pipe.mu.Lock() pipe.cmds = append(pipe.cmds, cmd) + pipe.mu.Unlock() } // Close closes the pipeline, releasing any open resources. func (pipe *Pipeline) Close() error { + atomic.StoreInt32(&pipe.closed, 1) pipe.Discard() - pipe.closed = true return nil } +func (pipe *Pipeline) isClosed() bool { + return atomic.LoadInt32(&pipe.closed) == 1 +} + // Discard resets the pipeline and discards queued commands. func (pipe *Pipeline) Discard() error { - if pipe.closed { + defer pipe.mu.Unlock() + pipe.mu.Lock() + if pipe.isClosed() { return errClosed } pipe.cmds = pipe.cmds[:0] return nil } +// Exec executes all previously queued commands using one +// client-server roundtrip. +// // Exec always returns list of commands and error of the first failed // command if any. func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { - if pipe.closed { + if pipe.isClosed() { return nil, errClosed } + + defer pipe.mu.Unlock() + pipe.mu.Lock() + if len(pipe.cmds) == 0 { return pipe.cmds, nil } diff --git a/pipeline_test.go b/pipeline_test.go index ddf7480b3d..ed01baf480 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -150,4 +150,25 @@ var _ = Describe("Pipelining", func() { wg.Wait() }) + It("should be thread-safe", func() { + const N = 1000 + + pipeline := client.Pipeline() + wg := &sync.WaitGroup{} + wg.Add(N) + for i := 0; i < N; i++ { + go func() { + pipeline.Ping() + wg.Done() + }() + } + wg.Wait() + + cmds, err := pipeline.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(N)) + + Expect(pipeline.Close()).NotTo(HaveOccurred()) + }) + }) From ade3425870003f0d0e85bd03214452693fa85a2f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 14 Nov 2015 14:44:16 +0200 Subject: [PATCH 0086/1746] multi: fix recovering from bad connection. --- cluster_pipeline.go | 2 +- error.go | 18 ++++++++++++++++-- multi.go | 29 ++++++++++++++++++++++------- multi_test.go | 26 ++++++++++++++++++++++++++ pipeline.go | 2 +- pool.go | 44 ++++++++++++++++++++++++-------------------- pool_test.go | 8 ++++---- pubsub.go | 12 +++++++----- pubsub_test.go | 2 +- redis.go | 12 ++++-------- redis_test.go | 2 +- ring.go | 2 +- 12 files changed, 108 insertions(+), 51 deletions(-) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 90687a8b39..3c93bbf0f7 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -63,7 +63,7 @@ func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { continue } - cn, err := client.conn() + cn, _, err := client.conn() if err != nil { setCmdsErr(cmds, err) retErr = err diff --git a/error.go b/error.go index 9e5d973ce1..1365ca1942 100644 --- a/error.go +++ b/error.go @@ -26,10 +26,24 @@ func (err redisError) Error() string { } func isNetworkError(err error) bool { - if _, ok := err.(net.Error); ok || err == io.EOF { + if err == io.EOF { return true } - return false + _, ok := err.(net.Error) + return ok +} + +func isBadConn(cn *conn, ei error) bool { + if cn.rd.Buffered() > 0 { + return true + } + if ei == nil { + return false + } + if _, ok := ei.(redisError); ok { + return false + } + return true } func isMovedError(err error) (moved bool, ask bool, addr string) { diff --git a/multi.go b/multi.go index d0859e6142..86be83f245 100644 --- a/multi.go +++ b/multi.go @@ -10,10 +10,10 @@ var errDiscard = errors.New("redis: Discard can be used only inside Exec") // Multi implements Redis transactions as described in // http://redis.io/topics/transactions. It's NOT safe for concurrent use -// by multiple goroutines, because Exec resets connection state. +// by multiple goroutines, because Exec resets list of watched keys. // If you don't need WATCH it is better to use Pipeline. // -// TODO(vmihailenco): rename to Tx +// TODO(vmihailenco): rename to Tx and rework API type Multi struct { commandable @@ -34,6 +34,18 @@ func (c *Client) Multi() *Multi { return multi } +func (c *Multi) putConn(cn *conn, ei error) { + var err error + if isBadConn(cn, ei) { + err = c.base.connPool.Remove(nil) // nil to force removal + } else { + err = c.base.connPool.Put(cn) + } + if err != nil { + log.Printf("redis: putConn failed: %s", err) + } +} + func (c *Multi) process(cmd Cmder) { if c.cmds == nil { c.base.process(cmd) @@ -112,15 +124,18 @@ func (c *Multi) Exec(f func() error) ([]Cmder, error) { return []Cmder{}, nil } - cn, err := c.base.conn() + // Strip MULTI and EXEC commands. + retCmds := cmds[1 : len(cmds)-1] + + cn, _, err := c.base.conn() if err != nil { - setCmdsErr(cmds[1:len(cmds)-1], err) - return cmds[1 : len(cmds)-1], err + setCmdsErr(retCmds, err) + return retCmds, err } err = c.execCmds(cn, cmds) - c.base.putConn(cn, err) - return cmds[1 : len(cmds)-1], err + c.putConn(cn, err) + return retCmds, err } func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { diff --git a/multi_test.go b/multi_test.go index b481a521ce..c95c703bd1 100644 --- a/multi_test.go +++ b/multi_test.go @@ -119,4 +119,30 @@ var _ = Describe("Multi", func() { Expect(get.Val()).To(Equal("20000")) }) + It("should recover from bad connection", func() { + // Put bad connection in the pool. + cn, _, err := client.Pool().Get() + Expect(err).NotTo(HaveOccurred()) + + cn.SetNetConn(&badConn{}) + err = client.Pool().Put(cn) + Expect(err).NotTo(HaveOccurred()) + + multi := client.Multi() + defer func() { + Expect(multi.Close()).NotTo(HaveOccurred()) + }() + + _, err = multi.Exec(func() error { + multi.Ping() + return nil + }) + Expect(err).To(MatchError("bad connection")) + + _, err = multi.Exec(func() error { + multi.Ping() + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) }) diff --git a/pipeline.go b/pipeline.go index 27c9f8eff9..56ff9654dd 100644 --- a/pipeline.go +++ b/pipeline.go @@ -88,7 +88,7 @@ func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { failedCmds := cmds for i := 0; i <= pipe.client.opt.MaxRetries; i++ { - cn, err := pipe.client.conn() + cn, _, err := pipe.client.conn() if err != nil { setCmdsErr(failedCmds, err) return cmds, err diff --git a/pool.go b/pool.go index d048468a42..a6ab2572c4 100644 --- a/pool.go +++ b/pool.go @@ -18,7 +18,7 @@ var ( type pool interface { First() *conn - Get() (*conn, error) + Get() (*conn, bool, error) Put(*conn) error Remove(*conn) error Len() int @@ -212,33 +212,36 @@ func (p *connPool) new() (*conn, error) { } // Get returns existed connection from the pool or creates a new one. -func (p *connPool) Get() (*conn, error) { +func (p *connPool) Get() (cn *conn, isNew bool, err error) { if p.closed() { - return nil, errClosed + err = errClosed + return } // Fetch first non-idle connection, if available. - if cn := p.First(); cn != nil { - return cn, nil + if cn = p.First(); cn != nil { + return } // Try to create a new one. if p.conns.Reserve() { - cn, err := p.new() + cn, err = p.new() if err != nil { p.conns.Remove(nil) - return nil, err + return } p.conns.Add(cn) - return cn, nil + isNew = true + return } // Otherwise, wait for the available connection. - if cn := p.wait(); cn != nil { - return cn, nil + if cn = p.wait(); cn != nil { + return } - return nil, errPoolTimeout + err = errPoolTimeout + return } func (p *connPool) Put(cn *conn) error { @@ -327,8 +330,8 @@ func (p *singleConnPool) First() *conn { return p.cn } -func (p *singleConnPool) Get() (*conn, error) { - return p.cn, nil +func (p *singleConnPool) Get() (*conn, bool, error) { + return p.cn, false, nil } func (p *singleConnPool) Put(cn *conn) error { @@ -382,24 +385,25 @@ func (p *stickyConnPool) First() *conn { return cn } -func (p *stickyConnPool) Get() (*conn, error) { +func (p *stickyConnPool) Get() (cn *conn, isNew bool, err error) { defer p.mx.Unlock() p.mx.Lock() if p.closed { - return nil, errClosed + err = errClosed + return } if p.cn != nil { - return p.cn, nil + cn = p.cn + return } - cn, err := p.pool.Get() + cn, isNew, err = p.pool.Get() if err != nil { - return nil, err + return } p.cn = cn - - return p.cn, nil + return } func (p *stickyConnPool) put() (err error) { diff --git a/pool_test.go b/pool_test.go index d59c7d2d04..9eb2c990dd 100644 --- a/pool_test.go +++ b/pool_test.go @@ -107,7 +107,7 @@ var _ = Describe("pool", func() { }) It("should remove broken connections", func() { - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn.Close()).NotTo(HaveOccurred()) Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) @@ -141,12 +141,12 @@ var _ = Describe("pool", func() { pool := client.Pool() // Reserve one connection. - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) // Reserve the rest of connections. for i := 0; i < 9; i++ { - _, err := client.Pool().Get() + _, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) } @@ -191,7 +191,7 @@ func BenchmarkPool(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - conn, err := pool.Get() + conn, _, err := pool.Get() if err != nil { b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) } diff --git a/pubsub.go b/pubsub.go index 8096c938d9..a3792d3f54 100644 --- a/pubsub.go +++ b/pubsub.go @@ -47,7 +47,7 @@ func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { } func (c *PubSub) subscribe(cmd string, channels ...string) error { - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { return err } @@ -112,7 +112,7 @@ func (c *PubSub) PUnsubscribe(patterns ...string) error { } func (c *PubSub) Ping(payload string) error { - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { return err } @@ -208,14 +208,16 @@ func newMessage(reply []interface{}) (interface{}, error) { // is not received in time. This is low-level API and most clients // should use ReceiveMessage. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { return nil, err } cn.ReadTimeout = timeout cmd := NewSliceCmd() - if err := cmd.readReply(cn); err != nil { + err = cmd.readReply(cn) + c.putConn(cn, err) + if err != nil { return nil, err } return newMessage(cmd.Val()) @@ -229,7 +231,7 @@ func (c *PubSub) Receive() (interface{}, error) { } func (c *PubSub) reconnect() { - c.connPool.Remove(nil) // close current connection + c.connPool.Remove(nil) // nil to force removal if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { log.Printf("redis: Subscribe failed: %s", err) diff --git a/pubsub_test.go b/pubsub_test.go index 5a7b0dad21..411c643170 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -254,7 +254,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - cn, err := pubsub.Pool().Get() + cn, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.SetNetConn(&badConn{ readErr: errTimeout, diff --git a/redis.go b/redis.go index 2d1076d2d4..6d654b1045 100644 --- a/redis.go +++ b/redis.go @@ -16,20 +16,16 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.opt.Addr, c.opt.DB) } -func (c *baseClient) conn() (*conn, error) { +func (c *baseClient) conn() (*conn, bool, error) { return c.connPool.Get() } func (c *baseClient) putConn(cn *conn, ei error) { var err error - if cn.rd.Buffered() > 0 { + if isBadConn(cn, ei) { err = c.connPool.Remove(cn) - } else if ei == nil { - err = c.connPool.Put(cn) - } else if _, ok := ei.(redisError); ok { - err = c.connPool.Put(cn) } else { - err = c.connPool.Remove(cn) + err = c.connPool.Put(cn) } if err != nil { log.Printf("redis: putConn failed: %s", err) @@ -42,7 +38,7 @@ func (c *baseClient) process(cmd Cmder) { cmd.reset() } - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { cmd.setErr(err) return diff --git a/redis_test.go b/redis_test.go index 3ad4ae256b..f1ebf62a24 100644 --- a/redis_test.go +++ b/redis_test.go @@ -157,7 +157,7 @@ var _ = Describe("Client", func() { }) // Put bad connection in the pool. - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.SetNetConn(&badConn{}) diff --git a/ring.go b/ring.go index 8005af97f0..facf3e61d4 100644 --- a/ring.go +++ b/ring.go @@ -313,7 +313,7 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { for name, cmds := range cmdsMap { client := pipe.ring.shards[name].Client - cn, err := client.conn() + cn, _, err := client.conn() if err != nil { setCmdsErr(cmds, err) if retErr == nil { From a242fa70275c80f27bdc79e6f2389d08224332d7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 14 Nov 2015 15:54:16 +0200 Subject: [PATCH 0087/1746] Try to make cluster tests more stable. --- cluster_test.go | 54 +++++++++++++++++++++++++++---------------------- main_test.go | 19 +++++++---------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 136340ccb6..bc395b4485 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -1,8 +1,10 @@ package redis_test import ( + "fmt" "math/rand" "net" + "strings" "testing" "time" @@ -53,7 +55,7 @@ func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.Cluste } func startCluster(scenario *clusterScenario) error { - // Start processes, connect individual clients + // Start processes and collect node ids for pos, port := range scenario.ports { process, err := startRedis(port, "--cluster-enabled", "yes") if err != nil { @@ -81,44 +83,48 @@ func startCluster(scenario *clusterScenario) error { // Bootstrap masters slots := []int{0, 5000, 10000, 16384} - for pos, client := range scenario.masters() { - err := client.ClusterAddSlotsRange(slots[pos], slots[pos+1]-1).Err() + for pos, master := range scenario.masters() { + err := master.ClusterAddSlotsRange(slots[pos], slots[pos+1]-1).Err() if err != nil { return err } } // Bootstrap slaves - for pos, client := range scenario.slaves() { - masterId := scenario.nodeIds[pos] - - // Wait for masters - err := waitForSubstring(func() string { - return client.ClusterNodes().Val() - }, masterId, 10*time.Second) - if err != nil { - return err - } - - err = client.ClusterReplicate(masterId).Err() + for idx, slave := range scenario.slaves() { + masterId := scenario.nodeIds[idx] + + // Wait until master is available + err := eventually(func() error { + s := slave.ClusterNodes().Val() + wanted := masterId + if !strings.Contains(s, wanted) { + return fmt.Errorf("%q does not contain %q", s, wanted) + } + return nil + }, 10*time.Second) if err != nil { return err } - // Wait for slaves - err = waitForSubstring(func() string { - return scenario.primary().ClusterNodes().Val() - }, "slave "+masterId, 10*time.Second) + err = slave.ClusterReplicate(masterId).Err() if err != nil { return err } } - // Wait for cluster state to turn OK + // Wait until all nodes have consistent info for _, client := range scenario.clients { - err := waitForSubstring(func() string { - return client.ClusterInfo().Val() - }, "cluster_state:ok", 10*time.Second) + err := eventually(func() error { + for _, masterId := range scenario.nodeIds[:3] { + s := client.ClusterNodes().Val() + wanted := "slave " + masterId + if !strings.Contains(s, wanted) { + return fmt.Errorf("%q does not contain %q", s, wanted) + } + } + return nil + }, 10*time.Second) if err != nil { return err } @@ -260,7 +266,6 @@ var _ = Describe("Cluster", func() { It("should perform multi-pipelines", func() { slot := redis.HashSlot("A") - Expect(client.SlotAddrs(slot)).To(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) pipe := client.Pipeline() @@ -288,6 +293,7 @@ var _ = Describe("Cluster", func() { }) It("should return error when there are no attempts left", func() { + Expect(client.Close()).NotTo(HaveOccurred()) client = cluster.clusterClient(&redis.ClusterOptions{ MaxRedirects: -1, }) diff --git a/main_test.go b/main_test.go index eafbeee5db..806d7d3d10 100644 --- a/main_test.go +++ b/main_test.go @@ -1,12 +1,10 @@ package redis_test import ( - "fmt" "net" "os" "os/exec" "path/filepath" - "strings" "sync/atomic" "syscall" "testing" @@ -100,17 +98,14 @@ func TestGinkgoSuite(t *testing.T) { //------------------------------------------------------------------------------ -// Replaces ginkgo's Eventually. -func waitForSubstring(fn func() string, substr string, timeout time.Duration) error { - var s string - - found := make(chan struct{}) +func eventually(fn func() error, timeout time.Duration) (err error) { + done := make(chan struct{}) var exit int32 go func() { for atomic.LoadInt32(&exit) == 0 { - s = fn() - if strings.Contains(s, substr) { - found <- struct{}{} + err = fn() + if err == nil { + close(done) return } time.Sleep(timeout / 100) @@ -118,12 +113,12 @@ func waitForSubstring(fn func() string, substr string, timeout time.Duration) er }() select { - case <-found: + case <-done: return nil case <-time.After(timeout): atomic.StoreInt32(&exit, 1) + return err } - return fmt.Errorf("%q does not contain %q", s, substr) } func execCmd(name string, args ...string) (*os.Process, error) { From ba5bd55d92872899c41a1d7293570f4199e4842b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 14 Nov 2015 15:55:54 +0200 Subject: [PATCH 0088/1746] travis: enable new infrastructure. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6ef52f7206..22cc20680e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: go +sudo: false services: - redis-server From 842ea553dcf9fe637cd33aaa5be232b8f18fe0d2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 14 Nov 2015 16:36:21 +0200 Subject: [PATCH 0089/1746] Fix GeoRadius reply parsing. --- command.go | 40 ++++++++++++++++--- commands.go | 35 +++-------------- commands_test.go | 96 +++++++++++++++++---------------------------- parser.go | 100 ++++++++++++++++++++++++----------------------- 4 files changed, 126 insertions(+), 145 deletions(-) diff --git a/command.go b/command.go index 1fbadc089f..25f9dc53b7 100644 --- a/command.go +++ b/command.go @@ -772,9 +772,9 @@ func (cmd *ClusterSlotCmd) readReply(cn *conn) error { // GeoLocation is used with GeoAdd to add geospatial location. type GeoLocation struct { - Name string - Longitude, Latitude, Distance float64 - GeoHash int64 + Name string + Longitude, Latitude, Dist float64 + GeoHash int64 } // GeoRadiusQuery is used with GeoRadius to query geospatial index. @@ -793,11 +793,39 @@ type GeoRadiusQuery struct { type GeoLocationCmd struct { baseCmd + q *GeoRadiusQuery locations []GeoLocation } -func NewGeoLocationCmd(args ...interface{}) *GeoLocationCmd { - return &GeoLocationCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} +func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { + args = append(args, q.Radius) + if q.Unit != "" { + args = append(args, q.Unit) + } else { + args = append(args, "km") + } + if q.WithCoord { + args = append(args, "WITHCOORD") + } + if q.WithDist { + args = append(args, "WITHDIST") + } + if q.WithGeoHash { + args = append(args, "WITHHASH") + } + if q.Count > 0 { + args = append(args, "COUNT", q.Count) + } + if q.Sort != "" { + args = append(args, q.Sort) + } + return &GeoLocationCmd{ + baseCmd: baseCmd{ + _args: args, + _clusterKeyPos: 1, + }, + q: q, + } } func (cmd *GeoLocationCmd) reset() { @@ -818,7 +846,7 @@ func (cmd *GeoLocationCmd) String() string { } func (cmd *GeoLocationCmd) readReply(cn *conn) error { - reply, err := readArrayReply(cn, geoLocationSliceParser) + reply, err := readArrayReply(cn, newGeoLocationSliceParser(cmd.q)) if err != nil { cmd.err = err return err diff --git a/commands.go b/commands.go index 694cf4e9e0..2a7f7922b9 100644 --- a/commands.go +++ b/commands.go @@ -1722,41 +1722,16 @@ func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { return cmd } -func (c *commandable) geoRadius(args []interface{}, query *GeoRadiusQuery) *GeoLocationCmd { - args = append(args, query.Radius) - if query.Unit != "" { - args = append(args, query.Unit) - } else { - args = append(args, "km") - } - if query.WithCoord { - args = append(args, "WITHCOORD") - } - if query.WithDist { - args = append(args, "WITHDIST") - } - if query.WithGeoHash { - args = append(args, "WITHHASH") - } - if query.Count > 0 { - args = append(args, "COUNT", query.Count) - } - if query.Sort != "" { - args = append(args, query.Sort) - } - cmd := NewGeoLocationCmd(args...) +func (c *commandable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { + cmd := NewGeoLocationCmd(query, "GEORADIUS", key, longitude, latitude) c.Process(cmd) return cmd } -func (c *commandable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { - args := []interface{}{"GEORADIUS", key, longitude, latitude} - return c.geoRadius(args, query) -} - func (c *commandable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { - args := []interface{}{"GEORADIUSBYMEMBER", key, member} - return c.geoRadius(args, query) + cmd := NewGeoLocationCmd(query, "GEORADIUSBYMEMBER", key, member) + c.Process(cmd) + return cmd } func (c *commandable) GeoDist(key string, member1, member2, unit string) *FloatCmd { diff --git a/commands_test.go b/commands_test.go index 3304a1ee57..b931d05e0e 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2553,17 +2553,7 @@ var _ = Describe("Commands", func() { }) Describe("Geo add and radius search", func() { - - It("should add one geo location", func() { - geoAdd := client.GeoAdd( - "Sicily", - &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - ) - Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(1))) - }) - - It("should add multiple geo locations", func() { + BeforeEach(func() { geoAdd := client.GeoAdd( "Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, @@ -2573,16 +2563,19 @@ var _ = Describe("Commands", func() { Expect(geoAdd.Val()).To(Equal(int64(2))) }) - It("should search geo radius", func() { + It("should not add same geo location", func() { geoAdd := client.GeoAdd( "Sicily", &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, ) Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(2))) + Expect(geoAdd.Val()).To(Equal(int64(0))) + }) - res, err := client.GeoRadius("Sicily", 15, 37, &redis.GeoRadiusQuery{Radius: 200}).Result() + It("should search geo radius", func() { + res, err := client.GeoRadius("Sicily", 15, 37, &redis.GeoRadiusQuery{ + Radius: 200, + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(2)) Expect(res[0].Name).To(Equal("Palermo")) @@ -2590,14 +2583,6 @@ var _ = Describe("Commands", func() { }) It("should search geo radius with options", func() { - locations := []*redis.GeoLocation{ - &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, - } - geoAdd := client.GeoAdd("Sicily", locations...) - Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(2))) - res, err := client.GeoRadius("Sicily", 15, 37, &redis.GeoRadiusQuery{ Radius: 200, Unit: "km", @@ -2610,27 +2595,41 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(2)) Expect(res[1].Name).To(Equal("Palermo")) - Expect(res[1].Distance).To(Equal(190.4424)) + Expect(res[1].Dist).To(Equal(190.4424)) Expect(res[1].GeoHash).To(Equal(int64(3479099956230698))) Expect(res[1].Longitude).To(Equal(13.361389338970184)) Expect(res[1].Latitude).To(Equal(38.115556395496299)) Expect(res[0].Name).To(Equal("Catania")) - Expect(res[0].Distance).To(Equal(56.4413)) + Expect(res[0].Dist).To(Equal(56.4413)) Expect(res[0].GeoHash).To(Equal(int64(3479447370796909))) Expect(res[0].Longitude).To(Equal(15.087267458438873)) Expect(res[0].Latitude).To(Equal(37.50266842333162)) }) - It("should search geo radius by member with options", func() { - locations := []*redis.GeoLocation{ - &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, - } - - geoAdd := client.GeoAdd("Sicily", locations...) - Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(2))) + It("should search geo radius with WithDist=false", func() { + res, err := client.GeoRadius("Sicily", 15, 37, &redis.GeoRadiusQuery{ + Radius: 200, + Unit: "km", + WithGeoHash: true, + WithCoord: true, + Count: 2, + Sort: "ASC", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(2)) + Expect(res[1].Name).To(Equal("Palermo")) + Expect(res[1].Dist).To(Equal(float64(0))) + Expect(res[1].GeoHash).To(Equal(int64(3479099956230698))) + Expect(res[1].Longitude).To(Equal(13.361389338970184)) + Expect(res[1].Latitude).To(Equal(38.115556395496299)) + Expect(res[0].Name).To(Equal("Catania")) + Expect(res[0].Dist).To(Equal(float64(0))) + Expect(res[0].GeoHash).To(Equal(int64(3479447370796909))) + Expect(res[0].Longitude).To(Equal(15.087267458438873)) + Expect(res[0].Latitude).To(Equal(37.50266842333162)) + }) + It("should search geo radius by member with options", func() { res, err := client.GeoRadiusByMember("Sicily", "Catania", &redis.GeoRadiusQuery{ Radius: 200, Unit: "km", @@ -2643,25 +2642,18 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(2)) Expect(res[0].Name).To(Equal("Catania")) - Expect(res[0].Distance).To(Equal(0.0)) + Expect(res[0].Dist).To(Equal(0.0)) Expect(res[0].GeoHash).To(Equal(int64(3479447370796909))) Expect(res[0].Longitude).To(Equal(15.087267458438873)) Expect(res[0].Latitude).To(Equal(37.50266842333162)) Expect(res[1].Name).To(Equal("Palermo")) - Expect(res[1].Distance).To(Equal(166.2742)) + Expect(res[1].Dist).To(Equal(166.2742)) Expect(res[1].GeoHash).To(Equal(int64(3479099956230698))) Expect(res[1].Longitude).To(Equal(13.361389338970184)) Expect(res[1].Latitude).To(Equal(38.115556395496299)) }) It("should search geo radius with no results", func() { - geoAdd := client.GeoAdd("Sicily", &redis.GeoLocation{ - Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, - ) - Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(2))) - res, err := client.GeoRadius("Sicily", 99, 37, &redis.GeoRadiusQuery{ Radius: 200, Unit: "km", @@ -2682,15 +2674,6 @@ var _ = Describe("Commands", func() { // "166274.15156960033" // GEODIST Sicily Palermo Catania km // "166.27415156960032" - locations := []*redis.GeoLocation{ - &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, - } - - geoAdd := client.GeoAdd("Sicily", locations...) - Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(2))) - geoDist := client.GeoDist("Sicily", "Palermo", "Catania", "km") Expect(geoDist.Err()).NotTo(HaveOccurred()) Expect(geoDist.Val()).To(Equal(166.27415156960032)) @@ -2701,20 +2684,11 @@ var _ = Describe("Commands", func() { }) It("should get geo hash in string representation", func() { - locations := []*redis.GeoLocation{ - &redis.GeoLocation{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, - &redis.GeoLocation{Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}, - } - geoAdd := client.GeoAdd("Sicily", locations...) - Expect(geoAdd.Err()).NotTo(HaveOccurred()) - Expect(geoAdd.Val()).To(Equal(int64(2))) - res, err := client.GeoHash("Sicily", "Palermo", "Catania").Result() Expect(err).NotTo(HaveOccurred()) Expect(res[0]).To(Equal("sqc8b49rny0")) Expect(res[1]).To(Equal("sqdtr74hyu0")) }) - }) Describe("marshaling/unmarshaling", func() { diff --git a/parser.go b/parser.go index dd242ca1a1..43274fc36d 100644 --- a/parser.go +++ b/parser.go @@ -629,65 +629,69 @@ func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { return infos, nil } -func geoLocationParser(cn *conn, n int64) (interface{}, error) { - loc := &GeoLocation{} +func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { + return func(cn *conn, n int64) (interface{}, error) { + var loc GeoLocation - var err error - loc.Name, err = readStringReply(cn) - if err != nil { - return nil, err - } - if n >= 2 { - loc.Distance, err = readFloatReply(cn) - if err != nil { - return nil, err - } - } - if n >= 3 { - loc.GeoHash, err = readIntReply(cn) + var err error + loc.Name, err = readStringReply(cn) if err != nil { return nil, err } - } - if n >= 4 { - n, err := readArrayHeader(cn) - if err != nil { - return nil, err + if q.WithDist { + loc.Dist, err = readFloatReply(cn) + if err != nil { + return nil, err + } } - if n != 2 { - return nil, fmt.Errorf("got %d coordinates, expected 2", n) + if q.WithGeoHash { + loc.GeoHash, err = readIntReply(cn) + if err != nil { + return nil, err + } } + if q.WithCoord { + n, err := readArrayHeader(cn) + if err != nil { + return nil, err + } + if n != 2 { + return nil, fmt.Errorf("got %d coordinates, expected 2", n) + } - loc.Longitude, err = readFloatReply(cn) - if err != nil { - return nil, err - } - loc.Latitude, err = readFloatReply(cn) - if err != nil { - return nil, err + loc.Longitude, err = readFloatReply(cn) + if err != nil { + return nil, err + } + loc.Latitude, err = readFloatReply(cn) + if err != nil { + return nil, err + } } - } - return loc, nil + return &loc, nil + } } -func geoLocationSliceParser(cn *conn, n int64) (interface{}, error) { - locs := make([]GeoLocation, 0, n) - for i := int64(0); i < n; i++ { - v, err := readReply(cn, geoLocationParser) - if err != nil { - return nil, err - } - switch vv := v.(type) { - case []byte: - locs = append(locs, GeoLocation{ - Name: string(vv), - }) - case *GeoLocation: - locs = append(locs, *vv) - default: - return nil, fmt.Errorf("got %T, expected string or *GeoLocation", v) +func newGeoLocationSliceParser(q *GeoRadiusQuery) multiBulkParser { + return func(cn *conn, n int64) (interface{}, error) { + locs := make([]GeoLocation, 0, n) + for i := int64(0); i < n; i++ { + v, err := readReply(cn, newGeoLocationParser(q)) + if err != nil { + return nil, err + } + switch vv := v.(type) { + case []byte: + locs = append(locs, GeoLocation{ + Name: string(vv), + }) + case *GeoLocation: + locs = append(locs, *vv) + default: + return nil, fmt.Errorf("got %T, expected string or *GeoLocation", v) + } } + return locs, nil } - return locs, nil } From f766eb02098fbbaff031f9edbe6c31fdfe7bb0ac Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 15 Nov 2015 10:23:00 +0200 Subject: [PATCH 0090/1746] Improve Tx example. --- example_test.go | 31 ++++++++++++++----------------- multi.go | 12 ++++++++++++ 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/example_test.go b/example_test.go index cb4b8f670a..1d8be98c9b 100644 --- a/example_test.go +++ b/example_test.go @@ -159,13 +159,16 @@ func ExamplePipeline() { // Output: 1 } -func ExampleMulti() { +func ExampleWatch() { + var incr func(string) error + // Transactionally increments key using GET and SET commands. - incr := func(tx *redis.Multi, key string) error { - err := tx.Watch(key).Err() + incr = func(key string) error { + tx, err := client.Watch(key) if err != nil { return err } + defer tx.Close() n, err := tx.Get(key).Int64() if err != nil && err != redis.Nil { @@ -176,27 +179,21 @@ func ExampleMulti() { tx.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) + if err == redis.TxFailedErr { + return incr(key) + } return err } var wg sync.WaitGroup - for i := 0; i < 10; i++ { + for i := 0; i < 100; i++ { wg.Add(1) go func() { defer wg.Done() - tx := client.Multi() - defer tx.Close() - - for { - err := incr(tx, "counter3") - if err == redis.TxFailedErr { - // Retry. - continue - } else if err != nil { - panic(err) - } - break + err := incr("counter3") + if err != nil { + panic(err) } }() } @@ -204,7 +201,7 @@ func ExampleMulti() { n, err := client.Get("counter3").Int64() fmt.Println(n, err) - // Output: 10 + // Output: 100 } func ExamplePubSub() { diff --git a/multi.go b/multi.go index 86be83f245..43db5e8f9e 100644 --- a/multi.go +++ b/multi.go @@ -23,6 +23,18 @@ type Multi struct { closed bool } +// Watch marks the keys to be watched for conditional execution +// of a transaction. +func (c *Client) Watch(keys ...string) (*Multi, error) { + tx := c.Multi() + if err := tx.Watch(keys...).Err(); err != nil { + tx.Close() + return nil, err + } + return tx, nil +} + +// Deprecated. Use Watch instead. func (c *Client) Multi() *Multi { multi := &Multi{ base: &baseClient{ From b792d8a4cfb90c976c81c693c37309358d60dffc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 18 Nov 2015 16:35:02 +0200 Subject: [PATCH 0091/1746] Fix doc comment for PoolTimeout. --- redis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.go b/redis.go index 6d654b1045..cd88cefd0b 100644 --- a/redis.go +++ b/redis.go @@ -121,7 +121,7 @@ type Options struct { PoolSize int // Specifies amount of time client waits for connection if all // connections are busy before returning an error. - // Default is 5 seconds. + // Default is 1 seconds. PoolTimeout time.Duration // Specifies amount of time after which client closes idle // connections. Should be less than server's timeout. From 98414ea72a1bfcb318bcbf00168b90d39540b675 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 21 Nov 2015 10:20:01 +0200 Subject: [PATCH 0092/1746] Increase test timeout. --- ring.go | 2 +- ring_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ring.go b/ring.go index facf3e61d4..ff77f4d6f2 100644 --- a/ring.go +++ b/ring.go @@ -95,7 +95,7 @@ func (shard *ringShard) Vote(up bool) bool { // keys across multiple Redis servers (shards). It's safe for // concurrent use by multiple goroutines. // -// It monitors the state of each shard and removes dead shards from +// Ring monitors the state of each shard and removes dead shards from // the ring. When shard comes online it is added back to the ring. This // gives you maximum availability and partition tolerance, but no // consistency between different shards or even clients. Each client diff --git a/ring_test.go b/ring_test.go index 5b52b3204d..55eb90dd62 100644 --- a/ring_test.go +++ b/ring_test.go @@ -56,7 +56,7 @@ var _ = Describe("Redis ring", func() { // Ring needs 5 * heartbeat time to detect that node is down. // Give it more to be sure. heartbeat := 100 * time.Millisecond - time.Sleep(5*heartbeat + heartbeat) + time.Sleep(5*heartbeat + 2*heartbeat) setRingKeys() From d3c6b6f35380202e5822e7455c5bdf49c22a4778 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 21 Nov 2015 13:16:13 +0200 Subject: [PATCH 0093/1746] tests: check Cluster node flags. --- cluster_test.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index bc395b4485..26d306e3bf 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -116,11 +116,22 @@ func startCluster(scenario *clusterScenario) error { // Wait until all nodes have consistent info for _, client := range scenario.clients { err := eventually(func() error { - for _, masterId := range scenario.nodeIds[:3] { - s := client.ClusterNodes().Val() - wanted := "slave " + masterId - if !strings.Contains(s, wanted) { - return fmt.Errorf("%q does not contain %q", s, wanted) + s := client.ClusterNodes().Val() + nodes := strings.Split(s, "\n") + if len(nodes) < 6 { + return fmt.Errorf("got %d nodes, wanted 6", len(nodes)) + } + for _, node := range nodes { + if node == "" { + continue + } + parts := strings.Split(node, " ") + var flags string + if len(parts) >= 3 { + flags = parts[2] + } + if !strings.Contains(flags, "master") && !strings.Contains(flags, "slave") { + return fmt.Errorf("node flags are %q", flags) } } return nil From ee3a8f1212e4b73316881180a5a522297b8c08f4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 21 Nov 2015 13:24:42 +0200 Subject: [PATCH 0094/1746] makefile: give Redis time to exit. --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1b43765b4f..bda28e3543 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,7 @@ all: testdeps go test ./... -test.v -test.cpu=1,2,4 - go test ./... -test.short -test.race - -test: testdeps - go test ./... -test.v=1 + sleep 3 # give Redis time to exit + go test ./... -test.v -test.short -test.race testdeps: .test/redis/src/redis-server From f130ab6161ce93f862a488acd388dfbc10699bd4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Nov 2015 14:24:57 +0200 Subject: [PATCH 0095/1746] travis: test on tip. --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22cc20680e..1d3148f707 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,18 @@ -language: go sudo: false +language: go services: -- redis-server + - redis-server go: - 1.3 - 1.4 - 1.5 + - tip + +matrix: + allow_failures: + - go: tip install: - go get gopkg.in/bsm/ratelimit.v1 From b6b689904a02c3fd4e716554161381f6c3c4d4af Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Nov 2015 14:44:38 +0200 Subject: [PATCH 0096/1746] Fix test. --- cluster_test.go | 31 ++++++++++++++++--------------- commands_test.go | 17 ++++++----------- example_test.go | 1 + pubsub.go | 5 +++++ pubsub_test.go | 9 +++++++-- ring_test.go | 2 +- 6 files changed, 36 insertions(+), 29 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 26d306e3bf..d875735f26 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -4,6 +4,7 @@ import ( "fmt" "math/rand" "net" + "reflect" "strings" "testing" @@ -116,23 +117,23 @@ func startCluster(scenario *clusterScenario) error { // Wait until all nodes have consistent info for _, client := range scenario.clients { err := eventually(func() error { - s := client.ClusterNodes().Val() - nodes := strings.Split(s, "\n") - if len(nodes) < 6 { - return fmt.Errorf("got %d nodes, wanted 6", len(nodes)) + res, err := client.ClusterSlots().Result() + if err != nil { + return err } - for _, node := range nodes { - if node == "" { - continue - } - parts := strings.Split(node, " ") - var flags string - if len(parts) >= 3 { - flags = parts[2] - } - if !strings.Contains(flags, "master") && !strings.Contains(flags, "slave") { - return fmt.Errorf("node flags are %q", flags) + wanted := []redis.ClusterSlotInfo{ + {0, 4999, []string{"127.0.0.1:8220", "127.0.0.1:8223"}}, + {5000, 9999, []string{"127.0.0.1:8221", "127.0.0.1:8224"}}, + {10000, 16383, []string{"127.0.0.1:8222", "127.0.0.1:8225"}}, + } + loop: + for _, info := range res { + for _, info2 := range wanted { + if reflect.DeepEqual(info, info2) { + continue loop + } } + return fmt.Errorf("cluster did not reach consistent state (%v)", res) } return nil }, 10*time.Second) diff --git a/commands_test.go b/commands_test.go index b931d05e0e..4e3488ddd0 100644 --- a/commands_test.go +++ b/commands_test.go @@ -20,10 +20,8 @@ var _ = Describe("Commands", func() { BeforeEach(func() { client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - ReadTimeout: 500 * time.Millisecond, - WriteTimeout: 500 * time.Millisecond, - PoolTimeout: 30 * time.Second, + Addr: redisAddr, + PoolTimeout: 30 * time.Second, }) }) @@ -81,13 +79,10 @@ var _ = Describe("Commands", func() { err := client.ClientPause(time.Second).Err() Expect(err).NotTo(HaveOccurred()) - Consistently(func() error { - return client.Ping().Err() - }, "400ms").Should(HaveOccurred()) // pause time - read timeout - - Eventually(func() error { - return client.Ping().Err() - }, "1s").ShouldNot(HaveOccurred()) + start := time.Now() + err = client.Ping().Err() + Expect(err).NotTo(HaveOccurred()) + Expect(time.Now()).To(BeTemporally("~", start.Add(time.Second), 800*time.Millisecond)) }) It("should ClientSetName and ClientGetName", func() { diff --git a/example_test.go b/example_test.go index 8de2082755..c796991448 100644 --- a/example_test.go +++ b/example_test.go @@ -238,6 +238,7 @@ func ExamplePubSub_Receive() { } for i := 0; i < 2; i++ { + // ReceiveTimeout is a low level API. Use ReceiveMessage instead. msgi, err := pubsub.ReceiveTimeout(100 * time.Millisecond) if err != nil { panic(err) diff --git a/pubsub.go b/pubsub.go index a3792d3f54..223f8b96fd 100644 --- a/pubsub.go +++ b/pubsub.go @@ -80,6 +80,9 @@ func (c *PubSub) PSubscribe(patterns ...string) error { } func remove(ss []string, es ...string) []string { + if len(es) == 0 { + return ss[:0] + } for _, e := range es { for i, s := range ss { if s == e { @@ -231,7 +234,9 @@ func (c *PubSub) Receive() (interface{}, error) { } func (c *PubSub) reconnect() { + // Close current connection. c.connPool.Remove(nil) // nil to force removal + if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { log.Printf("redis: Subscribe failed: %s", err) diff --git a/pubsub_test.go b/pubsub_test.go index 411c643170..1506a8eb23 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -2,6 +2,7 @@ package redis_test import ( "net" + "sync" "time" . "github.com/onsi/ginkgo" @@ -261,19 +262,23 @@ var _ = Describe("PubSub", func() { writeErr: errTimeout, }) + var wg sync.WaitGroup + wg.Add(1) go func() { defer GinkgoRecover() + defer wg.Done() time.Sleep(100 * time.Millisecond) - n, err := client.Publish("mychannel", "hello").Result() + err := client.Publish("mychannel", "hello").Err() Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(2))) }() msg, err := pubsub.ReceiveMessage() Expect(err).NotTo(HaveOccurred()) Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) + + wg.Wait() }) }) diff --git a/ring_test.go b/ring_test.go index 55eb90dd62..0de37dc4a9 100644 --- a/ring_test.go +++ b/ring_test.go @@ -56,7 +56,7 @@ var _ = Describe("Redis ring", func() { // Ring needs 5 * heartbeat time to detect that node is down. // Give it more to be sure. heartbeat := 100 * time.Millisecond - time.Sleep(5*heartbeat + 2*heartbeat) + time.Sleep(2 * 5 * heartbeat) setRingKeys() From d9278e3d745d4d1a4d8160acef2a27948030339b Mon Sep 17 00:00:00 2001 From: Leonid Shagabutdinov Date: Tue, 24 Nov 2015 13:09:53 +0600 Subject: [PATCH 0097/1746] add BLPop example --- example_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/example_test.go b/example_test.go index 8de2082755..f6cfe67eff 100644 --- a/example_test.go +++ b/example_test.go @@ -109,6 +109,21 @@ func ExampleClient_Incr() { // Output: 1 } +func ExampleClient_BLPop() { + if err := client.RPush("queue", "message").Err(); err != nil { + panic(err) + } + + // use `client.BLPop(0, "queue")` for infinite waiting time + result, err := client.BLPop(1*time.Second, "queue").Result() + if err != nil { + panic(err) + } + + fmt.Println(result[0], result[1]) + // Output: queue message +} + func ExampleClient_Scan() { client.FlushDb() for i := 0; i < 33; i++ { From 62ce55295973bde6bc3c5875e71c93a8b117d9fa Mon Sep 17 00:00:00 2001 From: Leonid Shagabutdinov Date: Thu, 26 Nov 2015 17:04:26 +0200 Subject: [PATCH 0098/1746] Fix PubSub panic on concurrent Close. --- pool.go | 12 ++++++------ pubsub_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/pool.go b/pool.go index a6ab2572c4..03bb1a000e 100644 --- a/pool.go +++ b/pool.go @@ -415,12 +415,12 @@ func (p *stickyConnPool) put() (err error) { func (p *stickyConnPool) Put(cn *conn) error { defer p.mx.Unlock() p.mx.Lock() - if p.cn != cn { - panic("p.cn != cn") - } if p.closed { return errClosed } + if p.cn != cn { + panic("p.cn != cn") + } return nil } @@ -433,15 +433,15 @@ func (p *stickyConnPool) remove() (err error) { func (p *stickyConnPool) Remove(cn *conn) error { defer p.mx.Unlock() p.mx.Lock() + if p.closed { + return errClosed + } if p.cn == nil { panic("p.cn == nil") } if cn != nil && p.cn != cn { panic("p.cn != cn") } - if p.closed { - return errClosed - } if cn == nil { return p.remove() } else { diff --git a/pubsub_test.go b/pubsub_test.go index 1506a8eb23..dd24bc64fc 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -235,8 +235,11 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() + var wg sync.WaitGroup + wg.Add(1) go func() { defer GinkgoRecover() + defer wg.Done() time.Sleep(readTimeout + 100*time.Millisecond) n, err := client.Publish("mychannel", "hello").Result() @@ -248,6 +251,8 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) + + wg.Wait() }) It("should reconnect on ReceiveMessage error", func() { @@ -281,4 +286,24 @@ var _ = Describe("PubSub", func() { wg.Wait() }) + It("should not panic on Close", func() { + pubsub, err := client.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer GinkgoRecover() + + wg.Done() + _, err := pubsub.ReceiveMessage() + Expect(err).To(MatchError("redis: client is closed")) + }() + wg.Wait() + + err = pubsub.Close() + Expect(err).NotTo(HaveOccurred()) + }) + }) From fb44c891ddccc854c418bca59d23bc58f751761a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 27 Nov 2015 13:35:30 +0200 Subject: [PATCH 0099/1746] Fix sporadic pool timeouts with IdleTimeout != 0. Fixes #195. --- cluster.go | 1 + pool.go | 34 +++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/cluster.go b/cluster.go index 463cae4f03..ab06890b51 100644 --- a/cluster.go +++ b/cluster.go @@ -146,6 +146,7 @@ func (c *ClusterClient) process(cmd Cmder) { pipe.Process(NewCmd("ASKING")) pipe.Process(cmd) _, _ = pipe.Exec() + pipe.Close() ask = false } else { client.Process(cmd) diff --git a/pool.go b/pool.go index 03bb1a000e..2c387e9c72 100644 --- a/pool.go +++ b/pool.go @@ -163,8 +163,12 @@ func (p *connPool) First() *conn { select { case cn := <-p.freeConns: if p.isIdle(cn) { - p.conns.Remove(cn) - continue + var err error + cn, err = p.replace(cn) + if err != nil { + log.Printf("redis: replace failed: %s", err) + continue + } } return cn default: @@ -181,8 +185,12 @@ func (p *connPool) wait() *conn { select { case cn := <-p.freeConns: if p.isIdle(cn) { - p.Remove(cn) - continue + var err error + cn, err = p.replace(cn) + if err != nil { + log.Printf("redis: replace failed: %s", err) + continue + } } return cn case <-deadline: @@ -257,16 +265,24 @@ func (p *connPool) Put(cn *conn) error { return nil } +func (p *connPool) replace(cn *conn) (*conn, error) { + newcn, err := p.new() + if err != nil { + _ = p.conns.Remove(cn) + return nil, err + } + _ = p.conns.Replace(cn, newcn) + return newcn, nil +} + func (p *connPool) Remove(cn *conn) error { // Replace existing connection with new one and unblock waiter. - newcn, err := p.new() + newcn, err := p.replace(cn) if err != nil { - log.Printf("redis: new failed: %s", err) - return p.conns.Remove(cn) + return err } - err = p.conns.Replace(cn, newcn) p.freeConns <- newcn - return err + return nil } // Len returns total number of connections. From 83b8c0a9c04292433a8fafe784958a28939a5073 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 1 Dec 2015 16:28:41 +0200 Subject: [PATCH 0100/1746] Fix ZStore Weight type. Fixes #206. --- commands.go | 12 ++++-------- commands_test.go | 4 ++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/commands.go b/commands.go index 2a7f7922b9..d3e274284c 100644 --- a/commands.go +++ b/commands.go @@ -990,15 +990,15 @@ func (c *commandable) SUnionStore(destination string, keys ...string) *IntCmd { //------------------------------------------------------------------------------ -// Sorted set member. +// Z represents sorted set member. type Z struct { Score float64 Member interface{} } -// Sorted set store operation. +// ZStore is used as an arg to ZInterStore and ZUnionStore. type ZStore struct { - Weights []int64 + Weights []float64 // Can be SUM, MIN or MAX. Aggregate string } @@ -1113,11 +1113,7 @@ func (c *commandable) ZIncrBy(key string, increment float64, member string) *Flo return cmd } -func (c *commandable) ZInterStore( - destination string, - store ZStore, - keys ...string, -) *IntCmd { +func (c *commandable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "ZINTERSTORE" args[1] = destination diff --git a/commands_test.go b/commands_test.go index 4e3488ddd0..327cc26405 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2082,7 +2082,7 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) zInterStore := client.ZInterStore( - "out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2") + "out", redis.ZStore{Weights: []float64{2, 3}}, "zset1", "zset2") Expect(zInterStore.Err()).NotTo(HaveOccurred()) Expect(zInterStore.Val()).To(Equal(int64(2))) @@ -2479,7 +2479,7 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) zUnionStore := client.ZUnionStore( - "out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2") + "out", redis.ZStore{Weights: []float64{2, 3}}, "zset1", "zset2") Expect(zUnionStore.Err()).NotTo(HaveOccurred()) Expect(zUnionStore.Val()).To(Equal(int64(3))) From 42141f11d1bcd031d7f0a8b45e644a932bbb167f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 2 Dec 2015 15:40:44 +0200 Subject: [PATCH 0101/1746] Improve ReceiveMessage. --- multi.go | 3 ++- pool.go | 17 +++++++++++------ pubsub.go | 42 ++++++++++++++++++++++-------------------- pubsub_test.go | 13 ++++++++++--- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/multi.go b/multi.go index 43db5e8f9e..bb4fafe7b8 100644 --- a/multi.go +++ b/multi.go @@ -49,7 +49,8 @@ func (c *Client) Multi() *Multi { func (c *Multi) putConn(cn *conn, ei error) { var err error if isBadConn(cn, ei) { - err = c.base.connPool.Remove(nil) // nil to force removal + // Close current connection. + c.base.connPool.(*stickyConnPool).Reset() } else { err = c.base.connPool.Put(cn) } diff --git a/pool.go b/pool.go index 2c387e9c72..9968bd7ec9 100644 --- a/pool.go +++ b/pool.go @@ -137,7 +137,7 @@ func newConnPool(opt *Options) *connPool { p := &connPool{ dialer: newConnDialer(opt), - rl: ratelimit.New(2*opt.getPoolSize(), time.Second), + rl: ratelimit.New(3*opt.getPoolSize(), time.Second), opt: opt, conns: newConnList(opt.getPoolSize()), freeConns: make(chan *conn, opt.getPoolSize()), @@ -458,11 +458,7 @@ func (p *stickyConnPool) Remove(cn *conn) error { if cn != nil && p.cn != cn { panic("p.cn != cn") } - if cn == nil { - return p.remove() - } else { - return nil - } + return nil } func (p *stickyConnPool) Len() int { @@ -483,6 +479,15 @@ func (p *stickyConnPool) FreeLen() int { return 0 } +func (p *stickyConnPool) Reset() (err error) { + p.mx.Lock() + if p.cn != nil { + err = p.remove() + } + p.mx.Unlock() + return err +} + func (p *stickyConnPool) Close() error { defer p.mx.Unlock() p.mx.Lock() diff --git a/pubsub.go b/pubsub.go index 223f8b96fd..3e20fe7209 100644 --- a/pubsub.go +++ b/pubsub.go @@ -235,7 +235,7 @@ func (c *PubSub) Receive() (interface{}, error) { func (c *PubSub) reconnect() { // Close current connection. - c.connPool.Remove(nil) // nil to force removal + c.connPool.(*stickyConnPool).Reset() if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { @@ -252,39 +252,41 @@ func (c *PubSub) reconnect() { // ReceiveMessage returns a message or error. It automatically // reconnects to Redis in case of network errors. func (c *PubSub) ReceiveMessage() (*Message, error) { - var badConn bool + var errNum int for { msgi, err := c.ReceiveTimeout(5 * time.Second) if err != nil { - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - if badConn { - c.reconnect() - badConn = false - continue - } + if !isNetworkError(err) { + return nil, err + } + + goodConn := errNum == 0 + errNum++ - err := c.Ping("") - if err != nil { - c.reconnect() - } else { - badConn = true + if goodConn { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + err := c.Ping("") + if err == nil { + continue + } + log.Printf("redis: PubSub.Ping failed: %s", err) } - continue } - if isNetworkError(err) { - c.reconnect() - continue + if errNum > 2 { + time.Sleep(time.Second) } - - return nil, err + c.reconnect() + continue } + // Reset error number. + errNum = 0 + switch msg := msgi.(type) { case *Subscription: // Ignore. case *Pong: - badConn = false // Ignore. case *Message: return msg, nil diff --git a/pubsub_test.go b/pubsub_test.go index dd24bc64fc..64938203b2 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -260,9 +260,9 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - cn, _, err := pubsub.Pool().Get() + cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{ + cn1.SetNetConn(&badConn{ readErr: errTimeout, writeErr: errTimeout, }) @@ -286,7 +286,7 @@ var _ = Describe("PubSub", func() { wg.Wait() }) - It("should not panic on Close", func() { + It("should return on Close", func() { pubsub, err := client.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() @@ -297,13 +297,20 @@ var _ = Describe("PubSub", func() { defer GinkgoRecover() wg.Done() + _, err := pubsub.ReceiveMessage() Expect(err).To(MatchError("redis: client is closed")) + + wg.Done() }() + wg.Wait() + wg.Add(1) err = pubsub.Close() Expect(err).NotTo(HaveOccurred()) + + wg.Wait() }) }) From 397440deca777211c19cb4fe3a033f9915f71735 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 9 Dec 2015 09:33:37 +0000 Subject: [PATCH 0102/1746] Added CLUSTER FORGET command --- commands.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/commands.go b/commands.go index d3e274284c..f023e48d32 100644 --- a/commands.go +++ b/commands.go @@ -1662,6 +1662,12 @@ func (c *commandable) ClusterMeet(host, port string) *StatusCmd { return cmd } +func (c *commandable) ClusterForget(nodeID string) *StatusCmd { + cmd := newKeylessStatusCmd("CLUSTER", "forget", nodeID) + c.Process(cmd) + return cmd +} + func (c *commandable) ClusterReplicate(nodeID string) *StatusCmd { cmd := newKeylessStatusCmd("CLUSTER", "replicate", nodeID) c.Process(cmd) From 401979b5976cbe8dfd45e727f7bd186db26f31a3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 10 Dec 2015 09:52:42 +0200 Subject: [PATCH 0103/1746] Accept interface{} values in list and set commands. --- commands.go | 44 ++++++++++++++++++-------------------------- commands_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/commands.go b/commands.go index f023e48d32..8b7a53e316 100644 --- a/commands.go +++ b/commands.go @@ -767,19 +767,17 @@ func (c *commandable) LPop(key string) *StringCmd { return cmd } -func (c *commandable) LPush(key string, values ...string) *IntCmd { - args := make([]interface{}, 2+len(values)) +func (c *commandable) LPush(key string, values ...interface{}) *IntCmd { + args := make([]interface{}, 2, 2+len(values)) args[0] = "LPUSH" args[1] = key - for i, value := range values { - args[2+i] = value - } + args = append(args, values...) cmd := NewIntCmd(args...) c.Process(cmd) return cmd } -func (c *commandable) LPushX(key, value string) *IntCmd { +func (c *commandable) LPushX(key, value interface{}) *IntCmd { cmd := NewIntCmd("LPUSHX", key, value) c.Process(cmd) return cmd @@ -796,13 +794,13 @@ func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { return cmd } -func (c *commandable) LRem(key string, count int64, value string) *IntCmd { +func (c *commandable) LRem(key string, count int64, value interface{}) *IntCmd { cmd := NewIntCmd("LREM", key, count, value) c.Process(cmd) return cmd } -func (c *commandable) LSet(key string, index int64, value string) *StatusCmd { +func (c *commandable) LSet(key string, index int64, value interface{}) *StatusCmd { cmd := NewStatusCmd("LSET", key, index, value) c.Process(cmd) return cmd @@ -831,19 +829,17 @@ func (c *commandable) RPopLPush(source, destination string) *StringCmd { return cmd } -func (c *commandable) RPush(key string, values ...string) *IntCmd { - args := make([]interface{}, 2+len(values)) +func (c *commandable) RPush(key string, values ...interface{}) *IntCmd { + args := make([]interface{}, 2, 2+len(values)) args[0] = "RPUSH" args[1] = key - for i, value := range values { - args[2+i] = value - } + args = append(args, values...) cmd := NewIntCmd(args...) c.Process(cmd) return cmd } -func (c *commandable) RPushX(key string, value string) *IntCmd { +func (c *commandable) RPushX(key string, value interface{}) *IntCmd { cmd := NewIntCmd("RPUSHX", key, value) c.Process(cmd) return cmd @@ -851,13 +847,11 @@ func (c *commandable) RPushX(key string, value string) *IntCmd { //------------------------------------------------------------------------------ -func (c *commandable) SAdd(key string, members ...string) *IntCmd { - args := make([]interface{}, 2+len(members)) +func (c *commandable) SAdd(key string, members ...interface{}) *IntCmd { + args := make([]interface{}, 2, 2+len(members)) args[0] = "SADD" args[1] = key - for i, member := range members { - args[2+i] = member - } + args = append(args, members...) cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -915,7 +909,7 @@ func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { return cmd } -func (c *commandable) SIsMember(key, member string) *BoolCmd { +func (c *commandable) SIsMember(key string, member interface{}) *BoolCmd { cmd := NewBoolCmd("SISMEMBER", key, member) c.Process(cmd) return cmd @@ -927,7 +921,7 @@ func (c *commandable) SMembers(key string) *StringSliceCmd { return cmd } -func (c *commandable) SMove(source, destination, member string) *BoolCmd { +func (c *commandable) SMove(source, destination string, member interface{}) *BoolCmd { cmd := NewBoolCmd("SMOVE", source, destination, member) c.Process(cmd) return cmd @@ -953,13 +947,11 @@ func (c *commandable) SRandMemberN(key string, count int64) *StringSliceCmd { return cmd } -func (c *commandable) SRem(key string, members ...string) *IntCmd { - args := make([]interface{}, 2+len(members)) +func (c *commandable) SRem(key string, members ...interface{}) *IntCmd { + args := make([]interface{}, 2, 2+len(members)) args[0] = "SREM" args[1] = key - for i, member := range members { - args[2+i] = member - } + args = append(args, members...) cmd := NewIntCmd(args...) c.Process(cmd) return cmd diff --git a/commands_test.go b/commands_test.go index 327cc26405..af49e8219f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1379,6 +1379,17 @@ var _ = Describe("Commands", func() { Expect(lRange.Val()).To(Equal([]string{"Hello", "World"})) }) + It("should LPush bytes", func() { + lPush := client.LPush("list", []byte("World")) + Expect(lPush.Err()).NotTo(HaveOccurred()) + lPush = client.LPush("list", []byte("Hello")) + Expect(lPush.Err()).NotTo(HaveOccurred()) + + lRange := client.LRange("list", 0, -1) + Expect(lRange.Err()).NotTo(HaveOccurred()) + Expect(lRange.Val()).To(Equal([]string{"Hello", "World"})) + }) + It("should LPushX", func() { lPush := client.LPush("list", "World") Expect(lPush.Err()).NotTo(HaveOccurred()) @@ -1578,6 +1589,24 @@ var _ = Describe("Commands", func() { Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) }) + It("should SAdd bytes", func() { + sAdd := client.SAdd("set", []byte("Hello")) + Expect(sAdd.Err()).NotTo(HaveOccurred()) + Expect(sAdd.Val()).To(Equal(int64(1))) + + sAdd = client.SAdd("set", []byte("World")) + Expect(sAdd.Err()).NotTo(HaveOccurred()) + Expect(sAdd.Val()).To(Equal(int64(1))) + + sAdd = client.SAdd("set", []byte("World")) + Expect(sAdd.Err()).NotTo(HaveOccurred()) + Expect(sAdd.Val()).To(Equal(int64(0))) + + sMembers := client.SMembers("set") + Expect(sMembers.Err()).NotTo(HaveOccurred()) + Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) + }) + It("should SCard", func() { sAdd := client.SAdd("set", "Hello") Expect(sAdd.Err()).NotTo(HaveOccurred()) From 12edede26a3200ae050bde1e17384af3811ab63a Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Sat, 12 Dec 2015 17:41:49 +0200 Subject: [PATCH 0104/1746] Revert "Accept interface{} values in list and set commands." This reverts commit 401979b5976cbe8dfd45e727f7bd186db26f31a3. --- commands.go | 44 ++++++++++++++++++++++++++------------------ commands_test.go | 29 ----------------------------- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/commands.go b/commands.go index 8b7a53e316..f023e48d32 100644 --- a/commands.go +++ b/commands.go @@ -767,17 +767,19 @@ func (c *commandable) LPop(key string) *StringCmd { return cmd } -func (c *commandable) LPush(key string, values ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(values)) +func (c *commandable) LPush(key string, values ...string) *IntCmd { + args := make([]interface{}, 2+len(values)) args[0] = "LPUSH" args[1] = key - args = append(args, values...) + for i, value := range values { + args[2+i] = value + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd } -func (c *commandable) LPushX(key, value interface{}) *IntCmd { +func (c *commandable) LPushX(key, value string) *IntCmd { cmd := NewIntCmd("LPUSHX", key, value) c.Process(cmd) return cmd @@ -794,13 +796,13 @@ func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { return cmd } -func (c *commandable) LRem(key string, count int64, value interface{}) *IntCmd { +func (c *commandable) LRem(key string, count int64, value string) *IntCmd { cmd := NewIntCmd("LREM", key, count, value) c.Process(cmd) return cmd } -func (c *commandable) LSet(key string, index int64, value interface{}) *StatusCmd { +func (c *commandable) LSet(key string, index int64, value string) *StatusCmd { cmd := NewStatusCmd("LSET", key, index, value) c.Process(cmd) return cmd @@ -829,17 +831,19 @@ func (c *commandable) RPopLPush(source, destination string) *StringCmd { return cmd } -func (c *commandable) RPush(key string, values ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(values)) +func (c *commandable) RPush(key string, values ...string) *IntCmd { + args := make([]interface{}, 2+len(values)) args[0] = "RPUSH" args[1] = key - args = append(args, values...) + for i, value := range values { + args[2+i] = value + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd } -func (c *commandable) RPushX(key string, value interface{}) *IntCmd { +func (c *commandable) RPushX(key string, value string) *IntCmd { cmd := NewIntCmd("RPUSHX", key, value) c.Process(cmd) return cmd @@ -847,11 +851,13 @@ func (c *commandable) RPushX(key string, value interface{}) *IntCmd { //------------------------------------------------------------------------------ -func (c *commandable) SAdd(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(members)) +func (c *commandable) SAdd(key string, members ...string) *IntCmd { + args := make([]interface{}, 2+len(members)) args[0] = "SADD" args[1] = key - args = append(args, members...) + for i, member := range members { + args[2+i] = member + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd @@ -909,7 +915,7 @@ func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { return cmd } -func (c *commandable) SIsMember(key string, member interface{}) *BoolCmd { +func (c *commandable) SIsMember(key, member string) *BoolCmd { cmd := NewBoolCmd("SISMEMBER", key, member) c.Process(cmd) return cmd @@ -921,7 +927,7 @@ func (c *commandable) SMembers(key string) *StringSliceCmd { return cmd } -func (c *commandable) SMove(source, destination string, member interface{}) *BoolCmd { +func (c *commandable) SMove(source, destination, member string) *BoolCmd { cmd := NewBoolCmd("SMOVE", source, destination, member) c.Process(cmd) return cmd @@ -947,11 +953,13 @@ func (c *commandable) SRandMemberN(key string, count int64) *StringSliceCmd { return cmd } -func (c *commandable) SRem(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(members)) +func (c *commandable) SRem(key string, members ...string) *IntCmd { + args := make([]interface{}, 2+len(members)) args[0] = "SREM" args[1] = key - args = append(args, members...) + for i, member := range members { + args[2+i] = member + } cmd := NewIntCmd(args...) c.Process(cmd) return cmd diff --git a/commands_test.go b/commands_test.go index af49e8219f..327cc26405 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1379,17 +1379,6 @@ var _ = Describe("Commands", func() { Expect(lRange.Val()).To(Equal([]string{"Hello", "World"})) }) - It("should LPush bytes", func() { - lPush := client.LPush("list", []byte("World")) - Expect(lPush.Err()).NotTo(HaveOccurred()) - lPush = client.LPush("list", []byte("Hello")) - Expect(lPush.Err()).NotTo(HaveOccurred()) - - lRange := client.LRange("list", 0, -1) - Expect(lRange.Err()).NotTo(HaveOccurred()) - Expect(lRange.Val()).To(Equal([]string{"Hello", "World"})) - }) - It("should LPushX", func() { lPush := client.LPush("list", "World") Expect(lPush.Err()).NotTo(HaveOccurred()) @@ -1589,24 +1578,6 @@ var _ = Describe("Commands", func() { Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) }) - It("should SAdd bytes", func() { - sAdd := client.SAdd("set", []byte("Hello")) - Expect(sAdd.Err()).NotTo(HaveOccurred()) - Expect(sAdd.Val()).To(Equal(int64(1))) - - sAdd = client.SAdd("set", []byte("World")) - Expect(sAdd.Err()).NotTo(HaveOccurred()) - Expect(sAdd.Val()).To(Equal(int64(1))) - - sAdd = client.SAdd("set", []byte("World")) - Expect(sAdd.Err()).NotTo(HaveOccurred()) - Expect(sAdd.Val()).To(Equal(int64(0))) - - sMembers := client.SMembers("set") - Expect(sMembers.Err()).NotTo(HaveOccurred()) - Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) - }) - It("should SCard", func() { sAdd := client.SAdd("set", "Hello") Expect(sAdd.Err()).NotTo(HaveOccurred()) From cbcdd97ca24b3f9f8c2967569d509b54b06df881 Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Sat, 12 Dec 2015 18:52:23 +0200 Subject: [PATCH 0105/1746] commands.go: fix input types. --- commands.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/commands.go b/commands.go index f023e48d32..f48906cf4a 100644 --- a/commands.go +++ b/commands.go @@ -779,7 +779,7 @@ func (c *commandable) LPush(key string, values ...string) *IntCmd { return cmd } -func (c *commandable) LPushX(key, value string) *IntCmd { +func (c *commandable) LPushX(key, value interface{}) *IntCmd { cmd := NewIntCmd("LPUSHX", key, value) c.Process(cmd) return cmd @@ -796,13 +796,13 @@ func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { return cmd } -func (c *commandable) LRem(key string, count int64, value string) *IntCmd { +func (c *commandable) LRem(key string, count int64, value interface{}) *IntCmd { cmd := NewIntCmd("LREM", key, count, value) c.Process(cmd) return cmd } -func (c *commandable) LSet(key string, index int64, value string) *StatusCmd { +func (c *commandable) LSet(key string, index int64, value interface{}) *StatusCmd { cmd := NewStatusCmd("LSET", key, index, value) c.Process(cmd) return cmd @@ -843,7 +843,7 @@ func (c *commandable) RPush(key string, values ...string) *IntCmd { return cmd } -func (c *commandable) RPushX(key string, value string) *IntCmd { +func (c *commandable) RPushX(key string, value interface{}) *IntCmd { cmd := NewIntCmd("RPUSHX", key, value) c.Process(cmd) return cmd @@ -915,7 +915,7 @@ func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { return cmd } -func (c *commandable) SIsMember(key, member string) *BoolCmd { +func (c *commandable) SIsMember(key string, member interface{}) *BoolCmd { cmd := NewBoolCmd("SISMEMBER", key, member) c.Process(cmd) return cmd @@ -927,7 +927,7 @@ func (c *commandable) SMembers(key string) *StringSliceCmd { return cmd } -func (c *commandable) SMove(source, destination, member string) *BoolCmd { +func (c *commandable) SMove(source, destination string, member interface{}) *BoolCmd { cmd := NewBoolCmd("SMOVE", source, destination, member) c.Process(cmd) return cmd From 9079a66323abc637fe1c48575ee0e2c9337a6400 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 16 Dec 2015 16:11:52 +0200 Subject: [PATCH 0106/1746] cluster: add Watch support. --- cluster.go | 11 ++++++++ cluster_pipeline.go | 5 ++-- cluster_test.go | 45 +++++++++++++++++++++++++++++++++ multi.go | 4 +-- multi_test.go | 61 ++++++++++++++++++++++++++++++--------------- 5 files changed, 102 insertions(+), 24 deletions(-) diff --git a/cluster.go b/cluster.go index ab06890b51..43bfc8df3d 100644 --- a/cluster.go +++ b/cluster.go @@ -44,6 +44,17 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return client } +// Watch creates new transaction and marks the keys to be watched +// for conditional execution of a transaction. +func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { + addr := c.slotMasterAddr(hashSlot(keys[0])) + client, err := c.getClient(addr) + if err != nil { + return nil, err + } + return client.Watch(keys...) +} + // Close closes the cluster client, releasing any open resources. // // It is rare to Close a ClusterClient, as the ClusterClient is meant diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 3c93bbf0f7..eb5cd2d0c1 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -4,9 +4,10 @@ package redis type ClusterPipeline struct { commandable - cmds []Cmder cluster *ClusterClient - closed bool + + cmds []Cmder + closed bool } // Pipeline creates a new pipeline which is able to execute commands diff --git a/cluster_test.go b/cluster_test.go index d875735f26..d409dcb515 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -5,7 +5,9 @@ import ( "math/rand" "net" "reflect" + "strconv" "strings" + "sync" "testing" "time" @@ -317,6 +319,49 @@ var _ = Describe("Cluster", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("MOVED")) }) + + It("should Watch", func() { + var incr func(string) error + + // Transactionally increments key using GET and SET commands. + incr = func(key string) error { + tx, err := client.Watch(key) + if err != nil { + return err + } + defer tx.Close() + + n, err := tx.Get(key).Int64() + if err != nil && err != redis.Nil { + return err + } + + _, err = tx.Exec(func() error { + tx.Set(key, strconv.FormatInt(n+1, 10), 0) + return nil + }) + if err == redis.TxFailedErr { + return incr(key) + } + return err + } + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + err := incr("key") + Expect(err).NotTo(HaveOccurred()) + }() + } + wg.Wait() + + n, err := client.Get("key").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(100))) + }) }) }) diff --git a/multi.go b/multi.go index bb4fafe7b8..9b9df75c4d 100644 --- a/multi.go +++ b/multi.go @@ -23,8 +23,8 @@ type Multi struct { closed bool } -// Watch marks the keys to be watched for conditional execution -// of a transaction. +// Watch creates new transaction and marks the keys to be watched +// for conditional execution of a transaction. func (c *Client) Watch(keys ...string) (*Multi, error) { tx := c.Multi() if err := tx.Watch(keys...).Err(); err != nil { diff --git a/multi_test.go b/multi_test.go index c95c703bd1..1e6f360366 100644 --- a/multi_test.go +++ b/multi_test.go @@ -1,6 +1,9 @@ package redis_test import ( + "strconv" + "sync" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -21,29 +24,47 @@ var _ = Describe("Multi", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("should exec", func() { - multi := client.Multi() - defer func() { - Expect(multi.Close()).NotTo(HaveOccurred()) - }() + It("should Watch", func() { + var incr func(string) error - var ( - set *redis.StatusCmd - get *redis.StringCmd - ) - cmds, err := multi.Exec(func() error { - set = multi.Set("key", "hello", 0) - get = multi.Get("key") - return nil - }) - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(2)) + // Transactionally increments key using GET and SET commands. + incr = func(key string) error { + tx, err := client.Watch(key) + if err != nil { + return err + } + defer tx.Close() - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) + n, err := tx.Get(key).Int64() + if err != nil && err != redis.Nil { + return err + } - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal("hello")) + _, err = tx.Exec(func() error { + tx.Set(key, strconv.FormatInt(n+1, 10), 0) + return nil + }) + if err == redis.TxFailedErr { + return incr(key) + } + return err + } + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + err := incr("key") + Expect(err).NotTo(HaveOccurred()) + }() + } + wg.Wait() + + n, err := client.Get("key").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(100))) }) It("should discard", func() { From e335934332e0e748414fa8a3c85e99d2c9567260 Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Mon, 21 Dec 2015 18:53:02 +0200 Subject: [PATCH 0107/1746] commands: add cluster reset function. --- commands.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/commands.go b/commands.go index 8b7a53e316..420cb71d87 100644 --- a/commands.go +++ b/commands.go @@ -1666,6 +1666,18 @@ func (c *commandable) ClusterReplicate(nodeID string) *StatusCmd { return cmd } +func (c *commandable) ClusterResetSoft() *StatusCmd { + cmd := newKeylessStatusCmd("CLUSTER", "reset", "soft") + c.Process(cmd) + return cmd +} + +func (c *commandable) ClusterResetHard() *StatusCmd { + cmd := newKeylessStatusCmd("CLUSTER", "reset", "hard") + c.Process(cmd) + return cmd +} + func (c *commandable) ClusterInfo() *StringCmd { cmd := NewStringCmd("CLUSTER", "info") cmd._clusterKeyPos = 0 From 9782e280cfd8e7abae94093ac559bad981815d9c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 22 Dec 2015 11:02:18 +0200 Subject: [PATCH 0108/1746] Improve nil reply parsing. --- commands_test.go | 13 ++++++++----- multi.go | 7 +++---- parser.go | 20 +++++++------------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/commands_test.go b/commands_test.go index 327cc26405..9796a37d22 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1298,12 +1298,15 @@ var _ = Describe("Commands", func() { }) It("should BRPopLPush", func() { - rPush := client.RPush("list1", "a", "b", "c") - Expect(rPush.Err()).NotTo(HaveOccurred()) + _, err := client.BRPopLPush("list1", "list2", time.Second).Result() + Expect(err).To(Equal(redis.Nil)) - bRPopLPush := client.BRPopLPush("list1", "list2", 0) - Expect(bRPopLPush.Err()).NotTo(HaveOccurred()) - Expect(bRPopLPush.Val()).To(Equal("c")) + err = client.RPush("list1", "a", "b", "c").Err() + Expect(err).NotTo(HaveOccurred()) + + v, err := client.BRPopLPush("list1", "list2", 0).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(v).To(Equal("c")) }) It("should LIndex", func() { diff --git a/multi.go b/multi.go index 9b9df75c4d..320e78eb11 100644 --- a/multi.go +++ b/multi.go @@ -174,6 +174,9 @@ func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { // Parse number of replies. line, err := readLine(cn) if err != nil { + if err == Nil { + err = TxFailedErr + } setCmdsErr(cmds[1:len(cmds)-1], err) return err } @@ -182,10 +185,6 @@ func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { setCmdsErr(cmds[1:len(cmds)-1], err) return err } - if len(line) == 3 && line[1] == '-' && line[2] == '1' { - setCmdsErr(cmds[1:len(cmds)-1], TxFailedErr) - return TxFailedErr - } var firstCmdErr error diff --git a/parser.go b/parser.go index 43274fc36d..94fa597979 100644 --- a/parser.go +++ b/parser.go @@ -231,9 +231,16 @@ func readLine(cn *conn) ([]byte, error) { if isPrefix { return line, errReaderTooSmall } + if isNilReply(line) { + return nil, Nil + } return line, nil } +func isNilReply(b []byte) bool { + return len(b) == 3 && (b[0] == '$' || b[0] == '*') && b[1] == '-' && b[2] == '1' +} + func readN(cn *conn, n int) ([]byte, error) { var b []byte if cap(cn.buf) < n { @@ -251,17 +258,6 @@ func parseErrorReply(cn *conn, line []byte) error { return errorf(string(line[1:])) } -func isNilReply(b []byte) bool { - return len(b) == 3 && b[1] == '-' && b[2] == '1' -} - -func parseNilReply(cn *conn, line []byte) error { - if isNilReply(line) { - return Nil - } - return fmt.Errorf("redis: can't parse nil reply: %.100", line) -} - func parseStatusReply(cn *conn, line []byte) ([]byte, error) { return line[1:], nil } @@ -282,8 +278,6 @@ func readIntReply(cn *conn) (int64, error) { switch line[0] { case errorReply: return 0, parseErrorReply(cn, line) - case stringReply: - return 0, parseNilReply(cn, line) case intReply: return parseIntReply(cn, line) default: From 36487d84626d74a6600ee7a06017b7e6d9fa1187 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 22 Dec 2015 11:44:49 +0200 Subject: [PATCH 0109/1746] Stabilize build. --- Makefile | 1 - cluster.go | 10 +++++----- cluster_client_test.go | 2 +- example_test.go | 2 +- main_test.go | 40 ++++++++++++++++++++++++++++------------ ring_test.go | 2 +- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index bda28e3543..e9fdd93b2b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ all: testdeps go test ./... -test.v -test.cpu=1,2,4 - sleep 3 # give Redis time to exit go test ./... -test.v -test.short -test.race testdeps: .test/redis/src/redis-server diff --git a/cluster.go b/cluster.go index 43bfc8df3d..418844e704 100644 --- a/cluster.go +++ b/cluster.go @@ -64,7 +64,7 @@ func (c *ClusterClient) Close() error { c.clientsMx.Lock() if c.closed { - return nil + return errClosed } c.closed = true c.resetClients() @@ -197,14 +197,14 @@ func (c *ClusterClient) process(cmd Cmder) { } // Closes all clients and returns last error if there are any. -func (c *ClusterClient) resetClients() (err error) { +func (c *ClusterClient) resetClients() (retErr error) { for addr, client := range c.clients { - if e := client.Close(); e != nil { - err = e + if err := client.Close(); err != nil && retErr == nil { + retErr = err } delete(c.clients, addr) } - return err + return retErr } func (c *ClusterClient) setSlots(slots []ClusterSlotInfo) { diff --git a/cluster_client_test.go b/cluster_client_test.go index c7f695d448..9502ba05b8 100644 --- a/cluster_client_test.go +++ b/cluster_client_test.go @@ -37,7 +37,7 @@ var _ = Describe("ClusterClient", func() { }) AfterEach(func() { - subject.Close() + _ = subject.Close() }) It("should initialize", func() { diff --git a/example_test.go b/example_test.go index ff99256214..bb98fdd5fb 100644 --- a/example_test.go +++ b/example_test.go @@ -254,7 +254,7 @@ func ExamplePubSub_Receive() { for i := 0; i < 2; i++ { // ReceiveTimeout is a low level API. Use ReceiveMessage instead. - msgi, err := pubsub.ReceiveTimeout(100 * time.Millisecond) + msgi, err := pubsub.ReceiveTimeout(500 * time.Millisecond) if err != nil { panic(err) } diff --git a/main_test.go b/main_test.go index 806d7d3d10..471c8ee956 100644 --- a/main_test.go +++ b/main_test.go @@ -1,6 +1,7 @@ package redis_test import ( + "errors" "net" "os" "os/exec" @@ -130,20 +131,19 @@ func execCmd(name string, args ...string) (*os.Process, error) { return cmd.Process, cmd.Start() } -func connectTo(port string) (client *redis.Client, err error) { - client = redis.NewClient(&redis.Options{ +func connectTo(port string) (*redis.Client, error) { + client := redis.NewClient(&redis.Options{ Addr: ":" + port, }) - deadline := time.Now().Add(3 * time.Second) - for time.Now().Before(deadline) { - if err = client.Ping().Err(); err == nil { - return client, nil - } - time.Sleep(250 * time.Millisecond) + err := eventually(func() error { + return client.Ping().Err() + }, 10*time.Second) + if err != nil { + return nil, err } - return nil, err + return client, nil } type redisProcess struct { @@ -152,8 +152,22 @@ type redisProcess struct { } func (p *redisProcess) Close() error { + if err := p.Kill(); err != nil { + return err + } + + err := eventually(func() error { + if err := p.Client.Ping().Err(); err != nil { + return nil + } + return errors.New("client is not shutdown") + }, 10*time.Second) + if err != nil { + return err + } + p.Client.Close() - return p.Kill() + return nil } var ( @@ -165,9 +179,11 @@ func redisDir(port string) (string, error) { dir, err := filepath.Abs(filepath.Join(".test", "instances", port)) if err != nil { return "", err - } else if err = os.RemoveAll(dir); err != nil { + } + if err := os.RemoveAll(dir); err != nil { return "", err - } else if err = os.MkdirAll(dir, 0775); err != nil { + } + if err := os.MkdirAll(dir, 0775); err != nil { return "", err } return dir, nil diff --git a/ring_test.go b/ring_test.go index 0de37dc4a9..7d5cd91cae 100644 --- a/ring_test.go +++ b/ring_test.go @@ -49,7 +49,7 @@ var _ = Describe("Redis ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) }) - It("uses one shard when other shard is down", func() { + It("uses single shard when one of the shards is down", func() { // Stop ringShard2. Expect(ringShard2.Close()).NotTo(HaveOccurred()) From d7c44c7899316631fdd417940d8ca18d0a018d43 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 22 Dec 2015 15:45:03 +0200 Subject: [PATCH 0110/1746] Better rate limited message. --- .travis.yml | 1 - multi.go | 15 +++++++-------- pool.go | 45 ++++++++++++++++++++++++++++++--------------- pool_test.go | 26 ++++++++++++++++++++++---- pubsub.go | 6 +++--- redis.go | 7 +++---- sentinel.go | 6 ++++-- 7 files changed, 69 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d3148f707..dc4191acb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ services: - redis-server go: - - 1.3 - 1.4 - 1.5 - tip diff --git a/multi.go b/multi.go index 320e78eb11..0e0281d346 100644 --- a/multi.go +++ b/multi.go @@ -46,16 +46,15 @@ func (c *Client) Multi() *Multi { return multi } -func (c *Multi) putConn(cn *conn, ei error) { - var err error - if isBadConn(cn, ei) { +func (c *Multi) putConn(cn *conn, err error) { + if isBadConn(cn, err) { // Close current connection. - c.base.connPool.(*stickyConnPool).Reset() + c.base.connPool.(*stickyConnPool).Reset(err) } else { - err = c.base.connPool.Put(cn) - } - if err != nil { - log.Printf("redis: putConn failed: %s", err) + err := c.base.connPool.Put(cn) + if err != nil { + log.Printf("redis: putConn failed: %s", err) + } } } diff --git a/pool.go b/pool.go index 9968bd7ec9..5ed720979c 100644 --- a/pool.go +++ b/pool.go @@ -20,7 +20,7 @@ type pool interface { First() *conn Get() (*conn, bool, error) Put(*conn) error - Remove(*conn) error + Remove(*conn, error) error Len() int FreeLen() int Close() error @@ -130,7 +130,7 @@ type connPool struct { _closed int32 - lastDialErr error + lastErr atomic.Value } func newConnPool(opt *Options) *connPool { @@ -204,15 +204,15 @@ func (p *connPool) wait() *conn { func (p *connPool) new() (*conn, error) { if p.rl.Limit() { err := fmt.Errorf( - "redis: you open connections too fast (last error: %v)", - p.lastDialErr, + "redis: you open connections too fast (last_error=%q)", + p.loadLastErr(), ) return nil, err } cn, err := p.dialer() if err != nil { - p.lastDialErr = err + p.storeLastErr(err.Error()) return nil, err } @@ -255,8 +255,9 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { func (p *connPool) Put(cn *conn) error { if cn.rd.Buffered() != 0 { b, _ := cn.rd.Peek(cn.rd.Buffered()) - log.Printf("redis: connection has unread data: %q", b) - return p.Remove(cn) + err := fmt.Errorf("redis: connection has unread data: %q", b) + log.Print(err) + return p.Remove(cn, err) } if p.opt.getIdleTimeout() > 0 { cn.usedAt = time.Now() @@ -275,7 +276,9 @@ func (p *connPool) replace(cn *conn) (*conn, error) { return newcn, nil } -func (p *connPool) Remove(cn *conn) error { +func (p *connPool) Remove(cn *conn, reason error) error { + p.storeLastErr(reason.Error()) + // Replace existing connection with new one and unblock waiter. newcn, err := p.replace(cn) if err != nil { @@ -330,6 +333,17 @@ func (p *connPool) reaper() { } } +func (p *connPool) storeLastErr(err string) { + p.lastErr.Store(err) +} + +func (p *connPool) loadLastErr() string { + if v := p.lastErr.Load(); v != nil { + return v.(string) + } + return "" +} + //------------------------------------------------------------------------------ type singleConnPool struct { @@ -357,7 +371,7 @@ func (p *singleConnPool) Put(cn *conn) error { return nil } -func (p *singleConnPool) Remove(cn *conn) error { +func (p *singleConnPool) Remove(cn *conn, _ error) error { if p.cn != cn { panic("p.cn != cn") } @@ -440,13 +454,13 @@ func (p *stickyConnPool) Put(cn *conn) error { return nil } -func (p *stickyConnPool) remove() (err error) { - err = p.pool.Remove(p.cn) +func (p *stickyConnPool) remove(reason error) (err error) { + err = p.pool.Remove(p.cn, reason) p.cn = nil return err } -func (p *stickyConnPool) Remove(cn *conn) error { +func (p *stickyConnPool) Remove(cn *conn, _ error) error { defer p.mx.Unlock() p.mx.Lock() if p.closed { @@ -479,10 +493,10 @@ func (p *stickyConnPool) FreeLen() int { return 0 } -func (p *stickyConnPool) Reset() (err error) { +func (p *stickyConnPool) Reset(reason error) (err error) { p.mx.Lock() if p.cn != nil { - err = p.remove() + err = p.remove(reason) } p.mx.Unlock() return err @@ -500,7 +514,8 @@ func (p *stickyConnPool) Close() error { if p.reusable { err = p.put() } else { - err = p.remove() + reason := errors.New("redis: sticky not reusable connection") + err = p.remove(reason) } } return err diff --git a/pool_test.go b/pool_test.go index 9eb2c990dd..4d787a6878 100644 --- a/pool_test.go +++ b/pool_test.go @@ -1,6 +1,7 @@ package redis_test import ( + "errors" "sync" "testing" "time" @@ -36,7 +37,6 @@ var _ = Describe("pool", func() { }) AfterEach(func() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -141,12 +141,12 @@ var _ = Describe("pool", func() { pool := client.Pool() // Reserve one connection. - cn, _, err := client.Pool().Get() + cn, _, err := pool.Get() Expect(err).NotTo(HaveOccurred()) // Reserve the rest of connections. for i := 0; i < 9; i++ { - _, _, err := client.Pool().Get() + _, _, err := pool.Get() Expect(err).NotTo(HaveOccurred()) } @@ -168,7 +168,8 @@ var _ = Describe("pool", func() { // ok } - Expect(pool.Remove(cn)).NotTo(HaveOccurred()) + err = pool.Remove(cn, errors.New("test")) + Expect(err).NotTo(HaveOccurred()) // Check that Ping is unblocked. select { @@ -179,6 +180,23 @@ var _ = Describe("pool", func() { } Expect(ping.Err()).NotTo(HaveOccurred()) }) + + It("should rate limit dial", func() { + pool := client.Pool() + + var rateErr error + for i := 0; i < 1000; i++ { + cn, _, err := pool.Get() + if err != nil { + rateErr = err + break + } + + _ = pool.Remove(cn, errors.New("test")) + } + + Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) + }) }) func BenchmarkPool(b *testing.B) { diff --git a/pubsub.go b/pubsub.go index 3e20fe7209..aea2bed7b8 100644 --- a/pubsub.go +++ b/pubsub.go @@ -233,9 +233,9 @@ func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } -func (c *PubSub) reconnect() { +func (c *PubSub) reconnect(reason error) { // Close current connection. - c.connPool.(*stickyConnPool).Reset() + c.connPool.(*stickyConnPool).Reset(reason) if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { @@ -276,7 +276,7 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { if errNum > 2 { time.Sleep(time.Second) } - c.reconnect() + c.reconnect(err) continue } diff --git a/redis.go b/redis.go index cd88cefd0b..087564fde4 100644 --- a/redis.go +++ b/redis.go @@ -20,10 +20,9 @@ func (c *baseClient) conn() (*conn, bool, error) { return c.connPool.Get() } -func (c *baseClient) putConn(cn *conn, ei error) { - var err error - if isBadConn(cn, ei) { - err = c.connPool.Remove(cn) +func (c *baseClient) putConn(cn *conn, err error) { + if isBadConn(cn, err) { + err = c.connPool.Remove(cn, err) } else { err = c.connPool.Put(cn) } diff --git a/sentinel.go b/sentinel.go index 63c011d471..7edd75f266 100644 --- a/sentinel.go +++ b/sentinel.go @@ -2,6 +2,7 @@ package redis import ( "errors" + "fmt" "log" "net" "strings" @@ -227,11 +228,12 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { break } if cn.RemoteAddr().String() != newMaster { - log.Printf( + err := fmt.Errorf( "redis-sentinel: closing connection to the old master %s", cn.RemoteAddr(), ) - d.pool.Remove(cn) + log.Print(err) + d.pool.Remove(cn, err) } else { cnsToPut = append(cnsToPut, cn) } From a6da93713abbcc558cb4a23435f7d2108f9c2c5f Mon Sep 17 00:00:00 2001 From: Rich Hong Date: Wed, 23 Dec 2015 17:24:42 -0500 Subject: [PATCH 0111/1746] Support multiple keys for the PFCOUNT command --- commands.go | 9 +++++++-- commands_test.go | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index d5350022ac..231d47215f 100644 --- a/commands.go +++ b/commands.go @@ -1340,8 +1340,13 @@ func (c *commandable) PFAdd(key string, fields ...string) *IntCmd { return cmd } -func (c *commandable) PFCount(key string) *IntCmd { - cmd := NewIntCmd("PFCOUNT", key) +func (c *commandable) PFCount(keys ...string) *IntCmd { + args := make([]interface{}, 1+len(keys)) + args[0] = "PFCOUNT" + for i, key := range keys { + args[1+i] = key + } + cmd := NewIntCmd(args...) c.Process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index 9796a37d22..c592090aca 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1203,6 +1203,10 @@ var _ = Describe("Commands", func() { pfCount = client.PFCount("hllMerged") Expect(pfCount.Err()).NotTo(HaveOccurred()) Expect(pfCount.Val()).To(Equal(int64(10))) + + pfCount = client.PFCount("hll1", "hll2") + Expect(pfCount.Err()).NotTo(HaveOccurred()) + Expect(pfCount.Val()).To(Equal(int64(10))) }) }) From 9b1148903efb856265efb32d3a562d21c06d928b Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Mon, 28 Dec 2015 18:58:04 +0200 Subject: [PATCH 0112/1746] commands.go: Add ClusterKeySlot function. --- cluster_test.go | 5 +++++ commands.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/cluster_test.go b/cluster_test.go index d409dcb515..a585ad78e6 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -229,6 +229,11 @@ var _ = Describe("Cluster", func() { Expect(res).To(ContainSubstring("cluster_known_nodes:6")) }) + It("should CLUSTER KEYSLOT", func() { + res, err := cluster.primary().ClusterKeySlot("somekey").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(int64(11058))) + }) }) Describe("Client", func() { diff --git a/commands.go b/commands.go index 231d47215f..ef615b0c57 100644 --- a/commands.go +++ b/commands.go @@ -1698,6 +1698,13 @@ func (c *commandable) ClusterInfo() *StringCmd { return cmd } +func (c *commandable) ClusterKeySlot(key string) *IntCmd { + cmd := NewIntCmd("CLUSTER", "keyslot", key) + cmd._clusterKeyPos = 2 + c.Process(cmd) + return cmd +} + func (c *commandable) ClusterFailover() *StatusCmd { cmd := newKeylessStatusCmd("CLUSTER", "failover") c.Process(cmd) From cbc5360e78f2d61bacb68d5e336493d6ff6ae491 Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Tue, 29 Dec 2015 18:16:14 +0200 Subject: [PATCH 0113/1746] commands.go: Add new functions to cluster. --- cluster_test.go | 48 ++++++++++++++++++++++++++++++++++++-- commands.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index a585ad78e6..fe004c9691 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -230,9 +230,53 @@ var _ = Describe("Cluster", func() { }) It("should CLUSTER KEYSLOT", func() { - res, err := cluster.primary().ClusterKeySlot("somekey").Result() + hashSlot, err := cluster.primary().ClusterKeySlot("somekey").Result() Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(int64(11058))) + Expect(hashSlot).To(Equal(int64(11058))) + }) + + It("should CLUSTER COUNT-FAILURE-REPORTS", func() { + n, err := cluster.primary().ClusterCountFailureReports(cluster.nodeIds[0]).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) + }) + + It("should CLUSTER COUNTKEYSINSLOT", func() { + n, err := cluster.primary().ClusterCountKeysInSlot(10).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) + }) + + It("should CLUSTER DELSLOTS", func() { + res, err := cluster.primary().ClusterDelSlotsRange(16000, 16384-1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal("OK")) + cluster.primary().ClusterAddSlotsRange(16000, 16384-1) + }) + + It("should CLUSTER SAVECONFIG", func() { + res, err := cluster.primary().ClusterSaveConfig().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal("OK")) + }) + + It("should CLUSTER SLAVES", func() { + nodesList, err := cluster.primary().ClusterSlaves(cluster.nodeIds[0]).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(nodesList).Should(ContainElement(ContainSubstring("slave"))) + Expect(nodesList).Should(HaveLen(1)) + }) + + It("should CLUSTER READONLY", func() { + res, err := cluster.primary().Readonly().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal("OK")) + }) + + It("should CLUSTER READWRITE", func() { + res, err := cluster.primary().ReadWrite().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal("OK")) }) }) diff --git a/commands.go b/commands.go index ef615b0c57..215f414c94 100644 --- a/commands.go +++ b/commands.go @@ -1705,6 +1705,68 @@ func (c *commandable) ClusterKeySlot(key string) *IntCmd { return cmd } +func (c *commandable) ClusterCountFailureReports(nodeID string) *IntCmd { + cmd := NewIntCmd("CLUSTER", "count-failure-reports", nodeID) + cmd._clusterKeyPos = 2 + c.Process(cmd) + return cmd +} + +func (c *commandable) ClusterCountKeysInSlot(slot int) *IntCmd { + cmd := NewIntCmd("CLUSTER", "countkeysinslot", slot) + cmd._clusterKeyPos = 2 + c.Process(cmd) + return cmd +} + +func (c *commandable) ClusterDelSlots(slots ...int) *StatusCmd { + args := make([]interface{}, 2+len(slots)) + args[0] = "CLUSTER" + args[1] = "DELSLOTS" + for i, slot := range slots { + args[2+i] = slot + } + cmd := newKeylessStatusCmd(args...) + c.Process(cmd) + return cmd +} + +func (c *commandable) ClusterDelSlotsRange(min, max int) *StatusCmd { + size := max - min + 1 + slots := make([]int, size) + for i := 0; i < size; i++ { + slots[i] = min + i + } + return c.ClusterDelSlots(slots...) +} + +func (c *commandable) ClusterSaveConfig() *StatusCmd { + cmd := newKeylessStatusCmd("CLUSTER", "saveconfig") + c.Process(cmd) + return cmd +} + +func (c *commandable) ClusterSlaves(nodeID string) *StringSliceCmd { + cmd := NewStringSliceCmd("CLUSTER", "SLAVES", nodeID) + cmd._clusterKeyPos = 2 + c.Process(cmd) + return cmd +} + +func (c *commandable) Readonly() *StatusCmd { + cmd := newKeylessStatusCmd("READONLY") + cmd._clusterKeyPos = 0 + c.Process(cmd) + return cmd +} + +func (c *commandable) ReadWrite() *StatusCmd { + cmd := newKeylessStatusCmd("READWRITE") + cmd._clusterKeyPos = 0 + c.Process(cmd) + return cmd +} + func (c *commandable) ClusterFailover() *StatusCmd { cmd := newKeylessStatusCmd("CLUSTER", "failover") c.Process(cmd) From 6eec22a5e51b8ba811da5928f0a2e7569acd3715 Mon Sep 17 00:00:00 2001 From: "Dolf Schimmel (Freeaqingme)" Date: Thu, 31 Dec 2015 00:43:28 +0100 Subject: [PATCH 0114/1746] Bugfix: pubsub/psubscribe should register a pattern rather than channel --- pubsub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub.go b/pubsub.go index 3e20fe7209..6de231ff51 100644 --- a/pubsub.go +++ b/pubsub.go @@ -74,7 +74,7 @@ func (c *PubSub) Subscribe(channels ...string) error { func (c *PubSub) PSubscribe(patterns ...string) error { err := c.subscribe("PSUBSCRIBE", patterns...) if err == nil { - c.channels = append(c.channels, patterns...) + c.patterns = append(c.patterns, patterns...) } return err } From 1739cd9380173ef4f513f792cb0ddb0d9e5c20f0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 31 Dec 2015 11:27:28 +0200 Subject: [PATCH 0115/1746] pubsub: add PSubscribe test. Updates #233. --- pubsub.go | 2 +- pubsub_test.go | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pubsub.go b/pubsub.go index 3ee00e88a4..031eb84380 100644 --- a/pubsub.go +++ b/pubsub.go @@ -244,7 +244,7 @@ func (c *PubSub) reconnect(reason error) { } if len(c.patterns) > 0 { if err := c.PSubscribe(c.patterns...); err != nil { - log.Printf("redis: Subscribe failed: %s", err) + log.Printf("redis: PSubscribe failed: %s", err) } } } diff --git a/pubsub_test.go b/pubsub_test.go index 64938203b2..bf940d4777 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -255,11 +255,7 @@ var _ = Describe("PubSub", func() { wg.Wait() }) - It("should reconnect on ReceiveMessage error", func() { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) - defer pubsub.Close() - + expectReceiveMessage := func(pubsub *redis.PubSub) { cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn1.SetNetConn(&badConn{ @@ -284,6 +280,22 @@ var _ = Describe("PubSub", func() { Expect(msg.Payload).To(Equal("hello")) wg.Wait() + } + + It("Subscribe should reconnect on ReceiveMessage error", func() { + pubsub, err := client.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + expectReceiveMessage(pubsub) + }) + + It("PSubscribe should reconnect on ReceiveMessage error", func() { + pubsub, err := client.PSubscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + expectReceiveMessage(pubsub) }) It("should return on Close", func() { From 0bf3759a6dbf6f0fbb93e5c4d625a9fd638d0769 Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Wed, 30 Dec 2015 15:53:45 +0200 Subject: [PATCH 0116/1746] Create hashtag package. --- cluster.go | 34 ++++--------------- cluster_pipeline.go | 6 +++- cluster_test.go | 13 +++---- export_test.go | 4 --- crc16.go => internal/hashtag/hashtag.go | 28 ++++++++++++++- .../hashtag/hashtag_test.go | 2 +- ring.go | 5 +-- 7 files changed, 49 insertions(+), 43 deletions(-) rename crc16.go => internal/hashtag/hashtag.go (83%) rename crc16_test.go => internal/hashtag/hashtag_test.go (96%) diff --git a/cluster.go b/cluster.go index 418844e704..2015a85e89 100644 --- a/cluster.go +++ b/cluster.go @@ -3,10 +3,11 @@ package redis import ( "log" "math/rand" - "strings" "sync" "sync/atomic" "time" + + "gopkg.in/redis.v3/internal/hashtag" ) // ClusterClient is a Redis Cluster client representing a pool of zero @@ -34,7 +35,7 @@ type ClusterClient struct { func NewClusterClient(opt *ClusterOptions) *ClusterClient { client := &ClusterClient{ addrs: opt.Addrs, - slots: make([][]string, hashSlots), + slots: make([][]string, hashtag.SlotNumber), clients: make(map[string]*Client), opt: opt, } @@ -47,7 +48,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { // Watch creates new transaction and marks the keys to be watched // for conditional execution of a transaction. func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { - addr := c.slotMasterAddr(hashSlot(keys[0])) + addr := c.slotMasterAddr(hashtag.Slot(keys[0])) client, err := c.getClient(addr) if err != nil { return nil, err @@ -138,7 +139,7 @@ func (c *ClusterClient) randomClient() (client *Client, err error) { func (c *ClusterClient) process(cmd Cmder) { var ask bool - slot := hashSlot(cmd.clusterKey()) + slot := hashtag.Slot(cmd.clusterKey()) addr := c.slotMasterAddr(slot) client, err := c.getClient(addr) @@ -215,7 +216,7 @@ func (c *ClusterClient) setSlots(slots []ClusterSlotInfo) { seen[addr] = struct{}{} } - for i := 0; i < hashSlots; i++ { + for i := 0; i < hashtag.SlotNumber; i++ { c.slots[i] = c.slots[i][:0] } for _, info := range slots { @@ -333,26 +334,3 @@ func (opt *ClusterOptions) clientOptions() *Options { IdleTimeout: opt.IdleTimeout, } } - -//------------------------------------------------------------------------------ - -const hashSlots = 16384 - -func hashKey(key string) string { - if s := strings.IndexByte(key, '{'); s > -1 { - if e := strings.IndexByte(key[s+1:], '}'); e > 0 { - return key[s+1 : s+e+1] - } - } - return key -} - -// hashSlot returns a consistent slot number between 0 and 16383 -// for any given string key. -func hashSlot(key string) int { - key = hashKey(key) - if key == "" { - return rand.Intn(hashSlots) - } - return int(crc16sum(key)) % hashSlots -} diff --git a/cluster_pipeline.go b/cluster_pipeline.go index eb5cd2d0c1..73d272bbf7 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -1,5 +1,9 @@ package redis +import ( + "gopkg.in/redis.v3/internal/hashtag" +) + // ClusterPipeline is not thread-safe. type ClusterPipeline struct { commandable @@ -48,7 +52,7 @@ func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - slot := hashSlot(cmd.clusterKey()) + slot := hashtag.Slot(cmd.clusterKey()) addr := pipe.cluster.slotMasterAddr(slot) cmdsMap[addr] = append(cmdsMap[addr], cmd) } diff --git a/cluster_test.go b/cluster_test.go index fe004c9691..8f6b5e64c7 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" "gopkg.in/redis.v3" + "gopkg.in/redis.v3/internal/hashtag" ) type clusterScenario struct { @@ -182,7 +183,7 @@ var _ = Describe("Cluster", func() { rand.Seed(100) for _, test := range tests { - Expect(redis.HashSlot(test.key)).To(Equal(test.slot), "for %s", test.key) + Expect(hashtag.Slot(test.key)).To(Equal(test.slot), "for %s", test.key) } }) @@ -198,7 +199,7 @@ var _ = Describe("Cluster", func() { } for _, test := range tests { - Expect(redis.HashSlot(test.one)).To(Equal(redis.HashSlot(test.two)), "for %s <-> %s", test.one, test.two) + Expect(hashtag.Slot(test.one)).To(Equal(hashtag.Slot(test.two)), "for %s <-> %s", test.one, test.two) } }) @@ -232,7 +233,7 @@ var _ = Describe("Cluster", func() { It("should CLUSTER KEYSLOT", func() { hashSlot, err := cluster.primary().ClusterKeySlot("somekey").Result() Expect(err).NotTo(HaveOccurred()) - Expect(hashSlot).To(Equal(int64(11058))) + Expect(hashSlot).To(Equal(int64(hashtag.Slot("somekey")))) }) It("should CLUSTER COUNT-FAILURE-REPORTS", func() { @@ -315,7 +316,7 @@ var _ = Describe("Cluster", func() { It("should follow redirects", func() { Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) - slot := redis.HashSlot("A") + slot := hashtag.Slot("A") Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) val, err := client.Get("A").Result() @@ -328,7 +329,7 @@ var _ = Describe("Cluster", func() { }) It("should perform multi-pipelines", func() { - slot := redis.HashSlot("A") + slot := hashtag.Slot("A") Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) pipe := client.Pipeline() @@ -361,7 +362,7 @@ var _ = Describe("Cluster", func() { MaxRedirects: -1, }) - slot := redis.HashSlot("A") + slot := hashtag.Slot("A") Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) err := client.Get("A").Err() diff --git a/export_test.go b/export_test.go index 66ccec25d0..e7b4b05669 100644 --- a/export_test.go +++ b/export_test.go @@ -11,7 +11,3 @@ var NewConnDialer = newConnDialer func (cn *conn) SetNetConn(netcn net.Conn) { cn.netcn = netcn } - -func HashSlot(key string) int { - return hashSlot(key) -} diff --git a/crc16.go b/internal/hashtag/hashtag.go similarity index 83% rename from crc16.go rename to internal/hashtag/hashtag.go index a7f3b569c1..2866488e59 100644 --- a/crc16.go +++ b/internal/hashtag/hashtag.go @@ -1,4 +1,11 @@ -package redis +package hashtag + +import ( + "math/rand" + "strings" +) + +const SlotNumber = 16384 // CRC16 implementation according to CCITT standards. // Copyright 2001-2010 Georges Menie (www.menie.org) @@ -39,6 +46,25 @@ var crc16tab = [256]uint16{ 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, } +func Key(key string) string { + if s := strings.IndexByte(key, '{'); s > -1 { + if e := strings.IndexByte(key[s+1:], '}'); e > 0 { + return key[s+1 : s+e+1] + } + } + return key +} + +// hashSlot returns a consistent slot number between 0 and 16383 +// for any given string key. +func Slot(key string) int { + key = Key(key) + if key == "" { + return rand.Intn(SlotNumber) + } + return int(crc16sum(key)) % SlotNumber +} + func crc16sum(key string) (crc uint16) { for i := 0; i < len(key); i++ { crc = (crc << 8) ^ crc16tab[(byte(crc>>8)^key[i])&0x00ff] diff --git a/crc16_test.go b/internal/hashtag/hashtag_test.go similarity index 96% rename from crc16_test.go rename to internal/hashtag/hashtag_test.go index a6b34162ce..8132878d79 100644 --- a/crc16_test.go +++ b/internal/hashtag/hashtag_test.go @@ -1,4 +1,4 @@ -package redis +package hashtag import ( . "github.com/onsi/ginkgo" diff --git a/ring.go b/ring.go index ff77f4d6f2..d6bcf8053b 100644 --- a/ring.go +++ b/ring.go @@ -8,6 +8,7 @@ import ( "time" "gopkg.in/redis.v3/internal/consistenthash" + "gopkg.in/redis.v3/internal/hashtag" ) var ( @@ -151,7 +152,7 @@ func (ring *Ring) getClient(key string) (*Client, error) { return nil, errClosed } - name := ring.hash.Get(hashKey(key)) + name := ring.hash.Get(hashtag.Key(key)) if name == "" { ring.mx.RUnlock() return nil, errRingShardsDown @@ -297,7 +298,7 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - name := pipe.ring.hash.Get(hashKey(cmd.clusterKey())) + name := pipe.ring.hash.Get(hashtag.Key(cmd.clusterKey())) if name == "" { cmd.setErr(errRingShardsDown) if retErr == nil { From c510761c7641fa99466564f1e9e5d1db4adb8026 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 4 Jan 2016 12:40:28 +0200 Subject: [PATCH 0117/1746] readme: add link to rate limiting package. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 25b9c819f5..e6232428a2 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Supports: - [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). - [Ring](http://godoc.org/gopkg.in/redis.v3#NewRing). - [Cache friendly](https://github.com/go-redis/cache). +- [Rate limiting](https://github.com/go-redis/rate). API docs: http://godoc.org/gopkg.in/redis.v3. Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. From be6b602480cb8ae627b5230855b477602595c852 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 6 Jan 2016 16:16:25 +0000 Subject: [PATCH 0118/1746] Mention distributed locks --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e6232428a2..13887132dc 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Supports: - [Ring](http://godoc.org/gopkg.in/redis.v3#NewRing). - [Cache friendly](https://github.com/go-redis/cache). - [Rate limiting](https://github.com/go-redis/rate). +- [Distributed Locks](https://github.com/bsm/redis-lock). API docs: http://godoc.org/gopkg.in/redis.v3. Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. From 602824623be4ccc422a7bcced725e54a760ba55d Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Fri, 8 Jan 2016 15:03:34 +0200 Subject: [PATCH 0119/1746] commands.go: add section parameter to Info function. --- commands.go | 9 ++++++--- commands_test.go | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 215f414c94..d6cb800dc7 100644 --- a/commands.go +++ b/commands.go @@ -1454,9 +1454,12 @@ func (c *commandable) FlushDb() *StatusCmd { return cmd } -func (c *commandable) Info() *StringCmd { - cmd := NewStringCmd("INFO") - cmd._clusterKeyPos = 0 +func (c *commandable) Info(section ...string) *StringCmd { + args := []interface{}{"INFO"} + if len(section) > 0 { + args = append(args, section[0]) + } + cmd := NewStringCmd(args...) c.Process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index c592090aca..f7d5502e2b 100644 --- a/commands_test.go +++ b/commands_test.go @@ -130,6 +130,13 @@ var _ = Describe("Commands", func() { Expect(info.Val()).NotTo(Equal("")) }) + It("should Info cpu", func() { + info := client.Info("cpu") + Expect(info.Err()).NotTo(HaveOccurred()) + Expect(info.Val()).NotTo(Equal("")) + Expect(info.Val()).To(ContainSubstring(`used_cpu_sys`)) + }) + It("should LastSave", func() { lastSave := client.LastSave() Expect(lastSave.Err()).NotTo(HaveOccurred()) From f7a4bd5023cc73904b5e3bd6355837a990d7d5e7 Mon Sep 17 00:00:00 2001 From: Anatolii Mihailenco Date: Tue, 19 Jan 2016 18:36:40 +0200 Subject: [PATCH 0120/1746] Add pool instrumentation. --- cluster.go | 17 +++++++++++++++++ cluster_test.go | 4 ++++ pool.go | 26 ++++++++++++++++++++++++++ redis.go | 6 ++++++ redis_test.go | 4 ++++ 5 files changed, 57 insertions(+) diff --git a/cluster.go b/cluster.go index 2015a85e89..d3139fd760 100644 --- a/cluster.go +++ b/cluster.go @@ -56,6 +56,22 @@ func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { return client.Watch(keys...) } +// PoolStats returns accumulated connection pool stats +func (c *ClusterClient) PoolStats() *PoolStats { + acc := PoolStats{} + c.clientsMx.RLock() + for _, client := range c.clients { + m := client.PoolStats() + acc.TotalConns += m.TotalConns + acc.FreeConns += m.FreeConns + acc.Requests += m.Requests + acc.Waits += m.Waits + acc.Timeouts += m.Timeouts + } + c.clientsMx.RUnlock() + return &acc +} + // Close closes the cluster client, releasing any open resources. // // It is rare to Close a ClusterClient, as the ClusterClient is meant @@ -306,6 +322,7 @@ type ClusterOptions struct { ReadTimeout time.Duration WriteTimeout time.Duration + // PoolSize applies per redis node and not for the whole cluster. PoolSize int PoolTimeout time.Duration IdleTimeout time.Duration diff --git a/cluster_test.go b/cluster_test.go index 8f6b5e64c7..80e1fbba0f 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -313,6 +313,10 @@ var _ = Describe("Cluster", func() { Expect(cnt).To(Equal(int64(1))) }) + It("should return pool stats", func() { + Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) + }) + It("should follow redirects", func() { Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) diff --git a/pool.go b/pool.go index 5ed720979c..d007f1a7ca 100644 --- a/pool.go +++ b/pool.go @@ -16,6 +16,16 @@ var ( errPoolTimeout = errors.New("redis: connection pool timeout") ) +// PoolStats contains pool state information and accumulated stats +type PoolStats struct { + Requests uint64 // number of times a connection was requested by the pool + Waits uint64 // number of times our pool had to wait for a connection to avail + Timeouts uint64 // number of times a wait timeout occurred + + TotalConns uint64 // the number of total connections in the pool + FreeConns uint64 // the number of free connections in the pool +} + type pool interface { First() *conn Get() (*conn, bool, error) @@ -24,6 +34,7 @@ type pool interface { Len() int FreeLen() int Close() error + Stats() PoolStats } type connList struct { @@ -127,6 +138,7 @@ type connPool struct { opt *Options conns *connList freeConns chan *conn + stats PoolStats _closed int32 @@ -226,6 +238,8 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { return } + atomic.AddUint64(&p.stats.Requests, 1) + // Fetch first non-idle connection, if available. if cn = p.First(); cn != nil { return @@ -244,10 +258,12 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { } // Otherwise, wait for the available connection. + atomic.AddUint64(&p.stats.Waits, 1) if cn = p.wait(); cn != nil { return } + atomic.AddUint64(&p.stats.Timeouts, 1) err = errPoolTimeout return } @@ -298,6 +314,12 @@ func (p *connPool) FreeLen() int { return len(p.freeConns) } +func (p *connPool) Stats() PoolStats { + p.stats.TotalConns = uint64(p.Len()) + p.stats.FreeConns = uint64(p.FreeLen()) + return p.stats +} + func (p *connPool) Close() (retErr error) { if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { return errClosed @@ -386,6 +408,8 @@ func (p *singleConnPool) FreeLen() int { return 0 } +func (p *singleConnPool) Stats() PoolStats { return PoolStats{} } + func (p *singleConnPool) Close() error { return nil } @@ -493,6 +517,8 @@ func (p *stickyConnPool) FreeLen() int { return 0 } +func (p *stickyConnPool) Stats() PoolStats { return PoolStats{} } + func (p *stickyConnPool) Reset(reason error) (err error) { p.mx.Lock() if p.cn != nil { diff --git a/redis.go b/redis.go index 087564fde4..abfbc6bb49 100644 --- a/redis.go +++ b/redis.go @@ -192,3 +192,9 @@ func NewClient(opt *Options) *Client { pool := newConnPool(opt) return newClient(opt, pool) } + +// PoolStats returns connection pool stats +func (c *Client) PoolStats() *PoolStats { + stats := c.baseClient.connPool.Stats() + return &stats +} diff --git a/redis_test.go b/redis_test.go index f1ebf62a24..38222025a8 100644 --- a/redis_test.go +++ b/redis_test.go @@ -35,6 +35,10 @@ var _ = Describe("Client", func() { Expect(val).To(Equal("PONG")) }) + It("should return pool stats", func() { + Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) + }) + It("should support custom dialers", func() { custom := redis.NewClient(&redis.Options{ Dialer: func() (net.Conn, error) { From 3ed364e92ae4c5a756349a48f0eba281ebe01180 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 22 Jan 2016 12:29:23 +0200 Subject: [PATCH 0121/1746] Sort can return nil reply. Fixes #246. --- commands.go | 14 +++++++++-- commands_test.go | 63 +++++++++++++++++++++++++++++++++++++++--------- parser.go | 7 ++++-- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/commands.go b/commands.go index d6cb800dc7..f8a80f5616 100644 --- a/commands.go +++ b/commands.go @@ -270,7 +270,7 @@ type Sort struct { Store string } -func (c *commandable) Sort(key string, sort Sort) *StringSliceCmd { +func (sort *Sort) args(key string) []interface{} { args := []interface{}{"SORT", key} if sort.By != "" { args = append(args, "BY", sort.By) @@ -290,7 +290,17 @@ func (c *commandable) Sort(key string, sort Sort) *StringSliceCmd { if sort.Store != "" { args = append(args, "STORE", sort.Store) } - cmd := NewStringSliceCmd(args...) + return args +} + +func (c *commandable) Sort(key string, sort Sort) *StringSliceCmd { + cmd := NewStringSliceCmd(sort.args(key)...) + c.Process(cmd) + return cmd +} + +func (c *commandable) SortInterfaces(key string, sort Sort) *SliceCmd { + cmd := NewSliceCmd(sort.args(key)...) c.Process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index f7d5502e2b..b82b019704 100644 --- a/commands_test.go +++ b/commands_test.go @@ -500,19 +500,58 @@ var _ = Describe("Commands", func() { }) It("should Sort", func() { - lPush := client.LPush("list", "1") - Expect(lPush.Err()).NotTo(HaveOccurred()) - Expect(lPush.Val()).To(Equal(int64(1))) - lPush = client.LPush("list", "3") - Expect(lPush.Err()).NotTo(HaveOccurred()) - Expect(lPush.Val()).To(Equal(int64(2))) - lPush = client.LPush("list", "2") - Expect(lPush.Err()).NotTo(HaveOccurred()) - Expect(lPush.Val()).To(Equal(int64(3))) + size, err := client.LPush("list", "1").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(1))) - sort := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}) - Expect(sort.Err()).NotTo(HaveOccurred()) - Expect(sort.Val()).To(Equal([]string{"1", "2"})) + size, err = client.LPush("list", "3").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(2))) + + size, err = client.LPush("list", "2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(3))) + + els, err := client.Sort("list", redis.Sort{ + Offset: 0, + Count: 2, + Order: "ASC", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(els).To(Equal([]string{"1", "2"})) + }) + + It("should Sort and Get", func() { + size, err := client.LPush("list", "1").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(1))) + + size, err = client.LPush("list", "3").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(2))) + + size, err = client.LPush("list", "2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(3))) + + err = client.Set("object_2", "value2", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + { + els, err := client.Sort("list", redis.Sort{ + Get: []string{"object_*"}, + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(els).To(Equal([]string{"", "value2", ""})) + } + + { + els, err := client.SortInterfaces("list", redis.Sort{ + Get: []string{"object_*"}, + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(els).To(Equal([]interface{}{nil, "value2", nil})) + } }) It("should TTL", func() { diff --git a/parser.go b/parser.go index 94fa597979..741c02a31e 100644 --- a/parser.go +++ b/parser.go @@ -491,10 +491,13 @@ func stringSliceParser(cn *conn, n int64) (interface{}, error) { ss := make([]string, 0, n) for i := int64(0); i < n; i++ { s, err := readStringReply(cn) - if err != nil { + if err == Nil { + ss = append(ss, "") + } else if err != nil { return nil, err + } else { + ss = append(ss, s) } - ss = append(ss, s) } return ss, nil } From 6c7b789b3a9c21d9e2c805a04e0f2651ebefb5cd Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 25 Jan 2016 15:57:09 +0200 Subject: [PATCH 0122/1746] Tweak pool stats. --- cluster.go | 8 ++++---- pool.go | 22 +++++++++++++--------- redis.go | 3 +-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cluster.go b/cluster.go index d3139fd760..7291cb3036 100644 --- a/cluster.go +++ b/cluster.go @@ -56,17 +56,17 @@ func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { return client.Watch(keys...) } -// PoolStats returns accumulated connection pool stats +// PoolStats returns accumulated connection pool stats. func (c *ClusterClient) PoolStats() *PoolStats { acc := PoolStats{} c.clientsMx.RLock() for _, client := range c.clients { m := client.PoolStats() - acc.TotalConns += m.TotalConns - acc.FreeConns += m.FreeConns acc.Requests += m.Requests acc.Waits += m.Waits acc.Timeouts += m.Timeouts + acc.TotalConns += m.TotalConns + acc.FreeConns += m.FreeConns } c.clientsMx.RUnlock() return &acc @@ -322,7 +322,7 @@ type ClusterOptions struct { ReadTimeout time.Duration WriteTimeout time.Duration - // PoolSize applies per redis node and not for the whole cluster. + // PoolSize applies per cluster node and not for the whole cluster. PoolSize int PoolTimeout time.Duration IdleTimeout time.Duration diff --git a/pool.go b/pool.go index d007f1a7ca..76ba10df32 100644 --- a/pool.go +++ b/pool.go @@ -16,10 +16,10 @@ var ( errPoolTimeout = errors.New("redis: connection pool timeout") ) -// PoolStats contains pool state information and accumulated stats +// PoolStats contains pool state information and accumulated stats. type PoolStats struct { Requests uint64 // number of times a connection was requested by the pool - Waits uint64 // number of times our pool had to wait for a connection to avail + Waits uint64 // number of times our pool had to wait for a connection Timeouts uint64 // number of times a wait timeout occurred TotalConns uint64 // the number of total connections in the pool @@ -34,7 +34,7 @@ type pool interface { Len() int FreeLen() int Close() error - Stats() PoolStats + Stats() *PoolStats } type connList struct { @@ -314,10 +314,14 @@ func (p *connPool) FreeLen() int { return len(p.freeConns) } -func (p *connPool) Stats() PoolStats { - p.stats.TotalConns = uint64(p.Len()) - p.stats.FreeConns = uint64(p.FreeLen()) - return p.stats +func (p *connPool) Stats() *PoolStats { + stats := p.stats + stats.Requests = atomic.LoadUint64(&p.stats.Requests) + stats.Waits = atomic.LoadUint64(&p.stats.Waits) + stats.Timeouts = atomic.LoadUint64(&p.stats.Timeouts) + stats.TotalConns = uint64(p.Len()) + stats.FreeConns = uint64(p.FreeLen()) + return &stats } func (p *connPool) Close() (retErr error) { @@ -408,7 +412,7 @@ func (p *singleConnPool) FreeLen() int { return 0 } -func (p *singleConnPool) Stats() PoolStats { return PoolStats{} } +func (p *singleConnPool) Stats() *PoolStats { return nil } func (p *singleConnPool) Close() error { return nil @@ -517,7 +521,7 @@ func (p *stickyConnPool) FreeLen() int { return 0 } -func (p *stickyConnPool) Stats() PoolStats { return PoolStats{} } +func (p *stickyConnPool) Stats() *PoolStats { return nil } func (p *stickyConnPool) Reset(reason error) (err error) { p.mx.Lock() diff --git a/redis.go b/redis.go index abfbc6bb49..db22af6e2e 100644 --- a/redis.go +++ b/redis.go @@ -195,6 +195,5 @@ func NewClient(opt *Options) *Client { // PoolStats returns connection pool stats func (c *Client) PoolStats() *PoolStats { - stats := c.baseClient.connPool.Stats() - return &stats + return c.connPool.Stats() } From d3ee281748d0ffc71c4c9d36e9c3f7d6eecdde2b Mon Sep 17 00:00:00 2001 From: Francisco Souza Date: Wed, 3 Feb 2016 12:30:39 -0500 Subject: [PATCH 0123/1746] Declare and use a package-level Logger This allow users of the API to override the Logger. Fix #250. --- cluster.go | 5 ++--- commands.go | 5 ++--- multi.go | 5 ++--- pool.go | 7 +++---- pubsub.go | 7 +++---- redis.go | 5 ++++- ring.go | 3 +-- sentinel.go | 27 +++++++++++++-------------- 8 files changed, 30 insertions(+), 34 deletions(-) diff --git a/cluster.go b/cluster.go index 7291cb3036..817cb90ff6 100644 --- a/cluster.go +++ b/cluster.go @@ -1,7 +1,6 @@ package redis import ( - "log" "math/rand" "sync" "sync/atomic" @@ -256,13 +255,13 @@ func (c *ClusterClient) reloadSlots() { client, err := c.randomClient() if err != nil { - log.Printf("redis: randomClient failed: %s", err) + Logger.Printf("redis: randomClient failed: %s", err) return } slots, err := client.ClusterSlots().Result() if err != nil { - log.Printf("redis: ClusterSlots failed: %s", err) + Logger.Printf("redis: ClusterSlots failed: %s", err) return } c.setSlots(slots) diff --git a/commands.go b/commands.go index f8a80f5616..a3495eeaad 100644 --- a/commands.go +++ b/commands.go @@ -2,7 +2,6 @@ package redis import ( "io" - "log" "strconv" "time" ) @@ -32,7 +31,7 @@ func usePrecise(dur time.Duration) bool { func formatMs(dur time.Duration) string { if dur > 0 && dur < time.Millisecond { - log.Printf( + Logger.Printf( "redis: specified duration is %s, but minimal supported value is %s", dur, time.Millisecond, ) @@ -42,7 +41,7 @@ func formatMs(dur time.Duration) string { func formatSec(dur time.Duration) string { if dur > 0 && dur < time.Second { - log.Printf( + Logger.Printf( "redis: specified duration is %s, but minimal supported value is %s", dur, time.Second, ) diff --git a/multi.go b/multi.go index 0e0281d346..498951e0f7 100644 --- a/multi.go +++ b/multi.go @@ -3,7 +3,6 @@ package redis import ( "errors" "fmt" - "log" ) var errDiscard = errors.New("redis: Discard can be used only inside Exec") @@ -53,7 +52,7 @@ func (c *Multi) putConn(cn *conn, err error) { } else { err := c.base.connPool.Put(cn) if err != nil { - log.Printf("redis: putConn failed: %s", err) + Logger.Printf("redis: putConn failed: %s", err) } } } @@ -70,7 +69,7 @@ func (c *Multi) process(cmd Cmder) { func (c *Multi) Close() error { c.closed = true if err := c.Unwatch().Err(); err != nil { - log.Printf("redis: Unwatch failed: %s", err) + Logger.Printf("redis: Unwatch failed: %s", err) } return c.base.Close() } diff --git a/pool.go b/pool.go index 76ba10df32..af7773fd4c 100644 --- a/pool.go +++ b/pool.go @@ -3,7 +3,6 @@ package redis import ( "errors" "fmt" - "log" "sync" "sync/atomic" "time" @@ -178,7 +177,7 @@ func (p *connPool) First() *conn { var err error cn, err = p.replace(cn) if err != nil { - log.Printf("redis: replace failed: %s", err) + Logger.Printf("redis: replace failed: %s", err) continue } } @@ -200,7 +199,7 @@ func (p *connPool) wait() *conn { var err error cn, err = p.replace(cn) if err != nil { - log.Printf("redis: replace failed: %s", err) + Logger.Printf("redis: replace failed: %s", err) continue } } @@ -272,7 +271,7 @@ func (p *connPool) Put(cn *conn) error { if cn.rd.Buffered() != 0 { b, _ := cn.rd.Peek(cn.rd.Buffered()) err := fmt.Errorf("redis: connection has unread data: %q", b) - log.Print(err) + Logger.Print(err) return p.Remove(cn, err) } if p.opt.getIdleTimeout() > 0 { diff --git a/pubsub.go b/pubsub.go index 031eb84380..6baaeaf9f6 100644 --- a/pubsub.go +++ b/pubsub.go @@ -2,7 +2,6 @@ package redis import ( "fmt" - "log" "net" "time" ) @@ -239,12 +238,12 @@ func (c *PubSub) reconnect(reason error) { if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { - log.Printf("redis: Subscribe failed: %s", err) + Logger.Printf("redis: Subscribe failed: %s", err) } } if len(c.patterns) > 0 { if err := c.PSubscribe(c.patterns...); err != nil { - log.Printf("redis: PSubscribe failed: %s", err) + Logger.Printf("redis: PSubscribe failed: %s", err) } } } @@ -269,7 +268,7 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { if err == nil { continue } - log.Printf("redis: PubSub.Ping failed: %s", err) + Logger.Printf("redis: PubSub.Ping failed: %s", err) } } diff --git a/redis.go b/redis.go index db22af6e2e..aac6fbe448 100644 --- a/redis.go +++ b/redis.go @@ -4,9 +4,12 @@ import ( "fmt" "log" "net" + "os" "time" ) +var Logger = log.New(os.Stderr, "", log.LstdFlags) + type baseClient struct { connPool pool opt *Options @@ -27,7 +30,7 @@ func (c *baseClient) putConn(cn *conn, err error) { err = c.connPool.Put(cn) } if err != nil { - log.Printf("redis: putConn failed: %s", err) + Logger.Printf("redis: putConn failed: %s", err) } } diff --git a/ring.go b/ring.go index d6bcf8053b..1601221be3 100644 --- a/ring.go +++ b/ring.go @@ -3,7 +3,6 @@ package redis import ( "errors" "fmt" - "log" "sync" "time" @@ -202,7 +201,7 @@ func (ring *Ring) heartbeat() { for _, shard := range ring.shards { err := shard.Client.Ping().Err() if shard.Vote(err == nil || err == errPoolTimeout) { - log.Printf("redis: ring shard state changed: %s", shard) + Logger.Printf("redis: ring shard state changed: %s", shard) rebalance = true } } diff --git a/sentinel.go b/sentinel.go index 7edd75f266..462c9d3643 100644 --- a/sentinel.go +++ b/sentinel.go @@ -3,7 +3,6 @@ package redis import ( "errors" "fmt" - "log" "net" "strings" "sync" @@ -145,11 +144,11 @@ func (d *sentinelFailover) MasterAddr() (string, error) { if d._sentinel != nil { addr, err := d._sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - log.Printf("redis-sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + Logger.Printf("redis-sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) d.resetSentinel() } else { addr := net.JoinHostPort(addr[0], addr[1]) - log.Printf("redis-sentinel: %q addr is %s", d.masterName, addr) + Logger.Printf("redis-sentinel: %q addr is %s", d.masterName, addr) return addr, nil } } @@ -168,7 +167,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { }) masterAddr, err := sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - log.Printf("redis-sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + Logger.Printf("redis-sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) sentinel.Close() continue } @@ -178,7 +177,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { d.setSentinel(sentinel) addr := net.JoinHostPort(masterAddr[0], masterAddr[1]) - log.Printf("redis-sentinel: %q addr is %s", d.masterName, addr) + Logger.Printf("redis-sentinel: %q addr is %s", d.masterName, addr) return addr, nil } @@ -194,7 +193,7 @@ func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinels, err := sentinel.Sentinels(d.masterName).Result() if err != nil { - log.Printf("redis-sentinel: Sentinels %q failed: %s", d.masterName, err) + Logger.Printf("redis-sentinel: Sentinels %q failed: %s", d.masterName, err) return } for _, sentinel := range sentinels { @@ -204,7 +203,7 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { if key == "name" { sentinelAddr := vals[i+1].(string) if !contains(d.sentinelAddrs, sentinelAddr) { - log.Printf( + Logger.Printf( "redis-sentinel: discovered new %q sentinel: %s", d.masterName, sentinelAddr, ) @@ -232,7 +231,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { "redis-sentinel: closing connection to the old master %s", cn.RemoteAddr(), ) - log.Print(err) + Logger.Print(err) d.pool.Remove(cn, err) } else { cnsToPut = append(cnsToPut, cn) @@ -250,7 +249,7 @@ func (d *sentinelFailover) listen() { if pubsub == nil { pubsub = d._sentinel.PubSub() if err := pubsub.Subscribe("+switch-master"); err != nil { - log.Printf("redis-sentinel: Subscribe failed: %s", err) + Logger.Printf("redis-sentinel: Subscribe failed: %s", err) d.lock.Lock() d.resetSentinel() d.lock.Unlock() @@ -260,7 +259,7 @@ func (d *sentinelFailover) listen() { msgIface, err := pubsub.Receive() if err != nil { - log.Printf("redis-sentinel: Receive failed: %s", err) + Logger.Printf("redis-sentinel: Receive failed: %s", err) pubsub.Close() return } @@ -271,23 +270,23 @@ func (d *sentinelFailover) listen() { case "+switch-master": parts := strings.Split(msg.Payload, " ") if parts[0] != d.masterName { - log.Printf("redis-sentinel: ignore new %s addr", parts[0]) + Logger.Printf("redis-sentinel: ignore new %s addr", parts[0]) continue } addr := net.JoinHostPort(parts[3], parts[4]) - log.Printf( + Logger.Printf( "redis-sentinel: new %q addr is %s", d.masterName, addr, ) d.closeOldConns(addr) default: - log.Printf("redis-sentinel: unsupported message: %s", msg) + Logger.Printf("redis-sentinel: unsupported message: %s", msg) } case *Subscription: // Ignore. default: - log.Printf("redis-sentinel: unsupported message: %s", msgIface) + Logger.Printf("redis-sentinel: unsupported message: %s", msgIface) } } } From 2b2a6805dd26257c4ca6ce4ac6e488a3218218df Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 6 Feb 2016 11:45:34 +0200 Subject: [PATCH 0124/1746] Fix cluster slots parsing. --- command.go | 2 ++ parser.go | 21 ++++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/command.go b/command.go index 25f9dc53b7..1986c66110 100644 --- a/command.go +++ b/command.go @@ -725,12 +725,14 @@ func (cmd *ScanCmd) readReply(cn *conn) error { //------------------------------------------------------------------------------ +// TODO: rename to ClusterSlot type ClusterSlotInfo struct { Start int End int Addrs []string } +// TODO: rename to ClusterSlotsCmd type ClusterSlotCmd struct { baseCmd diff --git a/parser.go b/parser.go index 741c02a31e..b99ee896d9 100644 --- a/parser.go +++ b/parser.go @@ -107,7 +107,7 @@ func appendArg(b []byte, val interface{}) ([]byte, error) { } func appendArgs(b []byte, args []interface{}) ([]byte, error) { - b = append(b, '*') + b = append(b, arrayReply) b = strconv.AppendUint(b, uint64(len(args)), 10) b = append(b, '\r', '\n') for _, arg := range args { @@ -238,7 +238,9 @@ func readLine(cn *conn) ([]byte, error) { } func isNilReply(b []byte) bool { - return len(b) == 3 && (b[0] == '$' || b[0] == '*') && b[1] == '-' && b[2] == '1' + return len(b) == 3 && + (b[0] == stringReply || b[0] == arrayReply) && + b[1] == '-' && b[2] == '1' } func readN(cn *conn, n int) ([]byte, error) { @@ -337,7 +339,7 @@ func readFloatReply(cn *conn) (float64, error) { } func parseArrayHeader(cn *conn, line []byte) (int64, error) { - if len(line) == 3 && line[1] == '-' && line[2] == '1' { + if isNilReply(line) { return 0, Nil } @@ -604,8 +606,9 @@ func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { if err != nil { return nil, err } - if n != 2 { - return nil, fmt.Errorf("got %d elements in cluster info address, expected 2", n) + if n != 2 && n != 3 { + err := fmt.Errorf("got %d elements in cluster info address, expected 2 or 3", n) + return nil, err } ip, err := readStringReply(cn) @@ -618,6 +621,14 @@ func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { return nil, err } + if n == 3 { + // TODO: expose id in ClusterSlotInfo + _, err := readStringReply(cn) + if err != nil { + return nil, err + } + } + info.Addrs[i] = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) } From ebf51e1a1bef4ea30bd17f645d436c505c64710b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 6 Feb 2016 12:16:09 +0200 Subject: [PATCH 0125/1746] Add prefix to package logger. --- cluster.go | 4 ++-- commands.go | 4 ++-- multi.go | 4 ++-- pool.go | 6 +++--- pubsub.go | 6 +++--- redis.go | 4 ++-- ring.go | 2 +- sentinel.go | 30 +++++++++++++++--------------- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cluster.go b/cluster.go index 817cb90ff6..6e79cb5e73 100644 --- a/cluster.go +++ b/cluster.go @@ -255,13 +255,13 @@ func (c *ClusterClient) reloadSlots() { client, err := c.randomClient() if err != nil { - Logger.Printf("redis: randomClient failed: %s", err) + Logger.Printf("randomClient failed: %s", err) return } slots, err := client.ClusterSlots().Result() if err != nil { - Logger.Printf("redis: ClusterSlots failed: %s", err) + Logger.Printf("ClusterSlots failed: %s", err) return } c.setSlots(slots) diff --git a/commands.go b/commands.go index a3495eeaad..5e202ad640 100644 --- a/commands.go +++ b/commands.go @@ -32,7 +32,7 @@ func usePrecise(dur time.Duration) bool { func formatMs(dur time.Duration) string { if dur > 0 && dur < time.Millisecond { Logger.Printf( - "redis: specified duration is %s, but minimal supported value is %s", + "specified duration is %s, but minimal supported value is %s", dur, time.Millisecond, ) } @@ -42,7 +42,7 @@ func formatMs(dur time.Duration) string { func formatSec(dur time.Duration) string { if dur > 0 && dur < time.Second { Logger.Printf( - "redis: specified duration is %s, but minimal supported value is %s", + "specified duration is %s, but minimal supported value is %s", dur, time.Second, ) } diff --git a/multi.go b/multi.go index 498951e0f7..236bd30c13 100644 --- a/multi.go +++ b/multi.go @@ -52,7 +52,7 @@ func (c *Multi) putConn(cn *conn, err error) { } else { err := c.base.connPool.Put(cn) if err != nil { - Logger.Printf("redis: putConn failed: %s", err) + Logger.Printf("pool.Put failed: %s", err) } } } @@ -69,7 +69,7 @@ func (c *Multi) process(cmd Cmder) { func (c *Multi) Close() error { c.closed = true if err := c.Unwatch().Err(); err != nil { - Logger.Printf("redis: Unwatch failed: %s", err) + Logger.Printf("Unwatch failed: %s", err) } return c.base.Close() } diff --git a/pool.go b/pool.go index af7773fd4c..494eb9ee76 100644 --- a/pool.go +++ b/pool.go @@ -177,7 +177,7 @@ func (p *connPool) First() *conn { var err error cn, err = p.replace(cn) if err != nil { - Logger.Printf("redis: replace failed: %s", err) + Logger.Printf("pool.replace failed: %s", err) continue } } @@ -199,7 +199,7 @@ func (p *connPool) wait() *conn { var err error cn, err = p.replace(cn) if err != nil { - Logger.Printf("redis: replace failed: %s", err) + Logger.Printf("pool.replace failed: %s", err) continue } } @@ -270,7 +270,7 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { func (p *connPool) Put(cn *conn) error { if cn.rd.Buffered() != 0 { b, _ := cn.rd.Peek(cn.rd.Buffered()) - err := fmt.Errorf("redis: connection has unread data: %q", b) + err := fmt.Errorf("connection has unread data: %q", b) Logger.Print(err) return p.Remove(cn, err) } diff --git a/pubsub.go b/pubsub.go index 6baaeaf9f6..5c2d2e8bdc 100644 --- a/pubsub.go +++ b/pubsub.go @@ -238,12 +238,12 @@ func (c *PubSub) reconnect(reason error) { if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { - Logger.Printf("redis: Subscribe failed: %s", err) + Logger.Printf("Subscribe failed: %s", err) } } if len(c.patterns) > 0 { if err := c.PSubscribe(c.patterns...); err != nil { - Logger.Printf("redis: PSubscribe failed: %s", err) + Logger.Printf("PSubscribe failed: %s", err) } } } @@ -268,7 +268,7 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { if err == nil { continue } - Logger.Printf("redis: PubSub.Ping failed: %s", err) + Logger.Printf("PubSub.Ping failed: %s", err) } } diff --git a/redis.go b/redis.go index aac6fbe448..8372963a35 100644 --- a/redis.go +++ b/redis.go @@ -8,7 +8,7 @@ import ( "time" ) -var Logger = log.New(os.Stderr, "", log.LstdFlags) +var Logger = log.New(os.Stderr, "redis: ", log.LstdFlags) type baseClient struct { connPool pool @@ -30,7 +30,7 @@ func (c *baseClient) putConn(cn *conn, err error) { err = c.connPool.Put(cn) } if err != nil { - Logger.Printf("redis: putConn failed: %s", err) + Logger.Printf("pool.Put failed: %s", err) } } diff --git a/ring.go b/ring.go index 1601221be3..1d9b90251b 100644 --- a/ring.go +++ b/ring.go @@ -201,7 +201,7 @@ func (ring *Ring) heartbeat() { for _, shard := range ring.shards { err := shard.Client.Ping().Err() if shard.Vote(err == nil || err == errPoolTimeout) { - Logger.Printf("redis: ring shard state changed: %s", shard) + Logger.Printf("ring shard state changed: %s", shard) rebalance = true } } diff --git a/sentinel.go b/sentinel.go index 462c9d3643..bb95064632 100644 --- a/sentinel.go +++ b/sentinel.go @@ -144,11 +144,11 @@ func (d *sentinelFailover) MasterAddr() (string, error) { if d._sentinel != nil { addr, err := d._sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - Logger.Printf("redis-sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + Logger.Printf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) d.resetSentinel() } else { addr := net.JoinHostPort(addr[0], addr[1]) - Logger.Printf("redis-sentinel: %q addr is %s", d.masterName, addr) + Logger.Printf("sentinel: %q addr is %s", d.masterName, addr) return addr, nil } } @@ -167,7 +167,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { }) masterAddr, err := sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - Logger.Printf("redis-sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + Logger.Printf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) sentinel.Close() continue } @@ -177,7 +177,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { d.setSentinel(sentinel) addr := net.JoinHostPort(masterAddr[0], masterAddr[1]) - Logger.Printf("redis-sentinel: %q addr is %s", d.masterName, addr) + Logger.Printf("sentinel: %q addr is %s", d.masterName, addr) return addr, nil } @@ -193,7 +193,7 @@ func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinels, err := sentinel.Sentinels(d.masterName).Result() if err != nil { - Logger.Printf("redis-sentinel: Sentinels %q failed: %s", d.masterName, err) + Logger.Printf("sentinel: Sentinels %q failed: %s", d.masterName, err) return } for _, sentinel := range sentinels { @@ -204,7 +204,7 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinelAddr := vals[i+1].(string) if !contains(d.sentinelAddrs, sentinelAddr) { Logger.Printf( - "redis-sentinel: discovered new %q sentinel: %s", + "sentinel: discovered new %q sentinel: %s", d.masterName, sentinelAddr, ) d.sentinelAddrs = append(d.sentinelAddrs, sentinelAddr) @@ -228,7 +228,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { } if cn.RemoteAddr().String() != newMaster { err := fmt.Errorf( - "redis-sentinel: closing connection to the old master %s", + "sentinel: closing connection to the old master %s", cn.RemoteAddr(), ) Logger.Print(err) @@ -249,7 +249,7 @@ func (d *sentinelFailover) listen() { if pubsub == nil { pubsub = d._sentinel.PubSub() if err := pubsub.Subscribe("+switch-master"); err != nil { - Logger.Printf("redis-sentinel: Subscribe failed: %s", err) + Logger.Printf("sentinel: Subscribe failed: %s", err) d.lock.Lock() d.resetSentinel() d.lock.Unlock() @@ -257,36 +257,36 @@ func (d *sentinelFailover) listen() { } } - msgIface, err := pubsub.Receive() + msg, err := pubsub.Receive() if err != nil { - Logger.Printf("redis-sentinel: Receive failed: %s", err) + Logger.Printf("sentinel: Receive failed: %s", err) pubsub.Close() return } - switch msg := msgIface.(type) { + switch msg := msg.(type) { case *Message: switch msg.Channel { case "+switch-master": parts := strings.Split(msg.Payload, " ") if parts[0] != d.masterName { - Logger.Printf("redis-sentinel: ignore new %s addr", parts[0]) + Logger.Printf("sentinel: ignore new %s addr", parts[0]) continue } addr := net.JoinHostPort(parts[3], parts[4]) Logger.Printf( - "redis-sentinel: new %q addr is %s", + "sentinel: new %q addr is %s", d.masterName, addr, ) d.closeOldConns(addr) default: - Logger.Printf("redis-sentinel: unsupported message: %s", msg) + Logger.Printf("sentinel: unsupported message: %s", msg) } case *Subscription: // Ignore. default: - Logger.Printf("redis-sentinel: unsupported message: %s", msgIface) + Logger.Printf("sentinel: unsupported message: %s", msg) } } } From e750d2b7e27a245b1cb64af101cc4b1005d38fe4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 18 Feb 2016 12:42:35 +0200 Subject: [PATCH 0126/1746] Use uint32 because uint64 requires manual alignment on 386 arch. Fixes #256. --- pool.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pool.go b/pool.go index 494eb9ee76..87eb5ed670 100644 --- a/pool.go +++ b/pool.go @@ -17,12 +17,12 @@ var ( // PoolStats contains pool state information and accumulated stats. type PoolStats struct { - Requests uint64 // number of times a connection was requested by the pool - Waits uint64 // number of times our pool had to wait for a connection - Timeouts uint64 // number of times a wait timeout occurred + Requests uint32 // number of times a connection was requested by the pool + Waits uint32 // number of times our pool had to wait for a connection + Timeouts uint32 // number of times a wait timeout occurred - TotalConns uint64 // the number of total connections in the pool - FreeConns uint64 // the number of free connections in the pool + TotalConns uint32 // the number of total connections in the pool + FreeConns uint32 // the number of free connections in the pool } type pool interface { @@ -237,7 +237,7 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { return } - atomic.AddUint64(&p.stats.Requests, 1) + atomic.AddUint32(&p.stats.Requests, 1) // Fetch first non-idle connection, if available. if cn = p.First(); cn != nil { @@ -257,12 +257,12 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { } // Otherwise, wait for the available connection. - atomic.AddUint64(&p.stats.Waits, 1) + atomic.AddUint32(&p.stats.Waits, 1) if cn = p.wait(); cn != nil { return } - atomic.AddUint64(&p.stats.Timeouts, 1) + atomic.AddUint32(&p.stats.Timeouts, 1) err = errPoolTimeout return } @@ -315,11 +315,11 @@ func (p *connPool) FreeLen() int { func (p *connPool) Stats() *PoolStats { stats := p.stats - stats.Requests = atomic.LoadUint64(&p.stats.Requests) - stats.Waits = atomic.LoadUint64(&p.stats.Waits) - stats.Timeouts = atomic.LoadUint64(&p.stats.Timeouts) - stats.TotalConns = uint64(p.Len()) - stats.FreeConns = uint64(p.FreeLen()) + stats.Requests = atomic.LoadUint32(&p.stats.Requests) + stats.Waits = atomic.LoadUint32(&p.stats.Waits) + stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts) + stats.TotalConns = uint32(p.Len()) + stats.FreeConns = uint32(p.FreeLen()) return &stats } From 110e93a8e4e53c2a60d662d414afbdc5b21dff70 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 1 Mar 2016 12:31:06 +0200 Subject: [PATCH 0127/1746] Simplify connection management with sticky connection pool. Fixes #260. --- error.go | 10 +++---- export_test.go | 4 +++ main_test.go | 5 ---- multi.go | 14 +--------- multi_test.go | 27 ++++++++++++++++++ pool.go | 20 ++++--------- pubsub.go | 76 ++++++++++++++++++++++++++++++-------------------- pubsub_test.go | 55 ++++++++++++++++++++++++++---------- redis.go | 15 ++++++---- sentinel.go | 2 +- 10 files changed, 139 insertions(+), 89 deletions(-) diff --git a/error.go b/error.go index 1365ca1942..dce10a37bd 100644 --- a/error.go +++ b/error.go @@ -33,14 +33,14 @@ func isNetworkError(err error) bool { return ok } -func isBadConn(cn *conn, ei error) bool { - if cn.rd.Buffered() > 0 { - return true +func isBadConn(err error) bool { + if err == nil { + return false } - if ei == nil { + if _, ok := err.(redisError); ok { return false } - if _, ok := ei.(redisError); ok { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { return false } return true diff --git a/export_test.go b/export_test.go index e7b4b05669..4a6de2c658 100644 --- a/export_test.go +++ b/export_test.go @@ -6,6 +6,10 @@ func (c *baseClient) Pool() pool { return c.connPool } +func (c *PubSub) Pool() pool { + return c.base.connPool +} + var NewConnDialer = newConnDialer func (cn *conn) SetNetConn(netcn net.Conn) { diff --git a/main_test.go b/main_test.go index 471c8ee956..b9b3e218b5 100644 --- a/main_test.go +++ b/main_test.go @@ -7,7 +7,6 @@ import ( "os/exec" "path/filepath" "sync/atomic" - "syscall" "testing" "time" @@ -243,10 +242,6 @@ func startSentinel(port, masterName, masterPort string) (*redisProcess, error) { //------------------------------------------------------------------------------ -var ( - errTimeout = syscall.ETIMEDOUT -) - type badConnError string func (e badConnError) Error() string { return string(e) } diff --git a/multi.go b/multi.go index 236bd30c13..7ffc7e05e3 100644 --- a/multi.go +++ b/multi.go @@ -45,18 +45,6 @@ func (c *Client) Multi() *Multi { return multi } -func (c *Multi) putConn(cn *conn, err error) { - if isBadConn(cn, err) { - // Close current connection. - c.base.connPool.(*stickyConnPool).Reset(err) - } else { - err := c.base.connPool.Put(cn) - if err != nil { - Logger.Printf("pool.Put failed: %s", err) - } - } -} - func (c *Multi) process(cmd Cmder) { if c.cmds == nil { c.base.process(cmd) @@ -145,7 +133,7 @@ func (c *Multi) Exec(f func() error) ([]Cmder, error) { } err = c.execCmds(cn, cmds) - c.putConn(cn, err) + c.base.putConn(cn, err) return retCmds, err } diff --git a/multi_test.go b/multi_test.go index 1e6f360366..459d0a6222 100644 --- a/multi_test.go +++ b/multi_test.go @@ -166,4 +166,31 @@ var _ = Describe("Multi", func() { }) Expect(err).NotTo(HaveOccurred()) }) + + It("should recover from bad connection when there are no commands", func() { + // Put bad connection in the pool. + cn, _, err := client.Pool().Get() + Expect(err).NotTo(HaveOccurred()) + + cn.SetNetConn(&badConn{}) + err = client.Pool().Put(cn) + Expect(err).NotTo(HaveOccurred()) + + { + tx, err := client.Watch("key") + Expect(err).To(MatchError("bad connection")) + Expect(tx).To(BeNil()) + } + + { + tx, err := client.Watch("key") + Expect(err).NotTo(HaveOccurred()) + + err = tx.Ping().Err() + Expect(err).NotTo(HaveOccurred()) + + err = tx.Close() + Expect(err).NotTo(HaveOccurred()) + } + }) }) diff --git a/pool.go b/pool.go index 87eb5ed670..e2f9f2b392 100644 --- a/pool.go +++ b/pool.go @@ -246,13 +246,14 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { // Try to create a new one. if p.conns.Reserve() { + isNew = true + cn, err = p.new() if err != nil { p.conns.Remove(nil) return } p.conns.Add(cn) - isNew = true return } @@ -481,13 +482,13 @@ func (p *stickyConnPool) Put(cn *conn) error { return nil } -func (p *stickyConnPool) remove(reason error) (err error) { - err = p.pool.Remove(p.cn, reason) +func (p *stickyConnPool) remove(reason error) error { + err := p.pool.Remove(p.cn, reason) p.cn = nil return err } -func (p *stickyConnPool) Remove(cn *conn, _ error) error { +func (p *stickyConnPool) Remove(cn *conn, reason error) error { defer p.mx.Unlock() p.mx.Lock() if p.closed { @@ -499,7 +500,7 @@ func (p *stickyConnPool) Remove(cn *conn, _ error) error { if cn != nil && p.cn != cn { panic("p.cn != cn") } - return nil + return p.remove(reason) } func (p *stickyConnPool) Len() int { @@ -522,15 +523,6 @@ func (p *stickyConnPool) FreeLen() int { func (p *stickyConnPool) Stats() *PoolStats { return nil } -func (p *stickyConnPool) Reset(reason error) (err error) { - p.mx.Lock() - if p.cn != nil { - err = p.remove(reason) - } - p.mx.Unlock() - return err -} - func (p *stickyConnPool) Close() error { defer p.mx.Unlock() p.mx.Lock() diff --git a/pubsub.go b/pubsub.go index 5c2d2e8bdc..bde81b5edd 100644 --- a/pubsub.go +++ b/pubsub.go @@ -17,16 +17,18 @@ func (c *Client) Publish(channel, message string) *IntCmd { // http://redis.io/topics/pubsub. It's NOT safe for concurrent use by // multiple goroutines. type PubSub struct { - *baseClient + base *baseClient channels []string patterns []string + + nsub int // number of active subscriptions } // Deprecated. Use Subscribe/PSubscribe instead. func (c *Client) PubSub() *PubSub { return &PubSub{ - baseClient: &baseClient{ + base: &baseClient{ opt: c.opt, connPool: newStickyConnPool(c.connPool, false), }, @@ -46,7 +48,7 @@ func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { } func (c *PubSub) subscribe(cmd string, channels ...string) error { - cn, _, err := c.conn() + cn, _, err := c.base.conn() if err != nil { return err } @@ -65,6 +67,7 @@ func (c *PubSub) Subscribe(channels ...string) error { err := c.subscribe("SUBSCRIBE", channels...) if err == nil { c.channels = append(c.channels, channels...) + c.nsub += len(channels) } return err } @@ -74,6 +77,7 @@ func (c *PubSub) PSubscribe(patterns ...string) error { err := c.subscribe("PSUBSCRIBE", patterns...) if err == nil { c.patterns = append(c.patterns, patterns...) + c.nsub += len(patterns) } return err } @@ -113,8 +117,12 @@ func (c *PubSub) PUnsubscribe(patterns ...string) error { return err } +func (c *PubSub) Close() error { + return c.base.Close() +} + func (c *PubSub) Ping(payload string) error { - cn, _, err := c.conn() + cn, _, err := c.base.conn() if err != nil { return err } @@ -178,7 +186,7 @@ func (p *Pong) String() string { return "Pong" } -func newMessage(reply []interface{}) (interface{}, error) { +func (c *PubSub) newMessage(reply []interface{}) (interface{}, error) { switch kind := reply[0].(string); kind { case "subscribe", "unsubscribe", "psubscribe", "punsubscribe": return &Subscription{ @@ -210,7 +218,11 @@ func newMessage(reply []interface{}) (interface{}, error) { // is not received in time. This is low-level API and most clients // should use ReceiveMessage. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { - cn, _, err := c.conn() + if c.nsub == 0 { + c.resubscribe() + } + + cn, _, err := c.base.conn() if err != nil { return nil, err } @@ -222,7 +234,8 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { if err != nil { return nil, err } - return newMessage(cmd.Val()) + + return c.newMessage(cmd.Val()) } // Receive returns a message as a Subscription, Message, PMessage, @@ -232,22 +245,6 @@ func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } -func (c *PubSub) reconnect(reason error) { - // Close current connection. - c.connPool.(*stickyConnPool).Reset(reason) - - if len(c.channels) > 0 { - if err := c.Subscribe(c.channels...); err != nil { - Logger.Printf("Subscribe failed: %s", err) - } - } - if len(c.patterns) > 0 { - if err := c.PSubscribe(c.patterns...); err != nil { - Logger.Printf("PSubscribe failed: %s", err) - } - } -} - // ReceiveMessage returns a message or error. It automatically // reconnects to Redis in case of network errors. func (c *PubSub) ReceiveMessage() (*Message, error) { @@ -259,10 +256,8 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { return nil, err } - goodConn := errNum == 0 errNum++ - - if goodConn { + if errNum < 3 { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { err := c.Ping("") if err == nil { @@ -270,16 +265,16 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { } Logger.Printf("PubSub.Ping failed: %s", err) } - } - - if errNum > 2 { + } else { + // 3 consequent errors - connection is bad + // and/or Redis Server is down. + // Sleep to not exceed max number of open connections. time.Sleep(time.Second) } - c.reconnect(err) continue } - // Reset error number. + // Reset error number, because we received a message. errNum = 0 switch msg := msgi.(type) { @@ -300,3 +295,22 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { } } } + +func (c *PubSub) putConn(cn *conn, err error) { + if !c.base.putConn(cn, err) { + c.nsub = 0 + } +} + +func (c *PubSub) resubscribe() { + if len(c.channels) > 0 { + if err := c.Subscribe(c.channels...); err != nil { + Logger.Printf("Subscribe failed: %s", err) + } + } + if len(c.patterns) > 0 { + if err := c.PSubscribe(c.patterns...); err != nil { + Logger.Printf("PSubscribe failed: %s", err) + } + } +} diff --git a/pubsub_test.go b/pubsub_test.go index bf940d4777..36c75c38bd 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -1,6 +1,7 @@ package redis_test import ( + "io" "net" "sync" "time" @@ -230,18 +231,41 @@ var _ = Describe("PubSub", func() { Expect(pong.Payload).To(Equal("hello")) }) - It("should ReceiveMessage", func() { + It("should multi-ReceiveMessage", func() { pubsub, err := client.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - var wg sync.WaitGroup - wg.Add(1) + err = client.Publish("mychannel", "hello").Err() + Expect(err).NotTo(HaveOccurred()) + + err = client.Publish("mychannel", "world").Err() + Expect(err).NotTo(HaveOccurred()) + + msg, err := pubsub.ReceiveMessage() + Expect(err).NotTo(HaveOccurred()) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("hello")) + + msg, err = pubsub.ReceiveMessage() + Expect(err).NotTo(HaveOccurred()) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("world")) + }) + + It("should ReceiveMessage after timeout", func() { + pubsub, err := client.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + done := make(chan bool, 1) go func() { defer GinkgoRecover() - defer wg.Done() + defer func() { + done <- true + }() - time.Sleep(readTimeout + 100*time.Millisecond) + time.Sleep(5*time.Second + 100*time.Millisecond) n, err := client.Publish("mychannel", "hello").Result() Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(1))) @@ -252,22 +276,23 @@ var _ = Describe("PubSub", func() { Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) - wg.Wait() + Eventually(done).Should(Receive()) }) - expectReceiveMessage := func(pubsub *redis.PubSub) { + expectReceiveMessageOnError := func(pubsub *redis.PubSub) { cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn1.SetNetConn(&badConn{ - readErr: errTimeout, - writeErr: errTimeout, + readErr: io.EOF, + writeErr: io.EOF, }) - var wg sync.WaitGroup - wg.Add(1) + done := make(chan bool, 1) go func() { defer GinkgoRecover() - defer wg.Done() + defer func() { + done <- true + }() time.Sleep(100 * time.Millisecond) err := client.Publish("mychannel", "hello").Err() @@ -279,7 +304,7 @@ var _ = Describe("PubSub", func() { Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) - wg.Wait() + Eventually(done).Should(Receive()) } It("Subscribe should reconnect on ReceiveMessage error", func() { @@ -287,7 +312,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - expectReceiveMessage(pubsub) + expectReceiveMessageOnError(pubsub) }) It("PSubscribe should reconnect on ReceiveMessage error", func() { @@ -295,7 +320,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - expectReceiveMessage(pubsub) + expectReceiveMessageOnError(pubsub) }) It("should return on Close", func() { diff --git a/redis.go b/redis.go index 8372963a35..f37c97b8bb 100644 --- a/redis.go +++ b/redis.go @@ -23,15 +23,20 @@ func (c *baseClient) conn() (*conn, bool, error) { return c.connPool.Get() } -func (c *baseClient) putConn(cn *conn, err error) { - if isBadConn(cn, err) { +func (c *baseClient) putConn(cn *conn, err error) bool { + if isBadConn(err) { err = c.connPool.Remove(cn, err) - } else { - err = c.connPool.Put(cn) + if err != nil { + log.Printf("pool.Remove failed: %s", err) + } + return false } + + err = c.connPool.Put(cn) if err != nil { - Logger.Printf("pool.Put failed: %s", err) + log.Printf("pool.Put failed: %s", err) } + return true } func (c *baseClient) process(cmd Cmder) { diff --git a/sentinel.go b/sentinel.go index bb95064632..175c57e86d 100644 --- a/sentinel.go +++ b/sentinel.go @@ -88,7 +88,7 @@ func newSentinel(opt *Options) *sentinelClient { func (c *sentinelClient) PubSub() *PubSub { return &PubSub{ - baseClient: &baseClient{ + base: &baseClient{ opt: c.opt, connPool: newStickyConnPool(c.connPool, false), }, From 9c261facceaf16adcd2567f1aeaf33e584d3e5fd Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 1 Mar 2016 14:14:43 +0200 Subject: [PATCH 0128/1746] travis: test Go 1.6. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index dc4191acb1..70d6eb4704 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ services: go: - 1.4 - 1.5 + - 1.6 - tip matrix: From 6b369a317f187102af75d5ba4eac2ec5e8afd462 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 1 Mar 2016 14:21:25 +0200 Subject: [PATCH 0129/1746] Compare number approximatively. --- commands_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands_test.go b/commands_test.go index b82b019704..334ddd18c5 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2724,11 +2724,11 @@ var _ = Describe("Commands", func() { // "166.27415156960032" geoDist := client.GeoDist("Sicily", "Palermo", "Catania", "km") Expect(geoDist.Err()).NotTo(HaveOccurred()) - Expect(geoDist.Val()).To(Equal(166.27415156960032)) + Expect(geoDist.Val()).To(BeNumerically("~", 166.27, 0.01)) geoDist = client.GeoDist("Sicily", "Palermo", "Catania", "m") Expect(geoDist.Err()).NotTo(HaveOccurred()) - Expect(geoDist.Val()).To(Equal(166274.15156960033)) + Expect(geoDist.Val()).To(BeNumerically("~", 166274.15, 0.01)) }) It("should get geo hash in string representation", func() { From 78d40d5bd72d988b912c8a32b4ff2505de86c1a8 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 2 Mar 2016 13:26:05 +0200 Subject: [PATCH 0130/1746] Update conn.UsedAt on Read/Write. Fixes #263. --- conn.go | 14 ++++++++------ pool.go | 5 +---- redis_test.go | 29 ++++++++++++++++++++++------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/conn.go b/conn.go index ec4abd1811..40d612ba41 100644 --- a/conn.go +++ b/conn.go @@ -9,7 +9,7 @@ import ( const defaultBufSize = 4096 var ( - zeroTime = time.Time{} + noTimeout = time.Time{} ) type conn struct { @@ -17,7 +17,7 @@ type conn struct { rd *bufio.Reader buf []byte - usedAt time.Time + UsedAt time.Time ReadTimeout time.Duration WriteTimeout time.Duration } @@ -76,19 +76,21 @@ func (cn *conn) writeCmds(cmds ...Cmder) error { } func (cn *conn) Read(b []byte) (int, error) { + cn.UsedAt = time.Now() if cn.ReadTimeout != 0 { - cn.netcn.SetReadDeadline(time.Now().Add(cn.ReadTimeout)) + cn.netcn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) } else { - cn.netcn.SetReadDeadline(zeroTime) + cn.netcn.SetReadDeadline(noTimeout) } return cn.netcn.Read(b) } func (cn *conn) Write(b []byte) (int, error) { + cn.UsedAt = time.Now() if cn.WriteTimeout != 0 { - cn.netcn.SetWriteDeadline(time.Now().Add(cn.WriteTimeout)) + cn.netcn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) } else { - cn.netcn.SetWriteDeadline(zeroTime) + cn.netcn.SetWriteDeadline(noTimeout) } return cn.netcn.Write(b) } diff --git a/pool.go b/pool.go index e2f9f2b392..bb713bf410 100644 --- a/pool.go +++ b/pool.go @@ -164,7 +164,7 @@ func (p *connPool) closed() bool { } func (p *connPool) isIdle(cn *conn) bool { - return p.opt.getIdleTimeout() > 0 && time.Since(cn.usedAt) > p.opt.getIdleTimeout() + return p.opt.getIdleTimeout() > 0 && time.Since(cn.UsedAt) > p.opt.getIdleTimeout() } // First returns first non-idle connection from the pool or nil if @@ -275,9 +275,6 @@ func (p *connPool) Put(cn *conn) error { Logger.Print(err) return p.Remove(cn, err) } - if p.opt.getIdleTimeout() > 0 { - cn.usedAt = time.Now() - } p.freeConns <- cn return nil } diff --git a/redis_test.go b/redis_test.go index 38222025a8..724d241ed4 100644 --- a/redis_test.go +++ b/redis_test.go @@ -55,21 +55,19 @@ var _ = Describe("Client", func() { It("should close", func() { Expect(client.Close()).NotTo(HaveOccurred()) err := client.Ping().Err() - Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("redis: client is closed")) }) - It("should close pubsub without closing the connection", func() { + It("should close pubsub without closing the client", func() { pubsub := client.PubSub() Expect(pubsub.Close()).NotTo(HaveOccurred()) _, err := pubsub.Receive() - Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("redis: client is closed")) Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) - It("should close multi without closing the connection", func() { + It("should close multi without closing the client", func() { multi := client.Multi() Expect(multi.Close()).NotTo(HaveOccurred()) @@ -77,19 +75,19 @@ var _ = Describe("Client", func() { multi.Ping() return nil }) - Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("redis: client is closed")) + Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) - It("should close pipeline without closing the connection", func() { + It("should close pipeline without closing the client", func() { pipeline := client.Pipeline() Expect(pipeline.Close()).NotTo(HaveOccurred()) pipeline.Ping() _, err := pipeline.Exec() - Expect(err).To(HaveOccurred()) Expect(err).To(MatchError("redis: client is closed")) + Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) @@ -171,6 +169,23 @@ var _ = Describe("Client", func() { err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) }) + + It("should maintain conn.UsedAt", func() { + cn, _, err := client.Pool().Get() + Expect(err).NotTo(HaveOccurred()) + Expect(cn.UsedAt).To(BeZero()) + + err = client.Pool().Put(cn) + Expect(err).NotTo(HaveOccurred()) + Expect(cn.UsedAt).To(BeZero()) + + err = client.Ping().Err() + Expect(err).NotTo(HaveOccurred()) + + cn = client.Pool().First() + Expect(cn).NotTo(BeNil()) + Expect(cn.UsedAt).To(BeTemporally("~", time.Now())) + }) }) //------------------------------------------------------------------------------ From 73ad84252c7afa2a2d24ba55ae51d6c972148e7c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 2 Mar 2016 13:37:28 +0200 Subject: [PATCH 0131/1746] Use package logger. --- redis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis.go b/redis.go index f37c97b8bb..5dbaac0a02 100644 --- a/redis.go +++ b/redis.go @@ -27,14 +27,14 @@ func (c *baseClient) putConn(cn *conn, err error) bool { if isBadConn(err) { err = c.connPool.Remove(cn, err) if err != nil { - log.Printf("pool.Remove failed: %s", err) + Logger.Printf("pool.Remove failed: %s", err) } return false } err = c.connPool.Put(cn) if err != nil { - log.Printf("pool.Put failed: %s", err) + Logger.Printf("pool.Put failed: %s", err) } return true } From 43aade818abc9807d0d44017cf56d3705b8c3541 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 4 Mar 2016 10:03:50 +0200 Subject: [PATCH 0132/1746] Set conn.UsedAt when connection is created. Fixes #263. --- conn.go | 13 ++++++++----- export_test.go | 15 ++++++++++++++- redis_test.go | 11 ++++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/conn.go b/conn.go index 40d612ba41..62a8bdf154 100644 --- a/conn.go +++ b/conn.go @@ -8,9 +8,10 @@ import ( const defaultBufSize = 4096 -var ( - noTimeout = time.Time{} -) +var noTimeout = time.Time{} + +// Stubbed in tests. +var now = time.Now type conn struct { netcn net.Conn @@ -32,6 +33,8 @@ func newConnDialer(opt *Options) func() (*conn, error) { cn := &conn{ netcn: netcn, buf: make([]byte, defaultBufSize), + + UsedAt: now(), } cn.rd = bufio.NewReader(cn) return cn, cn.init(opt) @@ -76,7 +79,7 @@ func (cn *conn) writeCmds(cmds ...Cmder) error { } func (cn *conn) Read(b []byte) (int, error) { - cn.UsedAt = time.Now() + cn.UsedAt = now() if cn.ReadTimeout != 0 { cn.netcn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) } else { @@ -86,7 +89,7 @@ func (cn *conn) Read(b []byte) (int, error) { } func (cn *conn) Write(b []byte) (int, error) { - cn.UsedAt = time.Now() + cn.UsedAt = now() if cn.WriteTimeout != 0 { cn.netcn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) } else { diff --git a/export_test.go b/export_test.go index 4a6de2c658..95715b5f85 100644 --- a/export_test.go +++ b/export_test.go @@ -1,6 +1,9 @@ package redis -import "net" +import ( + "net" + "time" +) func (c *baseClient) Pool() pool { return c.connPool @@ -15,3 +18,13 @@ var NewConnDialer = newConnDialer func (cn *conn) SetNetConn(netcn net.Conn) { cn.netcn = netcn } + +func SetTime(tm time.Time) { + now = func() time.Time { + return tm + } +} + +func RestoreTime() { + now = time.Now +} diff --git a/redis_test.go b/redis_test.go index 724d241ed4..8c846d00e0 100644 --- a/redis_test.go +++ b/redis_test.go @@ -173,18 +173,23 @@ var _ = Describe("Client", func() { It("should maintain conn.UsedAt", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - Expect(cn.UsedAt).To(BeZero()) + Expect(cn.UsedAt).NotTo(BeZero()) + createdAt := cn.UsedAt + + future := time.Now().Add(time.Hour) + redis.SetTime(future) + defer redis.RestoreTime() err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) - Expect(cn.UsedAt).To(BeZero()) + Expect(cn.UsedAt.Equal(createdAt)).To(BeTrue()) err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) cn = client.Pool().First() Expect(cn).NotTo(BeNil()) - Expect(cn.UsedAt).To(BeTemporally("~", time.Now())) + Expect(cn.UsedAt.Equal(future)).To(BeTrue()) }) }) From d3f33b67b9fcbaa0a5054b1a94ff4510e757d1aa Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 8 Mar 2016 16:00:33 +0200 Subject: [PATCH 0133/1746] Fix error formatting. --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index b99ee896d9..758ec8df05 100644 --- a/parser.go +++ b/parser.go @@ -415,7 +415,7 @@ func readScanReply(cn *conn) ([]string, int64, error) { return nil, 0, err } if n != 2 { - return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2") + return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n) } b, err := readBytesReply(cn) From 9f40911f28f065dfcbcea0f7d266458ff6937f04 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 9 Mar 2016 09:49:05 +0000 Subject: [PATCH 0134/1746] Correct method accessors --- commands.go | 4 ++-- pipeline.go | 2 +- redis.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/commands.go b/commands.go index 5e202ad640..c523534672 100644 --- a/commands.go +++ b/commands.go @@ -583,7 +583,7 @@ func (c *commandable) SetNX(key string, value interface{}, expiration time.Durat // Redis `SET key value [expiration] XX` command. // // Zero expiration means the key has no expiration time. -func (c *Client) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { +func (c *commandable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if usePrecise(expiration) { cmd = NewBoolCmd("SET", key, value, "PX", formatMs(expiration), "XX") @@ -1282,7 +1282,7 @@ func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSli return c.zRevRangeBy("ZREVRANGEBYSCORE", key, opt) } -func (c commandable) ZRevRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) ZRevRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { return c.zRevRangeBy("ZREVRANGEBYLEX", key, opt) } diff --git a/pipeline.go b/pipeline.go index 56ff9654dd..8c800be3d4 100644 --- a/pipeline.go +++ b/pipeline.go @@ -11,7 +11,7 @@ import ( type Pipeline struct { commandable - client *baseClient + client baseClient mu sync.Mutex // protects cmds cmds []Cmder diff --git a/redis.go b/redis.go index 5dbaac0a02..488bfb9a6d 100644 --- a/redis.go +++ b/redis.go @@ -183,12 +183,12 @@ func (opt *Options) getIdleTimeout() time.Duration { // underlying connections. It's safe for concurrent use by multiple // goroutines. type Client struct { - *baseClient + baseClient commandable } func newClient(opt *Options, pool pool) *Client { - base := &baseClient{opt: opt, connPool: pool} + base := baseClient{opt: opt, connPool: pool} return &Client{ baseClient: base, commandable: commandable{process: base.process}, From 673e99943140855a07481485418e1504df27b6d6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 8 Mar 2016 17:18:52 +0200 Subject: [PATCH 0135/1746] Close connection on network timeout. --- cluster_pipeline.go | 2 +- command.go | 11 +---------- error.go | 8 +++++--- multi.go | 2 +- pipeline.go | 2 +- pubsub.go | 2 +- redis.go | 20 ++++++++------------ ring.go | 2 +- 8 files changed, 19 insertions(+), 30 deletions(-) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 73d272bbf7..5299b5f562 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -79,7 +79,7 @@ func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { if err != nil { retErr = err } - client.putConn(cn, err) + client.putConn(cn, err, false) } cmdsMap = failedCmds diff --git a/command.go b/command.go index 1986c66110..31516f6067 100644 --- a/command.go +++ b/command.go @@ -32,7 +32,6 @@ type Cmder interface { setErr(error) reset() - writeTimeout() *time.Duration readTimeout() *time.Duration clusterKey() string @@ -82,7 +81,7 @@ type baseCmd struct { _clusterKeyPos int - _writeTimeout, _readTimeout *time.Duration + _readTimeout *time.Duration } func (cmd *baseCmd) Err() error { @@ -104,10 +103,6 @@ func (cmd *baseCmd) setReadTimeout(d time.Duration) { cmd._readTimeout = &d } -func (cmd *baseCmd) writeTimeout() *time.Duration { - return cmd._writeTimeout -} - func (cmd *baseCmd) clusterKey() string { if cmd._clusterKeyPos > 0 && cmd._clusterKeyPos < len(cmd._args) { return fmt.Sprint(cmd._args[cmd._clusterKeyPos]) @@ -115,10 +110,6 @@ func (cmd *baseCmd) clusterKey() string { return "" } -func (cmd *baseCmd) setWriteTimeout(d time.Duration) { - cmd._writeTimeout = &d -} - func (cmd *baseCmd) setErr(e error) { cmd.err = e } diff --git a/error.go b/error.go index dce10a37bd..3f2a560c0b 100644 --- a/error.go +++ b/error.go @@ -33,15 +33,17 @@ func isNetworkError(err error) bool { return ok } -func isBadConn(err error) bool { +func isBadConn(err error, allowTimeout bool) bool { if err == nil { return false } if _, ok := err.(redisError); ok { return false } - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return false + if allowTimeout { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return false + } } return true } diff --git a/multi.go b/multi.go index 7ffc7e05e3..1a13d04732 100644 --- a/multi.go +++ b/multi.go @@ -133,7 +133,7 @@ func (c *Multi) Exec(f func() error) ([]Cmder, error) { } err = c.execCmds(cn, cmds) - c.base.putConn(cn, err) + c.base.putConn(cn, err, false) return retCmds, err } diff --git a/pipeline.go b/pipeline.go index 8c800be3d4..8caae6bf4a 100644 --- a/pipeline.go +++ b/pipeline.go @@ -98,7 +98,7 @@ func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { resetCmds(failedCmds) } failedCmds, err = execCmds(cn, failedCmds) - pipe.client.putConn(cn, err) + pipe.client.putConn(cn, err, false) if err != nil && retErr == nil { retErr = err } diff --git a/pubsub.go b/pubsub.go index bde81b5edd..1b422ec826 100644 --- a/pubsub.go +++ b/pubsub.go @@ -297,7 +297,7 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { } func (c *PubSub) putConn(cn *conn, err error) { - if !c.base.putConn(cn, err) { + if !c.base.putConn(cn, err, true) { c.nsub = 0 } } diff --git a/redis.go b/redis.go index 488bfb9a6d..5af7d68308 100644 --- a/redis.go +++ b/redis.go @@ -23,8 +23,8 @@ func (c *baseClient) conn() (*conn, bool, error) { return c.connPool.Get() } -func (c *baseClient) putConn(cn *conn, err error) bool { - if isBadConn(err) { +func (c *baseClient) putConn(cn *conn, err error, allowTimeout bool) bool { + if isBadConn(err, allowTimeout) { err = c.connPool.Remove(cn, err) if err != nil { Logger.Printf("pool.Remove failed: %s", err) @@ -51,20 +51,16 @@ func (c *baseClient) process(cmd Cmder) { return } - if timeout := cmd.writeTimeout(); timeout != nil { - cn.WriteTimeout = *timeout - } else { - cn.WriteTimeout = c.opt.WriteTimeout - } - - if timeout := cmd.readTimeout(); timeout != nil { - cn.ReadTimeout = *timeout + readTimeout := cmd.readTimeout() + if readTimeout != nil { + cn.ReadTimeout = *readTimeout } else { cn.ReadTimeout = c.opt.ReadTimeout } + cn.WriteTimeout = c.opt.WriteTimeout if err := cn.writeCmds(cmd); err != nil { - c.putConn(cn, err) + c.putConn(cn, err, false) cmd.setErr(err) if shouldRetry(err) { continue @@ -73,7 +69,7 @@ func (c *baseClient) process(cmd Cmder) { } err = cmd.readReply(cn) - c.putConn(cn, err) + c.putConn(cn, err, readTimeout != nil) if shouldRetry(err) { continue } diff --git a/ring.go b/ring.go index 1d9b90251b..f1ae8adf49 100644 --- a/ring.go +++ b/ring.go @@ -326,7 +326,7 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { resetCmds(cmds) } failedCmds, err := execCmds(cn, cmds) - client.putConn(cn, err) + client.putConn(cn, err, false) if err != nil && retErr == nil { retErr = err } From 4edc7a059c71eec2003ec25559f0d2d32df99ba3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 9 Mar 2016 12:49:21 +0200 Subject: [PATCH 0136/1746] Fix race in tests. --- export_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/export_test.go b/export_test.go index 95715b5f85..36b2c547ae 100644 --- a/export_test.go +++ b/export_test.go @@ -2,6 +2,7 @@ package redis import ( "net" + "sync" "time" ) @@ -19,12 +20,18 @@ func (cn *conn) SetNetConn(netcn net.Conn) { cn.netcn = netcn } +var timeMu sync.Mutex + func SetTime(tm time.Time) { + timeMu.Lock() now = func() time.Time { return tm } + timeMu.Unlock() } func RestoreTime() { + timeMu.Lock() now = time.Now + timeMu.Unlock() } From 0db1d730c840987ca4688d3682acde52fdf0c097 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 9 Mar 2016 13:38:33 +0200 Subject: [PATCH 0137/1746] Improve pool tests by verifying number of created connections. --- commands_test.go | 3 +++ pool.go | 4 ++- pool_test.go | 12 +++++++++ pubsub.go | 12 ++++----- pubsub_test.go | 64 ++++++++++++++++++++++++++++-------------------- 5 files changed, 62 insertions(+), 33 deletions(-) diff --git a/commands_test.go b/commands_test.go index 334ddd18c5..49d8488b11 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1303,6 +1303,9 @@ var _ = Describe("Commands", func() { bLPop := client.BLPop(time.Second, "list1") Expect(bLPop.Val()).To(BeNil()) Expect(bLPop.Err()).To(Equal(redis.Nil)) + + stats := client.Pool().Stats() + Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(1))) }) It("should BRPop", func() { diff --git a/pool.go b/pool.go index bb713bf410..3725c4081b 100644 --- a/pool.go +++ b/pool.go @@ -18,7 +18,8 @@ var ( // PoolStats contains pool state information and accumulated stats. type PoolStats struct { Requests uint32 // number of times a connection was requested by the pool - Waits uint32 // number of times our pool had to wait for a connection + Hits uint32 // number of times free connection was found in the pool + Waits uint32 // number of times the pool had to wait for a connection Timeouts uint32 // number of times a wait timeout occurred TotalConns uint32 // the number of total connections in the pool @@ -241,6 +242,7 @@ func (p *connPool) Get() (cn *conn, isNew bool, err error) { // Fetch first non-idle connection, if available. if cn = p.First(); cn != nil { + atomic.AddUint32(&p.stats.Hits, 1) return } diff --git a/pool_test.go b/pool_test.go index 4d787a6878..bc88c5fcb2 100644 --- a/pool_test.go +++ b/pool_test.go @@ -123,6 +123,12 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(Equal(1)) Expect(pool.FreeLen()).To(Equal(1)) + + stats := pool.Stats() + Expect(stats.Requests).To(Equal(uint32(3))) + Expect(stats.Hits).To(Equal(uint32(2))) + Expect(stats.Waits).To(Equal(uint32(0))) + Expect(stats.Timeouts).To(Equal(uint32(0))) }) It("should reuse connections", func() { @@ -135,6 +141,12 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(Equal(1)) Expect(pool.FreeLen()).To(Equal(1)) + + stats := pool.Stats() + Expect(stats.Requests).To(Equal(uint32(100))) + Expect(stats.Hits).To(Equal(uint32(99))) + Expect(stats.Waits).To(Equal(uint32(0))) + Expect(stats.Timeouts).To(Equal(uint32(0))) }) It("should unblock client when connection is removed", func() { diff --git a/pubsub.go b/pubsub.go index 1b422ec826..c1fb4628a4 100644 --- a/pubsub.go +++ b/pubsub.go @@ -245,10 +245,11 @@ func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } -// ReceiveMessage returns a message or error. It automatically -// reconnects to Redis in case of network errors. +// ReceiveMessage returns a Message or error ignoring Subscription or Pong +// messages. It automatically reconnects to Redis Server and resubscribes +// to channels in case of network errors. func (c *PubSub) ReceiveMessage() (*Message, error) { - var errNum int + var errNum uint for { msgi, err := c.ReceiveTimeout(5 * time.Second) if err != nil { @@ -260,10 +261,9 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { if errNum < 3 { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { err := c.Ping("") - if err == nil { - continue + if err != nil { + Logger.Printf("PubSub.Ping failed: %s", err) } - Logger.Printf("PubSub.Ping failed: %s", err) } } else { // 3 consequent errors - connection is bad diff --git a/pubsub_test.go b/pubsub_test.go index 36c75c38bd..669c0737d8 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -33,12 +33,6 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - n, err := client.Publish("mychannel1", "hello").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(1))) - - Expect(pubsub.PUnsubscribe("mychannel*")).NotTo(HaveOccurred()) - { msgi, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) @@ -48,6 +42,18 @@ var _ = Describe("PubSub", func() { Expect(subscr.Count).To(Equal(1)) } + { + msgi, err := pubsub.ReceiveTimeout(time.Second) + Expect(err.(net.Error).Timeout()).To(Equal(true)) + Expect(msgi).To(BeNil()) + } + + n, err := client.Publish("mychannel1", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + Expect(pubsub.PUnsubscribe("mychannel*")).NotTo(HaveOccurred()) + { msgi, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) @@ -66,11 +72,8 @@ var _ = Describe("PubSub", func() { Expect(subscr.Count).To(Equal(0)) } - { - msgi, err := pubsub.ReceiveTimeout(time.Second) - Expect(err.(net.Error).Timeout()).To(Equal(true)) - Expect(msgi).NotTo(HaveOccurred()) - } + stats := client.Pool().Stats() + Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) }) It("should pub/sub channels", func() { @@ -128,16 +131,6 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() - n, err := client.Publish("mychannel", "hello").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(1))) - - n, err = client.Publish("mychannel2", "hello2").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(1))) - - Expect(pubsub.Unsubscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) - { msgi, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) @@ -156,6 +149,22 @@ var _ = Describe("PubSub", func() { Expect(subscr.Count).To(Equal(2)) } + { + msgi, err := pubsub.ReceiveTimeout(time.Second) + Expect(err.(net.Error).Timeout()).To(Equal(true)) + Expect(msgi).NotTo(HaveOccurred()) + } + + n, err := client.Publish("mychannel", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + n, err = client.Publish("mychannel2", "hello2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + Expect(pubsub.Unsubscribe("mychannel", "mychannel2")).NotTo(HaveOccurred()) + { msgi, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) @@ -190,11 +199,8 @@ var _ = Describe("PubSub", func() { Expect(subscr.Count).To(Equal(0)) } - { - msgi, err := pubsub.ReceiveTimeout(time.Second) - Expect(err.(net.Error).Timeout()).To(Equal(true)) - Expect(msgi).NotTo(HaveOccurred()) - } + stats := client.Pool().Stats() + Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) }) It("should ping/pong", func() { @@ -277,6 +283,9 @@ var _ = Describe("PubSub", func() { Expect(msg.Payload).To(Equal("hello")) Eventually(done).Should(Receive()) + + stats := client.Pool().Stats() + Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) }) expectReceiveMessageOnError := func(pubsub *redis.PubSub) { @@ -305,6 +314,9 @@ var _ = Describe("PubSub", func() { Expect(msg.Payload).To(Equal("hello")) Eventually(done).Should(Receive()) + + stats := client.Pool().Stats() + Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) } It("Subscribe should reconnect on ReceiveMessage error", func() { From 27635bbe4e50897f3fbd320ced4d00eb624386ec Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 9 Mar 2016 15:14:01 +0200 Subject: [PATCH 0138/1746] Fix FailoverClient to close connection to Sentinel. Fixes races build. --- export_test.go | 7 ---- main_test.go | 3 +- redis.go | 19 +++++++-- sentinel.go | 105 +++++++++++++++++++++++++++++-------------------- 4 files changed, 80 insertions(+), 54 deletions(-) diff --git a/export_test.go b/export_test.go index 36b2c547ae..95715b5f85 100644 --- a/export_test.go +++ b/export_test.go @@ -2,7 +2,6 @@ package redis import ( "net" - "sync" "time" ) @@ -20,18 +19,12 @@ func (cn *conn) SetNetConn(netcn net.Conn) { cn.netcn = netcn } -var timeMu sync.Mutex - func SetTime(tm time.Time) { - timeMu.Lock() now = func() time.Time { return tm } - timeMu.Unlock() } func RestoreTime() { - timeMu.Lock() now = time.Now - timeMu.Unlock() } diff --git a/main_test.go b/main_test.go index b9b3e218b5..d298dd21ea 100644 --- a/main_test.go +++ b/main_test.go @@ -98,9 +98,10 @@ func TestGinkgoSuite(t *testing.T) { //------------------------------------------------------------------------------ -func eventually(fn func() error, timeout time.Duration) (err error) { +func eventually(fn func() error, timeout time.Duration) error { done := make(chan struct{}) var exit int32 + var err error go func() { for atomic.LoadInt32(&exit) == 0 { err = fn() diff --git a/redis.go b/redis.go index 5af7d68308..5558ad102f 100644 --- a/redis.go +++ b/redis.go @@ -13,6 +13,8 @@ var Logger = log.New(os.Stderr, "redis: ", log.LstdFlags) type baseClient struct { connPool pool opt *Options + + onClose func() error // hook called when client is closed } func (c *baseClient) String() string { @@ -83,7 +85,16 @@ func (c *baseClient) process(cmd Cmder) { // It is rare to Close a Client, as the Client is meant to be // long-lived and shared between many goroutines. func (c *baseClient) Close() error { - return c.connPool.Close() + var retErr error + if c.onClose != nil { + if err := c.onClose(); err != nil && retErr == nil { + retErr = err + } + } + if err := c.connPool.Close(); err != nil && retErr == nil { + retErr = err + } + return retErr } //------------------------------------------------------------------------------ @@ -186,8 +197,10 @@ type Client struct { func newClient(opt *Options, pool pool) *Client { base := baseClient{opt: opt, connPool: pool} return &Client{ - baseClient: base, - commandable: commandable{process: base.process}, + baseClient: base, + commandable: commandable{ + process: base.process, + }, } } diff --git a/sentinel.go b/sentinel.go index 175c57e86d..db5db64d46 100644 --- a/sentinel.go +++ b/sentinel.go @@ -65,18 +65,31 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt: opt, } - return newClient(opt, failover.Pool()) + base := baseClient{ + opt: opt, + connPool: failover.Pool(), + + onClose: func() error { + return failover.Close() + }, + } + return &Client{ + baseClient: base, + commandable: commandable{ + process: base.process, + }, + } } //------------------------------------------------------------------------------ type sentinelClient struct { + baseClient commandable - *baseClient } func newSentinel(opt *Options) *sentinelClient { - base := &baseClient{ + base := baseClient{ opt: opt, connPool: newConnPool(opt), } @@ -116,8 +129,12 @@ type sentinelFailover struct { pool pool poolOnce sync.Once - lock sync.RWMutex - _sentinel *sentinelClient + mu sync.RWMutex + sentinel *sentinelClient +} + +func (d *sentinelFailover) Close() error { + return d.resetSentinel() } func (d *sentinelFailover) dial() (net.Conn, error) { @@ -137,15 +154,15 @@ func (d *sentinelFailover) Pool() pool { } func (d *sentinelFailover) MasterAddr() (string, error) { - defer d.lock.Unlock() - d.lock.Lock() + defer d.mu.Unlock() + d.mu.Lock() // Try last working sentinel. - if d._sentinel != nil { - addr, err := d._sentinel.GetMasterAddrByName(d.masterName).Result() + if d.sentinel != nil { + addr, err := d.sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { Logger.Printf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) - d.resetSentinel() + d._resetSentinel() } else { addr := net.JoinHostPort(addr[0], addr[1]) Logger.Printf("sentinel: %q addr is %s", d.masterName, addr) @@ -186,10 +203,26 @@ func (d *sentinelFailover) MasterAddr() (string, error) { func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { d.discoverSentinels(sentinel) - d._sentinel = sentinel + d.sentinel = sentinel go d.listen() } +func (d *sentinelFailover) resetSentinel() error { + d.mu.Lock() + err := d._resetSentinel() + d.mu.Unlock() + return err +} + +func (d *sentinelFailover) _resetSentinel() error { + var err error + if d.sentinel != nil { + err = d.sentinel.Close() + d.sentinel = nil + } + return err +} + func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinels, err := sentinel.Sentinels(d.masterName).Result() if err != nil { @@ -247,55 +280,41 @@ func (d *sentinelFailover) listen() { var pubsub *PubSub for { if pubsub == nil { - pubsub = d._sentinel.PubSub() + pubsub = d.sentinel.PubSub() if err := pubsub.Subscribe("+switch-master"); err != nil { Logger.Printf("sentinel: Subscribe failed: %s", err) - d.lock.Lock() d.resetSentinel() - d.lock.Unlock() return } } - msg, err := pubsub.Receive() + msg, err := pubsub.ReceiveMessage() if err != nil { - Logger.Printf("sentinel: Receive failed: %s", err) + Logger.Printf("sentinel: ReceiveMessage failed: %s", err) pubsub.Close() + d.resetSentinel() return } - switch msg := msg.(type) { - case *Message: - switch msg.Channel { - case "+switch-master": - parts := strings.Split(msg.Payload, " ") - if parts[0] != d.masterName { - Logger.Printf("sentinel: ignore new %s addr", parts[0]) - continue - } - addr := net.JoinHostPort(parts[3], parts[4]) - Logger.Printf( - "sentinel: new %q addr is %s", - d.masterName, addr, - ) - - d.closeOldConns(addr) - default: - Logger.Printf("sentinel: unsupported message: %s", msg) + switch msg.Channel { + case "+switch-master": + parts := strings.Split(msg.Payload, " ") + if parts[0] != d.masterName { + Logger.Printf("sentinel: ignore new %s addr", parts[0]) + continue } - case *Subscription: - // Ignore. - default: - Logger.Printf("sentinel: unsupported message: %s", msg) + + addr := net.JoinHostPort(parts[3], parts[4]) + Logger.Printf( + "sentinel: new %q addr is %s", + d.masterName, addr, + ) + + d.closeOldConns(addr) } } } -func (d *sentinelFailover) resetSentinel() { - d._sentinel.Close() - d._sentinel = nil -} - func contains(slice []string, str string) bool { for _, s := range slice { if s == str { From b90cea8e8dad46a1549cd83a517032b31555d0e4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 10 Mar 2016 11:10:47 +0200 Subject: [PATCH 0139/1746] More benchmarks. --- .travis.yml | 1 + README.md | 21 ++++ bench_test.go | 296 ++++++++++++++++++++++++++++++++++++++++++++++++++ conn.go | 6 +- pool_test.go | 22 ---- redis_test.go | 199 --------------------------------- 6 files changed, 321 insertions(+), 224 deletions(-) create mode 100644 bench_test.go diff --git a/.travis.yml b/.travis.yml index 70d6eb4704..a476a44d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ install: - go get gopkg.in/bsm/ratelimit.v1 - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega + - go get github.com/garyburd/redigo/redis - mkdir -p $HOME/gopath/src/gopkg.in - mv $HOME/gopath/src/github.com/go-redis/redis $HOME/gopath/src/gopkg.in/redis.v3 - cd $HOME/gopath/src/gopkg.in/redis.v3 diff --git a/README.md b/README.md index 13887132dc..bd9edfc99b 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,27 @@ Some corner cases: EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, []string{"hello"}).Result() +## Benchmark + +``` +BenchmarkSetGoRedis10Conns64Bytes-4 200000 7184 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns64Bytes-4 200000 7174 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns1KB-4 200000 7341 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns1KB-4 200000 7425 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns10KB-4 200000 9480 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns10KB-4 200000 9301 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns1MB-4 2000 590321 ns/op 2337 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns1MB-4 2000 588935 ns/op 2337 B/op 6 allocs/op +BenchmarkSetRedigo10Conns64Bytes-4 200000 7238 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns64Bytes-4 200000 7435 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns1KB-4 200000 7635 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns1KB-4 200000 7597 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns10KB-4 100000 17126 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns10KB-4 100000 17030 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns1MB-4 2000 675397 ns/op 226 B/op 7 allocs/op +BenchmarkSetRedigo100Conns1MB-4 2000 669053 ns/op 226 B/op 7 allocs/op +``` + ## Shameless plug Check my [PostgreSQL client for Go](https://github.com/go-pg/pg). diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000000..dfd2bf3e56 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,296 @@ +package redis_test + +import ( + "bytes" + "testing" + "time" + + redigo "github.com/garyburd/redigo/redis" + + "gopkg.in/redis.v3" +) + +func benchmarkRedisClient(poolSize int) *redis.Client { + client := redis.NewClient(&redis.Options{ + Addr: ":6379", + DialTimeout: time.Second, + ReadTimeout: time.Second, + WriteTimeout: time.Second, + PoolSize: poolSize, + }) + if err := client.FlushDb().Err(); err != nil { + panic(err) + } + return client +} + +func BenchmarkRedisPing(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Ping().Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkRedisSet(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + value := string(bytes.Repeat([]byte{'1'}, 10000)) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Set("key", value, 0).Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkRedisGetNil(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Get("key").Err(); err != redis.Nil { + b.Fatal(err) + } + } + }) +} + +func benchmarkSetGoRedis(b *testing.B, poolSize, payloadSize int) { + client := benchmarkRedisClient(poolSize) + defer client.Close() + + value := string(bytes.Repeat([]byte{'1'}, payloadSize)) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Set("key", value, 0).Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkSetGoRedis10Conns64Bytes(b *testing.B) { + benchmarkSetGoRedis(b, 10, 64) +} + +func BenchmarkSetGoRedis100Conns64Bytes(b *testing.B) { + benchmarkSetGoRedis(b, 100, 64) +} + +func BenchmarkSetGoRedis10Conns1KB(b *testing.B) { + benchmarkSetGoRedis(b, 10, 1024) +} + +func BenchmarkSetGoRedis100Conns1KB(b *testing.B) { + benchmarkSetGoRedis(b, 100, 1024) +} + +func BenchmarkSetGoRedis10Conns10KB(b *testing.B) { + benchmarkSetGoRedis(b, 10, 10*1024) +} + +func BenchmarkSetGoRedis100Conns10KB(b *testing.B) { + benchmarkSetGoRedis(b, 100, 10*1024) +} + +func BenchmarkSetGoRedis10Conns1MB(b *testing.B) { + benchmarkSetGoRedis(b, 10, 1024*1024) +} + +func BenchmarkSetGoRedis100Conns1MB(b *testing.B) { + benchmarkSetGoRedis(b, 100, 1024*1024) +} + +func benchmarkSetRedigo(b *testing.B, poolSize, payloadSize int) { + pool := &redigo.Pool{ + Dial: func() (redigo.Conn, error) { + return redigo.DialTimeout("tcp", ":6379", time.Second, time.Second, time.Second) + }, + MaxActive: poolSize, + MaxIdle: poolSize, + } + defer pool.Close() + + value := string(bytes.Repeat([]byte{'1'}, payloadSize)) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + conn := pool.Get() + if _, err := conn.Do("SET", "key", value); err != nil { + b.Fatal(err) + } + conn.Close() + } + }) +} + +func BenchmarkSetRedigo10Conns64Bytes(b *testing.B) { + benchmarkSetRedigo(b, 10, 64) +} + +func BenchmarkSetRedigo100Conns64Bytes(b *testing.B) { + benchmarkSetRedigo(b, 100, 64) +} + +func BenchmarkSetRedigo10Conns1KB(b *testing.B) { + benchmarkSetRedigo(b, 10, 1024) +} + +func BenchmarkSetRedigo100Conns1KB(b *testing.B) { + benchmarkSetRedigo(b, 100, 1024) +} + +func BenchmarkSetRedigo10Conns10KB(b *testing.B) { + benchmarkSetRedigo(b, 10, 10*1024) +} + +func BenchmarkSetRedigo100Conns10KB(b *testing.B) { + benchmarkSetRedigo(b, 100, 10*1024) +} + +func BenchmarkSetRedigo10Conns1MB(b *testing.B) { + benchmarkSetRedigo(b, 10, 1024*1024) +} + +func BenchmarkSetRedigo100Conns1MB(b *testing.B) { + benchmarkSetRedigo(b, 100, 1024*1024) +} + +func BenchmarkRedisSetGetBytes(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + value := bytes.Repeat([]byte{'1'}, 10000) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Set("key", value, 0).Err(); err != nil { + b.Fatal(err) + } + + got, err := client.Get("key").Bytes() + if err != nil { + b.Fatal(err) + } + if !bytes.Equal(got, value) { + b.Fatalf("got != value") + } + } + }) +} + +func BenchmarkRedisMGet(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + if err := client.MSet("key1", "hello1", "key2", "hello2").Err(); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.MGet("key1", "key2").Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkSetExpire(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Set("key", "hello", 0).Err(); err != nil { + b.Fatal(err) + } + if err := client.Expire("key", time.Second).Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkPipeline(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Set("key", "hello", 0) + pipe.Expire("key", time.Second) + return nil + }) + if err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkZAdd(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.ZAdd("key", redis.Z{float64(1), "hello"}).Err(); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkPool(b *testing.B) { + client := benchmarkRedisClient(10) + defer client.Close() + + pool := client.Pool() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + conn, _, err := pool.Get() + if err != nil { + b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) + } + if err = pool.Put(conn); err != nil { + b.Fatalf("no error expected on pool.Put but received: %s", err.Error()) + } + } + }) +} diff --git a/conn.go b/conn.go index 62a8bdf154..e7306d9498 100644 --- a/conn.go +++ b/conn.go @@ -65,16 +65,16 @@ func (cn *conn) init(opt *Options) error { } func (cn *conn) writeCmds(cmds ...Cmder) error { - buf := cn.buf[:0] + cn.buf = cn.buf[:0] for _, cmd := range cmds { var err error - buf, err = appendArgs(buf, cmd.args()) + cn.buf, err = appendArgs(cn.buf, cmd.args()) if err != nil { return err } } - _, err := cn.Write(buf) + _, err := cn.Write(cn.buf) return err } diff --git a/pool_test.go b/pool_test.go index bc88c5fcb2..2494e56ed5 100644 --- a/pool_test.go +++ b/pool_test.go @@ -3,7 +3,6 @@ package redis_test import ( "errors" "sync" - "testing" "time" . "github.com/onsi/ginkgo" @@ -210,24 +209,3 @@ var _ = Describe("pool", func() { Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) }) }) - -func BenchmarkPool(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - pool := client.Pool() - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - conn, _, err := pool.Get() - if err != nil { - b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) - } - if err = pool.Put(conn); err != nil { - b.Fatalf("no error expected on pool.Put but received: %s", err.Error()) - } - } - }) -} diff --git a/redis_test.go b/redis_test.go index 8c846d00e0..7b5197bcb1 100644 --- a/redis_test.go +++ b/redis_test.go @@ -1,9 +1,7 @@ package redis_test import ( - "bytes" "net" - "testing" "time" . "github.com/onsi/ginkgo" @@ -192,200 +190,3 @@ var _ = Describe("Client", func() { Expect(cn.UsedAt.Equal(future)).To(BeTrue()) }) }) - -//------------------------------------------------------------------------------ - -func benchRedisClient() *redis.Client { - client := redis.NewClient(&redis.Options{ - Addr: ":6379", - }) - if err := client.FlushDb().Err(); err != nil { - panic(err) - } - return client -} - -func BenchmarkRedisPing(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.Ping().Err(); err != nil { - b.Fatal(err) - } - } - }) -} - -func BenchmarkRedisSet(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - value := string(bytes.Repeat([]byte{'1'}, 10000)) - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.Set("key", value, 0).Err(); err != nil { - b.Fatal(err) - } - } - }) -} - -func BenchmarkRedisGetNil(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.Get("key").Err(); err != redis.Nil { - b.Fatal(err) - } - } - }) -} - -func benchmarkRedisSetGet(b *testing.B, size int) { - client := benchRedisClient() - defer client.Close() - - value := string(bytes.Repeat([]byte{'1'}, size)) - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.Set("key", value, 0).Err(); err != nil { - b.Fatal(err) - } - - got, err := client.Get("key").Result() - if err != nil { - b.Fatal(err) - } - if got != value { - b.Fatalf("got != value") - } - } - }) -} - -func BenchmarkRedisSetGet64Bytes(b *testing.B) { - benchmarkRedisSetGet(b, 64) -} - -func BenchmarkRedisSetGet1KB(b *testing.B) { - benchmarkRedisSetGet(b, 1024) -} - -func BenchmarkRedisSetGet10KB(b *testing.B) { - benchmarkRedisSetGet(b, 10*1024) -} - -func BenchmarkRedisSetGet1MB(b *testing.B) { - benchmarkRedisSetGet(b, 1024*1024) -} - -func BenchmarkRedisSetGetBytes(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - value := bytes.Repeat([]byte{'1'}, 10000) - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.Set("key", value, 0).Err(); err != nil { - b.Fatal(err) - } - - got, err := client.Get("key").Bytes() - if err != nil { - b.Fatal(err) - } - if !bytes.Equal(got, value) { - b.Fatalf("got != value") - } - } - }) -} - -func BenchmarkRedisMGet(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - if err := client.MSet("key1", "hello1", "key2", "hello2").Err(); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.MGet("key1", "key2").Err(); err != nil { - b.Fatal(err) - } - } - }) -} - -func BenchmarkSetExpire(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.Set("key", "hello", 0).Err(); err != nil { - b.Fatal(err) - } - if err := client.Expire("key", time.Second).Err(); err != nil { - b.Fatal(err) - } - } - }) -} - -func BenchmarkPipeline(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { - pipe.Set("key", "hello", 0) - pipe.Expire("key", time.Second) - return nil - }) - if err != nil { - b.Fatal(err) - } - } - }) -} - -func BenchmarkZAdd(b *testing.B) { - client := benchRedisClient() - defer client.Close() - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if err := client.ZAdd("key", redis.Z{float64(1), "hello"}).Err(); err != nil { - b.Fatal(err) - } - } - }) -} From ad0739be9976594d3ab71a4e61c5e8501fe11d66 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 12 Mar 2016 10:52:13 +0200 Subject: [PATCH 0140/1746] Extract pool package. Add pool benchmark. --- bench_test.go | 59 +++- cluster_pipeline.go | 5 +- command.go | 56 ++-- conn.go | 120 -------- conn_test.go | 25 -- error.go | 3 + export_test.go | 25 +- internal/pool/conn.go | 60 ++++ internal/pool/conn_list.go | 100 +++++++ internal/pool/pool.go | 284 ++++++++++++++++++ internal/pool/pool_single.go | 47 +++ internal/pool/pool_sticky.go | 128 +++++++++ multi.go | 8 +- multi_test.go | 4 +- options.go | 144 ++++++++++ parser.go | 71 ++--- parser_test.go | 8 +- pipeline.go | 6 +- pool.go | 542 ----------------------------------- pubsub.go | 17 +- pubsub_test.go | 4 +- redis.go | 130 ++------- redis_test.go | 8 +- ring.go | 3 +- sentinel.go | 10 +- 25 files changed, 969 insertions(+), 898 deletions(-) delete mode 100644 conn.go delete mode 100644 conn_test.go create mode 100644 internal/pool/conn.go create mode 100644 internal/pool/conn_list.go create mode 100644 internal/pool/pool.go create mode 100644 internal/pool/pool_single.go create mode 100644 internal/pool/pool_sticky.go create mode 100644 options.go delete mode 100644 pool.go diff --git a/bench_test.go b/bench_test.go index dfd2bf3e56..5d6fa37ff8 100644 --- a/bench_test.go +++ b/bench_test.go @@ -2,12 +2,15 @@ package redis_test import ( "bytes" + "errors" + "net" "testing" "time" redigo "github.com/garyburd/redigo/redis" "gopkg.in/redis.v3" + "gopkg.in/redis.v3/internal/pool" ) func benchmarkRedisClient(poolSize int) *redis.Client { @@ -274,11 +277,11 @@ func BenchmarkZAdd(b *testing.B) { }) } -func BenchmarkPool(b *testing.B) { - client := benchmarkRedisClient(10) - defer client.Close() - - pool := client.Pool() +func benchmarkPoolGetPut(b *testing.B, poolSize int) { + dial := func() (*pool.Conn, error) { + return pool.NewConn(&net.TCPConn{}), nil + } + pool := pool.NewConnPool(dial, poolSize, time.Second, 0) b.ResetTimer() @@ -294,3 +297,49 @@ func BenchmarkPool(b *testing.B) { } }) } + +func BenchmarkPoolGetPut10Conns(b *testing.B) { + benchmarkPoolGetPut(b, 10) +} + +func BenchmarkPoolGetPut100Conns(b *testing.B) { + benchmarkPoolGetPut(b, 100) +} + +func BenchmarkPoolGetPut1000Conns(b *testing.B) { + benchmarkPoolGetPut(b, 1000) +} + +func benchmarkPoolGetRemove(b *testing.B, poolSize int) { + dial := func() (*pool.Conn, error) { + return pool.NewConn(&net.TCPConn{}), nil + } + pool := pool.NewConnPool(dial, poolSize, time.Second, 0) + removeReason := errors.New("benchmark") + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + conn, _, err := pool.Get() + if err != nil { + b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) + } + if err = pool.Remove(conn, removeReason); err != nil { + b.Fatalf("no error expected on pool.Remove but received: %s", err.Error()) + } + } + }) +} + +func BenchmarkPoolGetRemove10Conns(b *testing.B) { + benchmarkPoolGetRemove(b, 10) +} + +func BenchmarkPoolGetRemove100Conns(b *testing.B) { + benchmarkPoolGetRemove(b, 100) +} + +func BenchmarkPoolGetRemove1000Conns(b *testing.B) { + benchmarkPoolGetRemove(b, 1000) +} diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 5299b5f562..7fa721c32e 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -2,6 +2,7 @@ package redis import ( "gopkg.in/redis.v3/internal/hashtag" + "gopkg.in/redis.v3/internal/pool" ) // ClusterPipeline is not thread-safe. @@ -96,9 +97,9 @@ func (pipe *ClusterPipeline) Close() error { } func (pipe *ClusterPipeline) execClusterCmds( - cn *conn, cmds []Cmder, failedCmds map[string][]Cmder, + cn *pool.Conn, cmds []Cmder, failedCmds map[string][]Cmder, ) (map[string][]Cmder, error) { - if err := cn.writeCmds(cmds...); err != nil { + if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) return failedCmds, err } diff --git a/command.go b/command.go index 31516f6067..6681ff530b 100644 --- a/command.go +++ b/command.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" "time" + + "gopkg.in/redis.v3/internal/pool" ) var ( @@ -28,7 +30,7 @@ var ( type Cmder interface { args() []interface{} - readReply(*conn) error + readReply(*pool.Conn) error setErr(error) reset() @@ -51,6 +53,20 @@ func resetCmds(cmds []Cmder) { } } +func writeCmd(cn *pool.Conn, cmds ...Cmder) error { + cn.Buf = cn.Buf[:0] + for _, cmd := range cmds { + var err error + cn.Buf, err = appendArgs(cn.Buf, cmd.args()) + if err != nil { + return err + } + } + + _, err := cn.Write(cn.Buf) + return err +} + func cmdString(cmd Cmder, val interface{}) string { var ss []string for _, arg := range cmd.args() { @@ -143,7 +159,7 @@ func (cmd *Cmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *Cmd) readReply(cn *conn) error { +func (cmd *Cmd) readReply(cn *pool.Conn) error { val, err := readReply(cn, sliceParser) if err != nil { cmd.err = err @@ -188,7 +204,7 @@ func (cmd *SliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *SliceCmd) readReply(cn *conn) error { +func (cmd *SliceCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, sliceParser) if err != nil { cmd.err = err @@ -231,7 +247,7 @@ func (cmd *StatusCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StatusCmd) readReply(cn *conn) error { +func (cmd *StatusCmd) readReply(cn *pool.Conn) error { cmd.val, cmd.err = readStringReply(cn) return cmd.err } @@ -265,7 +281,7 @@ func (cmd *IntCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *IntCmd) readReply(cn *conn) error { +func (cmd *IntCmd) readReply(cn *pool.Conn) error { cmd.val, cmd.err = readIntReply(cn) return cmd.err } @@ -303,7 +319,7 @@ func (cmd *DurationCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *DurationCmd) readReply(cn *conn) error { +func (cmd *DurationCmd) readReply(cn *pool.Conn) error { n, err := readIntReply(cn) if err != nil { cmd.err = err @@ -344,7 +360,7 @@ func (cmd *BoolCmd) String() string { var ok = []byte("OK") -func (cmd *BoolCmd) readReply(cn *conn) error { +func (cmd *BoolCmd) readReply(cn *pool.Conn) error { v, err := readReply(cn, nil) // `SET key value NX` returns nil when key already exists. But // `SETNX key value` returns bool (0/1). So convert nil to bool. @@ -430,13 +446,17 @@ func (cmd *StringCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringCmd) readReply(cn *conn) error { +func (cmd *StringCmd) readReply(cn *pool.Conn) error { b, err := readBytesReply(cn) if err != nil { cmd.err = err return err } - cmd.val = cn.copyBuf(b) + + new := make([]byte, len(b)) + copy(new, b) + cmd.val = new + return nil } @@ -469,7 +489,7 @@ func (cmd *FloatCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *FloatCmd) readReply(cn *conn) error { +func (cmd *FloatCmd) readReply(cn *pool.Conn) error { cmd.val, cmd.err = readFloatReply(cn) return cmd.err } @@ -503,7 +523,7 @@ func (cmd *StringSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringSliceCmd) readReply(cn *conn) error { +func (cmd *StringSliceCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, stringSliceParser) if err != nil { cmd.err = err @@ -542,7 +562,7 @@ func (cmd *BoolSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *BoolSliceCmd) readReply(cn *conn) error { +func (cmd *BoolSliceCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, boolSliceParser) if err != nil { cmd.err = err @@ -581,7 +601,7 @@ func (cmd *StringStringMapCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringStringMapCmd) readReply(cn *conn) error { +func (cmd *StringStringMapCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, stringStringMapParser) if err != nil { cmd.err = err @@ -620,7 +640,7 @@ func (cmd *StringIntMapCmd) reset() { cmd.err = nil } -func (cmd *StringIntMapCmd) readReply(cn *conn) error { +func (cmd *StringIntMapCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, stringIntMapParser) if err != nil { cmd.err = err @@ -659,7 +679,7 @@ func (cmd *ZSliceCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *ZSliceCmd) readReply(cn *conn) error { +func (cmd *ZSliceCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, zSliceParser) if err != nil { cmd.err = err @@ -703,7 +723,7 @@ func (cmd *ScanCmd) String() string { return cmdString(cmd, cmd.keys) } -func (cmd *ScanCmd) readReply(cn *conn) error { +func (cmd *ScanCmd) readReply(cn *pool.Conn) error { keys, cursor, err := readScanReply(cn) if err != nil { cmd.err = err @@ -751,7 +771,7 @@ func (cmd *ClusterSlotCmd) reset() { cmd.err = nil } -func (cmd *ClusterSlotCmd) readReply(cn *conn) error { +func (cmd *ClusterSlotCmd) readReply(cn *pool.Conn) error { v, err := readArrayReply(cn, clusterSlotInfoSliceParser) if err != nil { cmd.err = err @@ -838,7 +858,7 @@ func (cmd *GeoLocationCmd) String() string { return cmdString(cmd, cmd.locations) } -func (cmd *GeoLocationCmd) readReply(cn *conn) error { +func (cmd *GeoLocationCmd) readReply(cn *pool.Conn) error { reply, err := readArrayReply(cn, newGeoLocationSliceParser(cmd.q)) if err != nil { cmd.err = err diff --git a/conn.go b/conn.go deleted file mode 100644 index e7306d9498..0000000000 --- a/conn.go +++ /dev/null @@ -1,120 +0,0 @@ -package redis - -import ( - "bufio" - "net" - "time" -) - -const defaultBufSize = 4096 - -var noTimeout = time.Time{} - -// Stubbed in tests. -var now = time.Now - -type conn struct { - netcn net.Conn - rd *bufio.Reader - buf []byte - - UsedAt time.Time - ReadTimeout time.Duration - WriteTimeout time.Duration -} - -func newConnDialer(opt *Options) func() (*conn, error) { - dialer := opt.getDialer() - return func() (*conn, error) { - netcn, err := dialer() - if err != nil { - return nil, err - } - cn := &conn{ - netcn: netcn, - buf: make([]byte, defaultBufSize), - - UsedAt: now(), - } - cn.rd = bufio.NewReader(cn) - return cn, cn.init(opt) - } -} - -func (cn *conn) init(opt *Options) error { - if opt.Password == "" && opt.DB == 0 { - return nil - } - - // Temp client for Auth and Select. - client := newClient(opt, newSingleConnPool(cn)) - - if opt.Password != "" { - if err := client.Auth(opt.Password).Err(); err != nil { - return err - } - } - - if opt.DB > 0 { - if err := client.Select(opt.DB).Err(); err != nil { - return err - } - } - - return nil -} - -func (cn *conn) writeCmds(cmds ...Cmder) error { - cn.buf = cn.buf[:0] - for _, cmd := range cmds { - var err error - cn.buf, err = appendArgs(cn.buf, cmd.args()) - if err != nil { - return err - } - } - - _, err := cn.Write(cn.buf) - return err -} - -func (cn *conn) Read(b []byte) (int, error) { - cn.UsedAt = now() - if cn.ReadTimeout != 0 { - cn.netcn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) - } else { - cn.netcn.SetReadDeadline(noTimeout) - } - return cn.netcn.Read(b) -} - -func (cn *conn) Write(b []byte) (int, error) { - cn.UsedAt = now() - if cn.WriteTimeout != 0 { - cn.netcn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) - } else { - cn.netcn.SetWriteDeadline(noTimeout) - } - return cn.netcn.Write(b) -} - -func (cn *conn) RemoteAddr() net.Addr { - return cn.netcn.RemoteAddr() -} - -func (cn *conn) Close() error { - return cn.netcn.Close() -} - -func isSameSlice(s1, s2 []byte) bool { - return len(s1) > 0 && len(s2) > 0 && &s1[0] == &s2[0] -} - -func (cn *conn) copyBuf(b []byte) []byte { - if isSameSlice(b, cn.buf) { - new := make([]byte, len(b)) - copy(new, b) - return new - } - return b -} diff --git a/conn_test.go b/conn_test.go deleted file mode 100644 index 4f1eef6d73..0000000000 --- a/conn_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package redis_test - -import ( - "net" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "gopkg.in/redis.v3" -) - -var _ = Describe("newConnDialer with bad connection", func() { - It("should return an error", func() { - dialer := redis.NewConnDialer(&redis.Options{ - Dialer: func() (net.Conn, error) { - return &badConn{}, nil - }, - MaxRetries: 3, - Password: "password", - DB: 1, - }) - _, err := dialer() - Expect(err).To(MatchError("bad connection")) - }) -}) diff --git a/error.go b/error.go index 3f2a560c0b..e2430b4a94 100644 --- a/error.go +++ b/error.go @@ -1,12 +1,15 @@ package redis import ( + "errors" "fmt" "io" "net" "strings" ) +var errClosed = errors.New("redis: client is closed") + // Redis nil reply, .e.g. when key does not exist. var Nil = errorf("redis: nil") diff --git a/export_test.go b/export_test.go index 95715b5f85..cce779bb7e 100644 --- a/export_test.go +++ b/export_test.go @@ -1,30 +1,11 @@ package redis -import ( - "net" - "time" -) +import "gopkg.in/redis.v3/internal/pool" -func (c *baseClient) Pool() pool { +func (c *baseClient) Pool() pool.Pooler { return c.connPool } -func (c *PubSub) Pool() pool { +func (c *PubSub) Pool() pool.Pooler { return c.base.connPool } - -var NewConnDialer = newConnDialer - -func (cn *conn) SetNetConn(netcn net.Conn) { - cn.netcn = netcn -} - -func SetTime(tm time.Time) { - now = func() time.Time { - return tm - } -} - -func RestoreTime() { - now = time.Now -} diff --git a/internal/pool/conn.go b/internal/pool/conn.go new file mode 100644 index 0000000000..1e1e8611ce --- /dev/null +++ b/internal/pool/conn.go @@ -0,0 +1,60 @@ +package pool + +import ( + "bufio" + "net" + "time" +) + +const defaultBufSize = 4096 + +var noTimeout = time.Time{} + +type Conn struct { + NetConn net.Conn + Rd *bufio.Reader + Buf []byte + + UsedAt time.Time + ReadTimeout time.Duration + WriteTimeout time.Duration +} + +func NewConn(netConn net.Conn) *Conn { + cn := &Conn{ + NetConn: netConn, + Buf: make([]byte, defaultBufSize), + + UsedAt: time.Now(), + } + cn.Rd = bufio.NewReader(cn) + return cn +} + +func (cn *Conn) Read(b []byte) (int, error) { + cn.UsedAt = time.Now() + if cn.ReadTimeout != 0 { + cn.NetConn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) + } else { + cn.NetConn.SetReadDeadline(noTimeout) + } + return cn.NetConn.Read(b) +} + +func (cn *Conn) Write(b []byte) (int, error) { + cn.UsedAt = time.Now() + if cn.WriteTimeout != 0 { + cn.NetConn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) + } else { + cn.NetConn.SetWriteDeadline(noTimeout) + } + return cn.NetConn.Write(b) +} + +func (cn *Conn) RemoteAddr() net.Addr { + return cn.NetConn.RemoteAddr() +} + +func (cn *Conn) Close() error { + return cn.NetConn.Close() +} diff --git a/internal/pool/conn_list.go b/internal/pool/conn_list.go new file mode 100644 index 0000000000..f8b82ab230 --- /dev/null +++ b/internal/pool/conn_list.go @@ -0,0 +1,100 @@ +package pool + +import ( + "sync" + "sync/atomic" +) + +type connList struct { + cns []*Conn + mx sync.Mutex + len int32 // atomic + size int32 +} + +func newConnList(size int) *connList { + return &connList{ + cns: make([]*Conn, 0, size), + size: int32(size), + } +} + +func (l *connList) Len() int { + return int(atomic.LoadInt32(&l.len)) +} + +// Reserve reserves place in the list and returns true on success. The +// caller must add or remove connection if place was reserved. +func (l *connList) Reserve() bool { + len := atomic.AddInt32(&l.len, 1) + reserved := len <= l.size + if !reserved { + atomic.AddInt32(&l.len, -1) + } + return reserved +} + +// Add adds connection to the list. The caller must reserve place first. +func (l *connList) Add(cn *Conn) { + l.mx.Lock() + l.cns = append(l.cns, cn) + l.mx.Unlock() +} + +// Remove closes connection and removes it from the list. +func (l *connList) Remove(cn *Conn) error { + defer l.mx.Unlock() + l.mx.Lock() + + if cn == nil { + atomic.AddInt32(&l.len, -1) + return nil + } + + for i, c := range l.cns { + if c == cn { + l.cns = append(l.cns[:i], l.cns[i+1:]...) + atomic.AddInt32(&l.len, -1) + return cn.Close() + } + } + + if l.closed() { + return nil + } + panic("conn not found in the list") +} + +func (l *connList) Replace(cn, newcn *Conn) error { + defer l.mx.Unlock() + l.mx.Lock() + + for i, c := range l.cns { + if c == cn { + l.cns[i] = newcn + return cn.Close() + } + } + + if l.closed() { + return newcn.Close() + } + panic("conn not found in the list") +} + +func (l *connList) Close() (retErr error) { + l.mx.Lock() + for _, c := range l.cns { + if err := c.Close(); err != nil { + retErr = err + } + } + l.cns = nil + atomic.StoreInt32(&l.len, 0) + l.mx.Unlock() + return retErr +} + +func (l *connList) closed() bool { + return l.cns == nil +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go new file mode 100644 index 0000000000..ab03195da6 --- /dev/null +++ b/internal/pool/pool.go @@ -0,0 +1,284 @@ +package pool + +import ( + "errors" + "fmt" + "log" + "sync/atomic" + "time" + + "gopkg.in/bsm/ratelimit.v1" +) + +var Logger *log.Logger + +var ( + errClosed = errors.New("redis: client is closed") + ErrPoolTimeout = errors.New("redis: connection pool timeout") +) + +// PoolStats contains pool state information and accumulated stats. +type PoolStats struct { + Requests uint32 // number of times a connection was requested by the pool + Hits uint32 // number of times free connection was found in the pool + Waits uint32 // number of times the pool had to wait for a connection + Timeouts uint32 // number of times a wait timeout occurred + + TotalConns uint32 // the number of total connections in the pool + FreeConns uint32 // the number of free connections in the pool +} + +type Pooler interface { + First() *Conn + Get() (*Conn, bool, error) + Put(*Conn) error + Remove(*Conn, error) error + Len() int + FreeLen() int + Close() error + Stats() *PoolStats +} + +type dialer func() (*Conn, error) + +type ConnPool struct { + dial dialer + + poolTimeout time.Duration + idleTimeout time.Duration + + rl *ratelimit.RateLimiter + conns *connList + freeConns chan *Conn + stats PoolStats + + _closed int32 + + lastErr atomic.Value +} + +func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Duration) *ConnPool { + p := &ConnPool{ + dial: dial, + + poolTimeout: poolTimeout, + idleTimeout: idleTimeout, + + rl: ratelimit.New(3*poolSize, time.Second), + conns: newConnList(poolSize), + freeConns: make(chan *Conn, poolSize), + } + if idleTimeout > 0 { + go p.reaper() + } + return p +} + +func (p *ConnPool) closed() bool { + return atomic.LoadInt32(&p._closed) == 1 +} + +func (p *ConnPool) isIdle(cn *Conn) bool { + return p.idleTimeout > 0 && time.Since(cn.UsedAt) > p.idleTimeout +} + +// First returns first non-idle connection from the pool or nil if +// there are no connections. +func (p *ConnPool) First() *Conn { + for { + select { + case cn := <-p.freeConns: + if p.isIdle(cn) { + var err error + cn, err = p.replace(cn) + if err != nil { + Logger.Printf("pool.replace failed: %s", err) + continue + } + } + return cn + default: + return nil + } + } + panic("not reached") +} + +// wait waits for free non-idle connection. It returns nil on timeout. +func (p *ConnPool) wait() *Conn { + deadline := time.After(p.poolTimeout) + for { + select { + case cn := <-p.freeConns: + if p.isIdle(cn) { + var err error + cn, err = p.replace(cn) + if err != nil { + Logger.Printf("pool.replace failed: %s", err) + continue + } + } + return cn + case <-deadline: + return nil + } + } + panic("not reached") +} + +// Establish a new connection +func (p *ConnPool) new() (*Conn, error) { + if p.rl.Limit() { + err := fmt.Errorf( + "redis: you open connections too fast (last_error=%q)", + p.loadLastErr(), + ) + return nil, err + } + + cn, err := p.dial() + if err != nil { + p.storeLastErr(err.Error()) + return nil, err + } + + return cn, nil +} + +// Get returns existed connection from the pool or creates a new one. +func (p *ConnPool) Get() (cn *Conn, isNew bool, err error) { + if p.closed() { + err = errClosed + return + } + + atomic.AddUint32(&p.stats.Requests, 1) + + // Fetch first non-idle connection, if available. + if cn = p.First(); cn != nil { + atomic.AddUint32(&p.stats.Hits, 1) + return + } + + // Try to create a new one. + if p.conns.Reserve() { + isNew = true + + cn, err = p.new() + if err != nil { + p.conns.Remove(nil) + return + } + p.conns.Add(cn) + return + } + + // Otherwise, wait for the available connection. + atomic.AddUint32(&p.stats.Waits, 1) + if cn = p.wait(); cn != nil { + return + } + + atomic.AddUint32(&p.stats.Timeouts, 1) + err = ErrPoolTimeout + return +} + +func (p *ConnPool) Put(cn *Conn) error { + if cn.Rd.Buffered() != 0 { + b, _ := cn.Rd.Peek(cn.Rd.Buffered()) + err := fmt.Errorf("connection has unread data: %q", b) + Logger.Print(err) + return p.Remove(cn, err) + } + p.freeConns <- cn + return nil +} + +func (p *ConnPool) replace(cn *Conn) (*Conn, error) { + newcn, err := p.new() + if err != nil { + _ = p.conns.Remove(cn) + return nil, err + } + _ = p.conns.Replace(cn, newcn) + return newcn, nil +} + +func (p *ConnPool) Remove(cn *Conn, reason error) error { + p.storeLastErr(reason.Error()) + + // Replace existing connection with new one and unblock waiter. + newcn, err := p.replace(cn) + if err != nil { + return err + } + p.freeConns <- newcn + return nil +} + +// Len returns total number of connections. +func (p *ConnPool) Len() int { + return p.conns.Len() +} + +// FreeLen returns number of free connections. +func (p *ConnPool) FreeLen() int { + return len(p.freeConns) +} + +func (p *ConnPool) Stats() *PoolStats { + stats := p.stats + stats.Requests = atomic.LoadUint32(&p.stats.Requests) + stats.Waits = atomic.LoadUint32(&p.stats.Waits) + stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts) + stats.TotalConns = uint32(p.Len()) + stats.FreeConns = uint32(p.FreeLen()) + return &stats +} + +func (p *ConnPool) Close() (retErr error) { + if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { + return errClosed + } + // Wait for app to free connections, but don't close them immediately. + for i := 0; i < p.Len(); i++ { + if cn := p.wait(); cn == nil { + break + } + } + // Close all connections. + if err := p.conns.Close(); err != nil { + retErr = err + } + return retErr +} + +func (p *ConnPool) reaper() { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + for _ = range ticker.C { + if p.closed() { + break + } + + // pool.First removes idle connections from the pool and + // returns first non-idle connection. So just put returned + // connection back. + if cn := p.First(); cn != nil { + p.Put(cn) + } + } +} + +func (p *ConnPool) storeLastErr(err string) { + p.lastErr.Store(err) +} + +func (p *ConnPool) loadLastErr() string { + if v := p.lastErr.Load(); v != nil { + return v.(string) + } + return "" +} diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go new file mode 100644 index 0000000000..f2d58cf7f4 --- /dev/null +++ b/internal/pool/pool_single.go @@ -0,0 +1,47 @@ +package pool + +type SingleConnPool struct { + cn *Conn +} + +func NewSingleConnPool(cn *Conn) *SingleConnPool { + return &SingleConnPool{ + cn: cn, + } +} + +func (p *SingleConnPool) First() *Conn { + return p.cn +} + +func (p *SingleConnPool) Get() (*Conn, bool, error) { + return p.cn, false, nil +} + +func (p *SingleConnPool) Put(cn *Conn) error { + if p.cn != cn { + panic("p.cn != cn") + } + return nil +} + +func (p *SingleConnPool) Remove(cn *Conn, _ error) error { + if p.cn != cn { + panic("p.cn != cn") + } + return nil +} + +func (p *SingleConnPool) Len() int { + return 1 +} + +func (p *SingleConnPool) FreeLen() int { + return 0 +} + +func (p *SingleConnPool) Stats() *PoolStats { return nil } + +func (p *SingleConnPool) Close() error { + return nil +} diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go new file mode 100644 index 0000000000..c611c4b448 --- /dev/null +++ b/internal/pool/pool_sticky.go @@ -0,0 +1,128 @@ +package pool + +import ( + "errors" + "sync" +) + +type StickyConnPool struct { + pool *ConnPool + reusable bool + + cn *Conn + closed bool + mx sync.Mutex +} + +func NewStickyConnPool(pool *ConnPool, reusable bool) *StickyConnPool { + return &StickyConnPool{ + pool: pool, + reusable: reusable, + } +} + +func (p *StickyConnPool) First() *Conn { + p.mx.Lock() + cn := p.cn + p.mx.Unlock() + return cn +} + +func (p *StickyConnPool) Get() (cn *Conn, isNew bool, err error) { + defer p.mx.Unlock() + p.mx.Lock() + + if p.closed { + err = errClosed + return + } + if p.cn != nil { + cn = p.cn + return + } + + cn, isNew, err = p.pool.Get() + if err != nil { + return + } + p.cn = cn + return +} + +func (p *StickyConnPool) put() (err error) { + err = p.pool.Put(p.cn) + p.cn = nil + return err +} + +func (p *StickyConnPool) Put(cn *Conn) error { + defer p.mx.Unlock() + p.mx.Lock() + if p.closed { + return errClosed + } + if p.cn != cn { + panic("p.cn != cn") + } + return nil +} + +func (p *StickyConnPool) remove(reason error) error { + err := p.pool.Remove(p.cn, reason) + p.cn = nil + return err +} + +func (p *StickyConnPool) Remove(cn *Conn, reason error) error { + defer p.mx.Unlock() + p.mx.Lock() + if p.closed { + return errClosed + } + if p.cn == nil { + panic("p.cn == nil") + } + if cn != nil && p.cn != cn { + panic("p.cn != cn") + } + return p.remove(reason) +} + +func (p *StickyConnPool) Len() int { + defer p.mx.Unlock() + p.mx.Lock() + if p.cn == nil { + return 0 + } + return 1 +} + +func (p *StickyConnPool) FreeLen() int { + defer p.mx.Unlock() + p.mx.Lock() + if p.cn == nil { + return 1 + } + return 0 +} + +func (p *StickyConnPool) Stats() *PoolStats { return nil } + +func (p *StickyConnPool) Close() error { + defer p.mx.Unlock() + p.mx.Lock() + if p.closed { + return errClosed + } + p.closed = true + var err error + if p.cn != nil { + if p.reusable { + err = p.put() + } else { + reason := errors.New("redis: sticky not reusable connection") + err = p.remove(reason) + } + } + return err +} diff --git a/multi.go b/multi.go index 1a13d04732..6b43591f79 100644 --- a/multi.go +++ b/multi.go @@ -3,6 +3,8 @@ package redis import ( "errors" "fmt" + + "gopkg.in/redis.v3/internal/pool" ) var errDiscard = errors.New("redis: Discard can be used only inside Exec") @@ -38,7 +40,7 @@ func (c *Client) Multi() *Multi { multi := &Multi{ base: &baseClient{ opt: c.opt, - connPool: newStickyConnPool(c.connPool, true), + connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true), }, } multi.commandable.process = multi.process @@ -137,8 +139,8 @@ func (c *Multi) Exec(f func() error) ([]Cmder, error) { return retCmds, err } -func (c *Multi) execCmds(cn *conn, cmds []Cmder) error { - err := cn.writeCmds(cmds...) +func (c *Multi) execCmds(cn *pool.Conn, cmds []Cmder) error { + err := writeCmd(cn, cmds...) if err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) return err diff --git a/multi_test.go b/multi_test.go index 459d0a6222..fa532d1a42 100644 --- a/multi_test.go +++ b/multi_test.go @@ -145,7 +145,7 @@ var _ = Describe("Multi", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{}) + cn.NetConn = &badConn{} err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) @@ -172,7 +172,7 @@ var _ = Describe("Multi", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{}) + cn.NetConn = &badConn{} err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) diff --git a/options.go b/options.go new file mode 100644 index 0000000000..95a266633c --- /dev/null +++ b/options.go @@ -0,0 +1,144 @@ +package redis + +import ( + "net" + "time" + + "gopkg.in/redis.v3/internal/pool" +) + +type Options struct { + // The network type, either tcp or unix. + // Default is tcp. + Network string + // host:port address. + Addr string + + // Dialer creates new network connection and has priority over + // Network and Addr options. + Dialer func() (net.Conn, error) + + // An optional password. Must match the password specified in the + // requirepass server configuration option. + Password string + // A database to be selected after connecting to server. + DB int64 + + // The maximum number of retries before giving up. + // Default is to not retry failed commands. + MaxRetries int + + // Sets the deadline for establishing new connections. If reached, + // dial will fail with a timeout. + DialTimeout time.Duration + // Sets the deadline for socket reads. If reached, commands will + // fail with a timeout instead of blocking. + ReadTimeout time.Duration + // Sets the deadline for socket writes. If reached, commands will + // fail with a timeout instead of blocking. + WriteTimeout time.Duration + + // The maximum number of socket connections. + // Default is 10 connections. + PoolSize int + // Specifies amount of time client waits for connection if all + // connections are busy before returning an error. + // Default is 1 seconds. + PoolTimeout time.Duration + // Specifies amount of time after which client closes idle + // connections. Should be less than server's timeout. + // Default is to not close idle connections. + IdleTimeout time.Duration +} + +func (opt *Options) getNetwork() string { + if opt.Network == "" { + return "tcp" + } + return opt.Network +} + +func (opt *Options) getDialer() func() (net.Conn, error) { + if opt.Dialer == nil { + opt.Dialer = func() (net.Conn, error) { + return net.DialTimeout(opt.getNetwork(), opt.Addr, opt.getDialTimeout()) + } + } + return opt.Dialer +} + +func (opt *Options) getPoolDialer() func() (*pool.Conn, error) { + dial := opt.getDialer() + return func() (*pool.Conn, error) { + netcn, err := dial() + if err != nil { + return nil, err + } + cn := pool.NewConn(netcn) + return cn, opt.initConn(cn) + } +} + +func (opt *Options) getPoolSize() int { + if opt.PoolSize == 0 { + return 10 + } + return opt.PoolSize +} + +func (opt *Options) getDialTimeout() time.Duration { + if opt.DialTimeout == 0 { + return 5 * time.Second + } + return opt.DialTimeout +} + +func (opt *Options) getPoolTimeout() time.Duration { + if opt.PoolTimeout == 0 { + return 1 * time.Second + } + return opt.PoolTimeout +} + +func (opt *Options) getIdleTimeout() time.Duration { + return opt.IdleTimeout +} + +func (opt *Options) initConn(cn *pool.Conn) error { + if opt.Password == "" && opt.DB == 0 { + return nil + } + + // Temp client for Auth and Select. + client := newClient(opt, pool.NewSingleConnPool(cn)) + + if opt.Password != "" { + if err := client.Auth(opt.Password).Err(); err != nil { + return err + } + } + + if opt.DB > 0 { + if err := client.Select(opt.DB).Err(); err != nil { + return err + } + } + + return nil +} + +func newConnPool(opt *Options) *pool.ConnPool { + return pool.NewConnPool( + opt.getPoolDialer(), opt.getPoolSize(), opt.getPoolTimeout(), opt.getIdleTimeout()) +} + +// PoolStats contains pool state information and accumulated stats. +type PoolStats struct { + Requests uint32 // number of times a connection was requested by the pool + Hits uint32 // number of times free connection was found in the pool + Waits uint32 // number of times the pool had to wait for a connection + Timeouts uint32 // number of times a wait timeout occurred + + TotalConns uint32 // the number of total connections in the pool + FreeConns uint32 // the number of free connections in the pool +} diff --git a/parser.go b/parser.go index 758ec8df05..2496f66c46 100644 --- a/parser.go +++ b/parser.go @@ -6,6 +6,8 @@ import ( "io" "net" "strconv" + + "gopkg.in/redis.v3/internal/pool" ) const ( @@ -16,7 +18,7 @@ const ( arrayReply = '*' ) -type multiBulkParser func(cn *conn, n int64) (interface{}, error) +type multiBulkParser func(cn *pool.Conn, n int64) (interface{}, error) var ( errReaderTooSmall = errors.New("redis: reader is too small") @@ -223,8 +225,8 @@ func scan(b []byte, val interface{}) error { //------------------------------------------------------------------------------ -func readLine(cn *conn) ([]byte, error) { - line, isPrefix, err := cn.rd.ReadLine() +func readLine(cn *pool.Conn) ([]byte, error) { + line, isPrefix, err := cn.Rd.ReadLine() if err != nil { return line, err } @@ -243,28 +245,27 @@ func isNilReply(b []byte) bool { b[1] == '-' && b[2] == '1' } -func readN(cn *conn, n int) ([]byte, error) { - var b []byte - if cap(cn.buf) < n { - b = make([]byte, n) +func readN(cn *pool.Conn, n int) ([]byte, error) { + if d := n - cap(cn.Buf); d > 0 { + cn.Buf = append(cn.Buf, make([]byte, d)...) } else { - b = cn.buf[:n] + cn.Buf = cn.Buf[:n] } - _, err := io.ReadFull(cn.rd, b) - return b, err + _, err := io.ReadFull(cn.Rd, cn.Buf) + return cn.Buf, err } //------------------------------------------------------------------------------ -func parseErrorReply(cn *conn, line []byte) error { +func parseErrorReply(cn *pool.Conn, line []byte) error { return errorf(string(line[1:])) } -func parseStatusReply(cn *conn, line []byte) ([]byte, error) { +func parseStatusReply(cn *pool.Conn, line []byte) ([]byte, error) { return line[1:], nil } -func parseIntReply(cn *conn, line []byte) (int64, error) { +func parseIntReply(cn *pool.Conn, line []byte) (int64, error) { n, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) if err != nil { return 0, err @@ -272,7 +273,7 @@ func parseIntReply(cn *conn, line []byte) (int64, error) { return n, nil } -func readIntReply(cn *conn) (int64, error) { +func readIntReply(cn *pool.Conn) (int64, error) { line, err := readLine(cn) if err != nil { return 0, err @@ -287,7 +288,7 @@ func readIntReply(cn *conn) (int64, error) { } } -func parseBytesReply(cn *conn, line []byte) ([]byte, error) { +func parseBytesReply(cn *pool.Conn, line []byte) ([]byte, error) { if isNilReply(line) { return nil, Nil } @@ -305,7 +306,7 @@ func parseBytesReply(cn *conn, line []byte) ([]byte, error) { return b[:replyLen], nil } -func readBytesReply(cn *conn) ([]byte, error) { +func readBytesReply(cn *pool.Conn) ([]byte, error) { line, err := readLine(cn) if err != nil { return nil, err @@ -322,7 +323,7 @@ func readBytesReply(cn *conn) ([]byte, error) { } } -func readStringReply(cn *conn) (string, error) { +func readStringReply(cn *pool.Conn) (string, error) { b, err := readBytesReply(cn) if err != nil { return "", err @@ -330,7 +331,7 @@ func readStringReply(cn *conn) (string, error) { return string(b), nil } -func readFloatReply(cn *conn) (float64, error) { +func readFloatReply(cn *pool.Conn) (float64, error) { b, err := readBytesReply(cn) if err != nil { return 0, err @@ -338,7 +339,7 @@ func readFloatReply(cn *conn) (float64, error) { return strconv.ParseFloat(bytesToString(b), 64) } -func parseArrayHeader(cn *conn, line []byte) (int64, error) { +func parseArrayHeader(cn *pool.Conn, line []byte) (int64, error) { if isNilReply(line) { return 0, Nil } @@ -350,7 +351,7 @@ func parseArrayHeader(cn *conn, line []byte) (int64, error) { return n, nil } -func parseArrayReply(cn *conn, p multiBulkParser, line []byte) (interface{}, error) { +func parseArrayReply(cn *pool.Conn, p multiBulkParser, line []byte) (interface{}, error) { n, err := parseArrayHeader(cn, line) if err != nil { return nil, err @@ -358,7 +359,7 @@ func parseArrayReply(cn *conn, p multiBulkParser, line []byte) (interface{}, err return p(cn, n) } -func readArrayHeader(cn *conn) (int64, error) { +func readArrayHeader(cn *pool.Conn) (int64, error) { line, err := readLine(cn) if err != nil { return 0, err @@ -373,7 +374,7 @@ func readArrayHeader(cn *conn) (int64, error) { } } -func readArrayReply(cn *conn, p multiBulkParser) (interface{}, error) { +func readArrayReply(cn *pool.Conn, p multiBulkParser) (interface{}, error) { line, err := readLine(cn) if err != nil { return nil, err @@ -388,7 +389,7 @@ func readArrayReply(cn *conn, p multiBulkParser) (interface{}, error) { } } -func readReply(cn *conn, p multiBulkParser) (interface{}, error) { +func readReply(cn *pool.Conn, p multiBulkParser) (interface{}, error) { line, err := readLine(cn) if err != nil { return nil, err @@ -409,7 +410,7 @@ func readReply(cn *conn, p multiBulkParser) (interface{}, error) { return nil, fmt.Errorf("redis: can't parse %.100q", line) } -func readScanReply(cn *conn) ([]string, int64, error) { +func readScanReply(cn *pool.Conn) ([]string, int64, error) { n, err := readArrayHeader(cn) if err != nil { return nil, 0, err @@ -445,7 +446,7 @@ func readScanReply(cn *conn) ([]string, int64, error) { return keys, cursor, err } -func sliceParser(cn *conn, n int64) (interface{}, error) { +func sliceParser(cn *pool.Conn, n int64) (interface{}, error) { vals := make([]interface{}, 0, n) for i := int64(0); i < n; i++ { v, err := readReply(cn, sliceParser) @@ -465,7 +466,7 @@ func sliceParser(cn *conn, n int64) (interface{}, error) { return vals, nil } -func intSliceParser(cn *conn, n int64) (interface{}, error) { +func intSliceParser(cn *pool.Conn, n int64) (interface{}, error) { ints := make([]int64, 0, n) for i := int64(0); i < n; i++ { n, err := readIntReply(cn) @@ -477,7 +478,7 @@ func intSliceParser(cn *conn, n int64) (interface{}, error) { return ints, nil } -func boolSliceParser(cn *conn, n int64) (interface{}, error) { +func boolSliceParser(cn *pool.Conn, n int64) (interface{}, error) { bools := make([]bool, 0, n) for i := int64(0); i < n; i++ { n, err := readIntReply(cn) @@ -489,7 +490,7 @@ func boolSliceParser(cn *conn, n int64) (interface{}, error) { return bools, nil } -func stringSliceParser(cn *conn, n int64) (interface{}, error) { +func stringSliceParser(cn *pool.Conn, n int64) (interface{}, error) { ss := make([]string, 0, n) for i := int64(0); i < n; i++ { s, err := readStringReply(cn) @@ -504,7 +505,7 @@ func stringSliceParser(cn *conn, n int64) (interface{}, error) { return ss, nil } -func floatSliceParser(cn *conn, n int64) (interface{}, error) { +func floatSliceParser(cn *pool.Conn, n int64) (interface{}, error) { nn := make([]float64, 0, n) for i := int64(0); i < n; i++ { n, err := readFloatReply(cn) @@ -516,7 +517,7 @@ func floatSliceParser(cn *conn, n int64) (interface{}, error) { return nn, nil } -func stringStringMapParser(cn *conn, n int64) (interface{}, error) { +func stringStringMapParser(cn *pool.Conn, n int64) (interface{}, error) { m := make(map[string]string, n/2) for i := int64(0); i < n; i += 2 { key, err := readStringReply(cn) @@ -534,7 +535,7 @@ func stringStringMapParser(cn *conn, n int64) (interface{}, error) { return m, nil } -func stringIntMapParser(cn *conn, n int64) (interface{}, error) { +func stringIntMapParser(cn *pool.Conn, n int64) (interface{}, error) { m := make(map[string]int64, n/2) for i := int64(0); i < n; i += 2 { key, err := readStringReply(cn) @@ -552,7 +553,7 @@ func stringIntMapParser(cn *conn, n int64) (interface{}, error) { return m, nil } -func zSliceParser(cn *conn, n int64) (interface{}, error) { +func zSliceParser(cn *pool.Conn, n int64) (interface{}, error) { zz := make([]Z, n/2) for i := int64(0); i < n; i += 2 { var err error @@ -572,7 +573,7 @@ func zSliceParser(cn *conn, n int64) (interface{}, error) { return zz, nil } -func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { +func clusterSlotInfoSliceParser(cn *pool.Conn, n int64) (interface{}, error) { infos := make([]ClusterSlotInfo, 0, n) for i := int64(0); i < n; i++ { n, err := readArrayHeader(cn) @@ -638,7 +639,7 @@ func clusterSlotInfoSliceParser(cn *conn, n int64) (interface{}, error) { } func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { - return func(cn *conn, n int64) (interface{}, error) { + return func(cn *pool.Conn, n int64) (interface{}, error) { var loc GeoLocation var err error @@ -682,7 +683,7 @@ func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { } func newGeoLocationSliceParser(q *GeoRadiusQuery) multiBulkParser { - return func(cn *conn, n int64) (interface{}, error) { + return func(cn *pool.Conn, n int64) (interface{}, error) { locs := make([]GeoLocation, 0, n) for i := int64(0); i < n; i++ { v, err := readReply(cn, newGeoLocationParser(q)) diff --git a/parser_test.go b/parser_test.go index b1c7434439..77287a7aa8 100644 --- a/parser_test.go +++ b/parser_test.go @@ -4,6 +4,8 @@ import ( "bufio" "bytes" "testing" + + "gopkg.in/redis.v3/internal/pool" ) func BenchmarkParseReplyStatus(b *testing.B) { @@ -31,9 +33,9 @@ func benchmarkParseReply(b *testing.B, reply string, p multiBulkParser, wanterr for i := 0; i < b.N; i++ { buf.WriteString(reply) } - cn := &conn{ - rd: bufio.NewReader(buf), - buf: make([]byte, 0, defaultBufSize), + cn := &pool.Conn{ + Rd: bufio.NewReader(buf), + Buf: make([]byte, 4096), } b.ResetTimer() diff --git a/pipeline.go b/pipeline.go index 8caae6bf4a..098207c27e 100644 --- a/pipeline.go +++ b/pipeline.go @@ -3,6 +3,8 @@ package redis import ( "sync" "sync/atomic" + + "gopkg.in/redis.v3/internal/pool" ) // Pipeline implements pipelining as described in @@ -110,8 +112,8 @@ func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { return cmds, retErr } -func execCmds(cn *conn, cmds []Cmder) ([]Cmder, error) { - if err := cn.writeCmds(cmds...); err != nil { +func execCmds(cn *pool.Conn, cmds []Cmder) ([]Cmder, error) { + if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) return cmds, err } diff --git a/pool.go b/pool.go deleted file mode 100644 index 3725c4081b..0000000000 --- a/pool.go +++ /dev/null @@ -1,542 +0,0 @@ -package redis - -import ( - "errors" - "fmt" - "sync" - "sync/atomic" - "time" - - "gopkg.in/bsm/ratelimit.v1" -) - -var ( - errClosed = errors.New("redis: client is closed") - errPoolTimeout = errors.New("redis: connection pool timeout") -) - -// PoolStats contains pool state information and accumulated stats. -type PoolStats struct { - Requests uint32 // number of times a connection was requested by the pool - Hits uint32 // number of times free connection was found in the pool - Waits uint32 // number of times the pool had to wait for a connection - Timeouts uint32 // number of times a wait timeout occurred - - TotalConns uint32 // the number of total connections in the pool - FreeConns uint32 // the number of free connections in the pool -} - -type pool interface { - First() *conn - Get() (*conn, bool, error) - Put(*conn) error - Remove(*conn, error) error - Len() int - FreeLen() int - Close() error - Stats() *PoolStats -} - -type connList struct { - cns []*conn - mx sync.Mutex - len int32 // atomic - size int32 -} - -func newConnList(size int) *connList { - return &connList{ - cns: make([]*conn, 0, size), - size: int32(size), - } -} - -func (l *connList) Len() int { - return int(atomic.LoadInt32(&l.len)) -} - -// Reserve reserves place in the list and returns true on success. The -// caller must add or remove connection if place was reserved. -func (l *connList) Reserve() bool { - len := atomic.AddInt32(&l.len, 1) - reserved := len <= l.size - if !reserved { - atomic.AddInt32(&l.len, -1) - } - return reserved -} - -// Add adds connection to the list. The caller must reserve place first. -func (l *connList) Add(cn *conn) { - l.mx.Lock() - l.cns = append(l.cns, cn) - l.mx.Unlock() -} - -// Remove closes connection and removes it from the list. -func (l *connList) Remove(cn *conn) error { - defer l.mx.Unlock() - l.mx.Lock() - - if cn == nil { - atomic.AddInt32(&l.len, -1) - return nil - } - - for i, c := range l.cns { - if c == cn { - l.cns = append(l.cns[:i], l.cns[i+1:]...) - atomic.AddInt32(&l.len, -1) - return cn.Close() - } - } - - if l.closed() { - return nil - } - panic("conn not found in the list") -} - -func (l *connList) Replace(cn, newcn *conn) error { - defer l.mx.Unlock() - l.mx.Lock() - - for i, c := range l.cns { - if c == cn { - l.cns[i] = newcn - return cn.Close() - } - } - - if l.closed() { - return newcn.Close() - } - panic("conn not found in the list") -} - -func (l *connList) Close() (retErr error) { - l.mx.Lock() - for _, c := range l.cns { - if err := c.Close(); err != nil { - retErr = err - } - } - l.cns = nil - atomic.StoreInt32(&l.len, 0) - l.mx.Unlock() - return retErr -} - -func (l *connList) closed() bool { - return l.cns == nil -} - -type connPool struct { - dialer func() (*conn, error) - - rl *ratelimit.RateLimiter - opt *Options - conns *connList - freeConns chan *conn - stats PoolStats - - _closed int32 - - lastErr atomic.Value -} - -func newConnPool(opt *Options) *connPool { - p := &connPool{ - dialer: newConnDialer(opt), - - rl: ratelimit.New(3*opt.getPoolSize(), time.Second), - opt: opt, - conns: newConnList(opt.getPoolSize()), - freeConns: make(chan *conn, opt.getPoolSize()), - } - if p.opt.getIdleTimeout() > 0 { - go p.reaper() - } - return p -} - -func (p *connPool) closed() bool { - return atomic.LoadInt32(&p._closed) == 1 -} - -func (p *connPool) isIdle(cn *conn) bool { - return p.opt.getIdleTimeout() > 0 && time.Since(cn.UsedAt) > p.opt.getIdleTimeout() -} - -// First returns first non-idle connection from the pool or nil if -// there are no connections. -func (p *connPool) First() *conn { - for { - select { - case cn := <-p.freeConns: - if p.isIdle(cn) { - var err error - cn, err = p.replace(cn) - if err != nil { - Logger.Printf("pool.replace failed: %s", err) - continue - } - } - return cn - default: - return nil - } - } - panic("not reached") -} - -// wait waits for free non-idle connection. It returns nil on timeout. -func (p *connPool) wait() *conn { - deadline := time.After(p.opt.getPoolTimeout()) - for { - select { - case cn := <-p.freeConns: - if p.isIdle(cn) { - var err error - cn, err = p.replace(cn) - if err != nil { - Logger.Printf("pool.replace failed: %s", err) - continue - } - } - return cn - case <-deadline: - return nil - } - } - panic("not reached") -} - -// Establish a new connection -func (p *connPool) new() (*conn, error) { - if p.rl.Limit() { - err := fmt.Errorf( - "redis: you open connections too fast (last_error=%q)", - p.loadLastErr(), - ) - return nil, err - } - - cn, err := p.dialer() - if err != nil { - p.storeLastErr(err.Error()) - return nil, err - } - - return cn, nil -} - -// Get returns existed connection from the pool or creates a new one. -func (p *connPool) Get() (cn *conn, isNew bool, err error) { - if p.closed() { - err = errClosed - return - } - - atomic.AddUint32(&p.stats.Requests, 1) - - // Fetch first non-idle connection, if available. - if cn = p.First(); cn != nil { - atomic.AddUint32(&p.stats.Hits, 1) - return - } - - // Try to create a new one. - if p.conns.Reserve() { - isNew = true - - cn, err = p.new() - if err != nil { - p.conns.Remove(nil) - return - } - p.conns.Add(cn) - return - } - - // Otherwise, wait for the available connection. - atomic.AddUint32(&p.stats.Waits, 1) - if cn = p.wait(); cn != nil { - return - } - - atomic.AddUint32(&p.stats.Timeouts, 1) - err = errPoolTimeout - return -} - -func (p *connPool) Put(cn *conn) error { - if cn.rd.Buffered() != 0 { - b, _ := cn.rd.Peek(cn.rd.Buffered()) - err := fmt.Errorf("connection has unread data: %q", b) - Logger.Print(err) - return p.Remove(cn, err) - } - p.freeConns <- cn - return nil -} - -func (p *connPool) replace(cn *conn) (*conn, error) { - newcn, err := p.new() - if err != nil { - _ = p.conns.Remove(cn) - return nil, err - } - _ = p.conns.Replace(cn, newcn) - return newcn, nil -} - -func (p *connPool) Remove(cn *conn, reason error) error { - p.storeLastErr(reason.Error()) - - // Replace existing connection with new one and unblock waiter. - newcn, err := p.replace(cn) - if err != nil { - return err - } - p.freeConns <- newcn - return nil -} - -// Len returns total number of connections. -func (p *connPool) Len() int { - return p.conns.Len() -} - -// FreeLen returns number of free connections. -func (p *connPool) FreeLen() int { - return len(p.freeConns) -} - -func (p *connPool) Stats() *PoolStats { - stats := p.stats - stats.Requests = atomic.LoadUint32(&p.stats.Requests) - stats.Waits = atomic.LoadUint32(&p.stats.Waits) - stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts) - stats.TotalConns = uint32(p.Len()) - stats.FreeConns = uint32(p.FreeLen()) - return &stats -} - -func (p *connPool) Close() (retErr error) { - if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { - return errClosed - } - // Wait for app to free connections, but don't close them immediately. - for i := 0; i < p.Len(); i++ { - if cn := p.wait(); cn == nil { - break - } - } - // Close all connections. - if err := p.conns.Close(); err != nil { - retErr = err - } - return retErr -} - -func (p *connPool) reaper() { - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - - for _ = range ticker.C { - if p.closed() { - break - } - - // pool.First removes idle connections from the pool and - // returns first non-idle connection. So just put returned - // connection back. - if cn := p.First(); cn != nil { - p.Put(cn) - } - } -} - -func (p *connPool) storeLastErr(err string) { - p.lastErr.Store(err) -} - -func (p *connPool) loadLastErr() string { - if v := p.lastErr.Load(); v != nil { - return v.(string) - } - return "" -} - -//------------------------------------------------------------------------------ - -type singleConnPool struct { - cn *conn -} - -func newSingleConnPool(cn *conn) *singleConnPool { - return &singleConnPool{ - cn: cn, - } -} - -func (p *singleConnPool) First() *conn { - return p.cn -} - -func (p *singleConnPool) Get() (*conn, bool, error) { - return p.cn, false, nil -} - -func (p *singleConnPool) Put(cn *conn) error { - if p.cn != cn { - panic("p.cn != cn") - } - return nil -} - -func (p *singleConnPool) Remove(cn *conn, _ error) error { - if p.cn != cn { - panic("p.cn != cn") - } - return nil -} - -func (p *singleConnPool) Len() int { - return 1 -} - -func (p *singleConnPool) FreeLen() int { - return 0 -} - -func (p *singleConnPool) Stats() *PoolStats { return nil } - -func (p *singleConnPool) Close() error { - return nil -} - -//------------------------------------------------------------------------------ - -type stickyConnPool struct { - pool pool - reusable bool - - cn *conn - closed bool - mx sync.Mutex -} - -func newStickyConnPool(pool pool, reusable bool) *stickyConnPool { - return &stickyConnPool{ - pool: pool, - reusable: reusable, - } -} - -func (p *stickyConnPool) First() *conn { - p.mx.Lock() - cn := p.cn - p.mx.Unlock() - return cn -} - -func (p *stickyConnPool) Get() (cn *conn, isNew bool, err error) { - defer p.mx.Unlock() - p.mx.Lock() - - if p.closed { - err = errClosed - return - } - if p.cn != nil { - cn = p.cn - return - } - - cn, isNew, err = p.pool.Get() - if err != nil { - return - } - p.cn = cn - return -} - -func (p *stickyConnPool) put() (err error) { - err = p.pool.Put(p.cn) - p.cn = nil - return err -} - -func (p *stickyConnPool) Put(cn *conn) error { - defer p.mx.Unlock() - p.mx.Lock() - if p.closed { - return errClosed - } - if p.cn != cn { - panic("p.cn != cn") - } - return nil -} - -func (p *stickyConnPool) remove(reason error) error { - err := p.pool.Remove(p.cn, reason) - p.cn = nil - return err -} - -func (p *stickyConnPool) Remove(cn *conn, reason error) error { - defer p.mx.Unlock() - p.mx.Lock() - if p.closed { - return errClosed - } - if p.cn == nil { - panic("p.cn == nil") - } - if cn != nil && p.cn != cn { - panic("p.cn != cn") - } - return p.remove(reason) -} - -func (p *stickyConnPool) Len() int { - defer p.mx.Unlock() - p.mx.Lock() - if p.cn == nil { - return 0 - } - return 1 -} - -func (p *stickyConnPool) FreeLen() int { - defer p.mx.Unlock() - p.mx.Lock() - if p.cn == nil { - return 1 - } - return 0 -} - -func (p *stickyConnPool) Stats() *PoolStats { return nil } - -func (p *stickyConnPool) Close() error { - defer p.mx.Unlock() - p.mx.Lock() - if p.closed { - return errClosed - } - p.closed = true - var err error - if p.cn != nil { - if p.reusable { - err = p.put() - } else { - reason := errors.New("redis: sticky not reusable connection") - err = p.remove(reason) - } - } - return err -} diff --git a/pubsub.go b/pubsub.go index c1fb4628a4..f1c93c8f16 100644 --- a/pubsub.go +++ b/pubsub.go @@ -4,6 +4,8 @@ import ( "fmt" "net" "time" + + "gopkg.in/redis.v3/internal/pool" ) // Posts a message to the given channel. @@ -30,7 +32,7 @@ func (c *Client) PubSub() *PubSub { return &PubSub{ base: &baseClient{ opt: c.opt, - connPool: newStickyConnPool(c.connPool, false), + connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), }, } } @@ -47,19 +49,20 @@ func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { return pubsub, pubsub.PSubscribe(channels...) } -func (c *PubSub) subscribe(cmd string, channels ...string) error { +func (c *PubSub) subscribe(redisCmd string, channels ...string) error { cn, _, err := c.base.conn() if err != nil { return err } args := make([]interface{}, 1+len(channels)) - args[0] = cmd + args[0] = redisCmd for i, channel := range channels { args[1+i] = channel } - req := NewSliceCmd(args...) - return cn.writeCmds(req) + cmd := NewSliceCmd(args...) + + return writeCmd(cn, cmd) } // Subscribes the client to the specified channels. @@ -132,7 +135,7 @@ func (c *PubSub) Ping(payload string) error { args = append(args, payload) } cmd := NewCmd(args...) - return cn.writeCmds(cmd) + return writeCmd(cn, cmd) } // Message received after a successful subscription to channel. @@ -296,7 +299,7 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { } } -func (c *PubSub) putConn(cn *conn, err error) { +func (c *PubSub) putConn(cn *pool.Conn, err error) { if !c.base.putConn(cn, err, true) { c.nsub = 0 } diff --git a/pubsub_test.go b/pubsub_test.go index 669c0737d8..a8bb610be5 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -291,10 +291,10 @@ var _ = Describe("PubSub", func() { expectReceiveMessageOnError := func(pubsub *redis.PubSub) { cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn1.SetNetConn(&badConn{ + cn1.NetConn = &badConn{ readErr: io.EOF, writeErr: io.EOF, - }) + } done := make(chan bool, 1) go func() { diff --git a/redis.go b/redis.go index 5558ad102f..da4b41b377 100644 --- a/redis.go +++ b/redis.go @@ -3,15 +3,26 @@ package redis // import "gopkg.in/redis.v3" import ( "fmt" "log" - "net" "os" - "time" + "sync/atomic" + + "gopkg.in/redis.v3/internal/pool" ) -var Logger = log.New(os.Stderr, "redis: ", log.LstdFlags) +// Deprecated. Use SetLogger instead. +var Logger *log.Logger + +func init() { + SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags)) +} + +func SetLogger(logger *log.Logger) { + Logger = logger + pool.Logger = logger +} type baseClient struct { - connPool pool + connPool pool.Pooler opt *Options onClose func() error // hook called when client is closed @@ -21,11 +32,11 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.opt.Addr, c.opt.DB) } -func (c *baseClient) conn() (*conn, bool, error) { +func (c *baseClient) conn() (*pool.Conn, bool, error) { return c.connPool.Get() } -func (c *baseClient) putConn(cn *conn, err error, allowTimeout bool) bool { +func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { if isBadConn(err, allowTimeout) { err = c.connPool.Remove(cn, err) if err != nil { @@ -61,7 +72,7 @@ func (c *baseClient) process(cmd Cmder) { } cn.WriteTimeout = c.opt.WriteTimeout - if err := cn.writeCmds(cmd); err != nil { + if err := writeCmd(cn, cmd); err != nil { c.putConn(cn, err, false) cmd.setErr(err) if shouldRetry(err) { @@ -99,93 +110,6 @@ func (c *baseClient) Close() error { //------------------------------------------------------------------------------ -type Options struct { - // The network type, either tcp or unix. - // Default is tcp. - Network string - // host:port address. - Addr string - - // Dialer creates new network connection and has priority over - // Network and Addr options. - Dialer func() (net.Conn, error) - - // An optional password. Must match the password specified in the - // requirepass server configuration option. - Password string - // A database to be selected after connecting to server. - DB int64 - - // The maximum number of retries before giving up. - // Default is to not retry failed commands. - MaxRetries int - - // Sets the deadline for establishing new connections. If reached, - // dial will fail with a timeout. - DialTimeout time.Duration - // Sets the deadline for socket reads. If reached, commands will - // fail with a timeout instead of blocking. - ReadTimeout time.Duration - // Sets the deadline for socket writes. If reached, commands will - // fail with a timeout instead of blocking. - WriteTimeout time.Duration - - // The maximum number of socket connections. - // Default is 10 connections. - PoolSize int - // Specifies amount of time client waits for connection if all - // connections are busy before returning an error. - // Default is 1 seconds. - PoolTimeout time.Duration - // Specifies amount of time after which client closes idle - // connections. Should be less than server's timeout. - // Default is to not close idle connections. - IdleTimeout time.Duration -} - -func (opt *Options) getNetwork() string { - if opt.Network == "" { - return "tcp" - } - return opt.Network -} - -func (opt *Options) getDialer() func() (net.Conn, error) { - if opt.Dialer == nil { - opt.Dialer = func() (net.Conn, error) { - return net.DialTimeout(opt.getNetwork(), opt.Addr, opt.getDialTimeout()) - } - } - return opt.Dialer -} - -func (opt *Options) getPoolSize() int { - if opt.PoolSize == 0 { - return 10 - } - return opt.PoolSize -} - -func (opt *Options) getDialTimeout() time.Duration { - if opt.DialTimeout == 0 { - return 5 * time.Second - } - return opt.DialTimeout -} - -func (opt *Options) getPoolTimeout() time.Duration { - if opt.PoolTimeout == 0 { - return 1 * time.Second - } - return opt.PoolTimeout -} - -func (opt *Options) getIdleTimeout() time.Duration { - return opt.IdleTimeout -} - -//------------------------------------------------------------------------------ - // Client is a Redis client representing a pool of zero or more // underlying connections. It's safe for concurrent use by multiple // goroutines. @@ -194,7 +118,7 @@ type Client struct { commandable } -func newClient(opt *Options, pool pool) *Client { +func newClient(opt *Options, pool pool.Pooler) *Client { base := baseClient{opt: opt, connPool: pool} return &Client{ baseClient: base, @@ -206,11 +130,19 @@ func newClient(opt *Options, pool pool) *Client { // NewClient returns a client to the Redis Server specified by Options. func NewClient(opt *Options) *Client { - pool := newConnPool(opt) - return newClient(opt, pool) + return newClient(opt, newConnPool(opt)) } -// PoolStats returns connection pool stats +// PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { - return c.connPool.Stats() + s := c.connPool.Stats() + return &PoolStats{ + Requests: atomic.LoadUint32(&s.Requests), + Hits: atomic.LoadUint32(&s.Hits), + Waits: atomic.LoadUint32(&s.Waits), + Timeouts: atomic.LoadUint32(&s.Timeouts), + + TotalConns: atomic.LoadUint32(&s.TotalConns), + FreeConns: atomic.LoadUint32(&s.FreeConns), + } } diff --git a/redis_test.go b/redis_test.go index 7b5197bcb1..23c39009f8 100644 --- a/redis_test.go +++ b/redis_test.go @@ -160,7 +160,7 @@ var _ = Describe("Client", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{}) + cn.NetConn = &badConn{} err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) @@ -174,10 +174,6 @@ var _ = Describe("Client", func() { Expect(cn.UsedAt).NotTo(BeZero()) createdAt := cn.UsedAt - future := time.Now().Add(time.Hour) - redis.SetTime(future) - defer redis.RestoreTime() - err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) Expect(cn.UsedAt.Equal(createdAt)).To(BeTrue()) @@ -187,6 +183,6 @@ var _ = Describe("Client", func() { cn = client.Pool().First() Expect(cn).NotTo(BeNil()) - Expect(cn.UsedAt.Equal(future)).To(BeTrue()) + Expect(cn.UsedAt.After(createdAt)).To(BeTrue()) }) }) diff --git a/ring.go b/ring.go index f1ae8adf49..3b88d7ca51 100644 --- a/ring.go +++ b/ring.go @@ -8,6 +8,7 @@ import ( "gopkg.in/redis.v3/internal/consistenthash" "gopkg.in/redis.v3/internal/hashtag" + "gopkg.in/redis.v3/internal/pool" ) var ( @@ -200,7 +201,7 @@ func (ring *Ring) heartbeat() { for _, shard := range ring.shards { err := shard.Client.Ping().Err() - if shard.Vote(err == nil || err == errPoolTimeout) { + if shard.Vote(err == nil || err == pool.ErrPoolTimeout) { Logger.Printf("ring shard state changed: %s", shard) rebalance = true } diff --git a/sentinel.go b/sentinel.go index db5db64d46..5575e73e9f 100644 --- a/sentinel.go +++ b/sentinel.go @@ -7,6 +7,8 @@ import ( "strings" "sync" "time" + + "gopkg.in/redis.v3/internal/pool" ) //------------------------------------------------------------------------------ @@ -103,7 +105,7 @@ func (c *sentinelClient) PubSub() *PubSub { return &PubSub{ base: &baseClient{ opt: c.opt, - connPool: newStickyConnPool(c.connPool, false), + connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), }, } } @@ -126,7 +128,7 @@ type sentinelFailover struct { opt *Options - pool pool + pool *pool.ConnPool poolOnce sync.Once mu sync.RWMutex @@ -145,7 +147,7 @@ func (d *sentinelFailover) dial() (net.Conn, error) { return net.DialTimeout("tcp", addr, d.opt.DialTimeout) } -func (d *sentinelFailover) Pool() pool { +func (d *sentinelFailover) Pool() *pool.ConnPool { d.poolOnce.Do(func() { d.opt.Dialer = d.dial d.pool = newConnPool(d.opt) @@ -252,7 +254,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { // Good connections that should be put back to the pool. They // can't be put immediately, because pool.First will return them // again on next iteration. - cnsToPut := make([]*conn, 0) + cnsToPut := make([]*pool.Conn, 0) for { cn := d.pool.First() From e80f790e7640c34e347cf4235aa5f8348a7f4c9c Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Sat, 12 Mar 2016 10:25:59 +0000 Subject: [PATCH 0141/1746] Use go standard path for test data --- .gitignore | 2 +- Makefile | 6 +++--- main_test.go | 6 +++--- {.test => testdata}/redis.conf | 0 4 files changed, 7 insertions(+), 7 deletions(-) rename {.test => testdata}/redis.conf (100%) diff --git a/.gitignore b/.gitignore index 5959942e2b..ebfe903bcd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ *.rdb -.test/ +testdata/*/ diff --git a/Makefile b/Makefile index e9fdd93b2b..03d8959e2d 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,13 @@ all: testdeps go test ./... -test.v -test.cpu=1,2,4 go test ./... -test.v -test.short -test.race -testdeps: .test/redis/src/redis-server +testdeps: testdata/redis/src/redis-server .PHONY: all test testdeps -.test/redis: +testdata/redis: mkdir -p $@ wget -qO- https://github.com/antirez/redis/archive/unstable.tar.gz | tar xvz --strip-components=1 -C $@ -.test/redis/src/redis-server: .test/redis +testdata/redis/src/redis-server: testdata/redis cd $< && make all diff --git a/main_test.go b/main_test.go index d298dd21ea..e1d65c6a85 100644 --- a/main_test.go +++ b/main_test.go @@ -171,12 +171,12 @@ func (p *redisProcess) Close() error { } var ( - redisServerBin, _ = filepath.Abs(filepath.Join(".test", "redis", "src", "redis-server")) - redisServerConf, _ = filepath.Abs(filepath.Join(".test", "redis.conf")) + redisServerBin, _ = filepath.Abs(filepath.Join("testdata", "redis", "src", "redis-server")) + redisServerConf, _ = filepath.Abs(filepath.Join("testdata", "redis.conf")) ) func redisDir(port string) (string, error) { - dir, err := filepath.Abs(filepath.Join(".test", "instances", port)) + dir, err := filepath.Abs(filepath.Join("testdata", "instances", port)) if err != nil { return "", err } diff --git a/.test/redis.conf b/testdata/redis.conf similarity index 100% rename from .test/redis.conf rename to testdata/redis.conf From fdd0fdf678fbc2b4018c239922d01a2c2f10f5e7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 12 Mar 2016 12:41:02 +0200 Subject: [PATCH 0142/1746] Optimize pool.Remove. --- bench_test.go | 10 ++--- internal/pool/conn.go | 31 +++++++++------ internal/pool/conn_list.go | 76 ++++++++++++++---------------------- internal/pool/pool.go | 38 +++++++++++------- internal/pool/pool_single.go | 2 +- internal/pool/pool_sticky.go | 10 ++--- multi_test.go | 4 +- options.go | 37 +----------------- pool_test.go | 4 +- pubsub_test.go | 4 +- redis.go | 34 +++++++++++++++- redis_test.go | 2 +- sentinel.go | 2 +- 13 files changed, 126 insertions(+), 128 deletions(-) diff --git a/bench_test.go b/bench_test.go index 5d6fa37ff8..f4cffc1d63 100644 --- a/bench_test.go +++ b/bench_test.go @@ -278,8 +278,8 @@ func BenchmarkZAdd(b *testing.B) { } func benchmarkPoolGetPut(b *testing.B, poolSize int) { - dial := func() (*pool.Conn, error) { - return pool.NewConn(&net.TCPConn{}), nil + dial := func() (net.Conn, error) { + return &net.TCPConn{}, nil } pool := pool.NewConnPool(dial, poolSize, time.Second, 0) @@ -311,8 +311,8 @@ func BenchmarkPoolGetPut1000Conns(b *testing.B) { } func benchmarkPoolGetRemove(b *testing.B, poolSize int) { - dial := func() (*pool.Conn, error) { - return pool.NewConn(&net.TCPConn{}), nil + dial := func() (net.Conn, error) { + return &net.TCPConn{}, nil } pool := pool.NewConnPool(dial, poolSize, time.Second, 0) removeReason := errors.New("benchmark") @@ -325,7 +325,7 @@ func benchmarkPoolGetRemove(b *testing.B, poolSize int) { if err != nil { b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) } - if err = pool.Remove(conn, removeReason); err != nil { + if err = pool.Replace(conn, removeReason); err != nil { b.Fatalf("no error expected on pool.Remove but received: %s", err.Error()) } } diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 1e1e8611ce..c5a539bcd2 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -8,10 +8,12 @@ import ( const defaultBufSize = 4096 -var noTimeout = time.Time{} +var noDeadline = time.Time{} type Conn struct { - NetConn net.Conn + idx int + + netConn net.Conn Rd *bufio.Reader Buf []byte @@ -22,7 +24,9 @@ type Conn struct { func NewConn(netConn net.Conn) *Conn { cn := &Conn{ - NetConn: netConn, + idx: -1, + + netConn: netConn, Buf: make([]byte, defaultBufSize), UsedAt: time.Now(), @@ -31,30 +35,35 @@ func NewConn(netConn net.Conn) *Conn { return cn } +func (cn *Conn) SetNetConn(netConn net.Conn) { + cn.netConn = netConn + cn.UsedAt = time.Now() +} + func (cn *Conn) Read(b []byte) (int, error) { cn.UsedAt = time.Now() if cn.ReadTimeout != 0 { - cn.NetConn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) + cn.netConn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) } else { - cn.NetConn.SetReadDeadline(noTimeout) + cn.netConn.SetReadDeadline(noDeadline) } - return cn.NetConn.Read(b) + return cn.netConn.Read(b) } func (cn *Conn) Write(b []byte) (int, error) { cn.UsedAt = time.Now() if cn.WriteTimeout != 0 { - cn.NetConn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) + cn.netConn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) } else { - cn.NetConn.SetWriteDeadline(noTimeout) + cn.netConn.SetWriteDeadline(noDeadline) } - return cn.NetConn.Write(b) + return cn.netConn.Write(b) } func (cn *Conn) RemoteAddr() net.Addr { - return cn.NetConn.RemoteAddr() + return cn.netConn.RemoteAddr() } func (cn *Conn) Close() error { - return cn.NetConn.Close() + return cn.netConn.Close() } diff --git a/internal/pool/conn_list.go b/internal/pool/conn_list.go index f8b82ab230..e72dc91dc2 100644 --- a/internal/pool/conn_list.go +++ b/internal/pool/conn_list.go @@ -7,14 +7,14 @@ import ( type connList struct { cns []*Conn - mx sync.Mutex + mu sync.Mutex len int32 // atomic size int32 } func newConnList(size int) *connList { return &connList{ - cns: make([]*Conn, 0, size), + cns: make([]*Conn, size), size: int32(size), } } @@ -23,8 +23,8 @@ func (l *connList) Len() int { return int(atomic.LoadInt32(&l.len)) } -// Reserve reserves place in the list and returns true on success. The -// caller must add or remove connection if place was reserved. +// Reserve reserves place in the list and returns true on success. +// The caller must add or remove connection if place was reserved. func (l *connList) Reserve() bool { len := atomic.AddInt32(&l.len, 1) reserved := len <= l.size @@ -36,65 +36,49 @@ func (l *connList) Reserve() bool { // Add adds connection to the list. The caller must reserve place first. func (l *connList) Add(cn *Conn) { - l.mx.Lock() - l.cns = append(l.cns, cn) - l.mx.Unlock() + l.mu.Lock() + for i, c := range l.cns { + if c == nil { + cn.idx = i + l.cns[i] = cn + l.mu.Unlock() + return + } + } + panic("not reached") } // Remove closes connection and removes it from the list. func (l *connList) Remove(cn *Conn) error { - defer l.mx.Unlock() - l.mx.Lock() + atomic.AddInt32(&l.len, -1) - if cn == nil { - atomic.AddInt32(&l.len, -1) + if cn == nil { // free reserved place return nil } - for i, c := range l.cns { - if c == cn { - l.cns = append(l.cns[:i], l.cns[i+1:]...) - atomic.AddInt32(&l.len, -1) - return cn.Close() - } + l.mu.Lock() + if l.cns != nil { + l.cns[cn.idx] = nil + cn.idx = -1 } + l.mu.Unlock() - if l.closed() { - return nil - } - panic("conn not found in the list") + return nil } -func (l *connList) Replace(cn, newcn *Conn) error { - defer l.mx.Unlock() - l.mx.Lock() - - for i, c := range l.cns { - if c == cn { - l.cns[i] = newcn - return cn.Close() - } - } - - if l.closed() { - return newcn.Close() - } - panic("conn not found in the list") -} - -func (l *connList) Close() (retErr error) { - l.mx.Lock() +func (l *connList) Close() error { + var retErr error + l.mu.Lock() for _, c := range l.cns { - if err := c.Close(); err != nil { + if c == nil { + continue + } + if err := c.Close(); err != nil && retErr == nil { retErr = err } } l.cns = nil atomic.StoreInt32(&l.len, 0) - l.mx.Unlock() + l.mu.Unlock() return retErr } - -func (l *connList) closed() bool { - return l.cns == nil -} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index ab03195da6..bed6b46839 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "log" + "net" "sync/atomic" "time" @@ -32,17 +33,17 @@ type Pooler interface { First() *Conn Get() (*Conn, bool, error) Put(*Conn) error - Remove(*Conn, error) error + Replace(*Conn, error) error Len() int FreeLen() int Close() error Stats() *PoolStats } -type dialer func() (*Conn, error) +type dialer func() (net.Conn, error) type ConnPool struct { - dial dialer + _dial dialer poolTimeout time.Duration idleTimeout time.Duration @@ -59,7 +60,7 @@ type ConnPool struct { func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Duration) *ConnPool { p := &ConnPool{ - dial: dial, + _dial: dial, poolTimeout: poolTimeout, idleTimeout: idleTimeout, @@ -126,8 +127,7 @@ func (p *ConnPool) wait() *Conn { panic("not reached") } -// Establish a new connection -func (p *ConnPool) new() (*Conn, error) { +func (p *ConnPool) dial() (net.Conn, error) { if p.rl.Limit() { err := fmt.Errorf( "redis: you open connections too fast (last_error=%q)", @@ -136,15 +136,22 @@ func (p *ConnPool) new() (*Conn, error) { return nil, err } - cn, err := p.dial() + cn, err := p._dial() if err != nil { p.storeLastErr(err.Error()) return nil, err } - return cn, nil } +func (p *ConnPool) newConn() (*Conn, error) { + netConn, err := p.dial() + if err != nil { + return nil, err + } + return NewConn(netConn), nil +} + // Get returns existed connection from the pool or creates a new one. func (p *ConnPool) Get() (cn *Conn, isNew bool, err error) { if p.closed() { @@ -164,7 +171,7 @@ func (p *ConnPool) Get() (cn *Conn, isNew bool, err error) { if p.conns.Reserve() { isNew = true - cn, err = p.new() + cn, err = p.newConn() if err != nil { p.conns.Remove(nil) return @@ -189,23 +196,26 @@ func (p *ConnPool) Put(cn *Conn) error { b, _ := cn.Rd.Peek(cn.Rd.Buffered()) err := fmt.Errorf("connection has unread data: %q", b) Logger.Print(err) - return p.Remove(cn, err) + return p.Replace(cn, err) } p.freeConns <- cn return nil } func (p *ConnPool) replace(cn *Conn) (*Conn, error) { - newcn, err := p.new() + _ = cn.Close() + + netConn, err := p.dial() if err != nil { _ = p.conns.Remove(cn) return nil, err } - _ = p.conns.Replace(cn, newcn) - return newcn, nil + cn.SetNetConn(netConn) + + return cn, nil } -func (p *ConnPool) Remove(cn *Conn, reason error) error { +func (p *ConnPool) Replace(cn *Conn, reason error) error { p.storeLastErr(reason.Error()) // Replace existing connection with new one and unblock waiter. diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index f2d58cf7f4..e0ea868974 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -25,7 +25,7 @@ func (p *SingleConnPool) Put(cn *Conn) error { return nil } -func (p *SingleConnPool) Remove(cn *Conn, _ error) error { +func (p *SingleConnPool) Replace(cn *Conn, _ error) error { if p.cn != cn { panic("p.cn != cn") } diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index c611c4b448..8f4c324bdd 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -67,13 +67,13 @@ func (p *StickyConnPool) Put(cn *Conn) error { return nil } -func (p *StickyConnPool) remove(reason error) error { - err := p.pool.Remove(p.cn, reason) +func (p *StickyConnPool) replace(reason error) error { + err := p.pool.Replace(p.cn, reason) p.cn = nil return err } -func (p *StickyConnPool) Remove(cn *Conn, reason error) error { +func (p *StickyConnPool) Replace(cn *Conn, reason error) error { defer p.mx.Unlock() p.mx.Lock() if p.closed { @@ -85,7 +85,7 @@ func (p *StickyConnPool) Remove(cn *Conn, reason error) error { if cn != nil && p.cn != cn { panic("p.cn != cn") } - return p.remove(reason) + return p.replace(reason) } func (p *StickyConnPool) Len() int { @@ -121,7 +121,7 @@ func (p *StickyConnPool) Close() error { err = p.put() } else { reason := errors.New("redis: sticky not reusable connection") - err = p.remove(reason) + err = p.replace(reason) } } return err diff --git a/multi_test.go b/multi_test.go index fa532d1a42..459d0a6222 100644 --- a/multi_test.go +++ b/multi_test.go @@ -145,7 +145,7 @@ var _ = Describe("Multi", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.NetConn = &badConn{} + cn.SetNetConn(&badConn{}) err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) @@ -172,7 +172,7 @@ var _ = Describe("Multi", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.NetConn = &badConn{} + cn.SetNetConn(&badConn{}) err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) diff --git a/options.go b/options.go index 95a266633c..b05721241c 100644 --- a/options.go +++ b/options.go @@ -67,18 +67,6 @@ func (opt *Options) getDialer() func() (net.Conn, error) { return opt.Dialer } -func (opt *Options) getPoolDialer() func() (*pool.Conn, error) { - dial := opt.getDialer() - return func() (*pool.Conn, error) { - netcn, err := dial() - if err != nil { - return nil, err - } - cn := pool.NewConn(netcn) - return cn, opt.initConn(cn) - } -} - func (opt *Options) getPoolSize() int { if opt.PoolSize == 0 { return 10 @@ -104,32 +92,9 @@ func (opt *Options) getIdleTimeout() time.Duration { return opt.IdleTimeout } -func (opt *Options) initConn(cn *pool.Conn) error { - if opt.Password == "" && opt.DB == 0 { - return nil - } - - // Temp client for Auth and Select. - client := newClient(opt, pool.NewSingleConnPool(cn)) - - if opt.Password != "" { - if err := client.Auth(opt.Password).Err(); err != nil { - return err - } - } - - if opt.DB > 0 { - if err := client.Select(opt.DB).Err(); err != nil { - return err - } - } - - return nil -} - func newConnPool(opt *Options) *pool.ConnPool { return pool.NewConnPool( - opt.getPoolDialer(), opt.getPoolSize(), opt.getPoolTimeout(), opt.getIdleTimeout()) + opt.getDialer(), opt.getPoolSize(), opt.getPoolTimeout(), opt.getIdleTimeout()) } // PoolStats contains pool state information and accumulated stats. diff --git a/pool_test.go b/pool_test.go index 2494e56ed5..a5b07216c4 100644 --- a/pool_test.go +++ b/pool_test.go @@ -179,7 +179,7 @@ var _ = Describe("pool", func() { // ok } - err = pool.Remove(cn, errors.New("test")) + err = pool.Replace(cn, errors.New("test")) Expect(err).NotTo(HaveOccurred()) // Check that Ping is unblocked. @@ -203,7 +203,7 @@ var _ = Describe("pool", func() { break } - _ = pool.Remove(cn, errors.New("test")) + _ = pool.Replace(cn, errors.New("test")) } Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) diff --git a/pubsub_test.go b/pubsub_test.go index a8bb610be5..669c0737d8 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -291,10 +291,10 @@ var _ = Describe("PubSub", func() { expectReceiveMessageOnError := func(pubsub *redis.PubSub) { cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn1.NetConn = &badConn{ + cn1.SetNetConn(&badConn{ readErr: io.EOF, writeErr: io.EOF, - } + }) done := make(chan bool, 1) go func() { diff --git a/redis.go b/redis.go index da4b41b377..55f475727b 100644 --- a/redis.go +++ b/redis.go @@ -33,12 +33,19 @@ func (c *baseClient) String() string { } func (c *baseClient) conn() (*pool.Conn, bool, error) { - return c.connPool.Get() + cn, isNew, err := c.connPool.Get() + if err == nil && isNew { + err = c.initConn(cn) + if err != nil { + c.putConn(cn, err, false) + } + } + return cn, isNew, err } func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { if isBadConn(err, allowTimeout) { - err = c.connPool.Remove(cn, err) + err = c.connPool.Replace(cn, err) if err != nil { Logger.Printf("pool.Remove failed: %s", err) } @@ -52,6 +59,29 @@ func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { return true } +func (c *baseClient) initConn(cn *pool.Conn) error { + if c.opt.Password == "" && c.opt.DB == 0 { + return nil + } + + // Temp client for Auth and Select. + client := newClient(c.opt, pool.NewSingleConnPool(cn)) + + if c.opt.Password != "" { + if err := client.Auth(c.opt.Password).Err(); err != nil { + return err + } + } + + if c.opt.DB > 0 { + if err := client.Select(c.opt.DB).Err(); err != nil { + return err + } + } + + return nil +} + func (c *baseClient) process(cmd Cmder) { for i := 0; i <= c.opt.MaxRetries; i++ { if i > 0 { diff --git a/redis_test.go b/redis_test.go index 23c39009f8..1435b7a487 100644 --- a/redis_test.go +++ b/redis_test.go @@ -160,7 +160,7 @@ var _ = Describe("Client", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.NetConn = &badConn{} + cn.SetNetConn(&badConn{}) err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) diff --git a/sentinel.go b/sentinel.go index 5575e73e9f..694dd60251 100644 --- a/sentinel.go +++ b/sentinel.go @@ -267,7 +267,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { cn.RemoteAddr(), ) Logger.Print(err) - d.pool.Remove(cn, err) + d.pool.Replace(cn, err) } else { cnsToPut = append(cnsToPut, cn) } From 900913d1e5c5ca774b95795f1a8a117b6e1b07b1 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Sat, 12 Mar 2016 10:38:52 +0000 Subject: [PATCH 0143/1746] Move benchmark to relevant package. Added bench task --- Makefile | 5 ++- bench_test.go | 70 -------------------------------- internal/pool/bench_test.go | 80 +++++++++++++++++++++++++++++++++++++ internal/pool/pool.go | 10 ++--- 4 files changed, 89 insertions(+), 76 deletions(-) create mode 100644 internal/pool/bench_test.go diff --git a/Makefile b/Makefile index 03d8959e2d..42c86f2857 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,10 @@ all: testdeps testdeps: testdata/redis/src/redis-server -.PHONY: all test testdeps +bench: testdeps + go test ./... -test.run=NONE -test.bench=. -test.benchmem + +.PHONY: all test testdeps bench testdata/redis: mkdir -p $@ diff --git a/bench_test.go b/bench_test.go index f4cffc1d63..6a7edd627b 100644 --- a/bench_test.go +++ b/bench_test.go @@ -2,15 +2,12 @@ package redis_test import ( "bytes" - "errors" - "net" "testing" "time" redigo "github.com/garyburd/redigo/redis" "gopkg.in/redis.v3" - "gopkg.in/redis.v3/internal/pool" ) func benchmarkRedisClient(poolSize int) *redis.Client { @@ -276,70 +273,3 @@ func BenchmarkZAdd(b *testing.B) { } }) } - -func benchmarkPoolGetPut(b *testing.B, poolSize int) { - dial := func() (net.Conn, error) { - return &net.TCPConn{}, nil - } - pool := pool.NewConnPool(dial, poolSize, time.Second, 0) - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - conn, _, err := pool.Get() - if err != nil { - b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) - } - if err = pool.Put(conn); err != nil { - b.Fatalf("no error expected on pool.Put but received: %s", err.Error()) - } - } - }) -} - -func BenchmarkPoolGetPut10Conns(b *testing.B) { - benchmarkPoolGetPut(b, 10) -} - -func BenchmarkPoolGetPut100Conns(b *testing.B) { - benchmarkPoolGetPut(b, 100) -} - -func BenchmarkPoolGetPut1000Conns(b *testing.B) { - benchmarkPoolGetPut(b, 1000) -} - -func benchmarkPoolGetRemove(b *testing.B, poolSize int) { - dial := func() (net.Conn, error) { - return &net.TCPConn{}, nil - } - pool := pool.NewConnPool(dial, poolSize, time.Second, 0) - removeReason := errors.New("benchmark") - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - conn, _, err := pool.Get() - if err != nil { - b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) - } - if err = pool.Replace(conn, removeReason); err != nil { - b.Fatalf("no error expected on pool.Remove but received: %s", err.Error()) - } - } - }) -} - -func BenchmarkPoolGetRemove10Conns(b *testing.B) { - benchmarkPoolGetRemove(b, 10) -} - -func BenchmarkPoolGetRemove100Conns(b *testing.B) { - benchmarkPoolGetRemove(b, 100) -} - -func BenchmarkPoolGetRemove1000Conns(b *testing.B) { - benchmarkPoolGetRemove(b, 1000) -} diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go new file mode 100644 index 0000000000..ba6df0c00a --- /dev/null +++ b/internal/pool/bench_test.go @@ -0,0 +1,80 @@ +package pool_test + +import ( + "errors" + "net" + "testing" + "time" + + "gopkg.in/redis.v3/internal/pool" +) + +func benchmarkPoolGetPut(b *testing.B, poolSize int) { + dial := func() (net.Conn, error) { + return &net.TCPConn{}, nil + } + pool := pool.NewConnPool(dial, poolSize, time.Second, 0) + pool.DialLimiter = nil + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + conn, _, err := pool.Get() + if err != nil { + b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) + } + if err = pool.Put(conn); err != nil { + b.Fatalf("no error expected on pool.Put but received: %s", err.Error()) + } + } + }) +} + +func BenchmarkPoolGetPut10Conns(b *testing.B) { + benchmarkPoolGetPut(b, 10) +} + +func BenchmarkPoolGetPut100Conns(b *testing.B) { + benchmarkPoolGetPut(b, 100) +} + +func BenchmarkPoolGetPut1000Conns(b *testing.B) { + benchmarkPoolGetPut(b, 1000) +} + +func benchmarkPoolGetReplace(b *testing.B, poolSize int) { + dial := func() (net.Conn, error) { + return &net.TCPConn{}, nil + } + pool := pool.NewConnPool(dial, poolSize, time.Second, 0) + pool.DialLimiter = nil + + removeReason := errors.New("benchmark") + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + conn, _, err := pool.Get() + if err != nil { + b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) + } + if err = pool.Replace(conn, removeReason); err != nil { + b.Fatalf("no error expected on pool.Remove but received: %s", err.Error()) + } + } + }) +} + +func BenchmarkPoolGetReplace10Conns(b *testing.B) { + benchmarkPoolGetReplace(b, 10) +} + +func BenchmarkPoolGetReplace100Conns(b *testing.B) { + benchmarkPoolGetReplace(b, 100) +} + +func BenchmarkPoolGetReplace1000Conns(b *testing.B) { + benchmarkPoolGetReplace(b, 1000) +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index bed6b46839..60b9c402bf 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -43,12 +43,12 @@ type Pooler interface { type dialer func() (net.Conn, error) type ConnPool struct { - _dial dialer + _dial dialer + DialLimiter *ratelimit.RateLimiter poolTimeout time.Duration idleTimeout time.Duration - rl *ratelimit.RateLimiter conns *connList freeConns chan *Conn stats PoolStats @@ -60,12 +60,12 @@ type ConnPool struct { func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Duration) *ConnPool { p := &ConnPool{ - _dial: dial, + _dial: dial, + DialLimiter: ratelimit.New(3*poolSize, time.Second), poolTimeout: poolTimeout, idleTimeout: idleTimeout, - rl: ratelimit.New(3*poolSize, time.Second), conns: newConnList(poolSize), freeConns: make(chan *Conn, poolSize), } @@ -128,7 +128,7 @@ func (p *ConnPool) wait() *Conn { } func (p *ConnPool) dial() (net.Conn, error) { - if p.rl.Limit() { + if p.DialLimiter != nil && p.DialLimiter.Limit() { err := fmt.Errorf( "redis: you open connections too fast (last_error=%q)", p.loadLastErr(), From 1bb55e3a9a686c082ec9c8e44e8298a6ceb011b2 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Sat, 12 Mar 2016 13:39:50 +0200 Subject: [PATCH 0144/1746] Make free-connection stack a LIFO. --- internal/pool/conn.go | 4 +++ internal/pool/conn_stack.go | 72 +++++++++++++++++++++++++++++++++++++ internal/pool/pool.go | 64 ++++++++++++++------------------- 3 files changed, 103 insertions(+), 37 deletions(-) create mode 100644 internal/pool/conn_stack.go diff --git a/internal/pool/conn.go b/internal/pool/conn.go index c5a539bcd2..1f1d4a962e 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -35,6 +35,10 @@ func NewConn(netConn net.Conn) *Conn { return cn } +func (cn *Conn) IsStale(timeout time.Duration) bool { + return timeout > 0 && time.Since(cn.UsedAt) > timeout +} + func (cn *Conn) SetNetConn(netConn net.Conn) { cn.netConn = netConn cn.UsedAt = time.Now() diff --git a/internal/pool/conn_stack.go b/internal/pool/conn_stack.go new file mode 100644 index 0000000000..047bc85b52 --- /dev/null +++ b/internal/pool/conn_stack.go @@ -0,0 +1,72 @@ +package pool + +import ( + "sync" + "time" +) + +// connStack is used as a LIFO to maintain free connections +type connStack struct { + cns []*Conn + free chan struct{} + mu sync.Mutex +} + +func newConnStack(max int) *connStack { + return &connStack{ + cns: make([]*Conn, 0, max), + free: make(chan struct{}, max), + } +} + +func (s *connStack) Len() int { return len(s.free) } + +func (s *connStack) Push(cn *Conn) { + s.mu.Lock() + s.cns = append(s.cns, cn) + s.mu.Unlock() + s.free <- struct{}{} +} + +func (s *connStack) ShiftStale(timeout time.Duration) *Conn { + select { + case <-s.free: + s.mu.Lock() + defer s.mu.Unlock() + + if cn := s.cns[0]; cn.IsStale(timeout) { + copy(s.cns, s.cns[1:]) + s.cns = s.cns[:len(s.cns)-1] + return cn + } + return nil + default: + return nil + } +} + +func (s *connStack) Pop() *Conn { + select { + case <-s.free: + return s.pop() + default: + return nil + } +} + +func (s *connStack) PopWithTimeout(d time.Duration) *Conn { + select { + case <-s.free: + return s.pop() + case <-time.After(d): + return nil + } +} + +func (s *connStack) pop() (cn *Conn) { + s.mu.Lock() + ci := len(s.cns) - 1 + cn, s.cns = s.cns[ci], s.cns[:ci] + s.mu.Unlock() + return +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 60b9c402bf..d5c91db66b 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -50,7 +50,7 @@ type ConnPool struct { idleTimeout time.Duration conns *connList - freeConns chan *Conn + freeConns *connStack stats PoolStats _closed int32 @@ -67,7 +67,7 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Durati idleTimeout: idleTimeout, conns: newConnList(poolSize), - freeConns: make(chan *Conn, poolSize), + freeConns: newConnStack(poolSize), } if idleTimeout > 0 { go p.reaper() @@ -87,44 +87,33 @@ func (p *ConnPool) isIdle(cn *Conn) bool { // there are no connections. func (p *ConnPool) First() *Conn { for { - select { - case cn := <-p.freeConns: - if p.isIdle(cn) { - var err error - cn, err = p.replace(cn) - if err != nil { - Logger.Printf("pool.replace failed: %s", err) - continue - } + cn := p.freeConns.Pop() + if cn != nil && cn.IsStale(p.idleTimeout) { + var err error + cn, err = p.replace(cn) + if err != nil { + Logger.Printf("pool.replace failed: %s", err) + continue } - return cn - default: - return nil } + return cn } - panic("not reached") } // wait waits for free non-idle connection. It returns nil on timeout. func (p *ConnPool) wait() *Conn { - deadline := time.After(p.poolTimeout) for { - select { - case cn := <-p.freeConns: - if p.isIdle(cn) { - var err error - cn, err = p.replace(cn) - if err != nil { - Logger.Printf("pool.replace failed: %s", err) - continue - } + cn := p.freeConns.PopWithTimeout(p.poolTimeout) + if cn != nil && cn.IsStale(p.idleTimeout) { + var err error + cn, err = p.replace(cn) + if err != nil { + Logger.Printf("pool.replace failed: %s", err) + continue } - return cn - case <-deadline: - return nil } + return cn } - panic("not reached") } func (p *ConnPool) dial() (net.Conn, error) { @@ -198,7 +187,7 @@ func (p *ConnPool) Put(cn *Conn) error { Logger.Print(err) return p.Replace(cn, err) } - p.freeConns <- cn + p.freeConns.Push(cn) return nil } @@ -223,7 +212,7 @@ func (p *ConnPool) Replace(cn *Conn, reason error) error { if err != nil { return err } - p.freeConns <- newcn + p.freeConns.Push(newcn) return nil } @@ -234,7 +223,7 @@ func (p *ConnPool) Len() int { // FreeLen returns number of free connections. func (p *ConnPool) FreeLen() int { - return len(p.freeConns) + return p.freeConns.Len() } func (p *ConnPool) Stats() *PoolStats { @@ -273,11 +262,12 @@ func (p *ConnPool) reaper() { break } - // pool.First removes idle connections from the pool and - // returns first non-idle connection. So just put returned - // connection back. - if cn := p.First(); cn != nil { - p.Put(cn) + for { + cn := p.freeConns.ShiftStale(p.idleTimeout) + if cn == nil { + break + } + _ = p.conns.Remove(cn) } } } From ef5ccc12aef8eff5c5731537192b1b1cb8ba74c0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 12 Mar 2016 14:42:12 +0200 Subject: [PATCH 0145/1746] Add tests for conn reaper. --- example_test.go | 2 +- internal/pool/conn_stack.go | 6 ++- internal/pool/pool.go | 41 +++++++++++++--- internal/pool/pool_test.go | 93 +++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 internal/pool/pool_test.go diff --git a/example_test.go b/example_test.go index bb98fdd5fb..c93e0b703b 100644 --- a/example_test.go +++ b/example_test.go @@ -254,7 +254,7 @@ func ExamplePubSub_Receive() { for i := 0; i < 2; i++ { // ReceiveTimeout is a low level API. Use ReceiveMessage instead. - msgi, err := pubsub.ReceiveTimeout(500 * time.Millisecond) + msgi, err := pubsub.ReceiveTimeout(time.Second) if err != nil { panic(err) } diff --git a/internal/pool/conn_stack.go b/internal/pool/conn_stack.go index 047bc85b52..a26ab0ee6b 100644 --- a/internal/pool/conn_stack.go +++ b/internal/pool/conn_stack.go @@ -28,17 +28,19 @@ func (s *connStack) Push(cn *Conn) { s.free <- struct{}{} } -func (s *connStack) ShiftStale(timeout time.Duration) *Conn { +func (s *connStack) ShiftStale(idleTimeout time.Duration) *Conn { select { case <-s.free: s.mu.Lock() defer s.mu.Unlock() - if cn := s.cns[0]; cn.IsStale(timeout) { + if cn := s.cns[0]; cn.IsStale(idleTimeout) { copy(s.cns, s.cns[1:]) s.cns = s.cns[:len(s.cns)-1] return cn } + + s.free <- struct{}{} return nil default: return nil diff --git a/internal/pool/pool.go b/internal/pool/pool.go index d5c91db66b..243ebeafd3 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -83,6 +83,15 @@ func (p *ConnPool) isIdle(cn *Conn) bool { return p.idleTimeout > 0 && time.Since(cn.UsedAt) > p.idleTimeout } +func (p *ConnPool) Add(cn *Conn) bool { + if !p.conns.Reserve() { + return false + } + p.conns.Add(cn) + p.Put(cn) + return true +} + // First returns first non-idle connection from the pool or nil if // there are no connections. func (p *ConnPool) First() *Conn { @@ -216,6 +225,12 @@ func (p *ConnPool) Replace(cn *Conn, reason error) error { return nil } +func (p *ConnPool) Remove(cn *Conn, reason error) error { + p.storeLastErr(reason.Error()) + _ = cn.Close() + return p.conns.Remove(cn) +} + // Len returns total number of connections. func (p *ConnPool) Len() int { return p.conns.Len() @@ -253,6 +268,20 @@ func (p *ConnPool) Close() (retErr error) { return retErr } +func (p *ConnPool) ReapStaleConns() (n int, err error) { + for { + cn := p.freeConns.ShiftStale(p.idleTimeout) + if cn == nil { + break + } + if err = p.Remove(cn, errors.New("connection is stale")); err != nil { + return + } + n++ + } + return +} + func (p *ConnPool) reaper() { ticker := time.NewTicker(time.Minute) defer ticker.Stop() @@ -261,13 +290,11 @@ func (p *ConnPool) reaper() { if p.closed() { break } - - for { - cn := p.freeConns.ShiftStale(p.idleTimeout) - if cn == nil { - break - } - _ = p.conns.Remove(cn) + n, err := p.ReapStaleConns() + if err != nil { + Logger.Printf("ReapStaleConns failed: %s", err) + } else if n > 0 { + Logger.Printf("removed %d stale connections", n) } } } diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go new file mode 100644 index 0000000000..07d3a52505 --- /dev/null +++ b/internal/pool/pool_test.go @@ -0,0 +1,93 @@ +package pool_test + +import ( + "errors" + "net" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v3/internal/pool" +) + +func TestGinkgoSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pool") +} + +var _ = Describe("conns reapser", func() { + var connPool *pool.ConnPool + + BeforeEach(func() { + dial := func() (net.Conn, error) { + return &net.TCPConn{}, nil + } + connPool = pool.NewConnPool(dial, 10, 0, time.Minute) + + // add stale connections + for i := 0; i < 3; i++ { + cn := pool.NewConn(&net.TCPConn{}) + cn.UsedAt = time.Now().Add(-2 * time.Minute) + Expect(connPool.Add(cn)).To(BeTrue()) + } + + // add fresh connections + for i := 0; i < 3; i++ { + cn := pool.NewConn(&net.TCPConn{}) + Expect(connPool.Add(cn)).To(BeTrue()) + } + + Expect(connPool.Len()).To(Equal(6)) + Expect(connPool.FreeLen()).To(Equal(6)) + + n, err := connPool.ReapStaleConns() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(3)) + }) + + It("reaps stale connections", func() { + Expect(connPool.Len()).To(Equal(3)) + Expect(connPool.FreeLen()).To(Equal(3)) + }) + + It("pool is functional", func() { + for j := 0; j < 3; j++ { + var freeCns []*pool.Conn + for i := 0; i < 3; i++ { + cn := connPool.First() + Expect(cn).NotTo(BeNil()) + freeCns = append(freeCns, cn) + } + + Expect(connPool.Len()).To(Equal(3)) + Expect(connPool.FreeLen()).To(Equal(0)) + + cn := connPool.First() + Expect(cn).To(BeNil()) + + cn, isNew, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) + Expect(isNew).To(BeTrue()) + Expect(cn).NotTo(BeNil()) + + Expect(connPool.Len()).To(Equal(4)) + Expect(connPool.FreeLen()).To(Equal(0)) + + err = connPool.Remove(cn, errors.New("test")) + Expect(err).NotTo(HaveOccurred()) + + Expect(connPool.Len()).To(Equal(3)) + Expect(connPool.FreeLen()).To(Equal(0)) + + for _, cn := range freeCns { + err := connPool.Put(cn) + Expect(err).NotTo(HaveOccurred()) + } + + Expect(connPool.Len()).To(Equal(3)) + Expect(connPool.FreeLen()).To(Equal(3)) + } + }) +}) From 672fb9bb97c2587cbbae21fad2ab63df3cf23930 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 14 Mar 2016 12:36:18 +0200 Subject: [PATCH 0146/1746] Fix slice grow in readN. --- command_test.go | 13 +++++++++++-- parser.go | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/command_test.go b/command_test.go index 1218724e4e..088b178510 100644 --- a/command_test.go +++ b/command_test.go @@ -15,10 +15,14 @@ import ( var _ = Describe("Command", func() { var client *redis.Client - BeforeEach(func() { - client = redis.NewClient(&redis.Options{ + connect := func() *redis.Client { + return redis.NewClient(&redis.Options{ Addr: redisAddr, }) + } + + BeforeEach(func() { + client = connect() }) AfterEach(func() { @@ -63,8 +67,13 @@ var _ = Describe("Command", func() { Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) + // Reconnect to get new connection. + Expect(client.Close()).To(BeNil()) + client = connect() + get := client.Get("key") Expect(get.Err()).NotTo(HaveOccurred()) + Expect(len(get.Val())).To(Equal(len(val))) Expect(get.Val()).To(Equal(val)) }) diff --git a/parser.go b/parser.go index 2496f66c46..3d8742c983 100644 --- a/parser.go +++ b/parser.go @@ -247,6 +247,7 @@ func isNilReply(b []byte) bool { func readN(cn *pool.Conn, n int) ([]byte, error) { if d := n - cap(cn.Buf); d > 0 { + cn.Buf = cn.Buf[:cap(cn.Buf)] cn.Buf = append(cn.Buf, make([]byte, d)...) } else { cn.Buf = cn.Buf[:n] From 46790aa060f15fd4776bf336b7cb5db4b16d45e2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 14 Mar 2016 13:17:33 +0200 Subject: [PATCH 0147/1746] Add race test for big vals. Copy connection to avoid race with PubSub. --- Makefile | 4 +- cluster.go | 5 +- cluster_pipeline.go | 4 +- command_test.go | 146 +++++++++++++++++++++++++---------- error.go | 3 - example_test.go | 14 ++-- internal/pool/conn.go | 43 +++++++---- internal/pool/conn_list.go | 37 ++++----- internal/pool/pool.go | 41 ++++++---- internal/pool/pool_single.go | 10 ++- internal/pool/pool_sticky.go | 17 +++- internal/pool/pool_test.go | 2 + main_test.go | 17 +++- multi.go | 2 +- multi_test.go | 4 +- pipeline.go | 4 +- pool_test.go | 20 +---- pubsub.go | 4 + pubsub_test.go | 4 +- redis.go | 14 ++-- redis_test.go | 2 +- ring.go | 6 +- 22 files changed, 256 insertions(+), 147 deletions(-) diff --git a/Makefile b/Makefile index 42c86f2857..b7867b48eb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testdeps - go test ./... -test.v -test.cpu=1,2,4 - go test ./... -test.v -test.short -test.race + go test ./... -test.cpu=1,2,4 + go test ./... -test.short -test.race testdeps: testdata/redis/src/redis-server diff --git a/cluster.go b/cluster.go index 6e79cb5e73..0cd59d92f8 100644 --- a/cluster.go +++ b/cluster.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/redis.v3/internal/hashtag" + "gopkg.in/redis.v3/internal/pool" ) // ClusterClient is a Redis Cluster client representing a pool of zero @@ -80,7 +81,7 @@ func (c *ClusterClient) Close() error { c.clientsMx.Lock() if c.closed { - return errClosed + return pool.ErrClosed } c.closed = true c.resetClients() @@ -105,7 +106,7 @@ func (c *ClusterClient) getClient(addr string) (*Client, error) { c.clientsMx.Lock() if c.closed { c.clientsMx.Unlock() - return nil, errClosed + return nil, pool.ErrClosed } client, ok = c.clients[addr] diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 7fa721c32e..8641d3df60 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -34,7 +34,7 @@ func (pipe *ClusterPipeline) process(cmd Cmder) { // Discard resets the pipeline and discards queued commands. func (pipe *ClusterPipeline) Discard() error { if pipe.closed { - return errClosed + return pool.ErrClosed } pipe.cmds = pipe.cmds[:0] return nil @@ -42,7 +42,7 @@ func (pipe *ClusterPipeline) Discard() error { func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { if pipe.closed { - return nil, errClosed + return nil, pool.ErrClosed } if len(pipe.cmds) == 0 { return []Cmder{}, nil diff --git a/command_test.go b/command_test.go index 088b178510..064e7340cf 100644 --- a/command_test.go +++ b/command_test.go @@ -5,11 +5,13 @@ import ( "strconv" "sync" "testing" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "gopkg.in/redis.v3" + "gopkg.in/redis.v3/internal/pool" ) var _ = Describe("Command", func() { @@ -17,7 +19,8 @@ var _ = Describe("Command", func() { connect := func() *redis.Client { return redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: redisAddr, + PoolTimeout: time.Minute, }) } @@ -62,19 +65,19 @@ var _ = Describe("Command", func() { }) It("should handle big vals", func() { - val := string(bytes.Repeat([]byte{'*'}, 1<<16)) - set := client.Set("key", val, 0) - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) + bigVal := string(bytes.Repeat([]byte{'*'}, 1<<16)) + + err := client.Set("key", bigVal, 0).Err() + Expect(err).NotTo(HaveOccurred()) // Reconnect to get new connection. Expect(client.Close()).To(BeNil()) client = connect() - get := client.Get("key") - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(len(get.Val())).To(Equal(len(val))) - Expect(get.Val()).To(Equal(val)) + got, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(len(got)).To(Equal(len(bigVal))) + Expect(got).To(Equal(bigVal)) }) It("should handle many keys #1", func() { @@ -140,48 +143,111 @@ var _ = Describe("Command", func() { } It("should echo", func() { - wg := &sync.WaitGroup{} - for i := 0; i < C; i++ { - wg.Add(1) - - go func(i int) { - defer GinkgoRecover() - defer wg.Done() - - for j := 0; j < N; j++ { - msg := "echo" + strconv.Itoa(i) - echo := client.Echo(msg) - Expect(echo.Err()).NotTo(HaveOccurred()) - Expect(echo.Val()).To(Equal(msg)) - } - }(i) - } - wg.Wait() + perform(C, func() { + for i := 0; i < N; i++ { + msg := "echo" + strconv.Itoa(i) + echo, err := client.Echo(msg).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(echo).To(Equal(msg)) + } + }) }) It("should incr", func() { key := "TestIncrFromGoroutines" - wg := &sync.WaitGroup{} - for i := 0; i < C; i++ { - wg.Add(1) - go func() { - defer GinkgoRecover() - defer wg.Done() - - for j := 0; j < N; j++ { - err := client.Incr(key).Err() - Expect(err).NotTo(HaveOccurred()) - } - }() - } - wg.Wait() + perform(C, func() { + for i := 0; i < N; i++ { + err := client.Incr(key).Err() + Expect(err).NotTo(HaveOccurred()) + } + }) val, err := client.Get(key).Int64() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal(int64(C * N))) }) + It("should handle big vals", func() { + client2 := connect() + defer client2.Close() + + bigVal := string(bytes.Repeat([]byte{'*'}, 1<<16)) + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + perform(C, func() { + for i := 0; i < N; i++ { + got, err := client.Get("key").Result() + if err == redis.Nil { + continue + } + Expect(got).To(Equal(bigVal)) + } + }) + }() + + go func() { + defer wg.Done() + perform(C, func() { + for i := 0; i < N; i++ { + err := client2.Set("key", bigVal, 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + }) + }() + + wg.Wait() + }) + + It("should PubSub", func() { + connPool := client.Pool() + connPool.(*pool.ConnPool).DialLimiter = nil + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + perform(C, func() { + for i := 0; i < N; i++ { + pubsub, err := client.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + + time.Sleep(time.Millisecond) + err := pubsub.Close() + Expect(err).NotTo(HaveOccurred()) + }() + + _, err = pubsub.ReceiveMessage() + Expect(err.Error()).To(ContainSubstring("closed")) + } + }) + }() + + go func() { + defer wg.Done() + perform(C, func() { + for i := 0; i < N; i++ { + val := "echo" + strconv.Itoa(i) + echo, err := client.Echo(val).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(echo).To(Equal(val)) + } + }) + }() + + wg.Wait() + + Expect(connPool.Len()).To(Equal(connPool.FreeLen())) + Expect(connPool.Len()).To(BeNumerically("<=", 10)) + }) }) }) diff --git a/error.go b/error.go index e2430b4a94..3f2a560c0b 100644 --- a/error.go +++ b/error.go @@ -1,15 +1,12 @@ package redis import ( - "errors" "fmt" "io" "net" "strings" ) -var errClosed = errors.New("redis: client is closed") - // Redis nil reply, .e.g. when key does not exist. var Nil = errorf("redis: nil") diff --git a/example_test.go b/example_test.go index c93e0b703b..3034d86735 100644 --- a/example_test.go +++ b/example_test.go @@ -220,13 +220,13 @@ func ExampleClient_Watch() { } func ExamplePubSub() { - pubsub, err := client.Subscribe("mychannel") + pubsub, err := client.Subscribe("mychannel1") if err != nil { panic(err) } defer pubsub.Close() - err = client.Publish("mychannel", "hello").Err() + err = client.Publish("mychannel1", "hello").Err() if err != nil { panic(err) } @@ -237,17 +237,17 @@ func ExamplePubSub() { } fmt.Println(msg.Channel, msg.Payload) - // Output: mychannel hello + // Output: mychannel1 hello } func ExamplePubSub_Receive() { - pubsub, err := client.Subscribe("mychannel") + pubsub, err := client.Subscribe("mychannel2") if err != nil { panic(err) } defer pubsub.Close() - err = client.Publish("mychannel", "hello").Err() + err = client.Publish("mychannel2", "hello").Err() if err != nil { panic(err) } @@ -269,8 +269,8 @@ func ExamplePubSub_Receive() { } } - // Output: subscribe mychannel - // mychannel hello + // Output: subscribe mychannel2 + // mychannel2 hello } func ExampleScript() { diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 1f1d4a962e..cbe379b198 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -3,6 +3,7 @@ package pool import ( "bufio" "net" + "sync/atomic" "time" ) @@ -11,9 +12,9 @@ const defaultBufSize = 4096 var noDeadline = time.Time{} type Conn struct { - idx int + idx int32 - netConn net.Conn + NetConn net.Conn Rd *bufio.Reader Buf []byte @@ -26,7 +27,7 @@ func NewConn(netConn net.Conn) *Conn { cn := &Conn{ idx: -1, - netConn: netConn, + NetConn: netConn, Buf: make([]byte, defaultBufSize), UsedAt: time.Now(), @@ -35,39 +36,47 @@ func NewConn(netConn net.Conn) *Conn { return cn } -func (cn *Conn) IsStale(timeout time.Duration) bool { - return timeout > 0 && time.Since(cn.UsedAt) > timeout +func (cn *Conn) Index() int { + return int(atomic.LoadInt32(&cn.idx)) } -func (cn *Conn) SetNetConn(netConn net.Conn) { - cn.netConn = netConn - cn.UsedAt = time.Now() +func (cn *Conn) SetIndex(idx int) { + atomic.StoreInt32(&cn.idx, int32(idx)) +} + +func (cn *Conn) IsStale(timeout time.Duration) bool { + return timeout > 0 && time.Since(cn.UsedAt) > timeout } func (cn *Conn) Read(b []byte) (int, error) { cn.UsedAt = time.Now() if cn.ReadTimeout != 0 { - cn.netConn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) + cn.NetConn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) } else { - cn.netConn.SetReadDeadline(noDeadline) + cn.NetConn.SetReadDeadline(noDeadline) } - return cn.netConn.Read(b) + return cn.NetConn.Read(b) } func (cn *Conn) Write(b []byte) (int, error) { cn.UsedAt = time.Now() if cn.WriteTimeout != 0 { - cn.netConn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) + cn.NetConn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) } else { - cn.netConn.SetWriteDeadline(noDeadline) + cn.NetConn.SetWriteDeadline(noDeadline) } - return cn.netConn.Write(b) + return cn.NetConn.Write(b) } func (cn *Conn) RemoteAddr() net.Addr { - return cn.netConn.RemoteAddr() + return cn.NetConn.RemoteAddr() } -func (cn *Conn) Close() error { - return cn.netConn.Close() +func (cn *Conn) Close() int { + idx := cn.Index() + if !atomic.CompareAndSwapInt32(&cn.idx, int32(idx), -1) { + return -1 + } + _ = cn.NetConn.Close() + return idx } diff --git a/internal/pool/conn_list.go b/internal/pool/conn_list.go index e72dc91dc2..7e43ee74f0 100644 --- a/internal/pool/conn_list.go +++ b/internal/pool/conn_list.go @@ -24,7 +24,7 @@ func (l *connList) Len() int { } // Reserve reserves place in the list and returns true on success. -// The caller must add or remove connection if place was reserved. +// The caller must add connection or cancel reservation if it was reserved. func (l *connList) Reserve() bool { len := atomic.AddInt32(&l.len, 1) reserved := len <= l.size @@ -34,12 +34,16 @@ func (l *connList) Reserve() bool { return reserved } +func (l *connList) CancelReservation() { + atomic.AddInt32(&l.len, -1) +} + // Add adds connection to the list. The caller must reserve place first. func (l *connList) Add(cn *Conn) { l.mu.Lock() for i, c := range l.cns { if c == nil { - cn.idx = i + cn.SetIndex(i) l.cns[i] = cn l.mu.Unlock() return @@ -48,37 +52,34 @@ func (l *connList) Add(cn *Conn) { panic("not reached") } -// Remove closes connection and removes it from the list. -func (l *connList) Remove(cn *Conn) error { - atomic.AddInt32(&l.len, -1) - - if cn == nil { // free reserved place - return nil +func (l *connList) Replace(cn *Conn) { + l.mu.Lock() + if l.cns != nil { + l.cns[cn.idx] = cn } + l.mu.Unlock() +} +// Remove closes connection and removes it from the list. +func (l *connList) Remove(idx int) { l.mu.Lock() if l.cns != nil { - l.cns[cn.idx] = nil - cn.idx = -1 + l.cns[idx] = nil + l.len -= 1 } l.mu.Unlock() - - return nil } func (l *connList) Close() error { - var retErr error l.mu.Lock() for _, c := range l.cns { if c == nil { continue } - if err := c.Close(); err != nil && retErr == nil { - retErr = err - } + c.Close() } l.cns = nil - atomic.StoreInt32(&l.len, 0) + l.len = 0 l.mu.Unlock() - return retErr + return nil } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 243ebeafd3..4f2b2175b3 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -14,7 +14,8 @@ import ( var Logger *log.Logger var ( - errClosed = errors.New("redis: client is closed") + ErrClosed = errors.New("redis: client is closed") + errConnClosed = errors.New("redis: connection is closed") ErrPoolTimeout = errors.New("redis: connection pool timeout") ) @@ -36,8 +37,9 @@ type Pooler interface { Replace(*Conn, error) error Len() int FreeLen() int - Close() error Stats() *PoolStats + Close() error + Closed() bool } type dialer func() (net.Conn, error) @@ -58,6 +60,8 @@ type ConnPool struct { lastErr atomic.Value } +var _ Pooler = (*ConnPool)(nil) + func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Duration) *ConnPool { p := &ConnPool{ _dial: dial, @@ -75,7 +79,7 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Durati return p } -func (p *ConnPool) closed() bool { +func (p *ConnPool) Closed() bool { return atomic.LoadInt32(&p._closed) == 1 } @@ -152,8 +156,8 @@ func (p *ConnPool) newConn() (*Conn, error) { // Get returns existed connection from the pool or creates a new one. func (p *ConnPool) Get() (cn *Conn, isNew bool, err error) { - if p.closed() { - err = errClosed + if p.Closed() { + err = ErrClosed return } @@ -171,7 +175,7 @@ func (p *ConnPool) Get() (cn *Conn, isNew bool, err error) { cn, err = p.newConn() if err != nil { - p.conns.Remove(nil) + p.conns.CancelReservation() return } p.conns.Add(cn) @@ -201,14 +205,20 @@ func (p *ConnPool) Put(cn *Conn) error { } func (p *ConnPool) replace(cn *Conn) (*Conn, error) { - _ = cn.Close() + idx := cn.Close() + if idx == -1 { + return nil, errConnClosed + } netConn, err := p.dial() if err != nil { - _ = p.conns.Remove(cn) + p.conns.Remove(idx) return nil, err } - cn.SetNetConn(netConn) + + cn = NewConn(netConn) + cn.SetIndex(idx) + p.conns.Replace(cn) return cn, nil } @@ -226,9 +236,14 @@ func (p *ConnPool) Replace(cn *Conn, reason error) error { } func (p *ConnPool) Remove(cn *Conn, reason error) error { + idx := cn.Close() + if idx == -1 { + return errConnClosed + } + p.storeLastErr(reason.Error()) - _ = cn.Close() - return p.conns.Remove(cn) + p.conns.Remove(idx) + return nil } // Len returns total number of connections. @@ -253,7 +268,7 @@ func (p *ConnPool) Stats() *PoolStats { func (p *ConnPool) Close() (retErr error) { if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { - return errClosed + return ErrClosed } // Wait for app to free connections, but don't close them immediately. for i := 0; i < p.Len(); i++ { @@ -287,7 +302,7 @@ func (p *ConnPool) reaper() { defer ticker.Stop() for _ = range ticker.C { - if p.closed() { + if p.Closed() { break } n, err := p.ReapStaleConns() diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index e0ea868974..f9ebfa6d14 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -4,6 +4,8 @@ type SingleConnPool struct { cn *Conn } +var _ Pooler = (*SingleConnPool)(nil) + func NewSingleConnPool(cn *Conn) *SingleConnPool { return &SingleConnPool{ cn: cn, @@ -40,8 +42,14 @@ func (p *SingleConnPool) FreeLen() int { return 0 } -func (p *SingleConnPool) Stats() *PoolStats { return nil } +func (p *SingleConnPool) Stats() *PoolStats { + return nil +} func (p *SingleConnPool) Close() error { return nil } + +func (p *SingleConnPool) Closed() bool { + return false +} diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 8f4c324bdd..11a7ee4968 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -14,6 +14,8 @@ type StickyConnPool struct { mx sync.Mutex } +var _ Pooler = (*StickyConnPool)(nil) + func NewStickyConnPool(pool *ConnPool, reusable bool) *StickyConnPool { return &StickyConnPool{ pool: pool, @@ -33,7 +35,7 @@ func (p *StickyConnPool) Get() (cn *Conn, isNew bool, err error) { p.mx.Lock() if p.closed { - err = errClosed + err = ErrClosed return } if p.cn != nil { @@ -59,7 +61,7 @@ func (p *StickyConnPool) Put(cn *Conn) error { defer p.mx.Unlock() p.mx.Lock() if p.closed { - return errClosed + return ErrClosed } if p.cn != cn { panic("p.cn != cn") @@ -77,7 +79,7 @@ func (p *StickyConnPool) Replace(cn *Conn, reason error) error { defer p.mx.Unlock() p.mx.Lock() if p.closed { - return errClosed + return nil } if p.cn == nil { panic("p.cn == nil") @@ -112,7 +114,7 @@ func (p *StickyConnPool) Close() error { defer p.mx.Unlock() p.mx.Lock() if p.closed { - return errClosed + return ErrClosed } p.closed = true var err error @@ -126,3 +128,10 @@ func (p *StickyConnPool) Close() error { } return err } + +func (p *StickyConnPool) Closed() bool { + p.mx.Lock() + closed := p.closed + p.mx.Unlock() + return closed +} diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 07d3a52505..5dd7784ed4 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -31,12 +31,14 @@ var _ = Describe("conns reapser", func() { cn := pool.NewConn(&net.TCPConn{}) cn.UsedAt = time.Now().Add(-2 * time.Minute) Expect(connPool.Add(cn)).To(BeTrue()) + Expect(cn.Index()).To(Equal(i)) } // add fresh connections for i := 0; i < 3; i++ { cn := pool.NewConn(&net.TCPConn{}) Expect(connPool.Add(cn)).To(BeTrue()) + Expect(cn.Index()).To(Equal(3 + i)) } Expect(connPool.Len()).To(Equal(6)) diff --git a/main_test.go b/main_test.go index e1d65c6a85..e3e747fe6e 100644 --- a/main_test.go +++ b/main_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "sync" "sync/atomic" "testing" "time" @@ -98,6 +99,20 @@ func TestGinkgoSuite(t *testing.T) { //------------------------------------------------------------------------------ +func perform(n int, cb func()) { + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + + cb() + }() + } + wg.Wait() +} + func eventually(fn func() error, timeout time.Duration) error { done := make(chan struct{}) var exit int32 @@ -138,7 +153,7 @@ func connectTo(port string) (*redis.Client, error) { err := eventually(func() error { return client.Ping().Err() - }, 10*time.Second) + }, 30*time.Second) if err != nil { return nil, err } diff --git a/multi.go b/multi.go index 6b43591f79..a0498211de 100644 --- a/multi.go +++ b/multi.go @@ -109,7 +109,7 @@ func (c *Multi) Discard() error { // failed command or nil. func (c *Multi) Exec(f func() error) ([]Cmder, error) { if c.closed { - return nil, errClosed + return nil, pool.ErrClosed } c.cmds = []Cmder{NewStatusCmd("MULTI")} diff --git a/multi_test.go b/multi_test.go index 459d0a6222..fa532d1a42 100644 --- a/multi_test.go +++ b/multi_test.go @@ -145,7 +145,7 @@ var _ = Describe("Multi", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{}) + cn.NetConn = &badConn{} err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) @@ -172,7 +172,7 @@ var _ = Describe("Multi", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{}) + cn.NetConn = &badConn{} err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) diff --git a/pipeline.go b/pipeline.go index 098207c27e..842fad7bc2 100644 --- a/pipeline.go +++ b/pipeline.go @@ -62,7 +62,7 @@ func (pipe *Pipeline) Discard() error { defer pipe.mu.Unlock() pipe.mu.Lock() if pipe.isClosed() { - return errClosed + return pool.ErrClosed } pipe.cmds = pipe.cmds[:0] return nil @@ -75,7 +75,7 @@ func (pipe *Pipeline) Discard() error { // command if any. func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { if pipe.isClosed() { - return nil, errClosed + return nil, pool.ErrClosed } defer pipe.mu.Unlock() diff --git a/pool_test.go b/pool_test.go index a5b07216c4..225ad6adbd 100644 --- a/pool_test.go +++ b/pool_test.go @@ -2,7 +2,6 @@ package redis_test import ( "errors" - "sync" "time" . "github.com/onsi/ginkgo" @@ -14,20 +13,6 @@ import ( var _ = Describe("pool", func() { var client *redis.Client - var perform = func(n int, cb func()) { - wg := &sync.WaitGroup{} - for i := 0; i < n; i++ { - wg.Add(1) - go func() { - defer GinkgoRecover() - defer wg.Done() - - cb() - }() - } - wg.Wait() - } - BeforeEach(func() { client = redis.NewClient(&redis.Options{ Addr: redisAddr, @@ -108,12 +93,11 @@ var _ = Describe("pool", func() { It("should remove broken connections", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - Expect(cn.Close()).NotTo(HaveOccurred()) + cn.NetConn = &badConn{} Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) err = client.Ping().Err() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("use of closed network connection")) + Expect(err).To(MatchError("bad connection")) val, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) diff --git a/pubsub.go b/pubsub.go index f1c93c8f16..68b2aeb5fe 100644 --- a/pubsub.go +++ b/pubsub.go @@ -54,6 +54,7 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { if err != nil { return err } + c.putConn(cn, err) args := make([]interface{}, 1+len(channels)) args[0] = redisCmd @@ -306,6 +307,9 @@ func (c *PubSub) putConn(cn *pool.Conn, err error) { } func (c *PubSub) resubscribe() { + if c.base.closed() { + return + } if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { Logger.Printf("Subscribe failed: %s", err) diff --git a/pubsub_test.go b/pubsub_test.go index 669c0737d8..a8bb610be5 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -291,10 +291,10 @@ var _ = Describe("PubSub", func() { expectReceiveMessageOnError := func(pubsub *redis.PubSub) { cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn1.SetNetConn(&badConn{ + cn1.NetConn = &badConn{ readErr: io.EOF, writeErr: io.EOF, - }) + } done := make(chan bool, 1) go func() { diff --git a/redis.go b/redis.go index 55f475727b..aab5ba6331 100644 --- a/redis.go +++ b/redis.go @@ -45,17 +45,11 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { if isBadConn(err, allowTimeout) { - err = c.connPool.Replace(cn, err) - if err != nil { - Logger.Printf("pool.Remove failed: %s", err) - } + _ = c.connPool.Replace(cn, err) return false } - err = c.connPool.Put(cn) - if err != nil { - Logger.Printf("pool.Put failed: %s", err) - } + _ = c.connPool.Put(cn) return true } @@ -121,6 +115,10 @@ func (c *baseClient) process(cmd Cmder) { } } +func (c *baseClient) closed() bool { + return c.connPool.Closed() +} + // Close closes the client, releasing any open resources. // // It is rare to Close a Client, as the Client is meant to be diff --git a/redis_test.go b/redis_test.go index 1435b7a487..23c39009f8 100644 --- a/redis_test.go +++ b/redis_test.go @@ -160,7 +160,7 @@ var _ = Describe("Client", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{}) + cn.NetConn = &badConn{} err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) diff --git a/ring.go b/ring.go index 3b88d7ca51..c66a5bc1af 100644 --- a/ring.go +++ b/ring.go @@ -149,7 +149,7 @@ func (ring *Ring) getClient(key string) (*Client, error) { ring.mx.RLock() if ring.closed { - return nil, errClosed + return nil, pool.ErrClosed } name := ring.hash.Get(hashtag.Key(key)) @@ -277,7 +277,7 @@ func (pipe *RingPipeline) process(cmd Cmder) { // Discard resets the pipeline and discards queued commands. func (pipe *RingPipeline) Discard() error { if pipe.closed { - return errClosed + return pool.ErrClosed } pipe.cmds = pipe.cmds[:0] return nil @@ -287,7 +287,7 @@ func (pipe *RingPipeline) Discard() error { // command if any. func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { if pipe.closed { - return nil, errClosed + return nil, pool.ErrClosed } if len(pipe.cmds) == 0 { return pipe.cmds, nil From e37202e605475100e669f238fb813224ecaa03bc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 14 Mar 2016 16:51:46 +0200 Subject: [PATCH 0148/1746] Skip flaky tests. --- cluster_test.go | 2 +- command_test.go | 1 + commands_test.go | 12 ++++++++---- example_test.go | 21 ++++++++++++--------- options.go | 3 ++- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 80e1fbba0f..392a898d69 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -139,7 +139,7 @@ func startCluster(scenario *clusterScenario) error { return fmt.Errorf("cluster did not reach consistent state (%v)", res) } return nil - }, 10*time.Second) + }, 30*time.Second) if err != nil { return err } diff --git a/command_test.go b/command_test.go index 064e7340cf..e7ebc6020d 100644 --- a/command_test.go +++ b/command_test.go @@ -139,6 +139,7 @@ var _ = Describe("Command", func() { Describe("races", func() { var C, N = 10, 1000 if testing.Short() { + C = 3 N = 100 } diff --git a/commands_test.go b/commands_test.go index 49d8488b11..281396769a 100644 --- a/commands_test.go +++ b/commands_test.go @@ -57,16 +57,20 @@ var _ = Describe("Commands", func() { }) It("should BgRewriteAOF", func() { - r := client.BgRewriteAOF() - Expect(r.Err()).NotTo(HaveOccurred()) - Expect(r.Val()).To(ContainSubstring("Background append only file rewriting")) + Skip("flaky test") + + val, err := client.BgRewriteAOF().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(ContainSubstring("Background append only file rewriting")) }) It("should BgSave", func() { + Skip("flaky test") + // workaround for "ERR Can't BGSAVE while AOF log rewriting is in progress" Eventually(func() string { return client.BgSave().Val() - }, "10s").Should(Equal("Background saving started")) + }, "30s").Should(Equal("Background saving started")) }) It("should ClientKill", func() { diff --git a/example_test.go b/example_test.go index 3034d86735..1baaa111d8 100644 --- a/example_test.go +++ b/example_test.go @@ -13,7 +13,8 @@ var client *redis.Client func init() { client = redis.NewClient(&redis.Options{ - Addr: ":6379", + Addr: ":6379", + DialTimeout: 10 * time.Second, }) client.FlushDb() } @@ -247,30 +248,32 @@ func ExamplePubSub_Receive() { } defer pubsub.Close() - err = client.Publish("mychannel2", "hello").Err() + n, err := client.Publish("mychannel2", "hello").Result() if err != nil { panic(err) } + fmt.Println(n, "clients received message") - for i := 0; i < 2; i++ { + for { // ReceiveTimeout is a low level API. Use ReceiveMessage instead. msgi, err := pubsub.ReceiveTimeout(time.Second) if err != nil { - panic(err) + break } switch msg := msgi.(type) { case *redis.Subscription: - fmt.Println(msg.Kind, msg.Channel) + fmt.Println("subscribed to", msg.Channel) case *redis.Message: - fmt.Println(msg.Channel, msg.Payload) + fmt.Println("received", msg.Payload, "from", msg.Channel) default: - panic(fmt.Sprintf("unknown message: %#v", msgi)) + panic(fmt.Errorf("unknown message: %#v", msgi)) } } - // Output: subscribe mychannel2 - // mychannel2 hello + // Output: 1 clients received message + // subscribed to mychannel2 + // received hello from mychannel2 } func ExampleScript() { diff --git a/options.go b/options.go index b05721241c..de91d4e850 100644 --- a/options.go +++ b/options.go @@ -30,6 +30,7 @@ type Options struct { // Sets the deadline for establishing new connections. If reached, // dial will fail with a timeout. + // Default is 5 seconds. DialTimeout time.Duration // Sets the deadline for socket reads. If reached, commands will // fail with a timeout instead of blocking. @@ -43,7 +44,7 @@ type Options struct { PoolSize int // Specifies amount of time client waits for connection if all // connections are busy before returning an error. - // Default is 1 seconds. + // Default is 1 second. PoolTimeout time.Duration // Specifies amount of time after which client closes idle // connections. Should be less than server's timeout. From 707472c09bde3416534f28f79205a8b48c684b59 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 15 Mar 2016 14:04:35 +0200 Subject: [PATCH 0149/1746] Fix connection initialization. --- cluster_pipeline.go | 2 +- internal/pool/bench_test.go | 4 ++-- internal/pool/conn.go | 21 +++++++++++---------- internal/pool/conn_list.go | 3 ++- internal/pool/pool.go | 36 ++++++++++++++++++------------------ internal/pool/pool_single.go | 4 ++-- internal/pool/pool_sticky.go | 14 ++++++-------- internal/pool/pool_test.go | 3 +-- multi.go | 2 +- multi_test.go | 4 ++-- pipeline.go | 2 +- pool_test.go | 8 ++++---- pubsub.go | 6 +++--- pubsub_test.go | 2 +- redis.go | 21 +++++++++++++-------- redis_test.go | 4 ++-- ring.go | 2 +- 17 files changed, 71 insertions(+), 67 deletions(-) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 8641d3df60..6d55265983 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -69,7 +69,7 @@ func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { continue } - cn, _, err := client.conn() + cn, err := client.conn() if err != nil { setCmdsErr(cmds, err) retErr = err diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index ba6df0c00a..5acc5e2c68 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -20,7 +20,7 @@ func benchmarkPoolGetPut(b *testing.B, poolSize int) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - conn, _, err := pool.Get() + conn, err := pool.Get() if err != nil { b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) } @@ -56,7 +56,7 @@ func benchmarkPoolGetReplace(b *testing.B, poolSize int) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - conn, _, err := pool.Get() + conn, err := pool.Get() if err != nil { b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) } diff --git a/internal/pool/conn.go b/internal/pool/conn.go index cbe379b198..c376886231 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -18,7 +18,9 @@ type Conn struct { Rd *bufio.Reader Buf []byte - UsedAt time.Time + Inited bool + UsedAt time.Time + ReadTimeout time.Duration WriteTimeout time.Duration } @@ -40,8 +42,12 @@ func (cn *Conn) Index() int { return int(atomic.LoadInt32(&cn.idx)) } -func (cn *Conn) SetIndex(idx int) { - atomic.StoreInt32(&cn.idx, int32(idx)) +func (cn *Conn) SetIndex(newIdx int) int { + oldIdx := cn.Index() + if !atomic.CompareAndSwapInt32(&cn.idx, int32(oldIdx), int32(newIdx)) { + return -1 + } + return oldIdx } func (cn *Conn) IsStale(timeout time.Duration) bool { @@ -72,11 +78,6 @@ func (cn *Conn) RemoteAddr() net.Addr { return cn.NetConn.RemoteAddr() } -func (cn *Conn) Close() int { - idx := cn.Index() - if !atomic.CompareAndSwapInt32(&cn.idx, int32(idx), -1) { - return -1 - } - _ = cn.NetConn.Close() - return idx +func (cn *Conn) Close() error { + return cn.NetConn.Close() } diff --git a/internal/pool/conn_list.go b/internal/pool/conn_list.go index 7e43ee74f0..b3f58704e1 100644 --- a/internal/pool/conn_list.go +++ b/internal/pool/conn_list.go @@ -43,7 +43,7 @@ func (l *connList) Add(cn *Conn) { l.mu.Lock() for i, c := range l.cns { if c == nil { - cn.SetIndex(i) + cn.idx = int32(i) l.cns[i] = cn l.mu.Unlock() return @@ -76,6 +76,7 @@ func (l *connList) Close() error { if c == nil { continue } + c.idx = -1 c.Close() } l.cns = nil diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 4f2b2175b3..4de11fc6c5 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -32,7 +32,7 @@ type PoolStats struct { type Pooler interface { First() *Conn - Get() (*Conn, bool, error) + Get() (*Conn, error) Put(*Conn) error Replace(*Conn, error) error Len() int @@ -146,7 +146,7 @@ func (p *ConnPool) dial() (net.Conn, error) { return cn, nil } -func (p *ConnPool) newConn() (*Conn, error) { +func (p *ConnPool) NewConn() (*Conn, error) { netConn, err := p.dial() if err != nil { return nil, err @@ -155,42 +155,38 @@ func (p *ConnPool) newConn() (*Conn, error) { } // Get returns existed connection from the pool or creates a new one. -func (p *ConnPool) Get() (cn *Conn, isNew bool, err error) { +func (p *ConnPool) Get() (*Conn, error) { if p.Closed() { - err = ErrClosed - return + return nil, ErrClosed } atomic.AddUint32(&p.stats.Requests, 1) // Fetch first non-idle connection, if available. - if cn = p.First(); cn != nil { + if cn := p.First(); cn != nil { atomic.AddUint32(&p.stats.Hits, 1) - return + return cn, nil } // Try to create a new one. if p.conns.Reserve() { - isNew = true - - cn, err = p.newConn() + cn, err := p.NewConn() if err != nil { p.conns.CancelReservation() - return + return nil, err } p.conns.Add(cn) - return + return cn, nil } // Otherwise, wait for the available connection. atomic.AddUint32(&p.stats.Waits, 1) - if cn = p.wait(); cn != nil { - return + if cn := p.wait(); cn != nil { + return cn, nil } atomic.AddUint32(&p.stats.Timeouts, 1) - err = ErrPoolTimeout - return + return nil, ErrPoolTimeout } func (p *ConnPool) Put(cn *Conn) error { @@ -205,7 +201,9 @@ func (p *ConnPool) Put(cn *Conn) error { } func (p *ConnPool) replace(cn *Conn) (*Conn, error) { - idx := cn.Close() + _ = cn.Close() + + idx := cn.SetIndex(-1) if idx == -1 { return nil, errConnClosed } @@ -236,7 +234,9 @@ func (p *ConnPool) Replace(cn *Conn, reason error) error { } func (p *ConnPool) Remove(cn *Conn, reason error) error { - idx := cn.Close() + _ = cn.Close() + + idx := cn.SetIndex(-1) if idx == -1 { return errConnClosed } diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index f9ebfa6d14..39362c0211 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -16,8 +16,8 @@ func (p *SingleConnPool) First() *Conn { return p.cn } -func (p *SingleConnPool) Get() (*Conn, bool, error) { - return p.cn, false, nil +func (p *SingleConnPool) Get() (*Conn, error) { + return p.cn, nil } func (p *SingleConnPool) Put(cn *Conn) error { diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 11a7ee4968..8b76b6f668 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -30,25 +30,23 @@ func (p *StickyConnPool) First() *Conn { return cn } -func (p *StickyConnPool) Get() (cn *Conn, isNew bool, err error) { +func (p *StickyConnPool) Get() (*Conn, error) { defer p.mx.Unlock() p.mx.Lock() if p.closed { - err = ErrClosed - return + return nil, ErrClosed } if p.cn != nil { - cn = p.cn - return + return p.cn, nil } - cn, isNew, err = p.pool.Get() + cn, err := p.pool.Get() if err != nil { - return + return nil, err } p.cn = cn - return + return cn, nil } func (p *StickyConnPool) put() (err error) { diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 5dd7784ed4..1c591924c5 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -69,9 +69,8 @@ var _ = Describe("conns reapser", func() { cn := connPool.First() Expect(cn).To(BeNil()) - cn, isNew, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) - Expect(isNew).To(BeTrue()) Expect(cn).NotTo(BeNil()) Expect(connPool.Len()).To(Equal(4)) diff --git a/multi.go b/multi.go index a0498211de..79b7cb6df3 100644 --- a/multi.go +++ b/multi.go @@ -128,7 +128,7 @@ func (c *Multi) Exec(f func() error) ([]Cmder, error) { // Strip MULTI and EXEC commands. retCmds := cmds[1 : len(cmds)-1] - cn, _, err := c.base.conn() + cn, err := c.base.conn() if err != nil { setCmdsErr(retCmds, err) return retCmds, err diff --git a/multi_test.go b/multi_test.go index fa532d1a42..a82a347a5d 100644 --- a/multi_test.go +++ b/multi_test.go @@ -142,7 +142,7 @@ var _ = Describe("Multi", func() { It("should recover from bad connection", func() { // Put bad connection in the pool. - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} @@ -169,7 +169,7 @@ var _ = Describe("Multi", func() { It("should recover from bad connection when there are no commands", func() { // Put bad connection in the pool. - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} diff --git a/pipeline.go b/pipeline.go index 842fad7bc2..888d8c40de 100644 --- a/pipeline.go +++ b/pipeline.go @@ -90,7 +90,7 @@ func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { failedCmds := cmds for i := 0; i <= pipe.client.opt.MaxRetries; i++ { - cn, _, err := pipe.client.conn() + cn, err := pipe.client.conn() if err != nil { setCmdsErr(failedCmds, err) return cmds, err diff --git a/pool_test.go b/pool_test.go index 225ad6adbd..006ab0be84 100644 --- a/pool_test.go +++ b/pool_test.go @@ -91,7 +91,7 @@ var _ = Describe("pool", func() { }) It("should remove broken connections", func() { - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) @@ -136,12 +136,12 @@ var _ = Describe("pool", func() { pool := client.Pool() // Reserve one connection. - cn, _, err := pool.Get() + cn, err := pool.Get() Expect(err).NotTo(HaveOccurred()) // Reserve the rest of connections. for i := 0; i < 9; i++ { - _, _, err := pool.Get() + _, err := pool.Get() Expect(err).NotTo(HaveOccurred()) } @@ -181,7 +181,7 @@ var _ = Describe("pool", func() { var rateErr error for i := 0; i < 1000; i++ { - cn, _, err := pool.Get() + cn, err := pool.Get() if err != nil { rateErr = err break diff --git a/pubsub.go b/pubsub.go index 68b2aeb5fe..05e5921347 100644 --- a/pubsub.go +++ b/pubsub.go @@ -50,7 +50,7 @@ func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { } func (c *PubSub) subscribe(redisCmd string, channels ...string) error { - cn, _, err := c.base.conn() + cn, err := c.base.conn() if err != nil { return err } @@ -126,7 +126,7 @@ func (c *PubSub) Close() error { } func (c *PubSub) Ping(payload string) error { - cn, _, err := c.base.conn() + cn, err := c.base.conn() if err != nil { return err } @@ -226,7 +226,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { c.resubscribe() } - cn, _, err := c.base.conn() + cn, err := c.base.conn() if err != nil { return nil, err } diff --git a/pubsub_test.go b/pubsub_test.go index a8bb610be5..835d7c1a9c 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -289,7 +289,7 @@ var _ = Describe("PubSub", func() { }) expectReceiveMessageOnError := func(pubsub *redis.PubSub) { - cn1, _, err := pubsub.Pool().Get() + cn1, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn1.NetConn = &badConn{ readErr: io.EOF, diff --git a/redis.go b/redis.go index aab5ba6331..dc55572eff 100644 --- a/redis.go +++ b/redis.go @@ -32,15 +32,18 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.opt.Addr, c.opt.DB) } -func (c *baseClient) conn() (*pool.Conn, bool, error) { - cn, isNew, err := c.connPool.Get() - if err == nil && isNew { - err = c.initConn(cn) - if err != nil { - c.putConn(cn, err, false) +func (c *baseClient) conn() (*pool.Conn, error) { + cn, err := c.connPool.Get() + if err != nil { + return nil, err + } + if !cn.Inited { + if err := c.initConn(cn); err != nil { + _ = c.connPool.Replace(cn, err) + return nil, err } } - return cn, isNew, err + return cn, err } func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { @@ -54,6 +57,8 @@ func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { } func (c *baseClient) initConn(cn *pool.Conn) error { + cn.Inited = true + if c.opt.Password == "" && c.opt.DB == 0 { return nil } @@ -82,7 +87,7 @@ func (c *baseClient) process(cmd Cmder) { cmd.reset() } - cn, _, err := c.conn() + cn, err := c.conn() if err != nil { cmd.setErr(err) return diff --git a/redis_test.go b/redis_test.go index 23c39009f8..8b3d8dbd87 100644 --- a/redis_test.go +++ b/redis_test.go @@ -157,7 +157,7 @@ var _ = Describe("Client", func() { }) // Put bad connection in the pool. - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} @@ -169,7 +169,7 @@ var _ = Describe("Client", func() { }) It("should maintain conn.UsedAt", func() { - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn.UsedAt).NotTo(BeZero()) createdAt := cn.UsedAt diff --git a/ring.go b/ring.go index c66a5bc1af..32212161d4 100644 --- a/ring.go +++ b/ring.go @@ -314,7 +314,7 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { for name, cmds := range cmdsMap { client := pipe.ring.shards[name].Client - cn, _, err := client.conn() + cn, err := client.conn() if err != nil { setCmdsErr(cmds, err) if retErr == nil { From 956d6c508bde11ddf1263f84a8367f31d569b09f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 15 Mar 2016 14:45:04 +0200 Subject: [PATCH 0150/1746] Try to fix flaky test. --- example_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example_test.go b/example_test.go index 1baaa111d8..e5cb5b28e4 100644 --- a/example_test.go +++ b/example_test.go @@ -254,9 +254,9 @@ func ExamplePubSub_Receive() { } fmt.Println(n, "clients received message") - for { + for i := 0; i < 2; i++ { // ReceiveTimeout is a low level API. Use ReceiveMessage instead. - msgi, err := pubsub.ReceiveTimeout(time.Second) + msgi, err := pubsub.ReceiveTimeout(5 * time.Second) if err != nil { break } From f47fb47df0504eef87892242127de452af13769b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 16 Mar 2016 16:57:24 +0200 Subject: [PATCH 0151/1746] Extract race tests to separate file. Add more race tests. --- Makefile | 4 +- cluster_test.go | 5 +- command_test.go | 198 +--------------------------------- commands_test.go | 15 ++- example_test.go | 7 +- internal/pool/conn.go | 12 +++ internal/pool/conn_list.go | 19 ++-- internal/pool/pool.go | 58 +++++++--- main_test.go | 38 +++++-- multi_test.go | 7 +- options.go | 10 +- parser.go | 14 +-- pipeline_test.go | 6 +- pool_test.go | 32 +++--- pubsub_test.go | 6 +- race_test.go | 213 +++++++++++++++++++++++++++++++++++++ redis_test.go | 90 +++++++++------- sentinel.go | 6 +- sentinel_test.go | 1 + 19 files changed, 411 insertions(+), 330 deletions(-) create mode 100644 race_test.go diff --git a/Makefile b/Makefile index b7867b48eb..9ee35b2c69 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: testdeps - go test ./... -test.cpu=1,2,4 - go test ./... -test.short -test.race + go test ./... + go test ./... -short -race testdeps: testdata/redis/src/redis-server diff --git a/cluster_test.go b/cluster_test.go index 392a898d69..7423a7ebf2 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -66,7 +66,10 @@ func startCluster(scenario *clusterScenario) error { return err } - client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:" + port}) + client := redis.NewClient(&redis.Options{ + Addr: ":" + port, + }) + info, err := client.ClusterNodes().Result() if err != nil { return err diff --git a/command_test.go b/command_test.go index e7ebc6020d..2b218acf4d 100644 --- a/command_test.go +++ b/command_test.go @@ -1,35 +1,21 @@ package redis_test import ( - "bytes" - "strconv" - "sync" - "testing" - "time" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "gopkg.in/redis.v3" - "gopkg.in/redis.v3/internal/pool" ) var _ = Describe("Command", func() { var client *redis.Client - connect := func() *redis.Client { - return redis.NewClient(&redis.Options{ - Addr: redisAddr, - PoolTimeout: time.Minute, - }) - } - BeforeEach(func() { - client = connect() + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -54,64 +40,6 @@ var _ = Describe("Command", func() { Expect(set.Val()).To(Equal("OK")) }) - It("should escape special chars", func() { - set := client.Set("key", "hello1\r\nhello2\r\n", 0) - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) - - get := client.Get("key") - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal("hello1\r\nhello2\r\n")) - }) - - It("should handle big vals", func() { - bigVal := string(bytes.Repeat([]byte{'*'}, 1<<16)) - - err := client.Set("key", bigVal, 0).Err() - Expect(err).NotTo(HaveOccurred()) - - // Reconnect to get new connection. - Expect(client.Close()).To(BeNil()) - client = connect() - - got, err := client.Get("key").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(len(got)).To(Equal(len(bigVal))) - Expect(got).To(Equal(bigVal)) - }) - - It("should handle many keys #1", func() { - const n = 100000 - for i := 0; i < n; i++ { - client.Set("keys.key"+strconv.Itoa(i), "hello"+strconv.Itoa(i), 0) - } - keys := client.Keys("keys.*") - Expect(keys.Err()).NotTo(HaveOccurred()) - Expect(len(keys.Val())).To(Equal(n)) - }) - - It("should handle many keys #2", func() { - const n = 100000 - - keys := []string{"non-existent-key"} - for i := 0; i < n; i++ { - key := "keys.key" + strconv.Itoa(i) - client.Set(key, "hello"+strconv.Itoa(i), 0) - keys = append(keys, key) - } - keys = append(keys, "non-existent-key") - - mget := client.MGet(keys...) - Expect(mget.Err()).NotTo(HaveOccurred()) - Expect(len(mget.Val())).To(Equal(n + 2)) - vals := mget.Val() - for i := 0; i < n; i++ { - Expect(vals[i+1]).To(Equal("hello" + strconv.Itoa(i))) - } - Expect(vals[0]).To(BeNil()) - Expect(vals[n+1]).To(BeNil()) - }) - It("should convert strings via helpers", func() { set := client.Set("key", "10", 0) Expect(set.Err()).NotTo(HaveOccurred()) @@ -129,126 +57,4 @@ var _ = Describe("Command", func() { Expect(f).To(Equal(float64(10))) }) - It("Cmd should return string", func() { - cmd := redis.NewCmd("PING") - client.Process(cmd) - Expect(cmd.Err()).NotTo(HaveOccurred()) - Expect(cmd.Val()).To(Equal("PONG")) - }) - - Describe("races", func() { - var C, N = 10, 1000 - if testing.Short() { - C = 3 - N = 100 - } - - It("should echo", func() { - perform(C, func() { - for i := 0; i < N; i++ { - msg := "echo" + strconv.Itoa(i) - echo, err := client.Echo(msg).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(echo).To(Equal(msg)) - } - }) - }) - - It("should incr", func() { - key := "TestIncrFromGoroutines" - - perform(C, func() { - for i := 0; i < N; i++ { - err := client.Incr(key).Err() - Expect(err).NotTo(HaveOccurred()) - } - }) - - val, err := client.Get(key).Int64() - Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal(int64(C * N))) - }) - - It("should handle big vals", func() { - client2 := connect() - defer client2.Close() - - bigVal := string(bytes.Repeat([]byte{'*'}, 1<<16)) - - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - perform(C, func() { - for i := 0; i < N; i++ { - got, err := client.Get("key").Result() - if err == redis.Nil { - continue - } - Expect(got).To(Equal(bigVal)) - } - }) - }() - - go func() { - defer wg.Done() - perform(C, func() { - for i := 0; i < N; i++ { - err := client2.Set("key", bigVal, 0).Err() - Expect(err).NotTo(HaveOccurred()) - } - }) - }() - - wg.Wait() - }) - - It("should PubSub", func() { - connPool := client.Pool() - connPool.(*pool.ConnPool).DialLimiter = nil - - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - perform(C, func() { - for i := 0; i < N; i++ { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) - - go func() { - defer GinkgoRecover() - - time.Sleep(time.Millisecond) - err := pubsub.Close() - Expect(err).NotTo(HaveOccurred()) - }() - - _, err = pubsub.ReceiveMessage() - Expect(err.Error()).To(ContainSubstring("closed")) - } - }) - }() - - go func() { - defer wg.Done() - perform(C, func() { - for i := 0; i < N; i++ { - val := "echo" + strconv.Itoa(i) - echo, err := client.Echo(val).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(echo).To(Equal(val)) - } - }) - }() - - wg.Wait() - - Expect(connPool.Len()).To(Equal(connPool.FreeLen())) - Expect(connPool.Len()).To(BeNumerically("<=", 10)) - }) - }) - }) diff --git a/commands_test.go b/commands_test.go index 281396769a..6e2e6c1909 100644 --- a/commands_test.go +++ b/commands_test.go @@ -19,14 +19,11 @@ var _ = Describe("Commands", func() { var client *redis.Client BeforeEach(func() { - client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - PoolTimeout: 30 * time.Second, - }) + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -299,7 +296,7 @@ var _ = Describe("Commands", func() { }) It("should Move", func() { - move := client.Move("key", 1) + move := client.Move("key", 2) Expect(move.Err()).NotTo(HaveOccurred()) Expect(move.Val()).To(Equal(false)) @@ -307,7 +304,7 @@ var _ = Describe("Commands", func() { Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) - move = client.Move("key", 1) + move = client.Move("key", 2) Expect(move.Err()).NotTo(HaveOccurred()) Expect(move.Val()).To(Equal(true)) @@ -315,7 +312,7 @@ var _ = Describe("Commands", func() { Expect(get.Err()).To(Equal(redis.Nil)) Expect(get.Val()).To(Equal("")) - sel := client.Select(1) + sel := client.Select(2) Expect(sel.Err()).NotTo(HaveOccurred()) Expect(sel.Val()).To(Equal("OK")) @@ -323,7 +320,7 @@ var _ = Describe("Commands", func() { Expect(get.Err()).NotTo(HaveOccurred()) Expect(get.Val()).To(Equal("hello")) Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - Expect(client.Select(0).Err()).NotTo(HaveOccurred()) + Expect(client.Select(1).Err()).NotTo(HaveOccurred()) }) It("should Object", func() { diff --git a/example_test.go b/example_test.go index e5cb5b28e4..dc4e1bde4d 100644 --- a/example_test.go +++ b/example_test.go @@ -12,10 +12,9 @@ import ( var client *redis.Client func init() { - client = redis.NewClient(&redis.Options{ - Addr: ":6379", - DialTimeout: 10 * time.Second, - }) + opt := redisOptions() + opt.Addr = ":6379" + client = redis.NewClient(opt) client.FlushDb() } diff --git a/internal/pool/conn.go b/internal/pool/conn.go index c376886231..0fbb41931f 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -2,6 +2,7 @@ package pool import ( "bufio" + "io" "net" "sync/atomic" "time" @@ -78,6 +79,17 @@ func (cn *Conn) RemoteAddr() net.Addr { return cn.NetConn.RemoteAddr() } +func (cn *Conn) ReadN(n int) ([]byte, error) { + if d := n - cap(cn.Buf); d > 0 { + cn.Buf = cn.Buf[:cap(cn.Buf)] + cn.Buf = append(cn.Buf, make([]byte, d)...) + } else { + cn.Buf = cn.Buf[:n] + } + _, err := io.ReadFull(cn.Rd, cn.Buf) + return cn.Buf, err +} + func (cn *Conn) Close() error { return cn.NetConn.Close() } diff --git a/internal/pool/conn_list.go b/internal/pool/conn_list.go index b3f58704e1..61bf99ba33 100644 --- a/internal/pool/conn_list.go +++ b/internal/pool/conn_list.go @@ -43,7 +43,7 @@ func (l *connList) Add(cn *Conn) { l.mu.Lock() for i, c := range l.cns { if c == nil { - cn.idx = int32(i) + cn.SetIndex(i) l.cns[i] = cn l.mu.Unlock() return @@ -65,22 +65,25 @@ func (l *connList) Remove(idx int) { l.mu.Lock() if l.cns != nil { l.cns[idx] = nil - l.len -= 1 + atomic.AddInt32(&l.len, -1) } l.mu.Unlock() } -func (l *connList) Close() error { +func (l *connList) Reset() []*Conn { l.mu.Lock() - for _, c := range l.cns { - if c == nil { + + for _, cn := range l.cns { + if cn == nil { continue } - c.idx = -1 - c.Close() + cn.SetIndex(-1) } + + cns := l.cns l.cns = nil l.len = 0 + l.mu.Unlock() - return nil + return cns } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 4de11fc6c5..932146eab1 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -5,13 +5,14 @@ import ( "fmt" "log" "net" + "os" "sync/atomic" "time" "gopkg.in/bsm/ratelimit.v1" ) -var Logger *log.Logger +var Logger = log.New(os.Stderr, "pg: ", log.LstdFlags) var ( ErrClosed = errors.New("redis: client is closed") @@ -47,6 +48,7 @@ type dialer func() (net.Conn, error) type ConnPool struct { _dial dialer DialLimiter *ratelimit.RateLimiter + OnClose func(*Conn) error poolTimeout time.Duration idleTimeout time.Duration @@ -74,19 +76,11 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Durati freeConns: newConnStack(poolSize), } if idleTimeout > 0 { - go p.reaper() + go p.reaper(getIdleCheckFrequency()) } return p } -func (p *ConnPool) Closed() bool { - return atomic.LoadInt32(&p._closed) == 1 -} - -func (p *ConnPool) isIdle(cn *Conn) bool { - return p.idleTimeout > 0 && time.Since(cn.UsedAt) > p.idleTimeout -} - func (p *ConnPool) Add(cn *Conn) bool { if !p.conns.Reserve() { return false @@ -266,23 +260,43 @@ func (p *ConnPool) Stats() *PoolStats { return &stats } +func (p *ConnPool) Closed() bool { + return atomic.LoadInt32(&p._closed) == 1 +} + func (p *ConnPool) Close() (retErr error) { if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { return ErrClosed } + // Wait for app to free connections, but don't close them immediately. for i := 0; i < p.Len(); i++ { if cn := p.wait(); cn == nil { break } } + // Close all connections. - if err := p.conns.Close(); err != nil { - retErr = err + cns := p.conns.Reset() + for _, cn := range cns { + if cn == nil { + continue + } + if err := p.closeConn(cn); err != nil && retErr == nil { + retErr = err + } } + return retErr } +func (p *ConnPool) closeConn(cn *Conn) error { + if p.OnClose != nil { + _ = p.OnClose(cn) + } + return cn.Close() +} + func (p *ConnPool) ReapStaleConns() (n int, err error) { for { cn := p.freeConns.ShiftStale(p.idleTimeout) @@ -297,8 +311,8 @@ func (p *ConnPool) ReapStaleConns() (n int, err error) { return } -func (p *ConnPool) reaper() { - ticker := time.NewTicker(time.Minute) +func (p *ConnPool) reaper(frequency time.Duration) { + ticker := time.NewTicker(frequency) defer ticker.Stop() for _ = range ticker.C { @@ -324,3 +338,19 @@ func (p *ConnPool) loadLastErr() string { } return "" } + +//------------------------------------------------------------------------------ + +var idleCheckFrequency atomic.Value + +func SetIdleCheckFrequency(d time.Duration) { + idleCheckFrequency.Store(d) +} + +func getIdleCheckFrequency() time.Duration { + v := idleCheckFrequency.Load() + if v == nil { + return time.Minute + } + return v.(time.Duration) +} diff --git a/main_test.go b/main_test.go index e3e747fe6e..67905cf868 100644 --- a/main_test.go +++ b/main_test.go @@ -15,6 +15,7 @@ import ( . "github.com/onsi/gomega" "gopkg.in/redis.v3" + "gopkg.in/redis.v3/internal/pool" ) const ( @@ -52,6 +53,8 @@ var cluster = &clusterScenario{ var _ = BeforeSuite(func() { var err error + pool.SetIdleCheckFrequency(time.Second) // be aggressive in tests + redisMain, err = startRedis(redisPort) Expect(err).NotTo(HaveOccurred()) @@ -99,31 +102,49 @@ func TestGinkgoSuite(t *testing.T) { //------------------------------------------------------------------------------ -func perform(n int, cb func()) { +func redisOptions() *redis.Options { + return &redis.Options{ + Addr: redisAddr, + DB: 15, + DialTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + PoolSize: 10, + PoolTimeout: 30 * time.Second, + IdleTimeout: time.Second, // be aggressive in tests + } +} + +func perform(n int, cb func(int)) { var wg sync.WaitGroup for i := 0; i < n; i++ { wg.Add(1) - go func() { + go func(i int) { defer GinkgoRecover() defer wg.Done() - cb() - }() + cb(i) + }(i) } wg.Wait() } func eventually(fn func() error, timeout time.Duration) error { - done := make(chan struct{}) var exit int32 - var err error + var retErr error + var mu sync.Mutex + done := make(chan struct{}) + go func() { for atomic.LoadInt32(&exit) == 0 { - err = fn() + err := fn() if err == nil { close(done) return } + mu.Lock() + retErr = err + mu.Unlock() time.Sleep(timeout / 100) } }() @@ -133,6 +154,9 @@ func eventually(fn func() error, timeout time.Duration) error { return nil case <-time.After(timeout): atomic.StoreInt32(&exit, 1) + mu.Lock() + err := retErr + mu.Unlock() return err } } diff --git a/multi_test.go b/multi_test.go index a82a347a5d..e76c2b33a9 100644 --- a/multi_test.go +++ b/multi_test.go @@ -14,13 +14,11 @@ var _ = Describe("Multi", func() { var client *redis.Client BeforeEach(func() { - client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - }) + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -54,6 +52,7 @@ var _ = Describe("Multi", func() { for i := 0; i < 100; i++ { wg.Add(1) go func() { + defer GinkgoRecover() defer wg.Done() err := incr("key") diff --git a/options.go b/options.go index de91d4e850..935e756442 100644 --- a/options.go +++ b/options.go @@ -60,12 +60,12 @@ func (opt *Options) getNetwork() string { } func (opt *Options) getDialer() func() (net.Conn, error) { - if opt.Dialer == nil { - opt.Dialer = func() (net.Conn, error) { - return net.DialTimeout(opt.getNetwork(), opt.Addr, opt.getDialTimeout()) - } + if opt.Dialer != nil { + return opt.Dialer + } + return func() (net.Conn, error) { + return net.DialTimeout(opt.getNetwork(), opt.Addr, opt.getDialTimeout()) } - return opt.Dialer } func (opt *Options) getPoolSize() int { diff --git a/parser.go b/parser.go index 3d8742c983..0798857952 100644 --- a/parser.go +++ b/parser.go @@ -3,7 +3,6 @@ package redis import ( "errors" "fmt" - "io" "net" "strconv" @@ -245,17 +244,6 @@ func isNilReply(b []byte) bool { b[1] == '-' && b[2] == '1' } -func readN(cn *pool.Conn, n int) ([]byte, error) { - if d := n - cap(cn.Buf); d > 0 { - cn.Buf = cn.Buf[:cap(cn.Buf)] - cn.Buf = append(cn.Buf, make([]byte, d)...) - } else { - cn.Buf = cn.Buf[:n] - } - _, err := io.ReadFull(cn.Rd, cn.Buf) - return cn.Buf, err -} - //------------------------------------------------------------------------------ func parseErrorReply(cn *pool.Conn, line []byte) error { @@ -299,7 +287,7 @@ func parseBytesReply(cn *pool.Conn, line []byte) ([]byte, error) { return nil, err } - b, err := readN(cn, replyLen+2) + b, err := cn.ReadN(replyLen + 2) if err != nil { return nil, err } diff --git a/pipeline_test.go b/pipeline_test.go index ed01baf480..cfbed6e531 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -14,13 +14,11 @@ var _ = Describe("Pipelining", func() { var client *redis.Client BeforeEach(func() { - client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - }) + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) }) diff --git a/pool_test.go b/pool_test.go index 006ab0be84..bf1ae4aa06 100644 --- a/pool_test.go +++ b/pool_test.go @@ -8,16 +8,15 @@ import ( . "github.com/onsi/gomega" "gopkg.in/redis.v3" + "gopkg.in/redis.v3/internal/pool" ) var _ = Describe("pool", func() { var client *redis.Client BeforeEach(func() { - client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - PoolSize: 10, - }) + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -25,7 +24,7 @@ var _ = Describe("pool", func() { }) It("should respect max size", func() { - perform(1000, func() { + perform(1000, func(id int) { val, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("PONG")) @@ -38,7 +37,7 @@ var _ = Describe("pool", func() { }) It("should respect max on multi", func() { - perform(1000, func() { + perform(1000, func(id int) { var ping *redis.StatusCmd multi := client.Multi() @@ -60,7 +59,7 @@ var _ = Describe("pool", func() { }) It("should respect max on pipelines", func() { - perform(1000, func() { + perform(1000, func(id int) { pipe := client.Pipeline() ping := pipe.Ping() cmds, err := pipe.Exec() @@ -78,16 +77,17 @@ var _ = Describe("pool", func() { }) It("should respect max on pubsub", func() { - perform(10, func() { + connPool := client.Pool() + connPool.(*pool.ConnPool).DialLimiter = nil + + perform(1000, func(id int) { pubsub := client.PubSub() Expect(pubsub.Subscribe()).NotTo(HaveOccurred()) Expect(pubsub.Close()).NotTo(HaveOccurred()) }) - pool := client.Pool() - Expect(pool.Len()).To(BeNumerically("<=", 10)) - Expect(pool.FreeLen()).To(BeNumerically("<=", 10)) - Expect(pool.Len()).To(Equal(pool.FreeLen())) + Expect(connPool.Len()).To(Equal(connPool.FreeLen())) + Expect(connPool.Len()).To(BeNumerically("<=", 10)) }) It("should remove broken connections", func() { @@ -108,8 +108,8 @@ var _ = Describe("pool", func() { Expect(pool.FreeLen()).To(Equal(1)) stats := pool.Stats() - Expect(stats.Requests).To(Equal(uint32(3))) - Expect(stats.Hits).To(Equal(uint32(2))) + Expect(stats.Requests).To(Equal(uint32(4))) + Expect(stats.Hits).To(Equal(uint32(3))) Expect(stats.Waits).To(Equal(uint32(0))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) @@ -126,8 +126,8 @@ var _ = Describe("pool", func() { Expect(pool.FreeLen()).To(Equal(1)) stats := pool.Stats() - Expect(stats.Requests).To(Equal(uint32(100))) - Expect(stats.Hits).To(Equal(uint32(99))) + Expect(stats.Requests).To(Equal(uint32(101))) + Expect(stats.Hits).To(Equal(uint32(100))) Expect(stats.Waits).To(Equal(uint32(0))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) diff --git a/pubsub_test.go b/pubsub_test.go index 835d7c1a9c..ca1cdb259e 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -14,13 +14,9 @@ import ( var _ = Describe("PubSub", func() { var client *redis.Client - readTimeout := 3 * time.Second BeforeEach(func() { - client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - ReadTimeout: readTimeout, - }) + client = redis.NewClient(redisOptions()) Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) diff --git a/race_test.go b/race_test.go new file mode 100644 index 0000000000..0b942e114a --- /dev/null +++ b/race_test.go @@ -0,0 +1,213 @@ +package redis_test + +import ( + "bytes" + "fmt" + "net" + "strconv" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v3" + "gopkg.in/redis.v3/internal/pool" +) + +var _ = Describe("races", func() { + var client *redis.Client + + var C, N = 10, 1000 + if testing.Short() { + C = 4 + N = 100 + } + + BeforeEach(func() { + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).To(BeNil()) + }) + + AfterEach(func() { + err := client.Close() + Expect(err).NotTo(HaveOccurred()) + }) + + It("should echo", func() { + perform(C, func(id int) { + for i := 0; i < N; i++ { + msg := fmt.Sprintf("echo %d %d", id, i) + echo, err := client.Echo(msg).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(echo).To(Equal(msg)) + } + }) + }) + + It("should incr", func() { + key := "TestIncrFromGoroutines" + + perform(C, func(id int) { + for i := 0; i < N; i++ { + err := client.Incr(key).Err() + Expect(err).NotTo(HaveOccurred()) + } + }) + + val, err := client.Get(key).Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal(int64(C * N))) + }) + + It("should handle many keys", func() { + perform(C, func(id int) { + for i := 0; i < N; i++ { + err := client.Set( + fmt.Sprintf("keys.key-%d-%d", id, i), + fmt.Sprintf("hello-%d-%d", id, i), + 0, + ).Err() + Expect(err).NotTo(HaveOccurred()) + } + }) + + keys := client.Keys("keys.*") + Expect(keys.Err()).NotTo(HaveOccurred()) + Expect(len(keys.Val())).To(Equal(C * N)) + }) + + It("should handle many keys 2", func() { + perform(C, func(id int) { + keys := []string{"non-existent-key"} + for i := 0; i < N; i++ { + key := fmt.Sprintf("keys.key-%d", i) + keys = append(keys, key) + + err := client.Set(key, fmt.Sprintf("hello-%d", i), 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + keys = append(keys, "non-existent-key") + + vals, err := client.MGet(keys...).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(len(vals)).To(Equal(N + 2)) + + for i := 0; i < N; i++ { + Expect(vals[i+1]).To(Equal(fmt.Sprintf("hello-%d", i))) + } + + Expect(vals[0]).To(BeNil()) + Expect(vals[N+1]).To(BeNil()) + }) + }) + + It("should handle big vals in Get", func() { + bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb + + err := client.Set("key", bigVal, 0).Err() + Expect(err).NotTo(HaveOccurred()) + + // Reconnect to get new connection. + Expect(client.Close()).To(BeNil()) + client = redis.NewClient(redisOptions()) + + perform(C, func(id int) { + for i := 0; i < N; i++ { + got, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(bigVal)) + } + }) + + }) + + It("should handle big vals in Set", func() { + bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb + + perform(C, func(id int) { + for i := 0; i < N; i++ { + err := client.Set("key", bigVal, 0).Err() + Expect(err).NotTo(HaveOccurred()) + + got, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(bigVal)) + } + }) + }) + + It("should PubSub", func() { + connPool := client.Pool() + connPool.(*pool.ConnPool).DialLimiter = nil + + perform(C, func(id int) { + for i := 0; i < N; i++ { + pubsub, err := client.Subscribe(fmt.Sprintf("mychannel%d", id)) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + + time.Sleep(time.Millisecond) + err := pubsub.Close() + Expect(err).NotTo(HaveOccurred()) + }() + + _, err = pubsub.ReceiveMessage() + Expect(err.Error()).To(ContainSubstring("closed")) + + val := "echo" + strconv.Itoa(i) + echo, err := client.Echo(val).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(echo).To(Equal(val)) + } + }) + + Expect(connPool.Len()).To(Equal(connPool.FreeLen())) + Expect(connPool.Len()).To(BeNumerically("<=", 10)) + }) + + It("should select db", func() { + err := client.Set("db", 1, 0).Err() + Expect(err).NotTo(HaveOccurred()) + + perform(C, func(id int) { + opt := redisOptions() + opt.DB = int64(id) + client := redis.NewClient(opt) + for i := 0; i < N; i++ { + err := client.Set("db", id, 0).Err() + Expect(err).NotTo(HaveOccurred()) + + n, err := client.Get("db").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(id))) + } + err := client.Close() + Expect(err).NotTo(HaveOccurred()) + }) + + n, err := client.Get("db").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + }) + + It("should select DB with read timeout", func() { + perform(C, func(id int) { + opt := redisOptions() + opt.DB = int64(id) + opt.ReadTimeout = time.Nanosecond + client := redis.NewClient(opt) + + perform(C, func(id int) { + err := client.Ping().Err() + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + }) + + err := client.Close() + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/redis_test.go b/redis_test.go index 8b3d8dbd87..de297738dd 100644 --- a/redis_test.go +++ b/redis_test.go @@ -1,8 +1,8 @@ package redis_test import ( + "bytes" "net" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -14,9 +14,8 @@ var _ = Describe("Client", func() { var client *redis.Client BeforeEach(func() { - client = redis.NewClient(&redis.Options{ - Addr: redisAddr, - }) + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).To(BeNil()) }) AfterEach(func() { @@ -24,7 +23,7 @@ var _ = Describe("Client", func() { }) It("should Stringer", func() { - Expect(client.String()).To(Equal("Redis<:6380 db:0>")) + Expect(client.String()).To(Equal("Redis<:6380 db:15>")) }) It("should ping", func() { @@ -39,6 +38,7 @@ var _ = Describe("Client", func() { It("should support custom dialers", func() { custom := redis.NewClient(&redis.Options{ + Addr: ":1234", Dialer: func() (net.Conn, error) { return net.Dial("tcp", redisAddr) }, @@ -107,45 +107,30 @@ var _ = Describe("Client", func() { Expect(pipeline.Close()).NotTo(HaveOccurred()) }) - It("should support idle-timeouts", func() { - idle := redis.NewClient(&redis.Options{ - Addr: redisAddr, - IdleTimeout: 100 * time.Microsecond, - }) - defer idle.Close() - - Expect(idle.Ping().Err()).NotTo(HaveOccurred()) - time.Sleep(time.Millisecond) - Expect(idle.Ping().Err()).NotTo(HaveOccurred()) - }) - - It("should support DB selection", func() { - db1 := redis.NewClient(&redis.Options{ + It("should select DB", func() { + db2 := redis.NewClient(&redis.Options{ Addr: redisAddr, - DB: 1, + DB: 2, }) - defer db1.Close() + Expect(db2.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(db2.Get("db").Err()).To(Equal(redis.Nil)) + Expect(db2.Set("db", 2, 0).Err()).NotTo(HaveOccurred()) - Expect(db1.Get("key").Err()).To(Equal(redis.Nil)) - Expect(db1.Set("key", "value", 0).Err()).NotTo(HaveOccurred()) + n, err := db2.Get("db").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(2))) - Expect(client.Get("key").Err()).To(Equal(redis.Nil)) - Expect(db1.Get("key").Val()).To(Equal("value")) - Expect(db1.FlushDb().Err()).NotTo(HaveOccurred()) - }) + Expect(client.Get("db").Err()).To(Equal(redis.Nil)) - It("should support DB selection with read timeout (issue #135)", func() { - for i := 0; i < 100; i++ { - db1 := redis.NewClient(&redis.Options{ - Addr: redisAddr, - DB: 1, - ReadTimeout: time.Nanosecond, - }) + Expect(db2.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(db2.Close()).NotTo(HaveOccurred()) + }) - err := db1.Ping().Err() - Expect(err).To(HaveOccurred()) - Expect(err.(net.Error).Timeout()).To(BeTrue()) - } + It("should process custom commands", func() { + cmd := redis.NewCmd("PING") + client.Process(cmd) + Expect(cmd.Err()).NotTo(HaveOccurred()) + Expect(cmd.Val()).To(Equal("PONG")) }) It("should retry command on network error", func() { @@ -168,7 +153,7 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) }) - It("should maintain conn.UsedAt", func() { + It("should update conn.UsedAt on read/write", func() { cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn.UsedAt).NotTo(BeZero()) @@ -185,4 +170,31 @@ var _ = Describe("Client", func() { Expect(cn).NotTo(BeNil()) Expect(cn.UsedAt.After(createdAt)).To(BeTrue()) }) + + It("should escape special chars", func() { + set := client.Set("key", "hello1\r\nhello2\r\n", 0) + Expect(set.Err()).NotTo(HaveOccurred()) + Expect(set.Val()).To(Equal("OK")) + + get := client.Get("key") + Expect(get.Err()).NotTo(HaveOccurred()) + Expect(get.Val()).To(Equal("hello1\r\nhello2\r\n")) + }) + + It("should handle big vals", func() { + bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb + + err := client.Set("key", bigVal, 0).Err() + Expect(err).NotTo(HaveOccurred()) + + // Reconnect to get new connection. + Expect(client.Close()).To(BeNil()) + client = redis.NewClient(redisOptions()) + + got, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(len(got)).To(Equal(len(bigVal))) + Expect(got).To(Equal(bigVal)) + }) + }) diff --git a/sentinel.go b/sentinel.go index 694dd60251..887c5eb351 100644 --- a/sentinel.go +++ b/sentinel.go @@ -206,7 +206,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { d.discoverSentinels(sentinel) d.sentinel = sentinel - go d.listen() + go d.listen(sentinel) } func (d *sentinelFailover) resetSentinel() error { @@ -278,11 +278,11 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { } } -func (d *sentinelFailover) listen() { +func (d *sentinelFailover) listen(sentinel *sentinelClient) { var pubsub *PubSub for { if pubsub == nil { - pubsub = d.sentinel.PubSub() + pubsub = sentinel.PubSub() if err := pubsub.Subscribe("+switch-master"); err != nil { Logger.Printf("sentinel: Subscribe failed: %s", err) d.resetSentinel() diff --git a/sentinel_test.go b/sentinel_test.go index 14dcf834ba..693b957448 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -15,6 +15,7 @@ var _ = Describe("Sentinel", func() { MasterName: sentinelName, SentinelAddrs: []string{":" + sentinelPort}, }) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { From 93a7fe0de36f9ae3d3eeae646acaa4c0ee856961 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 17 Mar 2016 13:48:04 +0200 Subject: [PATCH 0152/1746] Move some tests to pool package. --- internal/pool/conn_stack.go | 4 +- internal/pool/main_test.go | 35 +++++++++ internal/pool/pool.go | 12 ++-- internal/pool/pool_test.go | 138 +++++++++++++++++++++++++++++++++--- pool_test.go | 64 ----------------- 5 files changed, 173 insertions(+), 80 deletions(-) create mode 100644 internal/pool/main_test.go diff --git a/internal/pool/conn_stack.go b/internal/pool/conn_stack.go index a26ab0ee6b..8b8f505786 100644 --- a/internal/pool/conn_stack.go +++ b/internal/pool/conn_stack.go @@ -32,13 +32,13 @@ func (s *connStack) ShiftStale(idleTimeout time.Duration) *Conn { select { case <-s.free: s.mu.Lock() - defer s.mu.Unlock() - if cn := s.cns[0]; cn.IsStale(idleTimeout) { copy(s.cns, s.cns[1:]) s.cns = s.cns[:len(s.cns)-1] + s.mu.Unlock() return cn } + s.mu.Unlock() s.free <- struct{}{} return nil diff --git a/internal/pool/main_test.go b/internal/pool/main_test.go new file mode 100644 index 0000000000..43afe3fa99 --- /dev/null +++ b/internal/pool/main_test.go @@ -0,0 +1,35 @@ +package pool_test + +import ( + "net" + "sync" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestGinkgoSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pool") +} + +func perform(n int, cbs ...func(int)) { + var wg sync.WaitGroup + for _, cb := range cbs { + for i := 0; i < n; i++ { + wg.Add(1) + go func(cb func(int), i int) { + defer GinkgoRecover() + defer wg.Done() + + cb(i) + }(cb, i) + } + } + wg.Wait() +} + +func dummyDialer() (net.Conn, error) { + return &net.TCPConn{}, nil +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 932146eab1..c3281c3d58 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -12,7 +12,7 @@ import ( "gopkg.in/bsm/ratelimit.v1" ) -var Logger = log.New(os.Stderr, "pg: ", log.LstdFlags) +var Logger = log.New(os.Stderr, "redis: ", log.LstdFlags) var ( ErrClosed = errors.New("redis: client is closed") @@ -108,9 +108,9 @@ func (p *ConnPool) First() *Conn { } // wait waits for free non-idle connection. It returns nil on timeout. -func (p *ConnPool) wait() *Conn { +func (p *ConnPool) wait(timeout time.Duration) *Conn { for { - cn := p.freeConns.PopWithTimeout(p.poolTimeout) + cn := p.freeConns.PopWithTimeout(timeout) if cn != nil && cn.IsStale(p.idleTimeout) { var err error cn, err = p.replace(cn) @@ -175,7 +175,7 @@ func (p *ConnPool) Get() (*Conn, error) { // Otherwise, wait for the available connection. atomic.AddUint32(&p.stats.Waits, 1) - if cn := p.wait(); cn != nil { + if cn := p.wait(p.poolTimeout); cn != nil { return cn, nil } @@ -270,8 +270,8 @@ func (p *ConnPool) Close() (retErr error) { } // Wait for app to free connections, but don't close them immediately. - for i := 0; i < p.Len(); i++ { - if cn := p.wait(); cn == nil { + for i := 0; i < p.Len()-p.FreeLen(); i++ { + if cn := p.wait(3 * time.Second); cn == nil { break } } diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 1c591924c5..d7a29883b6 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -12,19 +12,93 @@ import ( "gopkg.in/redis.v3/internal/pool" ) -func TestGinkgoSuite(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "pool") -} +var _ = Describe("ConnPool", func() { + var connPool *pool.ConnPool + + BeforeEach(func() { + pool.SetIdleCheckFrequency(time.Second) + connPool = pool.NewConnPool(dummyDialer, 10, time.Hour, time.Second) + }) + + AfterEach(func() { + connPool.Close() + }) + + It("rate limits dial", func() { + var rateErr error + for i := 0; i < 1000; i++ { + cn, err := connPool.Get() + if err != nil { + rateErr = err + break + } + + _ = connPool.Replace(cn, errors.New("test")) + } + + Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) + }) + + It("should unblock client when conn is removed", func() { + // Reserve one connection. + cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) + + // Reserve all other connections. + var cns []*pool.Conn + for i := 0; i < 9; i++ { + cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) + cns = append(cns, cn) + } + + started := make(chan bool, 1) + done := make(chan bool, 1) + go func() { + defer GinkgoRecover() + + started <- true + _, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) + done <- true + + err = connPool.Put(cn) + Expect(err).NotTo(HaveOccurred()) + }() + <-started + + // Check that Get is blocked. + select { + case <-done: + Fail("Get is not blocked") + default: + // ok + } + + err = connPool.Replace(cn, errors.New("test")) + Expect(err).NotTo(HaveOccurred()) + + // Check that Ping is unblocked. + select { + case <-done: + // ok + case <-time.After(time.Second): + Fail("Get is not unblocked") + } + + for _, cn := range cns { + err = connPool.Put(cn) + Expect(err).NotTo(HaveOccurred()) + } + }) +}) var _ = Describe("conns reapser", func() { var connPool *pool.ConnPool BeforeEach(func() { - dial := func() (net.Conn, error) { - return &net.TCPConn{}, nil - } - connPool = pool.NewConnPool(dial, 10, 0, time.Minute) + pool.SetIdleCheckFrequency(time.Hour) + connPool = pool.NewConnPool(dummyDialer, 10, 0, time.Minute) // add stale connections for i := 0; i < 3; i++ { @@ -49,6 +123,10 @@ var _ = Describe("conns reapser", func() { Expect(n).To(Equal(3)) }) + AfterEach(func() { + connPool.Close() + }) + It("reaps stale connections", func() { Expect(connPool.Len()).To(Equal(3)) Expect(connPool.FreeLen()).To(Equal(3)) @@ -92,3 +170,47 @@ var _ = Describe("conns reapser", func() { } }) }) + +var _ = Describe("race", func() { + var connPool *pool.ConnPool + + var C, N = 10, 1000 + if testing.Short() { + C = 4 + N = 100 + } + + BeforeEach(func() { + pool.SetIdleCheckFrequency(time.Second) + connPool = pool.NewConnPool(dummyDialer, 10, time.Second, time.Second) + }) + + AfterEach(func() { + connPool.Close() + }) + + It("does not happend", func() { + perform(C, func(id int) { + for i := 0; i < N; i++ { + cn, err := connPool.Get() + if err == nil { + connPool.Put(cn) + } + } + }, func(id int) { + for i := 0; i < N; i++ { + cn, err := connPool.Get() + if err == nil { + connPool.Replace(cn, errors.New("test")) + } + } + }, func(id int) { + for i := 0; i < N; i++ { + cn, err := connPool.Get() + if err == nil { + connPool.Remove(cn, errors.New("test")) + } + } + }) + }) +}) diff --git a/pool_test.go b/pool_test.go index bf1ae4aa06..ec5730c812 100644 --- a/pool_test.go +++ b/pool_test.go @@ -1,9 +1,6 @@ package redis_test import ( - "errors" - "time" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -131,65 +128,4 @@ var _ = Describe("pool", func() { Expect(stats.Waits).To(Equal(uint32(0))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) - - It("should unblock client when connection is removed", func() { - pool := client.Pool() - - // Reserve one connection. - cn, err := pool.Get() - Expect(err).NotTo(HaveOccurred()) - - // Reserve the rest of connections. - for i := 0; i < 9; i++ { - _, err := pool.Get() - Expect(err).NotTo(HaveOccurred()) - } - - var ping *redis.StatusCmd - started := make(chan bool, 1) - done := make(chan bool, 1) - go func() { - started <- true - ping = client.Ping() - done <- true - }() - <-started - - // Check that Ping is blocked. - select { - case <-done: - panic("Ping is not blocked") - default: - // ok - } - - err = pool.Replace(cn, errors.New("test")) - Expect(err).NotTo(HaveOccurred()) - - // Check that Ping is unblocked. - select { - case <-done: - // ok - case <-time.After(time.Second): - panic("Ping is not unblocked") - } - Expect(ping.Err()).NotTo(HaveOccurred()) - }) - - It("should rate limit dial", func() { - pool := client.Pool() - - var rateErr error - for i := 0; i < 1000; i++ { - cn, err := pool.Get() - if err != nil { - rateErr = err - break - } - - _ = pool.Replace(cn, errors.New("test")) - } - - Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) - }) }) From 6e1aef39ea16afa4b8a43cc7fe8cf15d825159f7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 17 Mar 2016 18:00:47 +0200 Subject: [PATCH 0153/1746] Faster and simpler pool. --- cluster.go | 108 +++++++------ cluster_test.go | 10 +- commands_test.go | 14 +- export_test.go | 10 +- internal/pool/bench_test.go | 46 +++--- internal/pool/conn.go | 17 -- internal/pool/conn_list.go | 89 ----------- internal/pool/conn_stack.go | 74 --------- internal/pool/pool.go | 298 +++++++++++++++++++---------------- internal/pool/pool_single.go | 2 +- internal/pool/pool_sticky.go | 10 +- internal/pool/pool_test.go | 81 ++++++---- main_test.go | 40 ++--- options.go | 17 +- pool_test.go | 2 +- pubsub.go | 4 +- pubsub_test.go | 19 ++- race_test.go | 18 +-- redis.go | 4 +- redis_test.go | 3 +- ring.go | 14 +- sentinel.go | 26 +-- 22 files changed, 416 insertions(+), 490 deletions(-) delete mode 100644 internal/pool/conn_list.go delete mode 100644 internal/pool/conn_stack.go diff --git a/cluster.go b/cluster.go index 0cd59d92f8..5ac079c707 100644 --- a/cluster.go +++ b/cluster.go @@ -16,15 +16,16 @@ import ( type ClusterClient struct { commandable + opt *ClusterOptions + + slotsMx sync.RWMutex // protects slots and addrs addrs []string slots [][]string - slotsMx sync.RWMutex // Protects slots and addrs. + clientsMx sync.RWMutex // protects clients and closed clients map[string]*Client - closed bool - clientsMx sync.RWMutex // Protects clients and closed. - opt *ClusterOptions + _closed int32 // atomic // Reports where slots reloading is in progress. reloading uint32 @@ -34,17 +35,29 @@ type ClusterClient struct { // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { client := &ClusterClient{ + opt: opt, addrs: opt.Addrs, slots: make([][]string, hashtag.SlotNumber), clients: make(map[string]*Client), - opt: opt, } client.commandable.process = client.process client.reloadSlots() - go client.reaper() return client } +// getClients returns a snapshot of clients for cluster nodes +// this ClusterClient has been working with recently. +// Note that snapshot can contain closed clients. +func (c *ClusterClient) getClients() map[string]*Client { + c.clientsMx.RLock() + clients := make(map[string]*Client, len(c.clients)) + for addr, client := range c.clients { + clients[addr] = client + } + c.clientsMx.RUnlock() + return clients +} + // Watch creates new transaction and marks the keys to be watched // for conditional execution of a transaction. func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { @@ -59,56 +72,56 @@ func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { // PoolStats returns accumulated connection pool stats. func (c *ClusterClient) PoolStats() *PoolStats { acc := PoolStats{} - c.clientsMx.RLock() - for _, client := range c.clients { - m := client.PoolStats() - acc.Requests += m.Requests - acc.Waits += m.Waits - acc.Timeouts += m.Timeouts - acc.TotalConns += m.TotalConns - acc.FreeConns += m.FreeConns + for _, client := range c.getClients() { + s := client.connPool.Stats() + acc.Requests += s.Requests + acc.Hits += s.Hits + acc.Waits += s.Waits + acc.Timeouts += s.Timeouts + acc.TotalConns += s.TotalConns + acc.FreeConns += s.FreeConns } - c.clientsMx.RUnlock() return &acc } +func (c *ClusterClient) closed() bool { + return atomic.LoadInt32(&c._closed) == 1 +} + // Close closes the cluster client, releasing any open resources. // // It is rare to Close a ClusterClient, as the ClusterClient is meant // to be long-lived and shared between many goroutines. func (c *ClusterClient) Close() error { - defer c.clientsMx.Unlock() - c.clientsMx.Lock() - - if c.closed { + if !atomic.CompareAndSwapInt32(&c._closed, 0, 1) { return pool.ErrClosed } - c.closed = true + + c.clientsMx.Lock() c.resetClients() + c.clientsMx.Unlock() c.setSlots(nil) return nil } // getClient returns a Client for a given address. func (c *ClusterClient) getClient(addr string) (*Client, error) { + if c.closed() { + return nil, pool.ErrClosed + } + if addr == "" { return c.randomClient() } c.clientsMx.RLock() client, ok := c.clients[addr] + c.clientsMx.RUnlock() if ok { - c.clientsMx.RUnlock() return client, nil } - c.clientsMx.RUnlock() c.clientsMx.Lock() - if c.closed { - c.clientsMx.Unlock() - return nil, pool.ErrClosed - } - client, ok = c.clients[addr] if !ok { opt := c.opt.clientOptions() @@ -276,28 +289,30 @@ func (c *ClusterClient) lazyReloadSlots() { } // reaper closes idle connections to the cluster. -func (c *ClusterClient) reaper() { - ticker := time.NewTicker(time.Minute) +func (c *ClusterClient) reaper(frequency time.Duration) { + ticker := time.NewTicker(frequency) defer ticker.Stop() - for _ = range ticker.C { - c.clientsMx.RLock() - if c.closed { - c.clientsMx.RUnlock() + for _ = range ticker.C { + if c.closed() { break } - for _, client := range c.clients { - pool := client.connPool - // pool.First removes idle connections from the pool and - // returns first non-idle connection. So just put returned - // connection back. - if cn := pool.First(); cn != nil { - pool.Put(cn) + var n int + for _, client := range c.getClients() { + nn, err := client.connPool.(*pool.ConnPool).ReapStaleConns() + if err != nil { + Logger.Printf("ReapStaleConns failed: %s", err) + } else { + n += nn } } - c.clientsMx.RUnlock() + s := c.PoolStats() + Logger.Printf( + "reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)", + n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts, + ) } } @@ -309,8 +324,7 @@ type ClusterOptions struct { // A seed list of host:port addresses of cluster nodes. Addrs []string - // The maximum number of MOVED/ASK redirects to follow before - // giving up. + // The maximum number of MOVED/ASK redirects to follow before giving up. // Default is 16 MaxRedirects int @@ -323,9 +337,10 @@ type ClusterOptions struct { WriteTimeout time.Duration // PoolSize applies per cluster node and not for the whole cluster. - PoolSize int - PoolTimeout time.Duration - IdleTimeout time.Duration + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration + IdleCheckFrequency time.Duration } func (opt *ClusterOptions) getMaxRedirects() int { @@ -349,5 +364,6 @@ func (opt *ClusterOptions) clientOptions() *Options { PoolSize: opt.PoolSize, PoolTimeout: opt.PoolTimeout, IdleTimeout: opt.IdleTimeout, + // IdleCheckFrequency is not copied to disable reaper } } diff --git a/cluster_test.go b/cluster_test.go index 7423a7ebf2..6a04212867 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -52,7 +52,15 @@ func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.Cluste addrs[i] = net.JoinHostPort("127.0.0.1", port) } if opt == nil { - opt = &redis.ClusterOptions{} + opt = &redis.ClusterOptions{ + DialTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + PoolSize: 10, + PoolTimeout: 30 * time.Second, + IdleTimeout: time.Second, + IdleCheckFrequency: time.Second, + } } opt.Addrs = addrs return redis.NewClusterClient(opt) diff --git a/commands_test.go b/commands_test.go index 6e2e6c1909..eb0d52372b 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1301,12 +1301,16 @@ var _ = Describe("Commands", func() { }) It("should BLPop timeout", func() { - bLPop := client.BLPop(time.Second, "list1") - Expect(bLPop.Val()).To(BeNil()) - Expect(bLPop.Err()).To(Equal(redis.Nil)) + val, err := client.BLPop(time.Second, "list1").Result() + Expect(err).To(Equal(redis.Nil)) + Expect(val).To(BeNil()) + + Expect(client.Ping().Err()).NotTo(HaveOccurred()) - stats := client.Pool().Stats() - Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(1))) + stats := client.PoolStats() + Expect(stats.Requests).To(Equal(uint32(3))) + Expect(stats.Hits).To(Equal(uint32(2))) + Expect(stats.Timeouts).To(Equal(uint32(0))) }) It("should BRPop", func() { diff --git a/export_test.go b/export_test.go index cce779bb7e..f071b540c7 100644 --- a/export_test.go +++ b/export_test.go @@ -1,6 +1,10 @@ package redis -import "gopkg.in/redis.v3/internal/pool" +import ( + "time" + + "gopkg.in/redis.v3/internal/pool" +) func (c *baseClient) Pool() pool.Pooler { return c.connPool @@ -9,3 +13,7 @@ func (c *baseClient) Pool() pool.Pooler { func (c *PubSub) Pool() pool.Pooler { return c.base.connPool } + +func SetReceiveMessageTimeout(d time.Duration) { + receiveMessageTimeout = d +} diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index 5acc5e2c68..ff102aa5f1 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -2,7 +2,6 @@ package pool_test import ( "errors" - "net" "testing" "time" @@ -10,22 +9,19 @@ import ( ) func benchmarkPoolGetPut(b *testing.B, poolSize int) { - dial := func() (net.Conn, error) { - return &net.TCPConn{}, nil - } - pool := pool.NewConnPool(dial, poolSize, time.Second, 0) - pool.DialLimiter = nil + connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) + connPool.DialLimiter = nil b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - conn, err := pool.Get() + cn, err := connPool.Get() if err != nil { - b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) + b.Fatal(err) } - if err = pool.Put(conn); err != nil { - b.Fatalf("no error expected on pool.Put but received: %s", err.Error()) + if err = connPool.Put(cn); err != nil { + b.Fatal(err) } } }) @@ -43,38 +39,34 @@ func BenchmarkPoolGetPut1000Conns(b *testing.B) { benchmarkPoolGetPut(b, 1000) } -func benchmarkPoolGetReplace(b *testing.B, poolSize int) { - dial := func() (net.Conn, error) { - return &net.TCPConn{}, nil - } - pool := pool.NewConnPool(dial, poolSize, time.Second, 0) - pool.DialLimiter = nil - +func benchmarkPoolGetRemove(b *testing.B, poolSize int) { + connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) + connPool.DialLimiter = nil removeReason := errors.New("benchmark") b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - conn, err := pool.Get() + cn, err := connPool.Get() if err != nil { - b.Fatalf("no error expected on pool.Get but received: %s", err.Error()) + b.Fatal(err) } - if err = pool.Replace(conn, removeReason); err != nil { - b.Fatalf("no error expected on pool.Remove but received: %s", err.Error()) + if err := connPool.Remove(cn, removeReason); err != nil { + b.Fatal(err) } } }) } -func BenchmarkPoolGetReplace10Conns(b *testing.B) { - benchmarkPoolGetReplace(b, 10) +func BenchmarkPoolGetRemove10Conns(b *testing.B) { + benchmarkPoolGetRemove(b, 10) } -func BenchmarkPoolGetReplace100Conns(b *testing.B) { - benchmarkPoolGetReplace(b, 100) +func BenchmarkPoolGetRemove100Conns(b *testing.B) { + benchmarkPoolGetRemove(b, 100) } -func BenchmarkPoolGetReplace1000Conns(b *testing.B) { - benchmarkPoolGetReplace(b, 1000) +func BenchmarkPoolGetRemove1000Conns(b *testing.B) { + benchmarkPoolGetRemove(b, 1000) } diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 0fbb41931f..497fd4e46c 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -4,7 +4,6 @@ import ( "bufio" "io" "net" - "sync/atomic" "time" ) @@ -13,8 +12,6 @@ const defaultBufSize = 4096 var noDeadline = time.Time{} type Conn struct { - idx int32 - NetConn net.Conn Rd *bufio.Reader Buf []byte @@ -28,8 +25,6 @@ type Conn struct { func NewConn(netConn net.Conn) *Conn { cn := &Conn{ - idx: -1, - NetConn: netConn, Buf: make([]byte, defaultBufSize), @@ -39,18 +34,6 @@ func NewConn(netConn net.Conn) *Conn { return cn } -func (cn *Conn) Index() int { - return int(atomic.LoadInt32(&cn.idx)) -} - -func (cn *Conn) SetIndex(newIdx int) int { - oldIdx := cn.Index() - if !atomic.CompareAndSwapInt32(&cn.idx, int32(oldIdx), int32(newIdx)) { - return -1 - } - return oldIdx -} - func (cn *Conn) IsStale(timeout time.Duration) bool { return timeout > 0 && time.Since(cn.UsedAt) > timeout } diff --git a/internal/pool/conn_list.go b/internal/pool/conn_list.go deleted file mode 100644 index 61bf99ba33..0000000000 --- a/internal/pool/conn_list.go +++ /dev/null @@ -1,89 +0,0 @@ -package pool - -import ( - "sync" - "sync/atomic" -) - -type connList struct { - cns []*Conn - mu sync.Mutex - len int32 // atomic - size int32 -} - -func newConnList(size int) *connList { - return &connList{ - cns: make([]*Conn, size), - size: int32(size), - } -} - -func (l *connList) Len() int { - return int(atomic.LoadInt32(&l.len)) -} - -// Reserve reserves place in the list and returns true on success. -// The caller must add connection or cancel reservation if it was reserved. -func (l *connList) Reserve() bool { - len := atomic.AddInt32(&l.len, 1) - reserved := len <= l.size - if !reserved { - atomic.AddInt32(&l.len, -1) - } - return reserved -} - -func (l *connList) CancelReservation() { - atomic.AddInt32(&l.len, -1) -} - -// Add adds connection to the list. The caller must reserve place first. -func (l *connList) Add(cn *Conn) { - l.mu.Lock() - for i, c := range l.cns { - if c == nil { - cn.SetIndex(i) - l.cns[i] = cn - l.mu.Unlock() - return - } - } - panic("not reached") -} - -func (l *connList) Replace(cn *Conn) { - l.mu.Lock() - if l.cns != nil { - l.cns[cn.idx] = cn - } - l.mu.Unlock() -} - -// Remove closes connection and removes it from the list. -func (l *connList) Remove(idx int) { - l.mu.Lock() - if l.cns != nil { - l.cns[idx] = nil - atomic.AddInt32(&l.len, -1) - } - l.mu.Unlock() -} - -func (l *connList) Reset() []*Conn { - l.mu.Lock() - - for _, cn := range l.cns { - if cn == nil { - continue - } - cn.SetIndex(-1) - } - - cns := l.cns - l.cns = nil - l.len = 0 - - l.mu.Unlock() - return cns -} diff --git a/internal/pool/conn_stack.go b/internal/pool/conn_stack.go deleted file mode 100644 index 8b8f505786..0000000000 --- a/internal/pool/conn_stack.go +++ /dev/null @@ -1,74 +0,0 @@ -package pool - -import ( - "sync" - "time" -) - -// connStack is used as a LIFO to maintain free connections -type connStack struct { - cns []*Conn - free chan struct{} - mu sync.Mutex -} - -func newConnStack(max int) *connStack { - return &connStack{ - cns: make([]*Conn, 0, max), - free: make(chan struct{}, max), - } -} - -func (s *connStack) Len() int { return len(s.free) } - -func (s *connStack) Push(cn *Conn) { - s.mu.Lock() - s.cns = append(s.cns, cn) - s.mu.Unlock() - s.free <- struct{}{} -} - -func (s *connStack) ShiftStale(idleTimeout time.Duration) *Conn { - select { - case <-s.free: - s.mu.Lock() - if cn := s.cns[0]; cn.IsStale(idleTimeout) { - copy(s.cns, s.cns[1:]) - s.cns = s.cns[:len(s.cns)-1] - s.mu.Unlock() - return cn - } - s.mu.Unlock() - - s.free <- struct{}{} - return nil - default: - return nil - } -} - -func (s *connStack) Pop() *Conn { - select { - case <-s.free: - return s.pop() - default: - return nil - } -} - -func (s *connStack) PopWithTimeout(d time.Duration) *Conn { - select { - case <-s.free: - return s.pop() - case <-time.After(d): - return nil - } -} - -func (s *connStack) pop() (cn *Conn) { - s.mu.Lock() - ci := len(s.cns) - 1 - cn, s.cns = s.cns[ci], s.cns[:ci] - s.mu.Unlock() - return -} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index c3281c3d58..10155f32b0 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -3,24 +3,34 @@ package pool import ( "errors" "fmt" + "io/ioutil" "log" "net" - "os" + "sync" "sync/atomic" "time" "gopkg.in/bsm/ratelimit.v1" ) -var Logger = log.New(os.Stderr, "redis: ", log.LstdFlags) +var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) var ( ErrClosed = errors.New("redis: client is closed") - errConnClosed = errors.New("redis: connection is closed") ErrPoolTimeout = errors.New("redis: connection pool timeout") + + errConnClosed = errors.New("connection is closed") + errConnStale = errors.New("connection is stale") ) +var timers = sync.Pool{ + New: func() interface{} { + return time.NewTimer(0) + }, +} + // PoolStats contains pool state information and accumulated stats. +// TODO: remove Waits type PoolStats struct { Requests uint32 // number of times a connection was requested by the pool Hits uint32 // number of times free connection was found in the pool @@ -32,10 +42,9 @@ type PoolStats struct { } type Pooler interface { - First() *Conn Get() (*Conn, error) Put(*Conn) error - Replace(*Conn, error) error + Remove(*Conn, error) error Len() int FreeLen() int Stats() *PoolStats @@ -53,18 +62,23 @@ type ConnPool struct { poolTimeout time.Duration idleTimeout time.Duration - conns *connList - freeConns *connStack - stats PoolStats + queue chan struct{} + + connsMu sync.Mutex + conns []*Conn + + freeConnsMu sync.Mutex + freeConns []*Conn - _closed int32 + stats PoolStats + _closed int32 // atomic lastErr atomic.Value } var _ Pooler = (*ConnPool)(nil) -func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Duration) *ConnPool { +func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout, idleCheckFrequency time.Duration) *ConnPool { p := &ConnPool{ _dial: dial, DialLimiter: ratelimit.New(3*poolSize, time.Second), @@ -72,55 +86,17 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout time.Durati poolTimeout: poolTimeout, idleTimeout: idleTimeout, - conns: newConnList(poolSize), - freeConns: newConnStack(poolSize), - } - if idleTimeout > 0 { - go p.reaper(getIdleCheckFrequency()) - } - return p -} - -func (p *ConnPool) Add(cn *Conn) bool { - if !p.conns.Reserve() { - return false + queue: make(chan struct{}, poolSize), + conns: make([]*Conn, 0, poolSize), + freeConns: make([]*Conn, 0, poolSize), } - p.conns.Add(cn) - p.Put(cn) - return true -} - -// First returns first non-idle connection from the pool or nil if -// there are no connections. -func (p *ConnPool) First() *Conn { - for { - cn := p.freeConns.Pop() - if cn != nil && cn.IsStale(p.idleTimeout) { - var err error - cn, err = p.replace(cn) - if err != nil { - Logger.Printf("pool.replace failed: %s", err) - continue - } - } - return cn + for i := 0; i < poolSize; i++ { + p.queue <- struct{}{} } -} - -// wait waits for free non-idle connection. It returns nil on timeout. -func (p *ConnPool) wait(timeout time.Duration) *Conn { - for { - cn := p.freeConns.PopWithTimeout(timeout) - if cn != nil && cn.IsStale(p.idleTimeout) { - var err error - cn, err = p.replace(cn) - if err != nil { - Logger.Printf("pool.replace failed: %s", err) - continue - } - } - return cn + if idleTimeout > 0 && idleCheckFrequency > 0 { + go p.reaper(idleCheckFrequency) } + return p } func (p *ConnPool) dial() (net.Conn, error) { @@ -148,6 +124,42 @@ func (p *ConnPool) NewConn() (*Conn, error) { return NewConn(netConn), nil } +func (p *ConnPool) PopFree() *Conn { + timer := timers.Get().(*time.Timer) + if !timer.Reset(p.poolTimeout) { + <-timer.C + } + + select { + case <-p.queue: + timers.Put(timer) + case <-timer.C: + timers.Put(timer) + atomic.AddUint32(&p.stats.Timeouts, 1) + return nil + } + + p.freeConnsMu.Lock() + cn := p.popFree() + p.freeConnsMu.Unlock() + + if cn == nil { + p.queue <- struct{}{} + } + return cn +} + +func (p *ConnPool) popFree() *Conn { + if len(p.freeConns) == 0 { + return nil + } + + idx := len(p.freeConns) - 1 + cn := p.freeConns[idx] + p.freeConns = p.freeConns[:idx] + return cn +} + // Get returns existed connection from the pool or creates a new one. func (p *ConnPool) Get() (*Conn, error) { if p.Closed() { @@ -156,31 +168,46 @@ func (p *ConnPool) Get() (*Conn, error) { atomic.AddUint32(&p.stats.Requests, 1) - // Fetch first non-idle connection, if available. - if cn := p.First(); cn != nil { - atomic.AddUint32(&p.stats.Hits, 1) - return cn, nil + timer := timers.Get().(*time.Timer) + if !timer.Reset(p.poolTimeout) { + <-timer.C } - // Try to create a new one. - if p.conns.Reserve() { - cn, err := p.NewConn() - if err != nil { - p.conns.CancelReservation() - return nil, err + select { + case <-p.queue: + timers.Put(timer) + case <-timer.C: + timers.Put(timer) + atomic.AddUint32(&p.stats.Timeouts, 1) + return nil, ErrPoolTimeout + } + + p.freeConnsMu.Lock() + cn := p.popFree() + p.freeConnsMu.Unlock() + + if cn != nil { + atomic.AddUint32(&p.stats.Hits, 1) + if !cn.IsStale(p.idleTimeout) { + return cn, nil } - p.conns.Add(cn) - return cn, nil + _ = cn.Close() + } + + newcn, err := p.NewConn() + if err != nil { + p.queue <- struct{}{} + return nil, err } - // Otherwise, wait for the available connection. - atomic.AddUint32(&p.stats.Waits, 1) - if cn := p.wait(p.poolTimeout); cn != nil { - return cn, nil + p.connsMu.Lock() + if cn != nil { + p.remove(cn, errConnStale) } + p.conns = append(p.conns, newcn) + p.connsMu.Unlock() - atomic.AddUint32(&p.stats.Timeouts, 1) - return nil, ErrPoolTimeout + return newcn, nil } func (p *ConnPool) Put(cn *Conn) error { @@ -188,71 +215,54 @@ func (p *ConnPool) Put(cn *Conn) error { b, _ := cn.Rd.Peek(cn.Rd.Buffered()) err := fmt.Errorf("connection has unread data: %q", b) Logger.Print(err) - return p.Replace(cn, err) + return p.Remove(cn, err) } - p.freeConns.Push(cn) + p.freeConnsMu.Lock() + p.freeConns = append(p.freeConns, cn) + p.freeConnsMu.Unlock() + p.queue <- struct{}{} return nil } -func (p *ConnPool) replace(cn *Conn) (*Conn, error) { +func (p *ConnPool) Remove(cn *Conn, reason error) error { _ = cn.Close() - - idx := cn.SetIndex(-1) - if idx == -1 { - return nil, errConnClosed - } - - netConn, err := p.dial() - if err != nil { - p.conns.Remove(idx) - return nil, err - } - - cn = NewConn(netConn) - cn.SetIndex(idx) - p.conns.Replace(cn) - - return cn, nil -} - -func (p *ConnPool) Replace(cn *Conn, reason error) error { - p.storeLastErr(reason.Error()) - - // Replace existing connection with new one and unblock waiter. - newcn, err := p.replace(cn) - if err != nil { - return err - } - p.freeConns.Push(newcn) + p.connsMu.Lock() + p.remove(cn, reason) + p.connsMu.Unlock() + p.queue <- struct{}{} return nil } -func (p *ConnPool) Remove(cn *Conn, reason error) error { - _ = cn.Close() - - idx := cn.SetIndex(-1) - if idx == -1 { - return errConnClosed - } - +func (p *ConnPool) remove(cn *Conn, reason error) { p.storeLastErr(reason.Error()) - p.conns.Remove(idx) - return nil + for i, c := range p.conns { + if c == cn { + p.conns = append(p.conns[:i], p.conns[i+1:]...) + break + } + } } // Len returns total number of connections. func (p *ConnPool) Len() int { - return p.conns.Len() + p.connsMu.Lock() + l := len(p.conns) + p.connsMu.Unlock() + return l } // FreeLen returns number of free connections. func (p *ConnPool) FreeLen() int { - return p.freeConns.Len() + p.freeConnsMu.Lock() + l := len(p.freeConns) + p.freeConnsMu.Unlock() + return l } func (p *ConnPool) Stats() *PoolStats { - stats := p.stats + stats := PoolStats{} stats.Requests = atomic.LoadUint32(&p.stats.Requests) + stats.Hits = atomic.LoadUint32(&p.stats.Hits) stats.Waits = atomic.LoadUint32(&p.stats.Waits) stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts) stats.TotalConns = uint32(p.Len()) @@ -269,16 +279,10 @@ func (p *ConnPool) Close() (retErr error) { return ErrClosed } - // Wait for app to free connections, but don't close them immediately. - for i := 0; i < p.Len()-p.FreeLen(); i++ { - if cn := p.wait(3 * time.Second); cn == nil { - break - } - } + p.connsMu.Lock() // Close all connections. - cns := p.conns.Reset() - for _, cn := range cns { + for _, cn := range p.conns { if cn == nil { continue } @@ -286,6 +290,12 @@ func (p *ConnPool) Close() (retErr error) { retErr = err } } + p.conns = nil + p.connsMu.Unlock() + + p.freeConnsMu.Lock() + p.freeConns = nil + p.freeConnsMu.Unlock() return retErr } @@ -298,16 +308,32 @@ func (p *ConnPool) closeConn(cn *Conn) error { } func (p *ConnPool) ReapStaleConns() (n int, err error) { - for { - cn := p.freeConns.ShiftStale(p.idleTimeout) - if cn == nil { + <-p.queue + p.freeConnsMu.Lock() + + if len(p.freeConns) == 0 { + p.freeConnsMu.Unlock() + p.queue <- struct{}{} + return + } + + var idx int + var cn *Conn + for idx, cn = range p.freeConns { + if !cn.IsStale(p.idleTimeout) { break } - if err = p.Remove(cn, errors.New("connection is stale")); err != nil { - return - } + p.connsMu.Lock() + p.remove(cn, errConnStale) + p.connsMu.Unlock() n++ } + if idx > 0 { + p.freeConns = append(p.freeConns[:0], p.freeConns[idx:]...) + } + + p.freeConnsMu.Unlock() + p.queue <- struct{}{} return } @@ -322,9 +348,13 @@ func (p *ConnPool) reaper(frequency time.Duration) { n, err := p.ReapStaleConns() if err != nil { Logger.Printf("ReapStaleConns failed: %s", err) - } else if n > 0 { - Logger.Printf("removed %d stale connections", n) + continue } + s := p.Stats() + Logger.Printf( + "reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)", + n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts, + ) } } diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index 39362c0211..cb1863eb13 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -27,7 +27,7 @@ func (p *SingleConnPool) Put(cn *Conn) error { return nil } -func (p *SingleConnPool) Replace(cn *Conn, _ error) error { +func (p *SingleConnPool) Remove(cn *Conn, _ error) error { if p.cn != cn { panic("p.cn != cn") } diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 8b76b6f668..24a4f755ff 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -67,13 +67,13 @@ func (p *StickyConnPool) Put(cn *Conn) error { return nil } -func (p *StickyConnPool) replace(reason error) error { - err := p.pool.Replace(p.cn, reason) +func (p *StickyConnPool) remove(reason error) error { + err := p.pool.Remove(p.cn, reason) p.cn = nil return err } -func (p *StickyConnPool) Replace(cn *Conn, reason error) error { +func (p *StickyConnPool) Remove(cn *Conn, reason error) error { defer p.mx.Unlock() p.mx.Lock() if p.closed { @@ -85,7 +85,7 @@ func (p *StickyConnPool) Replace(cn *Conn, reason error) error { if cn != nil && p.cn != cn { panic("p.cn != cn") } - return p.replace(reason) + return p.remove(reason) } func (p *StickyConnPool) Len() int { @@ -121,7 +121,7 @@ func (p *StickyConnPool) Close() error { err = p.put() } else { reason := errors.New("redis: sticky not reusable connection") - err = p.replace(reason) + err = p.remove(reason) } } return err diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index d7a29883b6..9d07cd597d 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -16,8 +16,8 @@ var _ = Describe("ConnPool", func() { var connPool *pool.ConnPool BeforeEach(func() { - pool.SetIdleCheckFrequency(time.Second) - connPool = pool.NewConnPool(dummyDialer, 10, time.Hour, time.Second) + connPool = pool.NewConnPool( + dummyDialer, 10, time.Hour, time.Millisecond, time.Millisecond) }) AfterEach(func() { @@ -33,7 +33,7 @@ var _ = Describe("ConnPool", func() { break } - _ = connPool.Replace(cn, errors.New("test")) + _ = connPool.Remove(cn, errors.New("test")) } Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) @@ -75,7 +75,7 @@ var _ = Describe("ConnPool", func() { // ok } - err = connPool.Replace(cn, errors.New("test")) + err = connPool.Remove(cn, errors.New("test")) Expect(err).NotTo(HaveOccurred()) // Check that Ping is unblocked. @@ -93,26 +93,33 @@ var _ = Describe("ConnPool", func() { }) }) -var _ = Describe("conns reapser", func() { +var _ = Describe("conns reaper", func() { var connPool *pool.ConnPool BeforeEach(func() { - pool.SetIdleCheckFrequency(time.Hour) - connPool = pool.NewConnPool(dummyDialer, 10, 0, time.Minute) + connPool = pool.NewConnPool( + dummyDialer, 10, time.Second, time.Millisecond, time.Hour) + + var cns []*pool.Conn // add stale connections for i := 0; i < 3; i++ { - cn := pool.NewConn(&net.TCPConn{}) + cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) cn.UsedAt = time.Now().Add(-2 * time.Minute) - Expect(connPool.Add(cn)).To(BeTrue()) - Expect(cn.Index()).To(Equal(i)) + cns = append(cns, cn) } // add fresh connections for i := 0; i < 3; i++ { cn := pool.NewConn(&net.TCPConn{}) - Expect(connPool.Add(cn)).To(BeTrue()) - Expect(cn.Index()).To(Equal(3 + i)) + cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) + cns = append(cns, cn) + } + + for _, cn := range cns { + Expect(connPool.Put(cn)).NotTo(HaveOccurred()) } Expect(connPool.Len()).To(Equal(6)) @@ -136,7 +143,8 @@ var _ = Describe("conns reapser", func() { for j := 0; j < 3; j++ { var freeCns []*pool.Conn for i := 0; i < 3; i++ { - cn := connPool.First() + cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) freeCns = append(freeCns, cn) } @@ -144,9 +152,6 @@ var _ = Describe("conns reapser", func() { Expect(connPool.Len()).To(Equal(3)) Expect(connPool.FreeLen()).To(Equal(0)) - cn := connPool.First() - Expect(cn).To(BeNil()) - cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) @@ -173,42 +178,60 @@ var _ = Describe("conns reapser", func() { var _ = Describe("race", func() { var connPool *pool.ConnPool - - var C, N = 10, 1000 - if testing.Short() { - C = 4 - N = 100 - } + var C, N int BeforeEach(func() { - pool.SetIdleCheckFrequency(time.Second) - connPool = pool.NewConnPool(dummyDialer, 10, time.Second, time.Second) + C, N = 10, 1000 + if testing.Short() { + C = 4 + N = 100 + } }) AfterEach(func() { connPool.Close() }) - It("does not happend", func() { + It("does not happen on Get, Put, and Remove", func() { + connPool = pool.NewConnPool( + dummyDialer, 10, time.Minute, time.Millisecond, time.Millisecond) + connPool.DialLimiter = nil + perform(C, func(id int) { for i := 0; i < N; i++ { cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) if err == nil { - connPool.Put(cn) + Expect(connPool.Put(cn)).NotTo(HaveOccurred()) } } }, func(id int) { for i := 0; i < N; i++ { cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) if err == nil { - connPool.Replace(cn, errors.New("test")) + Expect(connPool.Remove(cn, errors.New("test"))).NotTo(HaveOccurred()) } } - }, func(id int) { + }) + }) + + It("does not happen on Get and PopFree", func() { + connPool = pool.NewConnPool( + dummyDialer, 10, time.Minute, time.Second, time.Millisecond) + connPool.DialLimiter = nil + + perform(C, func(id int) { for i := 0; i < N; i++ { cn, err := connPool.Get() + Expect(err).NotTo(HaveOccurred()) if err == nil { - connPool.Remove(cn, errors.New("test")) + Expect(connPool.Put(cn)).NotTo(HaveOccurred()) + } + + cn = connPool.PopFree() + if cn != nil { + Expect(connPool.Put(cn)).NotTo(HaveOccurred()) } } }) diff --git a/main_test.go b/main_test.go index 67905cf868..cf6b181eea 100644 --- a/main_test.go +++ b/main_test.go @@ -15,7 +15,6 @@ import ( . "github.com/onsi/gomega" "gopkg.in/redis.v3" - "gopkg.in/redis.v3/internal/pool" ) const ( @@ -53,8 +52,6 @@ var cluster = &clusterScenario{ var _ = BeforeSuite(func() { var err error - pool.SetIdleCheckFrequency(time.Second) // be aggressive in tests - redisMain, err = startRedis(redisPort) Expect(err).NotTo(HaveOccurred()) @@ -104,27 +101,30 @@ func TestGinkgoSuite(t *testing.T) { func redisOptions() *redis.Options { return &redis.Options{ - Addr: redisAddr, - DB: 15, - DialTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - PoolSize: 10, - PoolTimeout: 30 * time.Second, - IdleTimeout: time.Second, // be aggressive in tests + Addr: redisAddr, + DB: 15, + DialTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + PoolSize: 10, + PoolTimeout: 30 * time.Second, + IdleTimeout: time.Second, + IdleCheckFrequency: time.Second, } } -func perform(n int, cb func(int)) { +func perform(n int, cbs ...func(int)) { var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer GinkgoRecover() - defer wg.Done() - - cb(i) - }(i) + for _, cb := range cbs { + for i := 0; i < n; i++ { + wg.Add(1) + go func(cb func(int), i int) { + defer GinkgoRecover() + defer wg.Done() + + cb(i) + }(cb, i) + } } wg.Wait() } diff --git a/options.go b/options.go index 935e756442..342e99d83e 100644 --- a/options.go +++ b/options.go @@ -50,6 +50,9 @@ type Options struct { // connections. Should be less than server's timeout. // Default is to not close idle connections. IdleTimeout time.Duration + // The frequency of idle checks. + // Default is 1 minute. + IdleCheckFrequency time.Duration } func (opt *Options) getNetwork() string { @@ -93,9 +96,21 @@ func (opt *Options) getIdleTimeout() time.Duration { return opt.IdleTimeout } +func (opt *Options) getIdleCheckFrequency() time.Duration { + if opt.IdleCheckFrequency == 0 { + return time.Minute + } + return opt.IdleCheckFrequency +} + func newConnPool(opt *Options) *pool.ConnPool { return pool.NewConnPool( - opt.getDialer(), opt.getPoolSize(), opt.getPoolTimeout(), opt.getIdleTimeout()) + opt.getDialer(), + opt.getPoolSize(), + opt.getPoolTimeout(), + opt.getIdleTimeout(), + opt.getIdleCheckFrequency(), + ) } // PoolStats contains pool state information and accumulated stats. diff --git a/pool_test.go b/pool_test.go index ec5730c812..793ff5031e 100644 --- a/pool_test.go +++ b/pool_test.go @@ -106,7 +106,7 @@ var _ = Describe("pool", func() { stats := pool.Stats() Expect(stats.Requests).To(Equal(uint32(4))) - Expect(stats.Hits).To(Equal(uint32(3))) + Expect(stats.Hits).To(Equal(uint32(2))) Expect(stats.Waits).To(Equal(uint32(0))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) diff --git a/pubsub.go b/pubsub.go index 05e5921347..1bf2f93356 100644 --- a/pubsub.go +++ b/pubsub.go @@ -8,6 +8,8 @@ import ( "gopkg.in/redis.v3/internal/pool" ) +var receiveMessageTimeout = 5 * time.Second + // Posts a message to the given channel. func (c *Client) Publish(channel, message string) *IntCmd { req := NewIntCmd("PUBLISH", channel, message) @@ -255,7 +257,7 @@ func (c *PubSub) Receive() (interface{}, error) { func (c *PubSub) ReceiveMessage() (*Message, error) { var errNum uint for { - msgi, err := c.ReceiveTimeout(5 * time.Second) + msgi, err := c.ReceiveTimeout(receiveMessageTimeout) if err != nil { if !isNetworkError(err) { return nil, err diff --git a/pubsub_test.go b/pubsub_test.go index ca1cdb259e..df0dd94b92 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -68,7 +68,7 @@ var _ = Describe("PubSub", func() { Expect(subscr.Count).To(Equal(0)) } - stats := client.Pool().Stats() + stats := client.PoolStats() Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) }) @@ -195,7 +195,7 @@ var _ = Describe("PubSub", func() { Expect(subscr.Count).To(Equal(0)) } - stats := client.Pool().Stats() + stats := client.PoolStats() Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) }) @@ -256,6 +256,9 @@ var _ = Describe("PubSub", func() { }) It("should ReceiveMessage after timeout", func() { + timeout := time.Second + redis.SetReceiveMessageTimeout(timeout) + pubsub, err := client.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) defer pubsub.Close() @@ -267,7 +270,7 @@ var _ = Describe("PubSub", func() { done <- true }() - time.Sleep(5*time.Second + 100*time.Millisecond) + time.Sleep(timeout + 100*time.Millisecond) n, err := client.Publish("mychannel", "hello").Result() Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(1))) @@ -280,8 +283,9 @@ var _ = Describe("PubSub", func() { Eventually(done).Should(Receive()) - stats := client.Pool().Stats() - Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) + stats := client.PoolStats() + Expect(stats.Requests).To(Equal(uint32(3))) + Expect(stats.Hits).To(Equal(uint32(1))) }) expectReceiveMessageOnError := func(pubsub *redis.PubSub) { @@ -311,8 +315,9 @@ var _ = Describe("PubSub", func() { Eventually(done).Should(Receive()) - stats := client.Pool().Stats() - Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) + stats := client.PoolStats() + Expect(stats.Requests).To(Equal(uint32(4))) + Expect(stats.Hits).To(Equal(uint32(1))) } It("Subscribe should reconnect on ReceiveMessage error", func() { diff --git a/race_test.go b/race_test.go index 0b942e114a..1ce8430a88 100644 --- a/race_test.go +++ b/race_test.go @@ -17,16 +17,17 @@ import ( var _ = Describe("races", func() { var client *redis.Client - - var C, N = 10, 1000 - if testing.Short() { - C = 4 - N = 100 - } + var C, N int BeforeEach(func() { client = redis.NewClient(redisOptions()) Expect(client.FlushDb().Err()).To(BeNil()) + + C, N = 10, 1000 + if testing.Short() { + C = 4 + N = 100 + } }) AfterEach(func() { @@ -123,16 +124,13 @@ var _ = Describe("races", func() { }) It("should handle big vals in Set", func() { + C, N = 4, 100 bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb perform(C, func(id int) { for i := 0; i < N; i++ { err := client.Set("key", bigVal, 0).Err() Expect(err).NotTo(HaveOccurred()) - - got, err := client.Get("key").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(got).To(Equal(bigVal)) } }) }) diff --git a/redis.go b/redis.go index dc55572eff..aaa0dac85c 100644 --- a/redis.go +++ b/redis.go @@ -39,7 +39,7 @@ func (c *baseClient) conn() (*pool.Conn, error) { } if !cn.Inited { if err := c.initConn(cn); err != nil { - _ = c.connPool.Replace(cn, err) + _ = c.connPool.Remove(cn, err) return nil, err } } @@ -48,7 +48,7 @@ func (c *baseClient) conn() (*pool.Conn, error) { func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { if isBadConn(err, allowTimeout) { - _ = c.connPool.Replace(cn, err) + _ = c.connPool.Remove(cn, err) return false } diff --git a/redis_test.go b/redis_test.go index de297738dd..cb8e3ac691 100644 --- a/redis_test.go +++ b/redis_test.go @@ -166,7 +166,8 @@ var _ = Describe("Client", func() { err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) - cn = client.Pool().First() + cn, err = client.Pool().Get() + Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) Expect(cn.UsedAt.After(createdAt)).To(BeTrue()) }) diff --git a/ring.go b/ring.go index 32212161d4..fd74b3b6ff 100644 --- a/ring.go +++ b/ring.go @@ -32,9 +32,10 @@ type RingOptions struct { ReadTimeout time.Duration WriteTimeout time.Duration - PoolSize int - PoolTimeout time.Duration - IdleTimeout time.Duration + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration + IdleCheckFrequency time.Duration } func (opt *RingOptions) clientOptions() *Options { @@ -46,9 +47,10 @@ func (opt *RingOptions) clientOptions() *Options { ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, - PoolSize: opt.PoolSize, - PoolTimeout: opt.PoolTimeout, - IdleTimeout: opt.IdleTimeout, + PoolSize: opt.PoolSize, + PoolTimeout: opt.PoolTimeout, + IdleTimeout: opt.IdleTimeout, + IdleCheckFrequency: opt.IdleCheckFrequency, } } diff --git a/sentinel.go b/sentinel.go index 887c5eb351..1f8ec136ed 100644 --- a/sentinel.go +++ b/sentinel.go @@ -26,15 +26,16 @@ type FailoverOptions struct { Password string DB int64 + MaxRetries int + DialTimeout time.Duration ReadTimeout time.Duration WriteTimeout time.Duration - PoolSize int - PoolTimeout time.Duration - IdleTimeout time.Duration - - MaxRetries int + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration + IdleCheckFrequency time.Duration } func (opt *FailoverOptions) options() *Options { @@ -44,15 +45,16 @@ func (opt *FailoverOptions) options() *Options { DB: opt.DB, Password: opt.Password, + MaxRetries: opt.MaxRetries, + DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, - PoolSize: opt.PoolSize, - PoolTimeout: opt.PoolTimeout, - IdleTimeout: opt.IdleTimeout, - - MaxRetries: opt.MaxRetries, + PoolSize: opt.PoolSize, + PoolTimeout: opt.PoolTimeout, + IdleTimeout: opt.IdleTimeout, + IdleCheckFrequency: opt.IdleCheckFrequency, } } @@ -257,7 +259,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { cnsToPut := make([]*pool.Conn, 0) for { - cn := d.pool.First() + cn := d.pool.PopFree() if cn == nil { break } @@ -267,7 +269,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { cn.RemoteAddr(), ) Logger.Print(err) - d.pool.Replace(cn, err) + d.pool.Remove(cn, err) } else { cnsToPut = append(cnsToPut, cn) } From 46e3a09455a2b17453fbc86ce489ccd9d89ca142 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 19 Mar 2016 15:45:28 +0200 Subject: [PATCH 0154/1746] Remove Go 1.4 from test build. Bump license. --- .travis.yml | 1 - LICENSE | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a476a44d3b..e0a2f98cb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ services: - redis-server go: - - 1.4 - 1.5 - 1.6 - tip diff --git a/LICENSE b/LICENSE index 6855a95feb..261f1ec375 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ -Copyright (c) 2012 The Redis Go Client Authors. All rights reserved. +Copyright (c) 2016 The github.com/go-redis/redis Contributors. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,9 +11,6 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT From d802f0e8bd28523c12371227e4f342a15bfca807 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 19 Mar 2016 16:10:34 +0200 Subject: [PATCH 0155/1746] Remove unneeded synchronization. --- redis.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/redis.go b/redis.go index aaa0dac85c..d3289ba269 100644 --- a/redis.go +++ b/redis.go @@ -4,7 +4,6 @@ import ( "fmt" "log" "os" - "sync/atomic" "gopkg.in/redis.v3/internal/pool" ) @@ -170,12 +169,12 @@ func NewClient(opt *Options) *Client { func (c *Client) PoolStats() *PoolStats { s := c.connPool.Stats() return &PoolStats{ - Requests: atomic.LoadUint32(&s.Requests), - Hits: atomic.LoadUint32(&s.Hits), - Waits: atomic.LoadUint32(&s.Waits), - Timeouts: atomic.LoadUint32(&s.Timeouts), + Requests: s.Requests, + Hits: s.Hits, + Waits: s.Waits, + Timeouts: s.Timeouts, - TotalConns: atomic.LoadUint32(&s.TotalConns), - FreeConns: atomic.LoadUint32(&s.FreeConns), + TotalConns: s.TotalConns, + FreeConns: s.FreeConns, } } From 30ce5ebd572a2b0bc978900c0d0de2d37a87ef9f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 19 Mar 2016 16:33:14 +0200 Subject: [PATCH 0156/1746] Cleanup error handling code. --- cluster.go | 4 ++-- error.go | 30 ++++++++++++++++-------------- redis.go | 4 ++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/cluster.go b/cluster.go index 5ac079c707..a31559bfed 100644 --- a/cluster.go +++ b/cluster.go @@ -195,12 +195,12 @@ func (c *ClusterClient) process(cmd Cmder) { // If there is no (real) error, we are done! err := cmd.Err() - if err == nil || err == Nil || err == TxFailedErr { + if err == nil { return } // On network errors try random node. - if isNetworkError(err) { + if shouldRetry(err) { client, err = c.randomClient() if err != nil { return diff --git a/error.go b/error.go index 3f2a560c0b..f0b27a8466 100644 --- a/error.go +++ b/error.go @@ -25,6 +25,11 @@ func (err redisError) Error() string { return err.s } +func isInternalError(err error) bool { + _, ok := err.(redisError) + return ok +} + func isNetworkError(err error) bool { if err == io.EOF { return true @@ -37,7 +42,7 @@ func isBadConn(err error, allowTimeout bool) bool { if err == nil { return false } - if _, ok := err.(redisError); ok { + if isInternalError(err) { return false } if allowTimeout { @@ -53,27 +58,24 @@ func isMovedError(err error) (moved bool, ask bool, addr string) { return } - parts := strings.SplitN(err.Error(), " ", 3) - if len(parts) != 3 { - return - } - - switch parts[0] { - case "MOVED": + s := err.Error() + if strings.HasPrefix(s, "MOVED ") { moved = true - addr = parts[2] - case "ASK": + } else if strings.HasPrefix(s, "ASK ") { ask = true - addr = parts[2] + } else { + return } + ind := strings.LastIndexByte(s, ' ') + if ind == -1 { + return false, false, "" + } + addr = s[ind+1:] return } // shouldRetry reports whether failed command should be retried. func shouldRetry(err error) bool { - if err == nil { - return false - } return isNetworkError(err) } diff --git a/redis.go b/redis.go index aaa0dac85c..27b7b61490 100644 --- a/redis.go +++ b/redis.go @@ -104,7 +104,7 @@ func (c *baseClient) process(cmd Cmder) { if err := writeCmd(cn, cmd); err != nil { c.putConn(cn, err, false) cmd.setErr(err) - if shouldRetry(err) { + if err != nil && shouldRetry(err) { continue } return @@ -112,7 +112,7 @@ func (c *baseClient) process(cmd Cmder) { err = cmd.readReply(cn) c.putConn(cn, err, readTimeout != nil) - if shouldRetry(err) { + if err != nil && shouldRetry(err) { continue } From 3e2da01875f95cbb4a688840366fb77a49fb8d56 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 19 Mar 2016 16:55:22 +0200 Subject: [PATCH 0157/1746] Add redis cluster benchmark results. --- README.md | 41 +++++++++++++++++++++++++---------------- example_test.go | 11 ++++++++--- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index bd9edfc99b..1d05c40108 100644 --- a/README.md +++ b/README.md @@ -94,23 +94,32 @@ Some corner cases: ## Benchmark +go-redis vs redigo: + +``` +BenchmarkSetGoRedis10Conns64Bytes-4 200000 7621 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns64Bytes-4 200000 7554 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns1KB-4 200000 7697 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns1KB-4 200000 7688 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns10KB-4 200000 9214 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns10KB-4 200000 9181 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns1MB-4 2000 583242 ns/op 2337 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns1MB-4 2000 583089 ns/op 2338 B/op 6 allocs/op +BenchmarkSetRedigo10Conns64Bytes-4 200000 7576 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns64Bytes-4 200000 7782 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns1KB-4 200000 7958 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns1KB-4 200000 7725 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns10KB-4 100000 18442 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns10KB-4 100000 18818 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns1MB-4 2000 668829 ns/op 226 B/op 7 allocs/op +BenchmarkSetRedigo100Conns1MB-4 2000 679542 ns/op 226 B/op 7 allocs/op +``` + +Redis Cluster: + ``` -BenchmarkSetGoRedis10Conns64Bytes-4 200000 7184 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns64Bytes-4 200000 7174 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis10Conns1KB-4 200000 7341 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns1KB-4 200000 7425 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis10Conns10KB-4 200000 9480 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns10KB-4 200000 9301 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis10Conns1MB-4 2000 590321 ns/op 2337 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns1MB-4 2000 588935 ns/op 2337 B/op 6 allocs/op -BenchmarkSetRedigo10Conns64Bytes-4 200000 7238 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo100Conns64Bytes-4 200000 7435 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo10Conns1KB-4 200000 7635 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo100Conns1KB-4 200000 7597 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo10Conns10KB-4 100000 17126 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo100Conns10KB-4 100000 17030 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo10Conns1MB-4 2000 675397 ns/op 226 B/op 7 allocs/op -BenchmarkSetRedigo100Conns1MB-4 2000 669053 ns/op 226 B/op 7 allocs/op +BenchmarkRedisPing-4 200000 6983 ns/op 116 B/op 4 allocs/op +BenchmarkRedisClusterPing-4 100000 11535 ns/op 117 B/op 4 allocs/op ``` ## Shameless plug diff --git a/example_test.go b/example_test.go index dc4e1bde4d..3380022845 100644 --- a/example_test.go +++ b/example_test.go @@ -12,9 +12,14 @@ import ( var client *redis.Client func init() { - opt := redisOptions() - opt.Addr = ":6379" - client = redis.NewClient(opt) + client = redis.NewClient(&redis.Options{ + Addr: ":6379", + DialTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + PoolSize: 10, + PoolTimeout: 30 * time.Second, + }) client.FlushDb() } From 04b6c9d3add890070824d840a1654f1bca5bd096 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 6 Apr 2016 13:13:03 +0300 Subject: [PATCH 0158/1746] Improve comment. --- cluster.go | 5 +++-- internal/pool/pool.go | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index a31559bfed..29b0cfbd81 100644 --- a/cluster.go +++ b/cluster.go @@ -324,8 +324,9 @@ type ClusterOptions struct { // A seed list of host:port addresses of cluster nodes. Addrs []string - // The maximum number of MOVED/ASK redirects to follow before giving up. - // Default is 16 + // The maximum number of retries before giving up. Command is retried + // on network errors and MOVED/ASK redirects. + // Default is 16. MaxRedirects int // Following options are copied from Options struct. diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 10155f32b0..700e660539 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -18,9 +18,7 @@ var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) var ( ErrClosed = errors.New("redis: client is closed") ErrPoolTimeout = errors.New("redis: connection pool timeout") - - errConnClosed = errors.New("connection is closed") - errConnStale = errors.New("connection is stale") + errConnStale = errors.New("connection is stale") ) var timers = sync.Pool{ From ebbeb40416e82629df1ff0c0353de367685a043f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 6 Apr 2016 13:19:00 +0300 Subject: [PATCH 0159/1746] Disable logging by default. --- redis.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/redis.go b/redis.go index 10d5092cf9..aed713cffa 100644 --- a/redis.go +++ b/redis.go @@ -2,18 +2,14 @@ package redis // import "gopkg.in/redis.v3" import ( "fmt" + "io/ioutil" "log" - "os" "gopkg.in/redis.v3/internal/pool" ) // Deprecated. Use SetLogger instead. -var Logger *log.Logger - -func init() { - SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags)) -} +var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) func SetLogger(logger *log.Logger) { Logger = logger From ec05edd08c936daca92183b94b0fc5fa47381e19 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 6 Apr 2016 14:01:08 +0300 Subject: [PATCH 0160/1746] Add cluster pipeline test. --- cluster_pipeline.go | 10 +++++ cluster_test.go | 96 ++++++++++++++++++++++++++++++++------------- 2 files changed, 78 insertions(+), 28 deletions(-) diff --git a/cluster_pipeline.go b/cluster_pipeline.go index 6d55265983..883c770828 100644 --- a/cluster_pipeline.go +++ b/cluster_pipeline.go @@ -27,6 +27,16 @@ func (c *ClusterClient) Pipeline() *ClusterPipeline { return pipe } +func (c *ClusterClient) Pipelined(fn func(*ClusterPipeline) error) ([]Cmder, error) { + pipe := c.Pipeline() + if err := fn(pipe); err != nil { + return nil, err + } + cmds, err := pipe.Exec() + _ = pipe.Close() + return cmds, err +} + func (pipe *ClusterPipeline) process(cmd Cmder) { pipe.cmds = append(pipe.cmds, cmd) } diff --git a/cluster_test.go b/cluster_test.go index 6a04212867..ff091a2cfd 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -191,6 +191,7 @@ var _ = Describe("Cluster", func() { {"", 5176}, {string([]byte{83, 153, 134, 118, 229, 214, 244, 75, 140, 37, 215, 215}), 5463}, } + // Empty keys receive random slot. rand.Seed(100) for _, test := range tests { @@ -343,34 +344,6 @@ var _ = Describe("Cluster", func() { }, "5s").Should(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) }) - It("should perform multi-pipelines", func() { - slot := hashtag.Slot("A") - Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) - - pipe := client.Pipeline() - defer pipe.Close() - - keys := []string{"A", "B", "C", "D", "E", "F", "G"} - for i, key := range keys { - pipe.Set(key, key+"_value", 0) - pipe.Expire(key, time.Duration(i+1)*time.Hour) - } - for _, key := range keys { - pipe.Get(key) - pipe.TTL(key) - } - - cmds, err := pipe.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(28)) - Expect(cmds[14].(*redis.StringCmd).Val()).To(Equal("A_value")) - Expect(cmds[15].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) - Expect(cmds[20].(*redis.StringCmd).Val()).To(Equal("D_value")) - Expect(cmds[21].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) - Expect(cmds[26].(*redis.StringCmd).Val()).To(Equal("G_value")) - Expect(cmds[27].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) - }) - It("should return error when there are no attempts left", func() { Expect(client.Close()).NotTo(HaveOccurred()) client = cluster.clusterClient(&redis.ClusterOptions{ @@ -428,6 +401,73 @@ var _ = Describe("Cluster", func() { Expect(n).To(Equal(int64(100))) }) }) + + Describe("pipeline", func() { + var client *redis.ClusterClient + + BeforeEach(func() { + client = cluster.clusterClient(nil) + }) + + AfterEach(func() { + for _, client := range cluster.masters() { + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + } + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + It("performs multi-pipelines", func() { + slot := hashtag.Slot("A") + Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + + pipe := client.Pipeline() + defer pipe.Close() + + keys := []string{"A", "B", "C", "D", "E", "F", "G"} + for i, key := range keys { + pipe.Set(key, key+"_value", 0) + pipe.Expire(key, time.Duration(i+1)*time.Hour) + } + for _, key := range keys { + pipe.Get(key) + pipe.TTL(key) + } + + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(28)) + Expect(cmds[14].(*redis.StringCmd).Val()).To(Equal("A_value")) + Expect(cmds[15].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) + Expect(cmds[20].(*redis.StringCmd).Val()).To(Equal("D_value")) + Expect(cmds[21].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) + Expect(cmds[26].(*redis.StringCmd).Val()).To(Equal("G_value")) + Expect(cmds[27].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) + }) + + It("works with missing keys", func() { + Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) + Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) + + var a, b, c *redis.StringCmd + cmds, err := client.Pipelined(func(pipe *redis.ClusterPipeline) error { + a = pipe.Get("A") + b = pipe.Get("B") + c = pipe.Get("C") + return nil + }) + Expect(err).To(Equal(redis.Nil)) + Expect(cmds).To(HaveLen(3)) + + Expect(a.Err()).NotTo(HaveOccurred()) + Expect(a.Val()).To(Equal("A_value")) + + Expect(b.Err()).To(Equal(redis.Nil)) + Expect(b.Val()).To(Equal("")) + + Expect(c.Err()).NotTo(HaveOccurred()) + Expect(c.Val()).To(Equal("C_value")) + }) + }) }) //------------------------------------------------------------------------------ From 3b051d2374dc02811b71c168bf32d67f834fffb1 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 10:47:15 +0300 Subject: [PATCH 0161/1746] Reuse single Pipeline type in Client, ClusterClient and Ring. --- cluster.go | 86 +++++++++++++++++++++++++++ cluster_pipeline.go | 140 -------------------------------------------- cluster_test.go | 2 +- pipeline.go | 55 ++++------------- redis.go | 37 ++++++++++++ ring.go | 71 ++++------------------ ring_test.go | 6 +- 7 files changed, 150 insertions(+), 247 deletions(-) delete mode 100644 cluster_pipeline.go diff --git a/cluster.go b/cluster.go index 29b0cfbd81..2922238945 100644 --- a/cluster.go +++ b/cluster.go @@ -316,6 +316,92 @@ func (c *ClusterClient) reaper(frequency time.Duration) { } } +func (c *ClusterClient) Pipeline() *Pipeline { + pipe := &Pipeline{ + exec: c.pipelineExec, + } + pipe.commandable.process = pipe.process + return pipe +} + +func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) +} + +func (c *ClusterClient) pipelineExec(cmds []Cmder) error { + var retErr error + + cmdsMap := make(map[string][]Cmder) + for _, cmd := range cmds { + slot := hashtag.Slot(cmd.clusterKey()) + addr := c.slotMasterAddr(slot) + cmdsMap[addr] = append(cmdsMap[addr], cmd) + } + + for attempt := 0; attempt <= c.opt.getMaxRedirects(); attempt++ { + failedCmds := make(map[string][]Cmder) + + for addr, cmds := range cmdsMap { + client, err := c.getClient(addr) + if err != nil { + setCmdsErr(cmds, err) + retErr = err + continue + } + + cn, err := client.conn() + if err != nil { + setCmdsErr(cmds, err) + retErr = err + continue + } + + failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) + if err != nil { + retErr = err + } + client.putConn(cn, err, false) + } + + cmdsMap = failedCmds + } + + return retErr +} + +func (c *ClusterClient) execClusterCmds( + cn *pool.Conn, cmds []Cmder, failedCmds map[string][]Cmder, +) (map[string][]Cmder, error) { + if err := writeCmd(cn, cmds...); err != nil { + setCmdsErr(cmds, err) + return failedCmds, err + } + + var firstCmdErr error + for i, cmd := range cmds { + err := cmd.readReply(cn) + if err == nil { + continue + } + if isNetworkError(err) { + cmd.reset() + failedCmds[""] = append(failedCmds[""], cmds[i:]...) + break + } else if moved, ask, addr := isMovedError(err); moved { + c.lazyReloadSlots() + cmd.reset() + failedCmds[addr] = append(failedCmds[addr], cmd) + } else if ask { + cmd.reset() + failedCmds[addr] = append(failedCmds[addr], NewCmd("ASKING"), cmd) + } else if firstCmdErr == nil { + firstCmdErr = err + } + } + + return failedCmds, firstCmdErr +} + //------------------------------------------------------------------------------ // ClusterOptions are used to configure a cluster client and should be diff --git a/cluster_pipeline.go b/cluster_pipeline.go deleted file mode 100644 index 883c770828..0000000000 --- a/cluster_pipeline.go +++ /dev/null @@ -1,140 +0,0 @@ -package redis - -import ( - "gopkg.in/redis.v3/internal/hashtag" - "gopkg.in/redis.v3/internal/pool" -) - -// ClusterPipeline is not thread-safe. -type ClusterPipeline struct { - commandable - - cluster *ClusterClient - - cmds []Cmder - closed bool -} - -// Pipeline creates a new pipeline which is able to execute commands -// against multiple shards. It's NOT safe for concurrent use by -// multiple goroutines. -func (c *ClusterClient) Pipeline() *ClusterPipeline { - pipe := &ClusterPipeline{ - cluster: c, - cmds: make([]Cmder, 0, 10), - } - pipe.commandable.process = pipe.process - return pipe -} - -func (c *ClusterClient) Pipelined(fn func(*ClusterPipeline) error) ([]Cmder, error) { - pipe := c.Pipeline() - if err := fn(pipe); err != nil { - return nil, err - } - cmds, err := pipe.Exec() - _ = pipe.Close() - return cmds, err -} - -func (pipe *ClusterPipeline) process(cmd Cmder) { - pipe.cmds = append(pipe.cmds, cmd) -} - -// Discard resets the pipeline and discards queued commands. -func (pipe *ClusterPipeline) Discard() error { - if pipe.closed { - return pool.ErrClosed - } - pipe.cmds = pipe.cmds[:0] - return nil -} - -func (pipe *ClusterPipeline) Exec() (cmds []Cmder, retErr error) { - if pipe.closed { - return nil, pool.ErrClosed - } - if len(pipe.cmds) == 0 { - return []Cmder{}, nil - } - - cmds = pipe.cmds - pipe.cmds = make([]Cmder, 0, 10) - - cmdsMap := make(map[string][]Cmder) - for _, cmd := range cmds { - slot := hashtag.Slot(cmd.clusterKey()) - addr := pipe.cluster.slotMasterAddr(slot) - cmdsMap[addr] = append(cmdsMap[addr], cmd) - } - - for attempt := 0; attempt <= pipe.cluster.opt.getMaxRedirects(); attempt++ { - failedCmds := make(map[string][]Cmder) - - for addr, cmds := range cmdsMap { - client, err := pipe.cluster.getClient(addr) - if err != nil { - setCmdsErr(cmds, err) - retErr = err - continue - } - - cn, err := client.conn() - if err != nil { - setCmdsErr(cmds, err) - retErr = err - continue - } - - failedCmds, err = pipe.execClusterCmds(cn, cmds, failedCmds) - if err != nil { - retErr = err - } - client.putConn(cn, err, false) - } - - cmdsMap = failedCmds - } - - return cmds, retErr -} - -// Close closes the pipeline, releasing any open resources. -func (pipe *ClusterPipeline) Close() error { - pipe.Discard() - pipe.closed = true - return nil -} - -func (pipe *ClusterPipeline) execClusterCmds( - cn *pool.Conn, cmds []Cmder, failedCmds map[string][]Cmder, -) (map[string][]Cmder, error) { - if err := writeCmd(cn, cmds...); err != nil { - setCmdsErr(cmds, err) - return failedCmds, err - } - - var firstCmdErr error - for i, cmd := range cmds { - err := cmd.readReply(cn) - if err == nil { - continue - } - if isNetworkError(err) { - cmd.reset() - failedCmds[""] = append(failedCmds[""], cmds[i:]...) - break - } else if moved, ask, addr := isMovedError(err); moved { - pipe.cluster.lazyReloadSlots() - cmd.reset() - failedCmds[addr] = append(failedCmds[addr], cmd) - } else if ask { - cmd.reset() - failedCmds[addr] = append(failedCmds[addr], NewCmd("ASKING"), cmd) - } else if firstCmdErr == nil { - firstCmdErr = err - } - } - - return failedCmds, firstCmdErr -} diff --git a/cluster_test.go b/cluster_test.go index ff091a2cfd..e7aa9b96d1 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -449,7 +449,7 @@ var _ = Describe("Cluster", func() { Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) var a, b, c *redis.StringCmd - cmds, err := client.Pipelined(func(pipe *redis.ClusterPipeline) error { + cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { a = pipe.Get("A") b = pipe.Get("B") c = pipe.Get("C") diff --git a/pipeline.go b/pipeline.go index 888d8c40de..95d389f277 100644 --- a/pipeline.go +++ b/pipeline.go @@ -13,7 +13,7 @@ import ( type Pipeline struct { commandable - client baseClient + exec func([]Cmder) error mu sync.Mutex // protects cmds cmds []Cmder @@ -21,25 +21,6 @@ type Pipeline struct { closed int32 } -func (c *Client) Pipeline() *Pipeline { - pipe := &Pipeline{ - client: c.baseClient, - cmds: make([]Cmder, 0, 10), - } - pipe.commandable.process = pipe.process - return pipe -} - -func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { - pipe := c.Pipeline() - if err := fn(pipe); err != nil { - return nil, err - } - cmds, err := pipe.Exec() - _ = pipe.Close() - return cmds, err -} - func (pipe *Pipeline) process(cmd Cmder) { pipe.mu.Lock() pipe.cmds = append(pipe.cmds, cmd) @@ -73,7 +54,7 @@ func (pipe *Pipeline) Discard() error { // // Exec always returns list of commands and error of the first failed // command if any. -func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { +func (pipe *Pipeline) Exec() ([]Cmder, error) { if pipe.isClosed() { return nil, pool.ErrClosed } @@ -85,31 +66,19 @@ func (pipe *Pipeline) Exec() (cmds []Cmder, retErr error) { return pipe.cmds, nil } - cmds = pipe.cmds - pipe.cmds = make([]Cmder, 0, 10) + cmds := pipe.cmds + pipe.cmds = nil - failedCmds := cmds - for i := 0; i <= pipe.client.opt.MaxRetries; i++ { - cn, err := pipe.client.conn() - if err != nil { - setCmdsErr(failedCmds, err) - return cmds, err - } + return cmds, pipe.exec(cmds) +} - if i > 0 { - resetCmds(failedCmds) - } - failedCmds, err = execCmds(cn, failedCmds) - pipe.client.putConn(cn, err, false) - if err != nil && retErr == nil { - retErr = err - } - if len(failedCmds) == 0 { - break - } +func (pipe *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + if err := fn(pipe); err != nil { + return nil, err } - - return cmds, retErr + cmds, err := pipe.Exec() + _ = pipe.Close() + return cmds, err } func execCmds(cn *pool.Conn, cmds []Cmder) ([]Cmder, error) { diff --git a/redis.go b/redis.go index aed713cffa..ee4e69a95b 100644 --- a/redis.go +++ b/redis.go @@ -174,3 +174,40 @@ func (c *Client) PoolStats() *PoolStats { FreeConns: s.FreeConns, } } + +func (c *Client) Pipeline() *Pipeline { + pipe := &Pipeline{ + exec: c.pipelineExec, + } + pipe.commandable.process = pipe.process + return pipe +} + +func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) +} + +func (c *Client) pipelineExec(cmds []Cmder) error { + var retErr error + failedCmds := cmds + for i := 0; i <= c.opt.MaxRetries; i++ { + cn, err := c.conn() + if err != nil { + setCmdsErr(failedCmds, err) + return err + } + + if i > 0 { + resetCmds(failedCmds) + } + failedCmds, err = execCmds(cn, failedCmds) + c.putConn(cn, err, false) + if err != nil && retErr == nil { + retErr = err + } + if len(failedCmds) == 0 { + break + } + } + return retErr +} diff --git a/ring.go b/ring.go index fd74b3b6ff..02be83c6c2 100644 --- a/ring.go +++ b/ring.go @@ -241,66 +241,24 @@ func (ring *Ring) Close() (retErr error) { return retErr } -// RingPipeline creates a new pipeline which is able to execute commands -// against multiple shards. It's NOT safe for concurrent use by -// multiple goroutines. -type RingPipeline struct { - commandable - - ring *Ring - - cmds []Cmder - closed bool -} - -func (ring *Ring) Pipeline() *RingPipeline { - pipe := &RingPipeline{ - ring: ring, - cmds: make([]Cmder, 0, 10), +func (ring *Ring) Pipeline() *Pipeline { + pipe := &Pipeline{ + exec: ring.pipelineExec, } pipe.commandable.process = pipe.process return pipe } -func (ring *Ring) Pipelined(fn func(*RingPipeline) error) ([]Cmder, error) { - pipe := ring.Pipeline() - if err := fn(pipe); err != nil { - return nil, err - } - cmds, err := pipe.Exec() - pipe.Close() - return cmds, err +func (ring *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return ring.Pipeline().pipelined(fn) } -func (pipe *RingPipeline) process(cmd Cmder) { - pipe.cmds = append(pipe.cmds, cmd) -} - -// Discard resets the pipeline and discards queued commands. -func (pipe *RingPipeline) Discard() error { - if pipe.closed { - return pool.ErrClosed - } - pipe.cmds = pipe.cmds[:0] - return nil -} - -// Exec always returns list of commands and error of the first failed -// command if any. -func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { - if pipe.closed { - return nil, pool.ErrClosed - } - if len(pipe.cmds) == 0 { - return pipe.cmds, nil - } - - cmds = pipe.cmds - pipe.cmds = make([]Cmder, 0, 10) +func (ring *Ring) pipelineExec(cmds []Cmder) error { + var retErr error cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - name := pipe.ring.hash.Get(hashtag.Key(cmd.clusterKey())) + name := ring.hash.Get(hashtag.Key(cmd.clusterKey())) if name == "" { cmd.setErr(errRingShardsDown) if retErr == nil { @@ -311,11 +269,11 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { cmdsMap[name] = append(cmdsMap[name], cmd) } - for i := 0; i <= pipe.ring.opt.MaxRetries; i++ { + for i := 0; i <= ring.opt.MaxRetries; i++ { failedCmdsMap := make(map[string][]Cmder) for name, cmds := range cmdsMap { - client := pipe.ring.shards[name].Client + client := ring.shards[name].Client cn, err := client.conn() if err != nil { setCmdsErr(cmds, err) @@ -344,12 +302,5 @@ func (pipe *RingPipeline) Exec() (cmds []Cmder, retErr error) { cmdsMap = failedCmdsMap } - return cmds, retErr -} - -// Close closes the pipeline, releasing any open resources. -func (pipe *RingPipeline) Close() error { - pipe.Discard() - pipe.closed = true - return nil + return retErr } diff --git a/ring_test.go b/ring_test.go index 7d5cd91cae..06e6c3036d 100644 --- a/ring_test.go +++ b/ring_test.go @@ -96,7 +96,7 @@ var _ = Describe("Redis ring", func() { Describe("pipelining", func() { It("returns an error when all shards are down", func() { ring := redis.NewRing(&redis.RingOptions{}) - _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { + _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { pipe.Ping() return nil }) @@ -133,7 +133,7 @@ var _ = Describe("Redis ring", func() { keys = append(keys, string(key)) } - _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { + _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { for _, key := range keys { pipe.Set(key, "value", 0).Err() } @@ -149,7 +149,7 @@ var _ = Describe("Redis ring", func() { }) It("supports hash tags", func() { - _, err := ring.Pipelined(func(pipe *redis.RingPipeline) error { + _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { for i := 0; i < 100; i++ { pipe.Set(fmt.Sprintf("key%d{tag}", i), "value", 0).Err() } From 5e5a540eb124a8994ad7302b741e89f1a1f2ef4b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 11:01:33 +0300 Subject: [PATCH 0162/1746] Accept interface{} in Eval. Fixes #243. --- commands.go | 4 ++-- example_test.go | 4 ++-- script.go | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/commands.go b/commands.go index c523534672..dc6dd08f41 100644 --- a/commands.go +++ b/commands.go @@ -1543,7 +1543,7 @@ func (c *commandable) Time() *StringSliceCmd { //------------------------------------------------------------------------------ -func (c *commandable) Eval(script string, keys []string, args []string) *Cmd { +func (c *commandable) Eval(script string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "EVAL" cmdArgs[1] = script @@ -1563,7 +1563,7 @@ func (c *commandable) Eval(script string, keys []string, args []string) *Cmd { return cmd } -func (c *commandable) EvalSha(sha1 string, keys []string, args []string) *Cmd { +func (c *commandable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "EVALSHA" cmdArgs[1] = sha1 diff --git a/example_test.go b/example_test.go index 3380022845..587ef78a40 100644 --- a/example_test.go +++ b/example_test.go @@ -288,7 +288,7 @@ func ExampleScript() { return false `) - n, err := IncrByXX.Run(client, []string{"xx_counter"}, []string{"2"}).Result() + n, err := IncrByXX.Run(client, []string{"xx_counter"}, 2).Result() fmt.Println(n, err) err = client.Set("xx_counter", "40", 0).Err() @@ -296,7 +296,7 @@ func ExampleScript() { panic(err) } - n, err = IncrByXX.Run(client, []string{"xx_counter"}, []string{"2"}).Result() + n, err = IncrByXX.Run(client, []string{"xx_counter"}, 2).Result() fmt.Println(n, err) // Output: redis: nil diff --git a/script.go b/script.go index 3f22f46959..c5fc1d2bdc 100644 --- a/script.go +++ b/script.go @@ -8,8 +8,8 @@ import ( ) type scripter interface { - Eval(script string, keys []string, args []string) *Cmd - EvalSha(sha1 string, keys []string, args []string) *Cmd + Eval(script string, keys []string, args ...interface{}) *Cmd + EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd ScriptExists(scripts ...string) *BoolSliceCmd ScriptLoad(script string) *StringCmd } @@ -35,18 +35,18 @@ func (s *Script) Exists(c scripter) *BoolSliceCmd { return c.ScriptExists(s.src) } -func (s *Script) Eval(c scripter, keys []string, args []string) *Cmd { - return c.Eval(s.src, keys, args) +func (s *Script) Eval(c scripter, keys []string, args ...interface{}) *Cmd { + return c.Eval(s.src, keys, args...) } -func (s *Script) EvalSha(c scripter, keys []string, args []string) *Cmd { - return c.EvalSha(s.hash, keys, args) +func (s *Script) EvalSha(c scripter, keys []string, args ...interface{}) *Cmd { + return c.EvalSha(s.hash, keys, args...) } -func (s *Script) Run(c scripter, keys []string, args []string) *Cmd { - r := s.EvalSha(c, keys, args) +func (s *Script) Run(c scripter, keys []string, args ...interface{}) *Cmd { + r := s.EvalSha(c, keys, args...) if err := r.Err(); err != nil && strings.HasPrefix(err.Error(), "NOSCRIPT ") { - return s.Eval(c, keys, args) + return s.Eval(c, keys, args...) } return r } From 7a03514d7fecc92d138a1e4d39358de1c1f87a0d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 11:23:58 +0300 Subject: [PATCH 0163/1746] Rename Multi to Tx to better reflect the purpose. Fixes #194. --- cluster.go | 2 +- commands_test.go | 60 ------------------------ pool_test.go | 10 ++-- race_test.go | 39 ++++++++++++++++ redis_test.go | 14 +++--- multi.go => tx.go | 91 ++++++++++++++++++------------------- multi_test.go => tx_test.go | 57 ++++++++++------------- 7 files changed, 124 insertions(+), 149 deletions(-) rename multi.go => tx.go (70%) rename multi_test.go => tx_test.go (77%) diff --git a/cluster.go b/cluster.go index 29b0cfbd81..bf160ee700 100644 --- a/cluster.go +++ b/cluster.go @@ -60,7 +60,7 @@ func (c *ClusterClient) getClients() map[string]*Client { // Watch creates new transaction and marks the keys to be watched // for conditional execution of a transaction. -func (c *ClusterClient) Watch(keys ...string) (*Multi, error) { +func (c *ClusterClient) Watch(keys ...string) (*Tx, error) { addr := c.slotMasterAddr(hashtag.Slot(keys[0])) client, err := c.getClient(addr) if err != nil { diff --git a/commands_test.go b/commands_test.go index eb0d52372b..dfa22ca07f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -4,9 +4,6 @@ import ( "encoding/json" "fmt" "reflect" - "strconv" - "sync" - "testing" "time" . "github.com/onsi/ginkgo" @@ -2551,63 +2548,6 @@ var _ = Describe("Commands", func() { }) - Describe("watch/unwatch", func() { - - It("should WatchUnwatch", func() { - var C, N = 10, 1000 - if testing.Short() { - N = 100 - } - - err := client.Set("key", "0", 0).Err() - Expect(err).NotTo(HaveOccurred()) - - wg := &sync.WaitGroup{} - for i := 0; i < C; i++ { - wg.Add(1) - - go func() { - defer GinkgoRecover() - defer wg.Done() - - multi := client.Multi() - defer multi.Close() - - for j := 0; j < N; j++ { - val, err := multi.Watch("key").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal("OK")) - - val, err = multi.Get("key").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(val).NotTo(Equal(redis.Nil)) - - num, err := strconv.ParseInt(val, 10, 64) - Expect(err).NotTo(HaveOccurred()) - - cmds, err := multi.Exec(func() error { - multi.Set("key", strconv.FormatInt(num+1, 10), 0) - return nil - }) - if err == redis.TxFailedErr { - j-- - continue - } - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(1)) - Expect(cmds[0].Err()).NotTo(HaveOccurred()) - } - }() - } - wg.Wait() - - val, err := client.Get("key").Int64() - Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal(int64(C * N))) - }) - - }) - Describe("Geo add and radius search", func() { BeforeEach(func() { geoAdd := client.GeoAdd( diff --git a/pool_test.go b/pool_test.go index 793ff5031e..942b9b979a 100644 --- a/pool_test.go +++ b/pool_test.go @@ -37,16 +37,18 @@ var _ = Describe("pool", func() { perform(1000, func(id int) { var ping *redis.StatusCmd - multi := client.Multi() - cmds, err := multi.Exec(func() error { - ping = multi.Ping() + tx, err := client.Watch() + Expect(err).NotTo(HaveOccurred()) + + cmds, err := tx.Exec(func() error { + ping = tx.Ping() return nil }) Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(1)) Expect(ping.Err()).NotTo(HaveOccurred()) Expect(ping.Val()).To(Equal("PONG")) - Expect(multi.Close()).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) }) pool := client.Pool() diff --git a/race_test.go b/race_test.go index 1ce8430a88..968f76e45b 100644 --- a/race_test.go +++ b/race_test.go @@ -208,4 +208,43 @@ var _ = Describe("races", func() { Expect(err).NotTo(HaveOccurred()) }) }) + + It("should Watch/Unwatch", func() { + err := client.Set("key", "0", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + perform(C, func(id int) { + for i := 0; i < N; i++ { + tx, err := client.Watch("key") + Expect(err).NotTo(HaveOccurred()) + + val, err := tx.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).NotTo(Equal(redis.Nil)) + + num, err := strconv.ParseInt(val, 10, 64) + Expect(err).NotTo(HaveOccurred()) + + cmds, err := tx.Exec(func() error { + tx.Set("key", strconv.FormatInt(num+1, 10), 0) + return nil + }) + if err == redis.TxFailedErr { + i-- + continue + } + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(1)) + Expect(cmds[0].Err()).NotTo(HaveOccurred()) + + err = tx.Close() + Expect(err).NotTo(HaveOccurred()) + } + }) + + val, err := client.Get("key").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal(int64(C * N))) + }) + }) diff --git a/redis_test.go b/redis_test.go index cb8e3ac691..fc0d702df6 100644 --- a/redis_test.go +++ b/redis_test.go @@ -66,11 +66,12 @@ var _ = Describe("Client", func() { }) It("should close multi without closing the client", func() { - multi := client.Multi() - Expect(multi.Close()).NotTo(HaveOccurred()) + tx, err := client.Watch() + Expect(err).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) - _, err := multi.Exec(func() error { - multi.Ping() + _, err = tx.Exec(func() error { + tx.Ping() return nil }) Expect(err).To(MatchError("redis: client is closed")) @@ -96,9 +97,10 @@ var _ = Describe("Client", func() { }) It("should close multi when client is closed", func() { - multi := client.Multi() + tx, err := client.Watch() + Expect(err).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) - Expect(multi.Close()).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) }) It("should close pipeline when client is closed", func() { diff --git a/multi.go b/tx.go similarity index 70% rename from multi.go rename to tx.go index 79b7cb6df3..51c8757424 100644 --- a/multi.go +++ b/tx.go @@ -9,13 +9,11 @@ import ( var errDiscard = errors.New("redis: Discard can be used only inside Exec") -// Multi implements Redis transactions as described in +// Tx implements Redis transactions as described in // http://redis.io/topics/transactions. It's NOT safe for concurrent use // by multiple goroutines, because Exec resets list of watched keys. // If you don't need WATCH it is better to use Pipeline. -// -// TODO(vmihailenco): rename to Tx and rework API -type Multi struct { +type Tx struct { commandable base *baseClient @@ -24,77 +22,78 @@ type Multi struct { closed bool } -// Watch creates new transaction and marks the keys to be watched -// for conditional execution of a transaction. -func (c *Client) Watch(keys ...string) (*Multi, error) { - tx := c.Multi() - if err := tx.Watch(keys...).Err(); err != nil { - tx.Close() - return nil, err - } - return tx, nil -} - -// Deprecated. Use Watch instead. -func (c *Client) Multi() *Multi { - multi := &Multi{ +func (c *Client) newTx() *Tx { + tx := &Tx{ base: &baseClient{ opt: c.opt, connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true), }, } - multi.commandable.process = multi.process - return multi + tx.commandable.process = tx.process + return tx +} + +// Watch creates new transaction and marks the keys to be watched +// for conditional execution of a transaction. +func (c *Client) Watch(keys ...string) (*Tx, error) { + tx := c.newTx() + if len(keys) > 0 { + if err := tx.Watch(keys...).Err(); err != nil { + tx.Close() + return nil, err + } + } + return tx, nil } -func (c *Multi) process(cmd Cmder) { - if c.cmds == nil { - c.base.process(cmd) +func (tx *Tx) process(cmd Cmder) { + if tx.cmds == nil { + tx.base.process(cmd) } else { - c.cmds = append(c.cmds, cmd) + tx.cmds = append(tx.cmds, cmd) } } -// Close closes the client, releasing any open resources. -func (c *Multi) Close() error { - c.closed = true - if err := c.Unwatch().Err(); err != nil { +// Close closes the transaction, releasing any open resources. +func (tx *Tx) Close() error { + tx.closed = true + if err := tx.Unwatch().Err(); err != nil { Logger.Printf("Unwatch failed: %s", err) } - return c.base.Close() + return tx.base.Close() } // Watch marks the keys to be watched for conditional execution // of a transaction. -func (c *Multi) Watch(keys ...string) *StatusCmd { +func (tx *Tx) Watch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) args[0] = "WATCH" for i, key := range keys { args[1+i] = key } cmd := NewStatusCmd(args...) - c.Process(cmd) + tx.Process(cmd) return cmd } // Unwatch flushes all the previously watched keys for a transaction. -func (c *Multi) Unwatch(keys ...string) *StatusCmd { +func (tx *Tx) Unwatch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) args[0] = "UNWATCH" for i, key := range keys { args[1+i] = key } cmd := NewStatusCmd(args...) - c.Process(cmd) + tx.Process(cmd) return cmd } // Discard discards queued commands. -func (c *Multi) Discard() error { - if c.cmds == nil { +func (tx *Tx) Discard() error { + if tx.cmds == nil { return errDiscard } - c.cmds = c.cmds[:1] + tx.cmds = tx.cmds[:1] return nil } @@ -107,19 +106,19 @@ func (c *Multi) Discard() error { // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. -func (c *Multi) Exec(f func() error) ([]Cmder, error) { - if c.closed { +func (tx *Tx) Exec(f func() error) ([]Cmder, error) { + if tx.closed { return nil, pool.ErrClosed } - c.cmds = []Cmder{NewStatusCmd("MULTI")} + tx.cmds = []Cmder{NewStatusCmd("MULTI")} if err := f(); err != nil { return nil, err } - c.cmds = append(c.cmds, NewSliceCmd("EXEC")) + tx.cmds = append(tx.cmds, NewSliceCmd("EXEC")) - cmds := c.cmds - c.cmds = nil + cmds := tx.cmds + tx.cmds = nil if len(cmds) == 2 { return []Cmder{}, nil @@ -128,18 +127,18 @@ func (c *Multi) Exec(f func() error) ([]Cmder, error) { // Strip MULTI and EXEC commands. retCmds := cmds[1 : len(cmds)-1] - cn, err := c.base.conn() + cn, err := tx.base.conn() if err != nil { setCmdsErr(retCmds, err) return retCmds, err } - err = c.execCmds(cn, cmds) - c.base.putConn(cn, err, false) + err = tx.execCmds(cn, cmds) + tx.base.putConn(cn, err, false) return retCmds, err } -func (c *Multi) execCmds(cn *pool.Conn, cmds []Cmder) error { +func (tx *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { err := writeCmd(cn, cmds...) if err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) diff --git a/multi_test.go b/tx_test.go similarity index 77% rename from multi_test.go rename to tx_test.go index e76c2b33a9..66ef6b0841 100644 --- a/multi_test.go +++ b/tx_test.go @@ -10,7 +10,7 @@ import ( "gopkg.in/redis.v3" ) -var _ = Describe("Multi", func() { +var _ = Describe("Tx", func() { var client *redis.Client BeforeEach(func() { @@ -67,15 +67,16 @@ var _ = Describe("Multi", func() { }) It("should discard", func() { - multi := client.Multi() + tx, err := client.Watch("key1", "key2") + Expect(err).NotTo(HaveOccurred()) defer func() { - Expect(multi.Close()).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) }() - cmds, err := multi.Exec(func() error { - multi.Set("key1", "hello1", 0) - multi.Discard() - multi.Set("key2", "hello2", 0) + cmds, err := tx.Exec(func() error { + tx.Set("key1", "hello1", 0) + tx.Discard() + tx.Set("key2", "hello2", 0) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -91,40 +92,31 @@ var _ = Describe("Multi", func() { }) It("should exec empty", func() { - multi := client.Multi() + tx, err := client.Watch() + Expect(err).NotTo(HaveOccurred()) defer func() { - Expect(multi.Close()).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) }() - cmds, err := multi.Exec(func() error { return nil }) + cmds, err := tx.Exec(func() error { return nil }) Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(0)) - ping := multi.Ping() + ping := tx.Ping() Expect(ping.Err()).NotTo(HaveOccurred()) Expect(ping.Val()).To(Equal("PONG")) }) - It("should exec empty queue", func() { - multi := client.Multi() - defer func() { - Expect(multi.Close()).NotTo(HaveOccurred()) - }() - - cmds, err := multi.Exec(func() error { return nil }) - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(0)) - }) - It("should exec bulks", func() { - multi := client.Multi() + tx, err := client.Watch() + Expect(err).NotTo(HaveOccurred()) defer func() { - Expect(multi.Close()).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) }() - cmds, err := multi.Exec(func() error { + cmds, err := tx.Exec(func() error { for i := int64(0); i < 20000; i++ { - multi.Incr("key") + tx.Incr("key") } return nil }) @@ -148,19 +140,20 @@ var _ = Describe("Multi", func() { err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) - multi := client.Multi() + tx, err := client.Watch() + Expect(err).NotTo(HaveOccurred()) defer func() { - Expect(multi.Close()).NotTo(HaveOccurred()) + Expect(tx.Close()).NotTo(HaveOccurred()) }() - _, err = multi.Exec(func() error { - multi.Ping() + _, err = tx.Exec(func() error { + tx.Ping() return nil }) Expect(err).To(MatchError("bad connection")) - _, err = multi.Exec(func() error { - multi.Ping() + _, err = tx.Exec(func() error { + tx.Ping() return nil }) Expect(err).NotTo(HaveOccurred()) From d79074eadb86ddedb17176798ba9331d7438d18c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 11:45:56 +0300 Subject: [PATCH 0164/1746] Remove PMessage. --- cluster.go | 1 - internal/pool/pool.go | 3 --- options.go | 1 - pool_test.go | 2 -- pubsub.go | 28 ++++------------------------ pubsub_test.go | 6 +++--- redis.go | 1 - 7 files changed, 7 insertions(+), 35 deletions(-) diff --git a/cluster.go b/cluster.go index baecb8f5dd..12b1757d14 100644 --- a/cluster.go +++ b/cluster.go @@ -76,7 +76,6 @@ func (c *ClusterClient) PoolStats() *PoolStats { s := client.connPool.Stats() acc.Requests += s.Requests acc.Hits += s.Hits - acc.Waits += s.Waits acc.Timeouts += s.Timeouts acc.TotalConns += s.TotalConns acc.FreeConns += s.FreeConns diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 700e660539..302a2f8b96 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -28,11 +28,9 @@ var timers = sync.Pool{ } // PoolStats contains pool state information and accumulated stats. -// TODO: remove Waits type PoolStats struct { Requests uint32 // number of times a connection was requested by the pool Hits uint32 // number of times free connection was found in the pool - Waits uint32 // number of times the pool had to wait for a connection Timeouts uint32 // number of times a wait timeout occurred TotalConns uint32 // the number of total connections in the pool @@ -261,7 +259,6 @@ func (p *ConnPool) Stats() *PoolStats { stats := PoolStats{} stats.Requests = atomic.LoadUint32(&p.stats.Requests) stats.Hits = atomic.LoadUint32(&p.stats.Hits) - stats.Waits = atomic.LoadUint32(&p.stats.Waits) stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts) stats.TotalConns = uint32(p.Len()) stats.FreeConns = uint32(p.FreeLen()) diff --git a/options.go b/options.go index 342e99d83e..6214496c0b 100644 --- a/options.go +++ b/options.go @@ -117,7 +117,6 @@ func newConnPool(opt *Options) *pool.ConnPool { type PoolStats struct { Requests uint32 // number of times a connection was requested by the pool Hits uint32 // number of times free connection was found in the pool - Waits uint32 // number of times the pool had to wait for a connection Timeouts uint32 // number of times a wait timeout occurred TotalConns uint32 // the number of total connections in the pool diff --git a/pool_test.go b/pool_test.go index 942b9b979a..bdf168a9c2 100644 --- a/pool_test.go +++ b/pool_test.go @@ -109,7 +109,6 @@ var _ = Describe("pool", func() { stats := pool.Stats() Expect(stats.Requests).To(Equal(uint32(4))) Expect(stats.Hits).To(Equal(uint32(2))) - Expect(stats.Waits).To(Equal(uint32(0))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) @@ -127,7 +126,6 @@ var _ = Describe("pool", func() { stats := pool.Stats() Expect(stats.Requests).To(Equal(uint32(101))) Expect(stats.Hits).To(Equal(uint32(100))) - Expect(stats.Waits).To(Equal(uint32(0))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) }) diff --git a/pubsub.go b/pubsub.go index 1bf2f93356..844dfea211 100644 --- a/pubsub.go +++ b/pubsub.go @@ -166,20 +166,6 @@ func (m *Message) String() string { return fmt.Sprintf("Message<%s: %s>", m.Channel, m.Payload) } -// TODO: remove PMessage if favor of Message - -// Message matching a pattern-matching subscription received as result -// of a PUBLISH command issued by another client. -type PMessage struct { - Channel string - Pattern string - Payload string -} - -func (m *PMessage) String() string { - return fmt.Sprintf("PMessage<%s: %s>", m.Channel, m.Payload) -} - // Pong received as result of a PING command issued by another client. type Pong struct { Payload string @@ -206,7 +192,7 @@ func (c *PubSub) newMessage(reply []interface{}) (interface{}, error) { Payload: reply[2].(string), }, nil case "pmessage": - return &PMessage{ + return &Message{ Pattern: reply[1].(string), Channel: reply[2].(string), Payload: reply[3].(string), @@ -244,9 +230,9 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { return c.newMessage(cmd.Val()) } -// Receive returns a message as a Subscription, Message, PMessage, -// Pong or error. See PubSub example for details. This is low-level -// API and most clients should use ReceiveMessage. +// Receive returns a message as a Subscription, Message, Pong or error. +// See PubSub example for details. This is low-level API and most clients +// should use ReceiveMessage. func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } @@ -290,12 +276,6 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { // Ignore. case *Message: return msg, nil - case *PMessage: - return &Message{ - Channel: msg.Channel, - Pattern: msg.Pattern, - Payload: msg.Payload, - }, nil default: return nil, fmt.Errorf("redis: unknown message: %T", msgi) } diff --git a/pubsub_test.go b/pubsub_test.go index df0dd94b92..95ab15383f 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -53,7 +53,7 @@ var _ = Describe("PubSub", func() { { msgi, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) - subscr := msgi.(*redis.PMessage) + subscr := msgi.(*redis.Message) Expect(subscr.Channel).To(Equal("mychannel1")) Expect(subscr.Pattern).To(Equal("mychannel*")) Expect(subscr.Payload).To(Equal("hello")) @@ -69,7 +69,7 @@ var _ = Describe("PubSub", func() { } stats := client.PoolStats() - Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) + Expect(stats.Requests - stats.Hits).To(Equal(uint32(2))) }) It("should pub/sub channels", func() { @@ -196,7 +196,7 @@ var _ = Describe("PubSub", func() { } stats := client.PoolStats() - Expect(stats.Requests - stats.Hits - stats.Waits).To(Equal(uint32(2))) + Expect(stats.Requests - stats.Hits).To(Equal(uint32(2))) }) It("should ping/pong", func() { diff --git a/redis.go b/redis.go index ee4e69a95b..8899848733 100644 --- a/redis.go +++ b/redis.go @@ -167,7 +167,6 @@ func (c *Client) PoolStats() *PoolStats { return &PoolStats{ Requests: s.Requests, Hits: s.Hits, - Waits: s.Waits, Timeouts: s.Timeouts, TotalConns: s.TotalConns, From 956758d3951b3440b5e757f0c180ae57d82c4f1b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 11:53:47 +0300 Subject: [PATCH 0165/1746] Don't convert bytes to string in Cmd (interface{} value). --- command.go | 8 +------- redis_test.go | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/command.go b/command.go index 6681ff530b..0c30bcb5f1 100644 --- a/command.go +++ b/command.go @@ -165,13 +165,7 @@ func (cmd *Cmd) readReply(cn *pool.Conn) error { cmd.err = err return cmd.err } - if v, ok := val.([]byte); ok { - // Convert to string to preserve old behaviour. - // TODO: remove in v4 - cmd.val = string(v) - } else { - cmd.val = val - } + cmd.val = val return nil } diff --git a/redis_test.go b/redis_test.go index fc0d702df6..9d4a360fd5 100644 --- a/redis_test.go +++ b/redis_test.go @@ -132,7 +132,7 @@ var _ = Describe("Client", func() { cmd := redis.NewCmd("PING") client.Process(cmd) Expect(cmd.Err()).NotTo(HaveOccurred()) - Expect(cmd.Val()).To(Equal("PONG")) + Expect(cmd.Val()).To(Equal([]byte("PONG"))) }) It("should retry command on network error", func() { From ae217e0444692de6672377a5c411d1ab4c1991d6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 12:52:43 +0300 Subject: [PATCH 0166/1746] Expose cluster node id in ClusterSlots. --- cluster.go | 21 +++++++++------ cluster_client_test.go | 10 +++---- cluster_test.go | 60 ++++++++++++++++++++++++++++-------------- command.go | 35 +++++++++++++----------- commands.go | 4 +-- parser.go | 31 ++++++++++------------ 6 files changed, 93 insertions(+), 68 deletions(-) diff --git a/cluster.go b/cluster.go index baecb8f5dd..9443a9f3af 100644 --- a/cluster.go +++ b/cluster.go @@ -237,7 +237,7 @@ func (c *ClusterClient) resetClients() (retErr error) { return retErr } -func (c *ClusterClient) setSlots(slots []ClusterSlotInfo) { +func (c *ClusterClient) setSlots(slots []ClusterSlot) { c.slotsMx.Lock() seen := make(map[string]struct{}) @@ -248,15 +248,20 @@ func (c *ClusterClient) setSlots(slots []ClusterSlotInfo) { for i := 0; i < hashtag.SlotNumber; i++ { c.slots[i] = c.slots[i][:0] } - for _, info := range slots { - for slot := info.Start; slot <= info.End; slot++ { - c.slots[slot] = info.Addrs + for _, slot := range slots { + var addrs []string + for _, node := range slot.Nodes { + addrs = append(addrs, node.Addr) } - for _, addr := range info.Addrs { - if _, ok := seen[addr]; !ok { - c.addrs = append(c.addrs, addr) - seen[addr] = struct{}{} + for i := slot.Start; i <= slot.End; i++ { + c.slots[i] = addrs + } + + for _, node := range slot.Nodes { + if _, ok := seen[node.Addr]; !ok { + c.addrs = append(c.addrs, node.Addr) + seen[node.Addr] = struct{}{} } } } diff --git a/cluster_client_test.go b/cluster_client_test.go index 9502ba05b8..560c438759 100644 --- a/cluster_client_test.go +++ b/cluster_client_test.go @@ -22,11 +22,11 @@ var _ = Describe("ClusterClient", func() { var subject *ClusterClient var populate = func() { - subject.setSlots([]ClusterSlotInfo{ - {0, 4095, []string{"127.0.0.1:7000", "127.0.0.1:7004"}}, - {12288, 16383, []string{"127.0.0.1:7003", "127.0.0.1:7007"}}, - {4096, 8191, []string{"127.0.0.1:7001", "127.0.0.1:7005"}}, - {8192, 12287, []string{"127.0.0.1:7002", "127.0.0.1:7006"}}, + subject.setSlots([]ClusterSlot{ + {0, 4095, []ClusterNode{{"", "127.0.0.1:7000"}, {"", "127.0.0.1:7004"}}}, + {12288, 16383, []ClusterNode{{"", "127.0.0.1:7003"}, {"", "127.0.0.1:7007"}}}, + {4096, 8191, []ClusterNode{{"", "127.0.0.1:7001"}, {"", "127.0.0.1:7005"}}}, + {8192, 12287, []ClusterNode{{"", "127.0.0.1:7002"}, {"", "127.0.0.1:7006"}}}, }) } diff --git a/cluster_test.go b/cluster_test.go index e7aa9b96d1..24d516bf6c 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -4,7 +4,6 @@ import ( "fmt" "math/rand" "net" - "reflect" "strconv" "strings" "sync" @@ -135,21 +134,12 @@ func startCluster(scenario *clusterScenario) error { if err != nil { return err } - wanted := []redis.ClusterSlotInfo{ - {0, 4999, []string{"127.0.0.1:8220", "127.0.0.1:8223"}}, - {5000, 9999, []string{"127.0.0.1:8221", "127.0.0.1:8224"}}, - {10000, 16383, []string{"127.0.0.1:8222", "127.0.0.1:8225"}}, + wanted := []redis.ClusterSlot{ + {0, 4999, []redis.ClusterNode{{"", "127.0.0.1:8220"}, {"", "127.0.0.1:8223"}}}, + {5000, 9999, []redis.ClusterNode{{"", "127.0.0.1:8221"}, {"", "127.0.0.1:8224"}}}, + {10000, 16383, []redis.ClusterNode{{"", "127.0.0.1:8222"}, {"", "127.0.0.1:8225"}}}, } - loop: - for _, info := range res { - for _, info2 := range wanted { - if reflect.DeepEqual(info, info2) { - continue loop - } - } - return fmt.Errorf("cluster did not reach consistent state (%v)", res) - } - return nil + return assertSlotsEqual(res, wanted) }, 30*time.Second) if err != nil { return err @@ -159,6 +149,34 @@ func startCluster(scenario *clusterScenario) error { return nil } +func assertSlotsEqual(slots, wanted []redis.ClusterSlot) error { +outer_loop: + for _, s2 := range wanted { + for _, s1 := range slots { + if slotEqual(s1, s2) { + continue outer_loop + } + } + return fmt.Errorf("%v not found in %v", s2, slots) + } + return nil +} + +func slotEqual(s1, s2 redis.ClusterSlot) bool { + if s1.Start != s2.Start { + return false + } + if s1.End != s2.End { + return false + } + for i, n1 := range s1.Nodes { + if n1.Addr != s2.Nodes[i].Addr { + return false + } + } + return true +} + func stopCluster(scenario *clusterScenario) error { for _, client := range scenario.clients { if err := client.Close(); err != nil { @@ -223,11 +241,13 @@ var _ = Describe("Cluster", func() { res, err := cluster.primary().ClusterSlots().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(3)) - Expect(res).To(ConsistOf([]redis.ClusterSlotInfo{ - {0, 4999, []string{"127.0.0.1:8220", "127.0.0.1:8223"}}, - {5000, 9999, []string{"127.0.0.1:8221", "127.0.0.1:8224"}}, - {10000, 16383, []string{"127.0.0.1:8222", "127.0.0.1:8225"}}, - })) + + wanted := []redis.ClusterSlot{ + {0, 4999, []redis.ClusterNode{{"", "127.0.0.1:8220"}, {"", "127.0.0.1:8223"}}}, + {5000, 9999, []redis.ClusterNode{{"", "127.0.0.1:8221"}, {"", "127.0.0.1:8224"}}}, + {10000, 16383, []redis.ClusterNode{{"", "127.0.0.1:8222"}, {"", "127.0.0.1:8225"}}}, + } + Expect(assertSlotsEqual(res, wanted)).NotTo(HaveOccurred()) }) It("should CLUSTER NODES", func() { diff --git a/command.go b/command.go index 0c30bcb5f1..b6a9b19573 100644 --- a/command.go +++ b/command.go @@ -25,7 +25,7 @@ var ( _ Cmder = (*StringIntMapCmd)(nil) _ Cmder = (*ZSliceCmd)(nil) _ Cmder = (*ScanCmd)(nil) - _ Cmder = (*ClusterSlotCmd)(nil) + _ Cmder = (*ClusterSlotsCmd)(nil) ) type Cmder interface { @@ -730,48 +730,51 @@ func (cmd *ScanCmd) readReply(cn *pool.Conn) error { //------------------------------------------------------------------------------ -// TODO: rename to ClusterSlot -type ClusterSlotInfo struct { +type ClusterNode struct { + Id string + Addr string +} + +type ClusterSlot struct { Start int End int - Addrs []string + Nodes []ClusterNode } -// TODO: rename to ClusterSlotsCmd -type ClusterSlotCmd struct { +type ClusterSlotsCmd struct { baseCmd - val []ClusterSlotInfo + val []ClusterSlot } -func NewClusterSlotCmd(args ...interface{}) *ClusterSlotCmd { - return &ClusterSlotCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} +func NewClusterSlotsCmd(args ...interface{}) *ClusterSlotsCmd { + return &ClusterSlotsCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} } -func (cmd *ClusterSlotCmd) Val() []ClusterSlotInfo { +func (cmd *ClusterSlotsCmd) Val() []ClusterSlot { return cmd.val } -func (cmd *ClusterSlotCmd) Result() ([]ClusterSlotInfo, error) { +func (cmd *ClusterSlotsCmd) Result() ([]ClusterSlot, error) { return cmd.Val(), cmd.Err() } -func (cmd *ClusterSlotCmd) String() string { +func (cmd *ClusterSlotsCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *ClusterSlotCmd) reset() { +func (cmd *ClusterSlotsCmd) reset() { cmd.val = nil cmd.err = nil } -func (cmd *ClusterSlotCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, clusterSlotInfoSliceParser) +func (cmd *ClusterSlotsCmd) readReply(cn *pool.Conn) error { + v, err := readArrayReply(cn, clusterSlotsParser) if err != nil { cmd.err = err return err } - cmd.val = v.([]ClusterSlotInfo) + cmd.val = v.([]ClusterSlot) return nil } diff --git a/commands.go b/commands.go index dc6dd08f41..41c1301971 100644 --- a/commands.go +++ b/commands.go @@ -1659,8 +1659,8 @@ func (c *commandable) PubSubNumPat() *IntCmd { //------------------------------------------------------------------------------ -func (c *commandable) ClusterSlots() *ClusterSlotCmd { - cmd := NewClusterSlotCmd("CLUSTER", "slots") +func (c *commandable) ClusterSlots() *ClusterSlotsCmd { + cmd := NewClusterSlotsCmd("CLUSTER", "slots") cmd._clusterKeyPos = 0 c.Process(cmd) return cmd diff --git a/parser.go b/parser.go index 0798857952..d96b5e86ef 100644 --- a/parser.go +++ b/parser.go @@ -562,9 +562,9 @@ func zSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return zz, nil } -func clusterSlotInfoSliceParser(cn *pool.Conn, n int64) (interface{}, error) { - infos := make([]ClusterSlotInfo, 0, n) - for i := int64(0); i < n; i++ { +func clusterSlotsParser(cn *pool.Conn, slotNum int64) (interface{}, error) { + slots := make([]ClusterSlot, slotNum) + for slotInd := 0; slotInd < len(slots); slotInd++ { n, err := readArrayHeader(cn) if err != nil { return nil, err @@ -584,14 +584,8 @@ func clusterSlotInfoSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return nil, err } - addrsn := n - 2 - info := ClusterSlotInfo{ - Start: int(start), - End: int(end), - Addrs: make([]string, addrsn), - } - - for i := int64(0); i < addrsn; i++ { + nodes := make([]ClusterNode, n-2) + for nodeInd := 0; nodeInd < len(nodes); nodeInd++ { n, err := readArrayHeader(cn) if err != nil { return nil, err @@ -610,21 +604,24 @@ func clusterSlotInfoSliceParser(cn *pool.Conn, n int64) (interface{}, error) { if err != nil { return nil, err } + nodes[nodeInd].Addr = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) if n == 3 { - // TODO: expose id in ClusterSlotInfo - _, err := readStringReply(cn) + id, err := readStringReply(cn) if err != nil { return nil, err } + nodes[nodeInd].Id = id } - - info.Addrs[i] = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) } - infos = append(infos, info) + slots[slotInd] = ClusterSlot{ + Start: int(start), + End: int(end), + Nodes: nodes, + } } - return infos, nil + return slots, nil } func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { From 51349cd90eb63b7ffc5134a0362589b68bd47389 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 13:15:33 +0300 Subject: [PATCH 0167/1746] Rename ZRangeByScore to ZRange since it is used in ZRangeByLex. --- commands.go | 19 +++++++++---------- commands_test.go | 44 ++++++++++++++++++++++---------------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/commands.go b/commands.go index dc6dd08f41..f511ddd3a6 100644 --- a/commands.go +++ b/commands.go @@ -1169,13 +1169,12 @@ func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd return cmd } -// TODO: Rename to something more generic in v4 -type ZRangeByScore struct { +type ZRangeBy struct { Min, Max string Offset, Count int64 } -func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeByScore, withScores bool) *StringSliceCmd { +func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Min, opt.Max} if withScores { args = append(args, "WITHSCORES") @@ -1193,15 +1192,15 @@ func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeByScore, withScores b return cmd } -func (c *commandable) ZRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { return c.zRangeBy("ZRANGEBYSCORE", key, opt, false) } -func (c *commandable) ZRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { return c.zRangeBy("ZRANGEBYLEX", key, opt, false) } -func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { +func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1263,7 +1262,7 @@ func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSlice return cmd } -func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Max, opt.Min} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1278,15 +1277,15 @@ func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeByScore) *StringSl return cmd } -func (c *commandable) ZRevRangeByScore(key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { return c.zRevRangeBy("ZREVRANGEBYSCORE", key, opt) } -func (c *commandable) ZRevRangeByLex(key string, opt ZRangeByScore) *StringSliceCmd { +func (c *commandable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { return c.zRevRangeBy("ZREVRANGEBYLEX", key, opt) } -func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeByScore) *ZSliceCmd { +func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min, "WITHSCORES"} if opt.Offset != 0 || opt.Count != 0 { args = append( diff --git a/commands_test.go b/commands_test.go index dfa22ca07f..5a2c112a3f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2199,28 +2199,28 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{3, "three"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) - zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "-inf", Max: "+inf", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"one", "two", "three"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "1", Max: "2", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"one", "two"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "(1", Max: "2", }) Expect(zRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRangeByScore.Val()).To(Equal([]string{"two"})) - zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeByScore{ + zRangeByScore = client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "(1", Max: "(2", }) @@ -2235,28 +2235,28 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) zAdd = client.ZAdd("zset", redis.Z{0, "c"}) - zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "-", Max: "+", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b", "c"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "[a", Max: "[b", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"a", "b"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "(a", Max: "[b", }) Expect(zRangeByLex.Err()).NotTo(HaveOccurred()) Expect(zRangeByLex.Val()).To(Equal([]string{"b"})) - zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeByScore{ + zRangeByLex = client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "(a", Max: "(b", }) @@ -2272,28 +2272,28 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{3, "three"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) - val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "-inf", Max: "+inf", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}, {3, "three"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "(1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{2, "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "(1", Max: "(2", }).Result() @@ -2420,17 +2420,17 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScore( - "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"three", "two", "one"})) vals, err = client.ZRevRangeByScore( - "zset", redis.ZRangeByScore{Max: "2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"two"})) vals, err = client.ZRevRangeByScore( - "zset", redis.ZRangeByScore{Max: "(2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{})) }) @@ -2444,17 +2444,17 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByLex( - "zset", redis.ZRangeByScore{Max: "+", Min: "-"}).Result() + "zset", redis.ZRangeBy{Max: "+", Min: "-"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"c", "b", "a"})) vals, err = client.ZRevRangeByLex( - "zset", redis.ZRangeByScore{Max: "[b", Min: "(a"}).Result() + "zset", redis.ZRangeBy{Max: "[b", Min: "(a"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{"b"})) vals, err = client.ZRevRangeByLex( - "zset", redis.ZRangeByScore{Max: "(b", Min: "(a"}).Result() + "zset", redis.ZRangeBy{Max: "(b", Min: "(a"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]string{})) }) @@ -2468,7 +2468,7 @@ var _ = Describe("Commands", func() { Expect(zadd.Err()).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(vals).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) }) @@ -2482,17 +2482,17 @@ var _ = Describe("Commands", func() { Expect(zAdd.Err()).NotTo(HaveOccurred()) val, err := client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "+inf", Min: "-inf"}).Result() + "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) val, err = client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{{2, "two"}})) val, err = client.ZRevRangeByScoreWithScores( - "zset", redis.ZRangeByScore{Max: "(2", Min: "(1"}).Result() + "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal([]redis.Z{})) }) From 38d30a4bab6120c990b676bfeb7d773db5473437 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 13:27:16 +0300 Subject: [PATCH 0168/1746] Use redis.v4 that is in alpha/beta state. --- .travis.yml | 4 ++-- bench_test.go | 2 +- cluster.go | 4 ++-- cluster_test.go | 4 ++-- command.go | 2 +- command_test.go | 2 +- commands_test.go | 2 +- example_test.go | 2 +- export_test.go | 2 +- internal/pool/bench_test.go | 2 +- internal/pool/pool_test.go | 2 +- main_test.go | 4 ++-- options.go | 2 +- parser.go | 2 +- parser_test.go | 2 +- pipeline.go | 2 +- pipeline_test.go | 2 +- pool_test.go | 4 ++-- pubsub.go | 2 +- pubsub_test.go | 2 +- race_test.go | 4 ++-- redis.go | 4 ++-- redis_test.go | 2 +- ring.go | 6 +++--- ring_test.go | 2 +- sentinel.go | 2 +- sentinel_test.go | 2 +- tx.go | 2 +- tx_test.go | 2 +- 29 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0a2f98cb2..afbe1a48c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,5 +19,5 @@ install: - go get github.com/onsi/gomega - go get github.com/garyburd/redigo/redis - mkdir -p $HOME/gopath/src/gopkg.in - - mv $HOME/gopath/src/github.com/go-redis/redis $HOME/gopath/src/gopkg.in/redis.v3 - - cd $HOME/gopath/src/gopkg.in/redis.v3 + - mv $HOME/gopath/src/github.com/go-redis/redis $HOME/gopath/src/gopkg.in/redis.v4 + - cd $HOME/gopath/src/gopkg.in/redis.v4 diff --git a/bench_test.go b/bench_test.go index 6a7edd627b..0c40cf3a45 100644 --- a/bench_test.go +++ b/bench_test.go @@ -7,7 +7,7 @@ import ( redigo "github.com/garyburd/redigo/redis" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) func benchmarkRedisClient(poolSize int) *redis.Client { diff --git a/cluster.go b/cluster.go index 75b37870fb..592db7b4a2 100644 --- a/cluster.go +++ b/cluster.go @@ -6,8 +6,8 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v3/internal/hashtag" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/hashtag" + "gopkg.in/redis.v4/internal/pool" ) // ClusterClient is a Redis Cluster client representing a pool of zero diff --git a/cluster_test.go b/cluster_test.go index 24d516bf6c..3a4ddb5847 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -14,8 +14,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" - "gopkg.in/redis.v3/internal/hashtag" + "gopkg.in/redis.v4" + "gopkg.in/redis.v4/internal/hashtag" ) type clusterScenario struct { diff --git a/command.go b/command.go index b6a9b19573..e10d2fb06c 100644 --- a/command.go +++ b/command.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) var ( diff --git a/command_test.go b/command_test.go index 2b218acf4d..ce7a9601fc 100644 --- a/command_test.go +++ b/command_test.go @@ -4,7 +4,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Command", func() { diff --git a/commands_test.go b/commands_test.go index dfa22ca07f..05728a9d6f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -9,7 +9,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Commands", func() { diff --git a/example_test.go b/example_test.go index 587ef78a40..2e1ea595b9 100644 --- a/example_test.go +++ b/example_test.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var client *redis.Client diff --git a/export_test.go b/export_test.go index f071b540c7..948cfb1623 100644 --- a/export_test.go +++ b/export_test.go @@ -3,7 +3,7 @@ package redis import ( "time" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) func (c *baseClient) Pool() pool.Pooler { diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index ff102aa5f1..878b20260b 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) func benchmarkPoolGetPut(b *testing.B, poolSize int) { diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 9d07cd597d..ceec428f82 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -9,7 +9,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) var _ = Describe("ConnPool", func() { diff --git a/main_test.go b/main_test.go index cf6b181eea..b0277e868c 100644 --- a/main_test.go +++ b/main_test.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) const ( @@ -94,7 +94,7 @@ var _ = AfterSuite(func() { func TestGinkgoSuite(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "gopkg.in/redis.v3") + RunSpecs(t, "gopkg.in/redis.v4") } //------------------------------------------------------------------------------ diff --git a/options.go b/options.go index 6214496c0b..dea2784c42 100644 --- a/options.go +++ b/options.go @@ -4,7 +4,7 @@ import ( "net" "time" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) type Options struct { diff --git a/parser.go b/parser.go index d96b5e86ef..4d8fee13eb 100644 --- a/parser.go +++ b/parser.go @@ -6,7 +6,7 @@ import ( "net" "strconv" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) const ( diff --git a/parser_test.go b/parser_test.go index 77287a7aa8..d8fa360955 100644 --- a/parser_test.go +++ b/parser_test.go @@ -5,7 +5,7 @@ import ( "bytes" "testing" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) func BenchmarkParseReplyStatus(b *testing.B) { diff --git a/pipeline.go b/pipeline.go index 95d389f277..fcddc61823 100644 --- a/pipeline.go +++ b/pipeline.go @@ -4,7 +4,7 @@ import ( "sync" "sync/atomic" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) // Pipeline implements pipelining as described in diff --git a/pipeline_test.go b/pipeline_test.go index cfbed6e531..939c8b8499 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Pipelining", func() { diff --git a/pool_test.go b/pool_test.go index bdf168a9c2..fcaf243ae6 100644 --- a/pool_test.go +++ b/pool_test.go @@ -4,8 +4,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4" + "gopkg.in/redis.v4/internal/pool" ) var _ = Describe("pool", func() { diff --git a/pubsub.go b/pubsub.go index 844dfea211..595c9d1b64 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,7 +5,7 @@ import ( "net" "time" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) var receiveMessageTimeout = 5 * time.Second diff --git a/pubsub_test.go b/pubsub_test.go index 95ab15383f..11b14b4849 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -9,7 +9,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("PubSub", func() { diff --git a/race_test.go b/race_test.go index 968f76e45b..c2d3dff2a2 100644 --- a/race_test.go +++ b/race_test.go @@ -11,8 +11,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4" + "gopkg.in/redis.v4/internal/pool" ) var _ = Describe("races", func() { diff --git a/redis.go b/redis.go index 8899848733..3e37a00e57 100644 --- a/redis.go +++ b/redis.go @@ -1,11 +1,11 @@ -package redis // import "gopkg.in/redis.v3" +package redis // import "gopkg.in/redis.v4" import ( "fmt" "io/ioutil" "log" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) // Deprecated. Use SetLogger instead. diff --git a/redis_test.go b/redis_test.go index 9d4a360fd5..5464a47533 100644 --- a/redis_test.go +++ b/redis_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Client", func() { diff --git a/ring.go b/ring.go index 02be83c6c2..0cd31214af 100644 --- a/ring.go +++ b/ring.go @@ -6,9 +6,9 @@ import ( "sync" "time" - "gopkg.in/redis.v3/internal/consistenthash" - "gopkg.in/redis.v3/internal/hashtag" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/consistenthash" + "gopkg.in/redis.v4/internal/hashtag" + "gopkg.in/redis.v4/internal/pool" ) var ( diff --git a/ring_test.go b/ring_test.go index 06e6c3036d..9e1576b757 100644 --- a/ring_test.go +++ b/ring_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Redis ring", func() { diff --git a/sentinel.go b/sentinel.go index 1f8ec136ed..cf8e838c61 100644 --- a/sentinel.go +++ b/sentinel.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) //------------------------------------------------------------------------------ diff --git a/sentinel_test.go b/sentinel_test.go index 693b957448..165cce66ea 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -4,7 +4,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Sentinel", func() { diff --git a/tx.go b/tx.go index 51c8757424..3273e2ca7f 100644 --- a/tx.go +++ b/tx.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "gopkg.in/redis.v3/internal/pool" + "gopkg.in/redis.v4/internal/pool" ) var errDiscard = errors.New("redis: Discard can be used only inside Exec") diff --git a/tx_test.go b/tx_test.go index 66ef6b0841..5df4b6afe7 100644 --- a/tx_test.go +++ b/tx_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v3" + "gopkg.in/redis.v4" ) var _ = Describe("Tx", func() { From 31abb18d9a79ad3d3de59c719f0aa92c9260a2e5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 14:52:01 +0300 Subject: [PATCH 0169/1746] Move logger to internal package. --- cluster.go | 9 +++++---- commands.go | 6 ++++-- export_test.go | 4 ++-- internal/log.go | 22 ++++++++++++++++++++++ internal/pool/pool.go | 12 +++++------- pubsub.go | 15 +++++++++------ pubsub_test.go | 5 ++--- redis.go | 8 +++----- ring.go | 3 ++- sentinel.go | 23 ++++++++++++----------- tx.go | 3 ++- 11 files changed, 68 insertions(+), 42 deletions(-) create mode 100644 internal/log.go diff --git a/cluster.go b/cluster.go index 592db7b4a2..2fb049122c 100644 --- a/cluster.go +++ b/cluster.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "time" + "gopkg.in/redis.v4/internal" "gopkg.in/redis.v4/internal/hashtag" "gopkg.in/redis.v4/internal/pool" ) @@ -273,13 +274,13 @@ func (c *ClusterClient) reloadSlots() { client, err := c.randomClient() if err != nil { - Logger.Printf("randomClient failed: %s", err) + internal.Logf("randomClient failed: %s", err) return } slots, err := client.ClusterSlots().Result() if err != nil { - Logger.Printf("ClusterSlots failed: %s", err) + internal.Logf("ClusterSlots failed: %s", err) return } c.setSlots(slots) @@ -306,14 +307,14 @@ func (c *ClusterClient) reaper(frequency time.Duration) { for _, client := range c.getClients() { nn, err := client.connPool.(*pool.ConnPool).ReapStaleConns() if err != nil { - Logger.Printf("ReapStaleConns failed: %s", err) + internal.Logf("ReapStaleConns failed: %s", err) } else { n += nn } } s := c.PoolStats() - Logger.Printf( + internal.Logf( "reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)", n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts, ) diff --git a/commands.go b/commands.go index ee2fe8633e..96bd1376aa 100644 --- a/commands.go +++ b/commands.go @@ -4,6 +4,8 @@ import ( "io" "strconv" "time" + + "gopkg.in/redis.v4/internal" ) func formatInt(i int64) string { @@ -31,7 +33,7 @@ func usePrecise(dur time.Duration) bool { func formatMs(dur time.Duration) string { if dur > 0 && dur < time.Millisecond { - Logger.Printf( + internal.Logf( "specified duration is %s, but minimal supported value is %s", dur, time.Millisecond, ) @@ -41,7 +43,7 @@ func formatMs(dur time.Duration) string { func formatSec(dur time.Duration) string { if dur > 0 && dur < time.Second { - Logger.Printf( + internal.Logf( "specified duration is %s, but minimal supported value is %s", dur, time.Second, ) diff --git a/export_test.go b/export_test.go index 948cfb1623..ff37e81746 100644 --- a/export_test.go +++ b/export_test.go @@ -14,6 +14,6 @@ func (c *PubSub) Pool() pool.Pooler { return c.base.connPool } -func SetReceiveMessageTimeout(d time.Duration) { - receiveMessageTimeout = d +func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) { + return c.receiveMessage(timeout) } diff --git a/internal/log.go b/internal/log.go new file mode 100644 index 0000000000..c1cdbf4ed9 --- /dev/null +++ b/internal/log.go @@ -0,0 +1,22 @@ +package internal + +import ( + "fmt" + "io/ioutil" + "log" +) + +var Debug bool + +var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) + +func Debugf(s string, args ...interface{}) { + if !Debug { + return + } + Logger.Output(2, fmt.Sprintf(s, args...)) +} + +func Logf(s string, args ...interface{}) { + Logger.Output(2, fmt.Sprintf(s, args...)) +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 302a2f8b96..330767c989 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -3,17 +3,15 @@ package pool import ( "errors" "fmt" - "io/ioutil" - "log" "net" "sync" "sync/atomic" "time" "gopkg.in/bsm/ratelimit.v1" -) -var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) + "gopkg.in/redis.v4/internal" +) var ( ErrClosed = errors.New("redis: client is closed") @@ -210,7 +208,7 @@ func (p *ConnPool) Put(cn *Conn) error { if cn.Rd.Buffered() != 0 { b, _ := cn.Rd.Peek(cn.Rd.Buffered()) err := fmt.Errorf("connection has unread data: %q", b) - Logger.Print(err) + internal.Logf(err.Error()) return p.Remove(cn, err) } p.freeConnsMu.Lock() @@ -342,11 +340,11 @@ func (p *ConnPool) reaper(frequency time.Duration) { } n, err := p.ReapStaleConns() if err != nil { - Logger.Printf("ReapStaleConns failed: %s", err) + internal.Logf("ReapStaleConns failed: %s", err) continue } s := p.Stats() - Logger.Printf( + internal.Logf( "reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)", n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts, ) diff --git a/pubsub.go b/pubsub.go index 595c9d1b64..aef4a76b08 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,11 +5,10 @@ import ( "net" "time" + "gopkg.in/redis.v4/internal" "gopkg.in/redis.v4/internal/pool" ) -var receiveMessageTimeout = 5 * time.Second - // Posts a message to the given channel. func (c *Client) Publish(channel, message string) *IntCmd { req := NewIntCmd("PUBLISH", channel, message) @@ -241,9 +240,13 @@ func (c *PubSub) Receive() (interface{}, error) { // messages. It automatically reconnects to Redis Server and resubscribes // to channels in case of network errors. func (c *PubSub) ReceiveMessage() (*Message, error) { + return c.receiveMessage(5 * time.Second) +} + +func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { var errNum uint for { - msgi, err := c.ReceiveTimeout(receiveMessageTimeout) + msgi, err := c.ReceiveTimeout(timeout) if err != nil { if !isNetworkError(err) { return nil, err @@ -254,7 +257,7 @@ func (c *PubSub) ReceiveMessage() (*Message, error) { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { err := c.Ping("") if err != nil { - Logger.Printf("PubSub.Ping failed: %s", err) + internal.Logf("PubSub.Ping failed: %s", err) } } } else { @@ -294,12 +297,12 @@ func (c *PubSub) resubscribe() { } if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { - Logger.Printf("Subscribe failed: %s", err) + internal.Logf("Subscribe failed: %s", err) } } if len(c.patterns) > 0 { if err := c.PSubscribe(c.patterns...); err != nil { - Logger.Printf("PSubscribe failed: %s", err) + internal.Logf("PSubscribe failed: %s", err) } } } diff --git a/pubsub_test.go b/pubsub_test.go index 11b14b4849..116099b5a6 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -256,8 +256,7 @@ var _ = Describe("PubSub", func() { }) It("should ReceiveMessage after timeout", func() { - timeout := time.Second - redis.SetReceiveMessageTimeout(timeout) + timeout := 100 * time.Millisecond pubsub, err := client.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) @@ -276,7 +275,7 @@ var _ = Describe("PubSub", func() { Expect(n).To(Equal(int64(1))) }() - msg, err := pubsub.ReceiveMessage() + msg, err := pubsub.ReceiveMessageTimeout(timeout) Expect(err).NotTo(HaveOccurred()) Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) diff --git a/redis.go b/redis.go index 3e37a00e57..be3aa02757 100644 --- a/redis.go +++ b/redis.go @@ -2,18 +2,16 @@ package redis // import "gopkg.in/redis.v4" import ( "fmt" - "io/ioutil" "log" + "gopkg.in/redis.v4/internal" "gopkg.in/redis.v4/internal/pool" ) -// Deprecated. Use SetLogger instead. -var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) +var Logger *log.Logger func SetLogger(logger *log.Logger) { - Logger = logger - pool.Logger = logger + internal.Logger = logger } type baseClient struct { diff --git a/ring.go b/ring.go index 0cd31214af..cac25d4e46 100644 --- a/ring.go +++ b/ring.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "gopkg.in/redis.v4/internal" "gopkg.in/redis.v4/internal/consistenthash" "gopkg.in/redis.v4/internal/hashtag" "gopkg.in/redis.v4/internal/pool" @@ -204,7 +205,7 @@ func (ring *Ring) heartbeat() { for _, shard := range ring.shards { err := shard.Client.Ping().Err() if shard.Vote(err == nil || err == pool.ErrPoolTimeout) { - Logger.Printf("ring shard state changed: %s", shard) + internal.Logf("ring shard state changed: %s", shard) rebalance = true } } diff --git a/sentinel.go b/sentinel.go index cf8e838c61..53c40a9aa2 100644 --- a/sentinel.go +++ b/sentinel.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "gopkg.in/redis.v4/internal" "gopkg.in/redis.v4/internal/pool" ) @@ -165,11 +166,11 @@ func (d *sentinelFailover) MasterAddr() (string, error) { if d.sentinel != nil { addr, err := d.sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - Logger.Printf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + internal.Logf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) d._resetSentinel() } else { addr := net.JoinHostPort(addr[0], addr[1]) - Logger.Printf("sentinel: %q addr is %s", d.masterName, addr) + internal.Logf("sentinel: %q addr is %s", d.masterName, addr) return addr, nil } } @@ -188,7 +189,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { }) masterAddr, err := sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - Logger.Printf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + internal.Logf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) sentinel.Close() continue } @@ -198,7 +199,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { d.setSentinel(sentinel) addr := net.JoinHostPort(masterAddr[0], masterAddr[1]) - Logger.Printf("sentinel: %q addr is %s", d.masterName, addr) + internal.Logf("sentinel: %q addr is %s", d.masterName, addr) return addr, nil } @@ -230,7 +231,7 @@ func (d *sentinelFailover) _resetSentinel() error { func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinels, err := sentinel.Sentinels(d.masterName).Result() if err != nil { - Logger.Printf("sentinel: Sentinels %q failed: %s", d.masterName, err) + internal.Logf("sentinel: Sentinels %q failed: %s", d.masterName, err) return } for _, sentinel := range sentinels { @@ -240,7 +241,7 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { if key == "name" { sentinelAddr := vals[i+1].(string) if !contains(d.sentinelAddrs, sentinelAddr) { - Logger.Printf( + internal.Logf( "sentinel: discovered new %q sentinel: %s", d.masterName, sentinelAddr, ) @@ -268,7 +269,7 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { "sentinel: closing connection to the old master %s", cn.RemoteAddr(), ) - Logger.Print(err) + internal.Logf(err.Error()) d.pool.Remove(cn, err) } else { cnsToPut = append(cnsToPut, cn) @@ -286,7 +287,7 @@ func (d *sentinelFailover) listen(sentinel *sentinelClient) { if pubsub == nil { pubsub = sentinel.PubSub() if err := pubsub.Subscribe("+switch-master"); err != nil { - Logger.Printf("sentinel: Subscribe failed: %s", err) + internal.Logf("sentinel: Subscribe failed: %s", err) d.resetSentinel() return } @@ -294,7 +295,7 @@ func (d *sentinelFailover) listen(sentinel *sentinelClient) { msg, err := pubsub.ReceiveMessage() if err != nil { - Logger.Printf("sentinel: ReceiveMessage failed: %s", err) + internal.Logf("sentinel: ReceiveMessage failed: %s", err) pubsub.Close() d.resetSentinel() return @@ -304,12 +305,12 @@ func (d *sentinelFailover) listen(sentinel *sentinelClient) { case "+switch-master": parts := strings.Split(msg.Payload, " ") if parts[0] != d.masterName { - Logger.Printf("sentinel: ignore new %s addr", parts[0]) + internal.Logf("sentinel: ignore new %s addr", parts[0]) continue } addr := net.JoinHostPort(parts[3], parts[4]) - Logger.Printf( + internal.Logf( "sentinel: new %q addr is %s", d.masterName, addr, ) diff --git a/tx.go b/tx.go index 3273e2ca7f..5274af4e47 100644 --- a/tx.go +++ b/tx.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "gopkg.in/redis.v4/internal" "gopkg.in/redis.v4/internal/pool" ) @@ -58,7 +59,7 @@ func (tx *Tx) process(cmd Cmder) { func (tx *Tx) Close() error { tx.closed = true if err := tx.Unwatch().Err(); err != nil { - Logger.Printf("Unwatch failed: %s", err) + internal.Logf("Unwatch failed: %s", err) } return tx.base.Close() } From 818785577eef1a91d480cfc660864d1b285b2a64 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 12 Apr 2016 19:41:56 +0300 Subject: [PATCH 0170/1746] Convert bytes to string in Cmd. --- command.go | 7 ++++++- command_test.go | 8 ++++---- redis_test.go | 8 ++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/command.go b/command.go index e10d2fb06c..b69f000f07 100644 --- a/command.go +++ b/command.go @@ -165,7 +165,12 @@ func (cmd *Cmd) readReply(cn *pool.Conn) error { cmd.err = err return cmd.err } - cmd.val = val + if b, ok := val.([]byte); ok { + // Bytes must be copied, because underlying memory is reused. + cmd.val = string(b) + } else { + cmd.val = val + } return nil } diff --git a/command_test.go b/command_test.go index ce7a9601fc..77391e466b 100644 --- a/command_test.go +++ b/command_test.go @@ -7,7 +7,7 @@ import ( "gopkg.in/redis.v4" ) -var _ = Describe("Command", func() { +var _ = Describe("Cmd", func() { var client *redis.Client BeforeEach(func() { @@ -19,7 +19,7 @@ var _ = Describe("Command", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("should implement Stringer", func() { + It("implements Stringer", func() { set := client.Set("foo", "bar", 0) Expect(set.String()).To(Equal("SET foo bar: OK")) @@ -27,7 +27,7 @@ var _ = Describe("Command", func() { Expect(get.String()).To(Equal("GET foo: bar")) }) - It("should have correct val/err states", func() { + It("has val/err", func() { set := client.Set("key", "hello", 0) Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) @@ -40,7 +40,7 @@ var _ = Describe("Command", func() { Expect(set.Val()).To(Equal("OK")) }) - It("should convert strings via helpers", func() { + It("has helpers", func() { set := client.Set("key", "10", 0) Expect(set.Err()).NotTo(HaveOccurred()) diff --git a/redis_test.go b/redis_test.go index 5464a47533..a11b103400 100644 --- a/redis_test.go +++ b/redis_test.go @@ -128,11 +128,15 @@ var _ = Describe("Client", func() { Expect(db2.Close()).NotTo(HaveOccurred()) }) - It("should process custom commands", func() { + It("processes custom commands", func() { cmd := redis.NewCmd("PING") client.Process(cmd) + + // Flush buffers. + Expect(client.Echo("hello").Err()).NotTo(HaveOccurred()) + Expect(cmd.Err()).NotTo(HaveOccurred()) - Expect(cmd.Val()).To(Equal([]byte("PONG"))) + Expect(cmd.Val()).To(Equal("PONG")) }) It("should retry command on network error", func() { From 7456a0e473166133ad86b50e8296efdcc45a4e81 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 13 Apr 2016 09:52:47 +0100 Subject: [PATCH 0171/1746] Add scan iterator. --- command.go | 19 ++++++----- commands.go | 28 ++++++++++----- example_test.go | 20 +++++++++++ iterator.go | 75 ++++++++++++++++++++++++++++++++++++++++ iterator_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ redis.go | 3 +- 6 files changed, 216 insertions(+), 18 deletions(-) create mode 100644 iterator.go create mode 100644 iterator_test.go diff --git a/command.go b/command.go index e10d2fb06c..8596c71626 100644 --- a/command.go +++ b/command.go @@ -689,41 +689,42 @@ type ScanCmd struct { baseCmd cursor int64 - keys []string + page []string } func NewScanCmd(args ...interface{}) *ScanCmd { - return &ScanCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + return &ScanCmd{ + baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}, + } } func (cmd *ScanCmd) reset() { cmd.cursor = 0 - cmd.keys = nil + cmd.page = nil cmd.err = nil } -// TODO: cursor should be string to match redis type // TODO: swap return values func (cmd *ScanCmd) Val() (int64, []string) { - return cmd.cursor, cmd.keys + return cmd.cursor, cmd.page } func (cmd *ScanCmd) Result() (int64, []string, error) { - return cmd.cursor, cmd.keys, cmd.err + return cmd.cursor, cmd.page, cmd.err } func (cmd *ScanCmd) String() string { - return cmdString(cmd, cmd.keys) + return cmdString(cmd, cmd.page) } func (cmd *ScanCmd) readReply(cn *pool.Conn) error { - keys, cursor, err := readScanReply(cn) + page, cursor, err := readScanReply(cn) if err != nil { cmd.err = err return cmd.err } - cmd.keys = keys + cmd.page = page cmd.cursor = cursor return nil } diff --git a/commands.go b/commands.go index 96bd1376aa..8ff6dc582d 100644 --- a/commands.go +++ b/commands.go @@ -318,7 +318,7 @@ func (c *commandable) Type(key string) *StatusCmd { return cmd } -func (c *commandable) Scan(cursor int64, match string, count int64) *ScanCmd { +func (c *commandable) Scan(cursor int64, match string, count int64) Scanner { args := []interface{}{"SCAN", cursor} if match != "" { args = append(args, "MATCH", match) @@ -328,10 +328,13 @@ func (c *commandable) Scan(cursor int64, match string, count int64) *ScanCmd { } cmd := NewScanCmd(args...) c.Process(cmd) - return cmd + return Scanner{ + client: c, + ScanCmd: cmd, + } } -func (c *commandable) SScan(key string, cursor int64, match string, count int64) *ScanCmd { +func (c *commandable) SScan(key string, cursor int64, match string, count int64) Scanner { args := []interface{}{"SSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) @@ -341,10 +344,13 @@ func (c *commandable) SScan(key string, cursor int64, match string, count int64) } cmd := NewScanCmd(args...) c.Process(cmd) - return cmd + return Scanner{ + client: c, + ScanCmd: cmd, + } } -func (c *commandable) HScan(key string, cursor int64, match string, count int64) *ScanCmd { +func (c *commandable) HScan(key string, cursor int64, match string, count int64) Scanner { args := []interface{}{"HSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) @@ -354,10 +360,13 @@ func (c *commandable) HScan(key string, cursor int64, match string, count int64) } cmd := NewScanCmd(args...) c.Process(cmd) - return cmd + return Scanner{ + client: c, + ScanCmd: cmd, + } } -func (c *commandable) ZScan(key string, cursor int64, match string, count int64) *ScanCmd { +func (c *commandable) ZScan(key string, cursor int64, match string, count int64) Scanner { args := []interface{}{"ZSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) @@ -367,7 +376,10 @@ func (c *commandable) ZScan(key string, cursor int64, match string, count int64) } cmd := NewScanCmd(args...) c.Process(cmd) - return cmd + return Scanner{ + client: c, + ScanCmd: cmd, + } } //------------------------------------------------------------------------------ diff --git a/example_test.go b/example_test.go index 2e1ea595b9..abac6af65c 100644 --- a/example_test.go +++ b/example_test.go @@ -314,3 +314,23 @@ func Example_customCommand() { fmt.Printf("%q %s", v, err) // Output: "" redis: nil } + +func ExampleScanIterator() { + iter := client.Scan(0, "", 0).Iterator() + for iter.Next() { + fmt.Println(iter.Val()) + } + if err := iter.Err(); err != nil { + panic(err) + } +} + +func ExampleScanCmd_Iterator() { + iter := client.Scan(0, "", 0).Iterator() + for iter.Next() { + fmt.Println(iter.Val()) + } + if err := iter.Err(); err != nil { + panic(err) + } +} diff --git a/iterator.go b/iterator.go new file mode 100644 index 0000000000..c57c5afdef --- /dev/null +++ b/iterator.go @@ -0,0 +1,75 @@ +package redis + +import "sync" + +type Scanner struct { + client *commandable + *ScanCmd +} + +// Iterator creates a new ScanIterator. +func (s Scanner) Iterator() *ScanIterator { + return &ScanIterator{ + Scanner: s, + } +} + +// ScanIterator is used to incrementally iterate over a collection of elements. +// It's safe for concurrent use by multiple goroutines. +type ScanIterator struct { + mu sync.Mutex // protects Scanner and pos + Scanner + pos int +} + +// Err returns the last iterator error, if any. +func (it *ScanIterator) Err() error { + it.mu.Lock() + err := it.ScanCmd.Err() + it.mu.Unlock() + return err +} + +// Next advances the cursor and returns true if more values can be read. +func (it *ScanIterator) Next() bool { + it.mu.Lock() + defer it.mu.Unlock() + + // Instantly return on errors. + if it.ScanCmd.Err() != nil { + return false + } + + // Advance cursor, check if we are still within range. + if it.pos < len(it.ScanCmd.page) { + it.pos++ + return true + } + + // Return if there is more data to fetch. + if it.ScanCmd.cursor == 0 { + return false + } + + // Fetch next page. + it.ScanCmd._args[1] = it.ScanCmd.cursor + it.ScanCmd.reset() + it.client.Process(it.ScanCmd) + if it.ScanCmd.Err() != nil { + return false + } + + it.pos = 1 + return len(it.ScanCmd.page) > 0 +} + +// Val returns the key/field at the current cursor position. +func (it *ScanIterator) Val() string { + var v string + it.mu.Lock() + if it.ScanCmd.Err() == nil && it.pos > 0 && it.pos <= len(it.ScanCmd.page) { + v = it.ScanCmd.page[it.pos-1] + } + it.mu.Unlock() + return v +} diff --git a/iterator_test.go b/iterator_test.go new file mode 100644 index 0000000000..327feeb5bd --- /dev/null +++ b/iterator_test.go @@ -0,0 +1,89 @@ +package redis_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v4" +) + +var _ = Describe("ScanIterator", func() { + var client *redis.Client + + var seed = func(n int) error { + pipe := client.Pipeline() + for i := 1; i <= n; i++ { + pipe.Set(fmt.Sprintf("K%02d", i), "x", 0).Err() + } + _, err := pipe.Exec() + return err + } + + BeforeEach(func() { + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + It("should scan across empty DBs", func() { + iter := client.Scan(0, "", 10).Iterator() + Expect(iter.Next()).To(BeFalse()) + Expect(iter.Err()).NotTo(HaveOccurred()) + }) + + It("should scan across one page", func() { + Expect(seed(7)).NotTo(HaveOccurred()) + + var vals []string + iter := client.Scan(0, "", 0).Iterator() + for iter.Next() { + vals = append(vals, iter.Val()) + } + Expect(iter.Err()).NotTo(HaveOccurred()) + Expect(vals).To(ConsistOf([]string{"K01", "K02", "K03", "K04", "K05", "K06", "K07"})) + }) + + It("should scan across multiple pages", func() { + Expect(seed(71)).NotTo(HaveOccurred()) + + var vals []string + iter := client.Scan(0, "", 10).Iterator() + for iter.Next() { + vals = append(vals, iter.Val()) + } + Expect(iter.Err()).NotTo(HaveOccurred()) + Expect(vals).To(HaveLen(71)) + Expect(vals).To(ContainElement("K01")) + Expect(vals).To(ContainElement("K71")) + }) + + It("should scan to page borders", func() { + Expect(seed(20)).NotTo(HaveOccurred()) + + var vals []string + iter := client.Scan(0, "", 10).Iterator() + for iter.Next() { + vals = append(vals, iter.Val()) + } + Expect(iter.Err()).NotTo(HaveOccurred()) + Expect(vals).To(HaveLen(20)) + }) + + It("should scan with match", func() { + Expect(seed(33)).NotTo(HaveOccurred()) + + var vals []string + iter := client.Scan(0, "K*2*", 10).Iterator() + for iter.Next() { + vals = append(vals, iter.Val()) + } + Expect(iter.Err()).NotTo(HaveOccurred()) + Expect(vals).To(HaveLen(13)) + }) + +}) diff --git a/redis.go b/redis.go index be3aa02757..9222db9105 100644 --- a/redis.go +++ b/redis.go @@ -146,12 +146,13 @@ type Client struct { func newClient(opt *Options, pool pool.Pooler) *Client { base := baseClient{opt: opt, connPool: pool} - return &Client{ + client := &Client{ baseClient: base, commandable: commandable{ process: base.process, }, } + return client } // NewClient returns a client to the Redis Server specified by Options. From 38be24b025c48fe605589473683100623f087302 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 9 Apr 2016 13:07:42 +0300 Subject: [PATCH 0172/1746] Scan: swap return values and change cursor type. --- command.go | 12 +++++------- commands.go | 8 ++++---- commands_test.go | 24 ++++++++++++------------ example_test.go | 4 ++-- parser.go | 4 ++-- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/command.go b/command.go index ed597c6164..248625e9fe 100644 --- a/command.go +++ b/command.go @@ -693,8 +693,8 @@ func (cmd *ZSliceCmd) readReply(cn *pool.Conn) error { type ScanCmd struct { baseCmd - cursor int64 page []string + cursor uint64 } func NewScanCmd(args ...interface{}) *ScanCmd { @@ -709,14 +709,12 @@ func (cmd *ScanCmd) reset() { cmd.err = nil } -// TODO: swap return values - -func (cmd *ScanCmd) Val() (int64, []string) { - return cmd.cursor, cmd.page +func (cmd *ScanCmd) Val() (keys []string, cursor uint64) { + return cmd.page, cmd.cursor } -func (cmd *ScanCmd) Result() (int64, []string, error) { - return cmd.cursor, cmd.page, cmd.err +func (cmd *ScanCmd) Result() (keys []string, cursor uint64, err error) { + return cmd.page, cmd.cursor, cmd.err } func (cmd *ScanCmd) String() string { diff --git a/commands.go b/commands.go index 8ff6dc582d..48a59f6e4c 100644 --- a/commands.go +++ b/commands.go @@ -318,7 +318,7 @@ func (c *commandable) Type(key string) *StatusCmd { return cmd } -func (c *commandable) Scan(cursor int64, match string, count int64) Scanner { +func (c *commandable) Scan(cursor uint64, match string, count int64) Scanner { args := []interface{}{"SCAN", cursor} if match != "" { args = append(args, "MATCH", match) @@ -334,7 +334,7 @@ func (c *commandable) Scan(cursor int64, match string, count int64) Scanner { } } -func (c *commandable) SScan(key string, cursor int64, match string, count int64) Scanner { +func (c *commandable) SScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"SSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) @@ -350,7 +350,7 @@ func (c *commandable) SScan(key string, cursor int64, match string, count int64) } } -func (c *commandable) HScan(key string, cursor int64, match string, count int64) Scanner { +func (c *commandable) HScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"HSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) @@ -366,7 +366,7 @@ func (c *commandable) HScan(key string, cursor int64, match string, count int64) } } -func (c *commandable) ZScan(key string, cursor int64, match string, count int64) Scanner { +func (c *commandable) ZScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"ZSCAN", key, cursor} if match != "" { args = append(args, "MATCH", match) diff --git a/commands_test.go b/commands_test.go index d1b00d9ca7..8f27974429 100644 --- a/commands_test.go +++ b/commands_test.go @@ -590,10 +590,10 @@ var _ = Describe("Commands", func() { Expect(set.Err()).NotTo(HaveOccurred()) } - cursor, keys, err := client.Scan(0, "", 0).Result() + keys, cursor, err := client.Scan(0, "", 0).Result() Expect(err).NotTo(HaveOccurred()) - Expect(cursor > 0).To(Equal(true)) - Expect(len(keys) > 0).To(Equal(true)) + Expect(keys).NotTo(BeEmpty()) + Expect(cursor).NotTo(BeZero()) }) It("should SScan", func() { @@ -602,10 +602,10 @@ var _ = Describe("Commands", func() { Expect(sadd.Err()).NotTo(HaveOccurred()) } - cursor, keys, err := client.SScan("myset", 0, "", 0).Result() + keys, cursor, err := client.SScan("myset", 0, "", 0).Result() Expect(err).NotTo(HaveOccurred()) - Expect(cursor > 0).To(Equal(true)) - Expect(len(keys) > 0).To(Equal(true)) + Expect(keys).NotTo(BeEmpty()) + Expect(cursor).NotTo(BeZero()) }) It("should HScan", func() { @@ -614,10 +614,10 @@ var _ = Describe("Commands", func() { Expect(sadd.Err()).NotTo(HaveOccurred()) } - cursor, keys, err := client.HScan("myhash", 0, "", 0).Result() + keys, cursor, err := client.HScan("myhash", 0, "", 0).Result() Expect(err).NotTo(HaveOccurred()) - Expect(cursor > 0).To(Equal(true)) - Expect(len(keys) > 0).To(Equal(true)) + Expect(keys).NotTo(BeEmpty()) + Expect(cursor).NotTo(BeZero()) }) It("should ZScan", func() { @@ -626,10 +626,10 @@ var _ = Describe("Commands", func() { Expect(sadd.Err()).NotTo(HaveOccurred()) } - cursor, keys, err := client.ZScan("myset", 0, "", 0).Result() + keys, cursor, err := client.ZScan("myset", 0, "", 0).Result() Expect(err).NotTo(HaveOccurred()) - Expect(cursor > 0).To(Equal(true)) - Expect(len(keys) > 0).To(Equal(true)) + Expect(keys).NotTo(BeEmpty()) + Expect(cursor).NotTo(BeZero()) }) }) diff --git a/example_test.go b/example_test.go index abac6af65c..4ab22f1053 100644 --- a/example_test.go +++ b/example_test.go @@ -138,12 +138,12 @@ func ExampleClient_Scan() { } } - var cursor int64 + var cursor uint64 var n int for { var keys []string var err error - cursor, keys, err = client.Scan(cursor, "", 10).Result() + keys, cursor, err = client.Scan(cursor, "", 10).Result() if err != nil { panic(err) } diff --git a/parser.go b/parser.go index 4d8fee13eb..7a7b6f3a63 100644 --- a/parser.go +++ b/parser.go @@ -399,7 +399,7 @@ func readReply(cn *pool.Conn, p multiBulkParser) (interface{}, error) { return nil, fmt.Errorf("redis: can't parse %.100q", line) } -func readScanReply(cn *pool.Conn) ([]string, int64, error) { +func readScanReply(cn *pool.Conn) ([]string, uint64, error) { n, err := readArrayHeader(cn) if err != nil { return nil, 0, err @@ -413,7 +413,7 @@ func readScanReply(cn *pool.Conn) ([]string, int64, error) { return nil, 0, err } - cursor, err := strconv.ParseInt(bytesToString(b), 10, 64) + cursor, err := strconv.ParseUint(bytesToString(b), 10, 64) if err != nil { return nil, 0, err } From 9cbb0c42df59407f84e40494f26cc22596e3ca3d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 2 May 2016 14:50:59 +0300 Subject: [PATCH 0173/1746] Change HGetAll and HMSet to return/accept map[string]string. --- commands.go | 21 +++++++---------- commands_test.go | 60 +++++++++++++++++++++--------------------------- 2 files changed, 34 insertions(+), 47 deletions(-) diff --git a/commands.go b/commands.go index 48a59f6e4c..e6c7e8ad25 100644 --- a/commands.go +++ b/commands.go @@ -646,13 +646,7 @@ func (c *commandable) HGet(key, field string) *StringCmd { return cmd } -func (c *commandable) HGetAll(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("HGETALL", key) - c.Process(cmd) - return cmd -} - -func (c *commandable) HGetAllMap(key string) *StringStringMapCmd { +func (c *commandable) HGetAll(key string) *StringStringMapCmd { cmd := NewStringStringMapCmd("HGETALL", key) c.Process(cmd) return cmd @@ -694,14 +688,15 @@ func (c *commandable) HMGet(key string, fields ...string) *SliceCmd { return cmd } -func (c *commandable) HMSet(key, field, value string, pairs ...string) *StatusCmd { - args := make([]interface{}, 4+len(pairs)) +func (c *commandable) HMSet(key string, fields map[string]string) *StatusCmd { + args := make([]interface{}, 2+len(fields)*2) args[0] = "HMSET" args[1] = key - args[2] = field - args[3] = value - for i, pair := range pairs { - args[4+i] = pair + i := 2 + for k, v := range fields { + args[i] = k + args[i+1] = v + i += 2 } cmd := NewStatusCmd(args...) c.Process(cmd) diff --git a/commands_test.go b/commands_test.go index 8f27974429..d4f25ff863 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1085,25 +1085,14 @@ var _ = Describe("Commands", func() { }) It("should HGetAll", func() { - hSet := client.HSet("hash", "key1", "hello1") - Expect(hSet.Err()).NotTo(HaveOccurred()) - hSet = client.HSet("hash", "key2", "hello2") - Expect(hSet.Err()).NotTo(HaveOccurred()) - - hGetAll := client.HGetAll("hash") - Expect(hGetAll.Err()).NotTo(HaveOccurred()) - Expect(hGetAll.Val()).To(Equal([]string{"key1", "hello1", "key2", "hello2"})) - }) - - It("should HGetAllMap", func() { - hSet := client.HSet("hash", "key1", "hello1") - Expect(hSet.Err()).NotTo(HaveOccurred()) - hSet = client.HSet("hash", "key2", "hello2") - Expect(hSet.Err()).NotTo(HaveOccurred()) + err := client.HSet("hash", "key1", "hello1").Err() + Expect(err).NotTo(HaveOccurred()) + err = client.HSet("hash", "key2", "hello2").Err() + Expect(err).NotTo(HaveOccurred()) - hGetAll := client.HGetAllMap("hash") - Expect(hGetAll.Err()).NotTo(HaveOccurred()) - Expect(hGetAll.Val()).To(Equal(map[string]string{"key1": "hello1", "key2": "hello2"})) + m, err := client.HGetAll("hash").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(m).To(Equal(map[string]string{"key1": "hello1", "key2": "hello2"})) }) It("should HIncrBy", func() { @@ -1168,28 +1157,31 @@ var _ = Describe("Commands", func() { }) It("should HMGet", func() { - hSet := client.HSet("hash", "key1", "hello1") - Expect(hSet.Err()).NotTo(HaveOccurred()) - hSet = client.HSet("hash", "key2", "hello2") - Expect(hSet.Err()).NotTo(HaveOccurred()) + err := client.HSet("hash", "key1", "hello1").Err() + Expect(err).NotTo(HaveOccurred()) + err = client.HSet("hash", "key2", "hello2").Err() + Expect(err).NotTo(HaveOccurred()) - hMGet := client.HMGet("hash", "key1", "key2", "_") - Expect(hMGet.Err()).NotTo(HaveOccurred()) - Expect(hMGet.Val()).To(Equal([]interface{}{"hello1", "hello2", nil})) + vals, err := client.HMGet("hash", "key1", "key2", "_").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]interface{}{"hello1", "hello2", nil})) }) It("should HMSet", func() { - hMSet := client.HMSet("hash", "key1", "hello1", "key2", "hello2") - Expect(hMSet.Err()).NotTo(HaveOccurred()) - Expect(hMSet.Val()).To(Equal("OK")) + ok, err := client.HMSet("hash", map[string]string{ + "key1": "hello1", + "key2": "hello2", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(Equal("OK")) - hGet := client.HGet("hash", "key1") - Expect(hGet.Err()).NotTo(HaveOccurred()) - Expect(hGet.Val()).To(Equal("hello1")) + v, err := client.HGet("hash", "key1").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(v).To(Equal("hello1")) - hGet = client.HGet("hash", "key2") - Expect(hGet.Err()).NotTo(HaveOccurred()) - Expect(hGet.Val()).To(Equal("hello2")) + v, err = client.HGet("hash", "key2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(v).To(Equal("hello2")) }) It("should HSet", func() { From 092698ecd376299b99184d4636abffb78e55d482 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 2 May 2016 15:54:15 +0300 Subject: [PATCH 0174/1746] Tweak transaction API. --- cluster.go | 8 +-- cluster_test.go | 25 ++++---- example_test.go | 23 ++++---- pool_test.go | 17 +++--- race_test.go | 30 +++++----- redis_test.go | 24 +++----- tx.go | 27 +++++---- tx_test.go | 149 +++++++++++++++++++++++------------------------- 8 files changed, 140 insertions(+), 163 deletions(-) diff --git a/cluster.go b/cluster.go index 2fb049122c..5d56a4fc24 100644 --- a/cluster.go +++ b/cluster.go @@ -59,15 +59,13 @@ func (c *ClusterClient) getClients() map[string]*Client { return clients } -// Watch creates new transaction and marks the keys to be watched -// for conditional execution of a transaction. -func (c *ClusterClient) Watch(keys ...string) (*Tx, error) { +func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { addr := c.slotMasterAddr(hashtag.Slot(keys[0])) client, err := c.getClient(addr) if err != nil { - return nil, err + return err } - return client.Watch(keys...) + return client.Watch(fn, keys...) } // PoolStats returns accumulated connection pool stats. diff --git a/cluster_test.go b/cluster_test.go index 3a4ddb5847..69ffa7a7bc 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -383,21 +383,18 @@ var _ = Describe("Cluster", func() { // Transactionally increments key using GET and SET commands. incr = func(key string) error { - tx, err := client.Watch(key) - if err != nil { + err := client.Watch(func(tx *redis.Tx) error { + n, err := tx.Get(key).Int64() + if err != nil && err != redis.Nil { + return err + } + + _, err = tx.MultiExec(func() error { + tx.Set(key, strconv.FormatInt(n+1, 10), 0) + return nil + }) return err - } - defer tx.Close() - - n, err := tx.Get(key).Int64() - if err != nil && err != redis.Nil { - return err - } - - _, err = tx.Exec(func() error { - tx.Set(key, strconv.FormatInt(n+1, 10), 0) - return nil - }) + }, key) if err == redis.TxFailedErr { return incr(key) } diff --git a/example_test.go b/example_test.go index 4ab22f1053..71d06c4cc1 100644 --- a/example_test.go +++ b/example_test.go @@ -184,21 +184,18 @@ func ExampleClient_Watch() { // Transactionally increments key using GET and SET commands. incr = func(key string) error { - tx, err := client.Watch(key) - if err != nil { - return err - } - defer tx.Close() + err := client.Watch(func(tx *redis.Tx) error { + n, err := tx.Get(key).Int64() + if err != nil && err != redis.Nil { + return err + } - n, err := tx.Get(key).Int64() - if err != nil && err != redis.Nil { + _, err = tx.MultiExec(func() error { + tx.Set(key, strconv.FormatInt(n+1, 10), 0) + return nil + }) return err - } - - _, err = tx.Exec(func() error { - tx.Set(key, strconv.FormatInt(n+1, 10), 0) - return nil - }) + }, key) if err == redis.TxFailedErr { return incr(key) } diff --git a/pool_test.go b/pool_test.go index fcaf243ae6..31bf9687ab 100644 --- a/pool_test.go +++ b/pool_test.go @@ -37,18 +37,19 @@ var _ = Describe("pool", func() { perform(1000, func(id int) { var ping *redis.StatusCmd - tx, err := client.Watch() - Expect(err).NotTo(HaveOccurred()) - - cmds, err := tx.Exec(func() error { - ping = tx.Ping() - return nil + err := client.Watch(func(tx *redis.Tx) error { + cmds, err := tx.MultiExec(func() error { + ping = tx.Ping() + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(1)) + return err }) Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(1)) + Expect(ping.Err()).NotTo(HaveOccurred()) Expect(ping.Val()).To(Equal("PONG")) - Expect(tx.Close()).NotTo(HaveOccurred()) }) pool := client.Pool() diff --git a/race_test.go b/race_test.go index c2d3dff2a2..f654f87ab2 100644 --- a/race_test.go +++ b/race_test.go @@ -215,30 +215,26 @@ var _ = Describe("races", func() { perform(C, func(id int) { for i := 0; i < N; i++ { - tx, err := client.Watch("key") - Expect(err).NotTo(HaveOccurred()) - - val, err := tx.Get("key").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(val).NotTo(Equal(redis.Nil)) + err := client.Watch(func(tx *redis.Tx) error { + val, err := tx.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).NotTo(Equal(redis.Nil)) - num, err := strconv.ParseInt(val, 10, 64) - Expect(err).NotTo(HaveOccurred()) + num, err := strconv.ParseInt(val, 10, 64) + Expect(err).NotTo(HaveOccurred()) - cmds, err := tx.Exec(func() error { - tx.Set("key", strconv.FormatInt(num+1, 10), 0) - return nil - }) + cmds, err := tx.MultiExec(func() error { + tx.Set("key", strconv.FormatInt(num+1, 10), 0) + return nil + }) + Expect(cmds).To(HaveLen(1)) + return err + }, "key") if err == redis.TxFailedErr { i-- continue } Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(1)) - Expect(cmds[0].Err()).NotTo(HaveOccurred()) - - err = tx.Close() - Expect(err).NotTo(HaveOccurred()) } }) diff --git a/redis_test.go b/redis_test.go index a11b103400..bbb00e0a31 100644 --- a/redis_test.go +++ b/redis_test.go @@ -65,16 +65,15 @@ var _ = Describe("Client", func() { Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) - It("should close multi without closing the client", func() { - tx, err := client.Watch() - Expect(err).NotTo(HaveOccurred()) - Expect(tx.Close()).NotTo(HaveOccurred()) - - _, err = tx.Exec(func() error { - tx.Ping() - return nil + It("should close Tx without closing the client", func() { + err := client.Watch(func(tx *redis.Tx) error { + _, err := tx.MultiExec(func() error { + tx.Ping() + return nil + }) + return err }) - Expect(err).To(MatchError("redis: client is closed")) + Expect(err).NotTo(HaveOccurred()) Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) @@ -96,13 +95,6 @@ var _ = Describe("Client", func() { Expect(pubsub.Close()).NotTo(HaveOccurred()) }) - It("should close multi when client is closed", func() { - tx, err := client.Watch() - Expect(err).NotTo(HaveOccurred()) - Expect(client.Close()).NotTo(HaveOccurred()) - Expect(tx.Close()).NotTo(HaveOccurred()) - }) - It("should close pipeline when client is closed", func() { pipeline := client.Pipeline() Expect(client.Close()).NotTo(HaveOccurred()) diff --git a/tx.go b/tx.go index 5274af4e47..34e828bc57 100644 --- a/tx.go +++ b/tx.go @@ -34,17 +34,19 @@ func (c *Client) newTx() *Tx { return tx } -// Watch creates new transaction and marks the keys to be watched -// for conditional execution of a transaction. -func (c *Client) Watch(keys ...string) (*Tx, error) { +func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { tx := c.newTx() if len(keys) > 0 { if err := tx.Watch(keys...).Err(); err != nil { - tx.Close() - return nil, err + tx.close() + return err } } - return tx, nil + retErr := fn(tx) + if err := tx.close(); err != nil && retErr == nil { + retErr = err + } + return retErr } func (tx *Tx) process(cmd Cmder) { @@ -55,8 +57,11 @@ func (tx *Tx) process(cmd Cmder) { } } -// Close closes the transaction, releasing any open resources. -func (tx *Tx) Close() error { +// close closes the transaction, releasing any open resources. +func (tx *Tx) close() error { + if tx.closed { + return nil + } tx.closed = true if err := tx.Unwatch().Err(); err != nil { internal.Logf("Unwatch failed: %s", err) @@ -98,7 +103,7 @@ func (tx *Tx) Discard() error { return nil } -// Exec executes all previously queued commands in a transaction +// MultiExec executes all previously queued commands in a transaction // and restores the connection state to normal. // // When using WATCH, EXEC will execute commands only if the watched keys @@ -107,13 +112,13 @@ func (tx *Tx) Discard() error { // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. -func (tx *Tx) Exec(f func() error) ([]Cmder, error) { +func (tx *Tx) MultiExec(fn func() error) ([]Cmder, error) { if tx.closed { return nil, pool.ErrClosed } tx.cmds = []Cmder{NewStatusCmd("MULTI")} - if err := f(); err != nil { + if err := fn(); err != nil { return nil, err } tx.cmds = append(tx.cmds, NewSliceCmd("EXEC")) diff --git a/tx_test.go b/tx_test.go index 5df4b6afe7..7ff84dd09e 100644 --- a/tx_test.go +++ b/tx_test.go @@ -27,21 +27,18 @@ var _ = Describe("Tx", func() { // Transactionally increments key using GET and SET commands. incr = func(key string) error { - tx, err := client.Watch(key) - if err != nil { + err := client.Watch(func(tx *redis.Tx) error { + n, err := tx.Get(key).Int64() + if err != nil && err != redis.Nil { + return err + } + + _, err = tx.MultiExec(func() error { + tx.Set(key, strconv.FormatInt(n+1, 10), 0) + return nil + }) return err - } - defer tx.Close() - - n, err := tx.Get(key).Int64() - if err != nil && err != redis.Nil { - return err - } - - _, err = tx.Exec(func() error { - tx.Set(key, strconv.FormatInt(n+1, 10), 0) - return nil - }) + }, key) if err == redis.TxFailedErr { return incr(key) } @@ -67,20 +64,18 @@ var _ = Describe("Tx", func() { }) It("should discard", func() { - tx, err := client.Watch("key1", "key2") - Expect(err).NotTo(HaveOccurred()) - defer func() { - Expect(tx.Close()).NotTo(HaveOccurred()) - }() - - cmds, err := tx.Exec(func() error { - tx.Set("key1", "hello1", 0) - tx.Discard() - tx.Set("key2", "hello2", 0) - return nil - }) + err := client.Watch(func(tx *redis.Tx) error { + cmds, err := tx.MultiExec(func() error { + tx.Set("key1", "hello1", 0) + tx.Discard() + tx.Set("key2", "hello2", 0) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(1)) + return err + }, "key1", "key2") Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(1)) get := client.Get("key1") Expect(get.Err()).To(Equal(redis.Nil)) @@ -92,43 +87,41 @@ var _ = Describe("Tx", func() { }) It("should exec empty", func() { - tx, err := client.Watch() + err := client.Watch(func(tx *redis.Tx) error { + cmds, err := tx.MultiExec(func() error { return nil }) + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(0)) + return err + }) Expect(err).NotTo(HaveOccurred()) - defer func() { - Expect(tx.Close()).NotTo(HaveOccurred()) - }() - cmds, err := tx.Exec(func() error { return nil }) + v, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(0)) - - ping := tx.Ping() - Expect(ping.Err()).NotTo(HaveOccurred()) - Expect(ping.Val()).To(Equal("PONG")) + Expect(v).To(Equal("PONG")) }) It("should exec bulks", func() { - tx, err := client.Watch() - Expect(err).NotTo(HaveOccurred()) - defer func() { - Expect(tx.Close()).NotTo(HaveOccurred()) - }() + const N = 20000 - cmds, err := tx.Exec(func() error { - for i := int64(0); i < 20000; i++ { - tx.Incr("key") + err := client.Watch(func(tx *redis.Tx) error { + cmds, err := tx.MultiExec(func() error { + for i := 0; i < N; i++ { + tx.Incr("key") + } + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(len(cmds)).To(Equal(N)) + for _, cmd := range cmds { + Expect(cmd.Err()).NotTo(HaveOccurred()) } - return nil + return err }) Expect(err).NotTo(HaveOccurred()) - Expect(len(cmds)).To(Equal(20000)) - for _, cmd := range cmds { - Expect(cmd.Err()).NotTo(HaveOccurred()) - } - get := client.Get("key") - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal("20000")) + num, err := client.Get("key").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(num).To(Equal(int64(N))) }) It("should recover from bad connection", func() { @@ -140,22 +133,21 @@ var _ = Describe("Tx", func() { err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) - tx, err := client.Watch() - Expect(err).NotTo(HaveOccurred()) - defer func() { - Expect(tx.Close()).NotTo(HaveOccurred()) - }() + do := func() error { + err := client.Watch(func(tx *redis.Tx) error { + _, err := tx.MultiExec(func() error { + tx.Ping() + return nil + }) + return err + }) + return err + } - _, err = tx.Exec(func() error { - tx.Ping() - return nil - }) + err = do() Expect(err).To(MatchError("bad connection")) - _, err = tx.Exec(func() error { - tx.Ping() - return nil - }) + err = do() Expect(err).NotTo(HaveOccurred()) }) @@ -168,21 +160,20 @@ var _ = Describe("Tx", func() { err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) - { - tx, err := client.Watch("key") - Expect(err).To(MatchError("bad connection")) - Expect(tx).To(BeNil()) + do := func() error { + err := client.Watch(func(tx *redis.Tx) error { + _, err := tx.MultiExec(func() error { + return nil + }) + return err + }, "key") + return err } - { - tx, err := client.Watch("key") - Expect(err).NotTo(HaveOccurred()) - - err = tx.Ping().Err() - Expect(err).NotTo(HaveOccurred()) + err = do() + Expect(err).To(MatchError("bad connection")) - err = tx.Close() - Expect(err).NotTo(HaveOccurred()) - } + err = do() + Expect(err).NotTo(HaveOccurred()) }) }) From 487feebef16306e9bc31cbedff76be1840f092a2 Mon Sep 17 00:00:00 2001 From: Joris Minjat Date: Fri, 6 May 2016 11:12:31 -0700 Subject: [PATCH 0175/1746] Add latency based routing to Redis Cluster client. --- cluster.go | 396 +++++++++++++++++++------------- cluster_client_test.go | 50 ++-- cluster_test.go | 10 +- command.go | 133 ++++++++--- command_test.go | 4 +- commands.go | 509 +++++++++++++++++++---------------------- commands_test.go | 21 +- options.go | 3 + parser.go | 69 +++++- redis.go | 8 +- ring.go | 60 +++-- 11 files changed, 757 insertions(+), 506 deletions(-) diff --git a/cluster.go b/cluster.go index 5d56a4fc24..cd433e7b63 100644 --- a/cluster.go +++ b/cluster.go @@ -11,6 +11,12 @@ import ( "gopkg.in/redis.v4/internal/pool" ) +type clusterNode struct { + Addr string + Latency int + Client *Client +} + // ClusterClient is a Redis Cluster client representing a pool of zero // or more underlying connections. It's safe for concurrent use by // multiple goroutines. @@ -19,14 +25,14 @@ type ClusterClient struct { opt *ClusterOptions - slotsMx sync.RWMutex // protects slots and addrs - addrs []string - slots [][]string - - clientsMx sync.RWMutex // protects clients and closed - clients map[string]*Client + mu sync.RWMutex + addrs []string + nodes map[string]*clusterNode + slots [][]*clusterNode + closed bool - _closed int32 // atomic + cmdsInfo map[string]*CommandInfo + cmdsInfoOnce *sync.Once // Reports where slots reloading is in progress. reloading uint32 @@ -35,44 +41,63 @@ type ClusterClient struct { // NewClusterClient returns a Redis Cluster client as described in // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { + if opt.RouteByLatency { + opt.ReadOnly = true + } + client := &ClusterClient{ - opt: opt, - addrs: opt.Addrs, - slots: make([][]string, hashtag.SlotNumber), - clients: make(map[string]*Client), + opt: opt, + nodes: make(map[string]*clusterNode), + + cmdsInfoOnce: new(sync.Once), } client.commandable.process = client.process + + for _, addr := range opt.Addrs { + _ = client.nodeByAddr(addr) + } client.reloadSlots() + return client } -// getClients returns a snapshot of clients for cluster nodes -// this ClusterClient has been working with recently. -// Note that snapshot can contain closed clients. -func (c *ClusterClient) getClients() map[string]*Client { - c.clientsMx.RLock() - clients := make(map[string]*Client, len(c.clients)) - for addr, client := range c.clients { - clients[addr] = client - } - c.clientsMx.RUnlock() - return clients +func (c *ClusterClient) cmdInfo(name string) *CommandInfo { + c.cmdsInfoOnce.Do(func() { + for _, node := range c.nodes { + cmdsInfo, err := node.Client.Command().Result() + if err == nil { + c.cmdsInfo = cmdsInfo + return + } + } + c.cmdsInfoOnce = &sync.Once{} + }) + return c.cmdsInfo[name] } -func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { - addr := c.slotMasterAddr(hashtag.Slot(keys[0])) - client, err := c.getClient(addr) - if err != nil { - return err +func (c *ClusterClient) getNodes() map[string]*clusterNode { + c.mu.RLock() + var nodes map[string]*clusterNode + if !c.closed { + nodes = make(map[string]*clusterNode, len(c.nodes)) + for addr, node := range c.nodes { + nodes[addr] = node + } } - return client.Watch(fn, keys...) + c.mu.RUnlock() + return nodes +} + +func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { + node := c.slotMasterNode(hashtag.Slot(keys[0])) + return node.Client.Watch(fn, keys...) } // PoolStats returns accumulated connection pool stats. func (c *ClusterClient) PoolStats() *PoolStats { acc := PoolStats{} - for _, client := range c.getClients() { - s := client.connPool.Stats() + for _, node := range c.getNodes() { + s := node.Client.connPool.Stats() acc.Requests += s.Requests acc.Hits += s.Hits acc.Timeouts += s.Timeouts @@ -82,113 +107,163 @@ func (c *ClusterClient) PoolStats() *PoolStats { return &acc } -func (c *ClusterClient) closed() bool { - return atomic.LoadInt32(&c._closed) == 1 -} - // Close closes the cluster client, releasing any open resources. // // It is rare to Close a ClusterClient, as the ClusterClient is meant // to be long-lived and shared between many goroutines. func (c *ClusterClient) Close() error { - if !atomic.CompareAndSwapInt32(&c._closed, 0, 1) { - return pool.ErrClosed + c.mu.Lock() + if !c.closed { + c.closeClients() + c.addrs = nil + c.nodes = nil + c.slots = nil + c.cmdsInfo = nil } - - c.clientsMx.Lock() - c.resetClients() - c.clientsMx.Unlock() - c.setSlots(nil) + c.closed = true + c.mu.Unlock() return nil } -// getClient returns a Client for a given address. -func (c *ClusterClient) getClient(addr string) (*Client, error) { - if c.closed() { - return nil, pool.ErrClosed +func (c *ClusterClient) nodeByAddr(addr string) *clusterNode { + c.mu.RLock() + node, ok := c.nodes[addr] + c.mu.RUnlock() + if ok { + return node } - if addr == "" { - return c.randomClient() + c.mu.Lock() + if !c.closed { + node, ok = c.nodes[addr] + if !ok { + node = c.newNode(addr) + c.nodes[addr] = node + c.addrs = append(c.addrs, node.Addr) + } } + c.mu.Unlock() - c.clientsMx.RLock() - client, ok := c.clients[addr] - c.clientsMx.RUnlock() - if ok { - return client, nil - } + return node +} - c.clientsMx.Lock() - client, ok = c.clients[addr] - if !ok { - opt := c.opt.clientOptions() - opt.Addr = addr - client = NewClient(opt) - c.clients[addr] = client +func (c *ClusterClient) newNode(addr string) *clusterNode { + opt := c.opt.clientOptions() + opt.Addr = addr + return &clusterNode{ + Addr: addr, + Client: NewClient(opt), } - c.clientsMx.Unlock() +} - return client, nil +func (c *ClusterClient) slotNodes(slot int) []*clusterNode { + c.mu.RLock() + nodes := c.slots[slot] + c.mu.RUnlock() + return nodes } -func (c *ClusterClient) slotAddrs(slot int) []string { - c.slotsMx.RLock() - addrs := c.slots[slot] - c.slotsMx.RUnlock() - return addrs +// randomNode returns random live node. +func (c *ClusterClient) randomNode() *clusterNode { + var node *clusterNode + for i := 0; i < 10; i++ { + c.mu.RLock() + addrs := c.addrs + c.mu.RUnlock() + + if len(addrs) == 0 { + return nil + } + + n := rand.Intn(len(addrs)) + node = c.nodeByAddr(addrs[n]) + + if node.Client.ClusterInfo().Err() == nil { + return node + } + } + return node } -func (c *ClusterClient) slotMasterAddr(slot int) string { - addrs := c.slotAddrs(slot) - if len(addrs) > 0 { - return addrs[0] +func (c *ClusterClient) slotMasterNode(slot int) *clusterNode { + nodes := c.slotNodes(slot) + if len(nodes) == 0 { + return c.randomNode() } - return "" + return nodes[0] } -// randomClient returns a Client for the first live node. -func (c *ClusterClient) randomClient() (client *Client, err error) { - for i := 0; i < 10; i++ { - n := rand.Intn(len(c.addrs)) - client, err = c.getClient(c.addrs[n]) - if err != nil { - continue - } - err = client.ClusterInfo().Err() - if err == nil { - return client, nil +func (c *ClusterClient) slotSlaveNode(slot int) *clusterNode { + nodes := c.slotNodes(slot) + switch len(nodes) { + case 0: + return c.randomNode() + case 1: + return nodes[0] + case 2: + return nodes[1] + default: + n := rand.Intn(len(nodes)-1) + 1 + return nodes[n] + } + +} + +func (c *ClusterClient) slotClosestNode(slot int) *clusterNode { + nodes := c.slotNodes(slot) + var node *clusterNode + for _, n := range nodes { + if node == nil || n.Latency < node.Latency { + node = n } } - return nil, err + return node } -func (c *ClusterClient) process(cmd Cmder) { - var ask bool +func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode) { + cmdInfo := c.cmdInfo(cmd.arg(0)) + if cmdInfo == nil { + return 0, c.randomNode() + } - slot := hashtag.Slot(cmd.clusterKey()) + if cmdInfo.FirstKeyPos == -1 { + return 0, c.randomNode() + } - addr := c.slotMasterAddr(slot) - client, err := c.getClient(addr) - if err != nil { - cmd.setErr(err) - return + firstKey := cmd.arg(int(cmdInfo.FirstKeyPos)) + slot := hashtag.Slot(firstKey) + + if cmdInfo.ReadOnly && c.opt.ReadOnly { + if c.opt.RouteByLatency { + return slot, c.slotClosestNode(slot) + } + return slot, c.slotSlaveNode(slot) } + return slot, c.slotMasterNode(slot) +} + +func (c *ClusterClient) process(cmd Cmder) { + var ask bool + slot, node := c.cmdSlotAndNode(cmd) for attempt := 0; attempt <= c.opt.getMaxRedirects(); attempt++ { if attempt > 0 { cmd.reset() } + if node == nil { + cmd.setErr(pool.ErrClosed) + return + } if ask { - pipe := client.Pipeline() + pipe := node.Client.Pipeline() pipe.Process(NewCmd("ASKING")) pipe.Process(cmd) _, _ = pipe.Exec() pipe.Close() ask = false } else { - client.Process(cmd) + node.Client.Process(cmd) } // If there is no (real) error, we are done! @@ -199,10 +274,7 @@ func (c *ClusterClient) process(cmd Cmder) { // On network errors try random node. if shouldRetry(err) { - client, err = c.randomClient() - if err != nil { - return - } + node = c.randomNode() continue } @@ -210,13 +282,11 @@ func (c *ClusterClient) process(cmd Cmder) { var addr string moved, ask, addr = isMovedError(err) if moved || ask { - if moved && c.slotMasterAddr(slot) != addr { + if moved && c.slotMasterNode(slot).Addr != addr { c.lazyReloadSlots() } - client, err = c.getClient(addr) - if err != nil { - return - } + + node = c.nodeByAddr(addr) continue } @@ -224,64 +294,71 @@ func (c *ClusterClient) process(cmd Cmder) { } } -// Closes all clients and returns last error if there are any. -func (c *ClusterClient) resetClients() (retErr error) { - for addr, client := range c.clients { - if err := client.Close(); err != nil && retErr == nil { +// closeClients closes all clients and returns the first error if there are any. +func (c *ClusterClient) closeClients() error { + var retErr error + for _, node := range c.nodes { + if err := node.Client.Close(); err != nil && retErr == nil { retErr = err } - delete(c.clients, addr) } return retErr } -func (c *ClusterClient) setSlots(slots []ClusterSlot) { - c.slotsMx.Lock() - - seen := make(map[string]struct{}) - for _, addr := range c.addrs { - seen[addr] = struct{}{} - } - +func (c *ClusterClient) setSlots(cs []ClusterSlot) { + slots := make([][]*clusterNode, hashtag.SlotNumber) for i := 0; i < hashtag.SlotNumber; i++ { - c.slots[i] = c.slots[i][:0] + slots[i] = nil } - for _, slot := range slots { - var addrs []string - for _, node := range slot.Nodes { - addrs = append(addrs, node.Addr) + for _, s := range cs { + var nodes []*clusterNode + for _, n := range s.Nodes { + nodes = append(nodes, c.nodeByAddr(n.Addr)) } - for i := slot.Start; i <= slot.End; i++ { - c.slots[i] = addrs + for i := s.Start; i <= s.End; i++ { + slots[i] = nodes } + } - for _, node := range slot.Nodes { - if _, ok := seen[node.Addr]; !ok { - c.addrs = append(c.addrs, node.Addr) - seen[node.Addr] = struct{}{} - } - } + c.mu.Lock() + if !c.closed { + c.slots = slots } + c.mu.Unlock() +} - c.slotsMx.Unlock() +func (c *ClusterClient) setNodesLatency() { + nodes := c.getNodes() + for _, node := range nodes { + var latency int + for i := 0; i < 10; i++ { + t1 := time.Now() + node.Client.Ping() + latency += int(time.Since(t1) / time.Millisecond) + } + node.Latency = latency + } } func (c *ClusterClient) reloadSlots() { defer atomic.StoreUint32(&c.reloading, 0) - client, err := c.randomClient() - if err != nil { - internal.Logf("randomClient failed: %s", err) + node := c.randomNode() + if node == nil { return } - slots, err := client.ClusterSlots().Result() + slots, err := node.Client.ClusterSlots().Result() if err != nil { - internal.Logf("ClusterSlots failed: %s", err) + internal.Logf("ClusterSlots on addr=%q failed: %s", node.Addr, err) return } + c.setSlots(slots) + if c.opt.RouteByLatency { + c.setNodesLatency() + } } func (c *ClusterClient) lazyReloadSlots() { @@ -297,13 +374,14 @@ func (c *ClusterClient) reaper(frequency time.Duration) { defer ticker.Stop() for _ = range ticker.C { - if c.closed() { + nodes := c.getNodes() + if nodes == nil { break } var n int - for _, client := range c.getClients() { - nn, err := client.connPool.(*pool.ConnPool).ReapStaleConns() + for _, node := range nodes { + nn, err := node.Client.connPool.(*pool.ConnPool).ReapStaleConns() if err != nil { internal.Logf("ReapStaleConns failed: %s", err) } else { @@ -334,25 +412,21 @@ func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { func (c *ClusterClient) pipelineExec(cmds []Cmder) error { var retErr error - cmdsMap := make(map[string][]Cmder) + cmdsMap := make(map[*clusterNode][]Cmder) for _, cmd := range cmds { - slot := hashtag.Slot(cmd.clusterKey()) - addr := c.slotMasterAddr(slot) - cmdsMap[addr] = append(cmdsMap[addr], cmd) + _, node := c.cmdSlotAndNode(cmd) + cmdsMap[node] = append(cmdsMap[node], cmd) } for attempt := 0; attempt <= c.opt.getMaxRedirects(); attempt++ { - failedCmds := make(map[string][]Cmder) + failedCmds := make(map[*clusterNode][]Cmder) - for addr, cmds := range cmdsMap { - client, err := c.getClient(addr) - if err != nil { - setCmdsErr(cmds, err) - retErr = err - continue + for node, cmds := range cmdsMap { + if node == nil { + node = c.randomNode() } - cn, err := client.conn() + cn, err := node.Client.conn() if err != nil { setCmdsErr(cmds, err) retErr = err @@ -363,7 +437,7 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { if err != nil { retErr = err } - client.putConn(cn, err, false) + node.Client.putConn(cn, err, false) } cmdsMap = failedCmds @@ -373,8 +447,8 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { } func (c *ClusterClient) execClusterCmds( - cn *pool.Conn, cmds []Cmder, failedCmds map[string][]Cmder, -) (map[string][]Cmder, error) { + cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, +) (map[*clusterNode][]Cmder, error) { if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) return failedCmds, err @@ -388,15 +462,17 @@ func (c *ClusterClient) execClusterCmds( } if isNetworkError(err) { cmd.reset() - failedCmds[""] = append(failedCmds[""], cmds[i:]...) + failedCmds[nil] = append(failedCmds[nil], cmds[i:]...) break } else if moved, ask, addr := isMovedError(err); moved { c.lazyReloadSlots() cmd.reset() - failedCmds[addr] = append(failedCmds[addr], cmd) + node := c.nodeByAddr(addr) + failedCmds[node] = append(failedCmds[node], cmd) } else if ask { cmd.reset() - failedCmds[addr] = append(failedCmds[addr], NewCmd("ASKING"), cmd) + node := c.nodeByAddr(addr) + failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) } else if firstCmdErr == nil { firstCmdErr = err } @@ -418,6 +494,12 @@ type ClusterOptions struct { // Default is 16. MaxRedirects int + // Enables read queries for a connection to a Redis Cluster slave node. + ReadOnly bool + + // Enables routing read-only queries to the closest master or slave node. + RouteByLatency bool + // Following options are copied from Options struct. Password string @@ -446,6 +528,7 @@ func (opt *ClusterOptions) getMaxRedirects() int { func (opt *ClusterOptions) clientOptions() *Options { return &Options{ Password: opt.Password, + ReadOnly: opt.ReadOnly, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, @@ -454,6 +537,7 @@ func (opt *ClusterOptions) clientOptions() *Options { PoolSize: opt.PoolSize, PoolTimeout: opt.PoolTimeout, IdleTimeout: opt.IdleTimeout, + // IdleCheckFrequency is not copied to disable reaper } } diff --git a/cluster_client_test.go b/cluster_client_test.go index 560c438759..8277abdfb6 100644 --- a/cluster_client_test.go +++ b/cluster_client_test.go @@ -6,16 +6,21 @@ import ( ) func (c *ClusterClient) SlotAddrs(slot int) []string { - return c.slotAddrs(slot) + var addrs []string + for _, n := range c.slotNodes(slot) { + addrs = append(addrs, n.Addr) + } + return addrs } // SwapSlot swaps a slot's master/slave address // for testing MOVED redirects -func (c *ClusterClient) SwapSlot(pos int) []string { - c.slotsMx.Lock() - defer c.slotsMx.Unlock() - c.slots[pos][0], c.slots[pos][1] = c.slots[pos][1], c.slots[pos][0] - return c.slots[pos] +func (c *ClusterClient) SwapSlotNodes(slot int) []string { + c.mu.Lock() + nodes := c.slots[slot] + nodes[0], nodes[1] = nodes[1], nodes[0] + c.mu.Unlock() + return c.SlotAddrs(slot) } var _ = Describe("ClusterClient", func() { @@ -42,19 +47,26 @@ var _ = Describe("ClusterClient", func() { It("should initialize", func() { Expect(subject.addrs).To(HaveLen(3)) - Expect(subject.slots).To(HaveLen(16384)) }) It("should update slots cache", func() { populate() - Expect(subject.slots[0]).To(Equal([]string{"127.0.0.1:7000", "127.0.0.1:7004"})) - Expect(subject.slots[4095]).To(Equal([]string{"127.0.0.1:7000", "127.0.0.1:7004"})) - Expect(subject.slots[4096]).To(Equal([]string{"127.0.0.1:7001", "127.0.0.1:7005"})) - Expect(subject.slots[8191]).To(Equal([]string{"127.0.0.1:7001", "127.0.0.1:7005"})) - Expect(subject.slots[8192]).To(Equal([]string{"127.0.0.1:7002", "127.0.0.1:7006"})) - Expect(subject.slots[12287]).To(Equal([]string{"127.0.0.1:7002", "127.0.0.1:7006"})) - Expect(subject.slots[12288]).To(Equal([]string{"127.0.0.1:7003", "127.0.0.1:7007"})) - Expect(subject.slots[16383]).To(Equal([]string{"127.0.0.1:7003", "127.0.0.1:7007"})) + Expect(subject.slots[0][0].Addr).To(Equal("127.0.0.1:7000")) + Expect(subject.slots[0][1].Addr).To(Equal("127.0.0.1:7004")) + Expect(subject.slots[4095][0].Addr).To(Equal("127.0.0.1:7000")) + Expect(subject.slots[4095][1].Addr).To(Equal("127.0.0.1:7004")) + Expect(subject.slots[4096][0].Addr).To(Equal("127.0.0.1:7001")) + Expect(subject.slots[4096][1].Addr).To(Equal("127.0.0.1:7005")) + Expect(subject.slots[8191][0].Addr).To(Equal("127.0.0.1:7001")) + Expect(subject.slots[8191][1].Addr).To(Equal("127.0.0.1:7005")) + Expect(subject.slots[8192][0].Addr).To(Equal("127.0.0.1:7002")) + Expect(subject.slots[8192][1].Addr).To(Equal("127.0.0.1:7006")) + Expect(subject.slots[12287][0].Addr).To(Equal("127.0.0.1:7002")) + Expect(subject.slots[12287][1].Addr).To(Equal("127.0.0.1:7006")) + Expect(subject.slots[12288][0].Addr).To(Equal("127.0.0.1:7003")) + Expect(subject.slots[12288][1].Addr).To(Equal("127.0.0.1:7007")) + Expect(subject.slots[16383][0].Addr).To(Equal("127.0.0.1:7003")) + Expect(subject.slots[16383][1].Addr).To(Equal("127.0.0.1:7007")) Expect(subject.addrs).To(Equal([]string{ "127.0.0.1:6379", "127.0.0.1:7003", @@ -71,11 +83,9 @@ var _ = Describe("ClusterClient", func() { It("should close", func() { populate() Expect(subject.Close()).NotTo(HaveOccurred()) - Expect(subject.clients).To(BeEmpty()) - Expect(subject.slots[0]).To(BeEmpty()) - Expect(subject.slots[8191]).To(BeEmpty()) - Expect(subject.slots[8192]).To(BeEmpty()) - Expect(subject.slots[16383]).To(BeEmpty()) + Expect(subject.addrs).To(BeEmpty()) + Expect(subject.nodes).To(BeEmpty()) + Expect(subject.slots).To(BeEmpty()) Expect(subject.Ping().Err().Error()).To(Equal("redis: client is closed")) }) }) diff --git a/cluster_test.go b/cluster_test.go index 69ffa7a7bc..6d081dfdc0 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -301,7 +301,7 @@ var _ = Describe("Cluster", func() { }) It("should CLUSTER READONLY", func() { - res, err := cluster.primary().Readonly().Result() + res, err := cluster.primary().ReadOnly().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(Equal("OK")) }) @@ -353,7 +353,7 @@ var _ = Describe("Cluster", func() { Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) slot := hashtag.Slot("A") - Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) val, err := client.Get("A").Result() Expect(err).NotTo(HaveOccurred()) @@ -361,7 +361,7 @@ var _ = Describe("Cluster", func() { Eventually(func() []string { return client.SlotAddrs(slot) - }, "5s").Should(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) + }, "10s").Should(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) }) It("should return error when there are no attempts left", func() { @@ -371,7 +371,7 @@ var _ = Describe("Cluster", func() { }) slot := hashtag.Slot("A") - Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) err := client.Get("A").Err() Expect(err).To(HaveOccurred()) @@ -435,7 +435,7 @@ var _ = Describe("Cluster", func() { It("performs multi-pipelines", func() { slot := hashtag.Slot("A") - Expect(client.SwapSlot(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) pipe := client.Pipeline() defer pipe.Close() diff --git a/command.go b/command.go index 248625e9fe..2db8fa2f5d 100644 --- a/command.go +++ b/command.go @@ -30,12 +30,12 @@ var ( type Cmder interface { args() []interface{} + arg(int) string readReply(*pool.Conn) error setErr(error) reset() readTimeout() *time.Duration - clusterKey() string Err() error fmt.Stringer @@ -92,10 +92,7 @@ func cmdString(cmd Cmder, val interface{}) string { type baseCmd struct { _args []interface{} - - err error - - _clusterKeyPos int + err error _readTimeout *time.Duration } @@ -111,6 +108,15 @@ func (cmd *baseCmd) args() []interface{} { return cmd._args } +func (cmd *baseCmd) arg(pos int) string { + if len(cmd._args) > pos { + if s, ok := cmd._args[pos].(string); ok { + return s + } + } + return "" +} + func (cmd *baseCmd) readTimeout() *time.Duration { return cmd._readTimeout } @@ -119,17 +125,14 @@ func (cmd *baseCmd) setReadTimeout(d time.Duration) { cmd._readTimeout = &d } -func (cmd *baseCmd) clusterKey() string { - if cmd._clusterKeyPos > 0 && cmd._clusterKeyPos < len(cmd._args) { - return fmt.Sprint(cmd._args[cmd._clusterKeyPos]) - } - return "" -} - func (cmd *baseCmd) setErr(e error) { cmd.err = e } +func newBaseCmd(args []interface{}) baseCmd { + return baseCmd{_args: args} +} + //------------------------------------------------------------------------------ type Cmd struct { @@ -139,7 +142,7 @@ type Cmd struct { } func NewCmd(args ...interface{}) *Cmd { - return &Cmd{baseCmd: baseCmd{_args: args}} + return &Cmd{baseCmd: newBaseCmd(args)} } func (cmd *Cmd) reset() { @@ -183,7 +186,8 @@ type SliceCmd struct { } func NewSliceCmd(args ...interface{}) *SliceCmd { - return &SliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &SliceCmd{baseCmd: cmd} } func (cmd *SliceCmd) reset() { @@ -222,11 +226,8 @@ type StatusCmd struct { } func NewStatusCmd(args ...interface{}) *StatusCmd { - return &StatusCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} -} - -func newKeylessStatusCmd(args ...interface{}) *StatusCmd { - return &StatusCmd{baseCmd: baseCmd{_args: args}} + cmd := newBaseCmd(args) + return &StatusCmd{baseCmd: cmd} } func (cmd *StatusCmd) reset() { @@ -260,7 +261,8 @@ type IntCmd struct { } func NewIntCmd(args ...interface{}) *IntCmd { - return &IntCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &IntCmd{baseCmd: cmd} } func (cmd *IntCmd) reset() { @@ -295,9 +297,10 @@ type DurationCmd struct { } func NewDurationCmd(precision time.Duration, args ...interface{}) *DurationCmd { + cmd := newBaseCmd(args) return &DurationCmd{ precision: precision, - baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}, + baseCmd: cmd, } } @@ -337,7 +340,8 @@ type BoolCmd struct { } func NewBoolCmd(args ...interface{}) *BoolCmd { - return &BoolCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &BoolCmd{baseCmd: cmd} } func (cmd *BoolCmd) reset() { @@ -393,7 +397,8 @@ type StringCmd struct { } func NewStringCmd(args ...interface{}) *StringCmd { - return &StringCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &StringCmd{baseCmd: cmd} } func (cmd *StringCmd) reset() { @@ -468,7 +473,8 @@ type FloatCmd struct { } func NewFloatCmd(args ...interface{}) *FloatCmd { - return &FloatCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &FloatCmd{baseCmd: cmd} } func (cmd *FloatCmd) reset() { @@ -502,7 +508,8 @@ type StringSliceCmd struct { } func NewStringSliceCmd(args ...interface{}) *StringSliceCmd { - return &StringSliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &StringSliceCmd{baseCmd: cmd} } func (cmd *StringSliceCmd) reset() { @@ -541,7 +548,8 @@ type BoolSliceCmd struct { } func NewBoolSliceCmd(args ...interface{}) *BoolSliceCmd { - return &BoolSliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &BoolSliceCmd{baseCmd: cmd} } func (cmd *BoolSliceCmd) reset() { @@ -580,7 +588,8 @@ type StringStringMapCmd struct { } func NewStringStringMapCmd(args ...interface{}) *StringStringMapCmd { - return &StringStringMapCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &StringStringMapCmd{baseCmd: cmd} } func (cmd *StringStringMapCmd) reset() { @@ -619,7 +628,8 @@ type StringIntMapCmd struct { } func NewStringIntMapCmd(args ...interface{}) *StringIntMapCmd { - return &StringIntMapCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &StringIntMapCmd{baseCmd: cmd} } func (cmd *StringIntMapCmd) Val() map[string]int64 { @@ -658,7 +668,8 @@ type ZSliceCmd struct { } func NewZSliceCmd(args ...interface{}) *ZSliceCmd { - return &ZSliceCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &ZSliceCmd{baseCmd: cmd} } func (cmd *ZSliceCmd) reset() { @@ -698,8 +709,9 @@ type ScanCmd struct { } func NewScanCmd(args ...interface{}) *ScanCmd { + cmd := newBaseCmd(args) return &ScanCmd{ - baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}, + baseCmd: cmd, } } @@ -752,7 +764,8 @@ type ClusterSlotsCmd struct { } func NewClusterSlotsCmd(args ...interface{}) *ClusterSlotsCmd { - return &ClusterSlotsCmd{baseCmd: baseCmd{_args: args, _clusterKeyPos: 1}} + cmd := newBaseCmd(args) + return &ClusterSlotsCmd{baseCmd: cmd} } func (cmd *ClusterSlotsCmd) Val() []ClusterSlot { @@ -833,12 +846,10 @@ func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { if q.Sort != "" { args = append(args, q.Sort) } + cmd := newBaseCmd(args) return &GeoLocationCmd{ - baseCmd: baseCmd{ - _args: args, - _clusterKeyPos: 1, - }, - q: q, + baseCmd: cmd, + q: q, } } @@ -868,3 +879,53 @@ func (cmd *GeoLocationCmd) readReply(cn *pool.Conn) error { cmd.locations = reply.([]GeoLocation) return nil } + +//------------------------------------------------------------------------------ + +type CommandInfo struct { + Name string + Arity int8 + Flags []string + FirstKeyPos int8 + LastKeyPos int8 + StepCount int8 + ReadOnly bool +} + +type CommandsInfoCmd struct { + baseCmd + + val map[string]*CommandInfo +} + +func NewCommandsInfoCmd(args ...interface{}) *CommandsInfoCmd { + cmd := newBaseCmd(args) + return &CommandsInfoCmd{baseCmd: cmd} +} + +func (cmd *CommandsInfoCmd) Val() map[string]*CommandInfo { + return cmd.val +} + +func (cmd *CommandsInfoCmd) Result() (map[string]*CommandInfo, error) { + return cmd.Val(), cmd.Err() +} + +func (cmd *CommandsInfoCmd) String() string { + return cmdString(cmd, cmd.val) +} + +func (cmd *CommandsInfoCmd) reset() { + cmd.val = nil + cmd.err = nil +} + +func (cmd *CommandsInfoCmd) readReply(cn *pool.Conn) error { + v, err := readArrayReply(cn, commandInfoSliceParser) + if err != nil { + cmd.err = err + return err + } + cmd.val = v.(map[string]*CommandInfo) + return nil +} diff --git a/command_test.go b/command_test.go index 77391e466b..ab31f3f186 100644 --- a/command_test.go +++ b/command_test.go @@ -21,10 +21,10 @@ var _ = Describe("Cmd", func() { It("implements Stringer", func() { set := client.Set("foo", "bar", 0) - Expect(set.String()).To(Equal("SET foo bar: OK")) + Expect(set.String()).To(Equal("set foo bar: OK")) get := client.Get("foo") - Expect(get.String()).To(Equal("GET foo: bar")) + Expect(get.String()).To(Equal("get foo: bar")) }) It("has val/err", func() { diff --git a/commands.go b/commands.go index e6c7e8ad25..00b047f084 100644 --- a/commands.go +++ b/commands.go @@ -62,20 +62,19 @@ func (c *commandable) Process(cmd Cmder) { //------------------------------------------------------------------------------ func (c *commandable) Auth(password string) *StatusCmd { - cmd := newKeylessStatusCmd("AUTH", password) + cmd := NewStatusCmd("auth", password) c.Process(cmd) return cmd } func (c *commandable) Echo(message string) *StringCmd { - cmd := NewStringCmd("ECHO", message) - cmd._clusterKeyPos = 0 + cmd := NewStringCmd("echo", message) c.Process(cmd) return cmd } func (c *commandable) Ping() *StatusCmd { - cmd := newKeylessStatusCmd("PING") + cmd := NewStatusCmd("ping") c.Process(cmd) return cmd } @@ -85,7 +84,7 @@ func (c *commandable) Quit() *StatusCmd { } func (c *commandable) Select(index int64) *StatusCmd { - cmd := newKeylessStatusCmd("SELECT", index) + cmd := NewStatusCmd("select", index) c.Process(cmd) return cmd } @@ -104,110 +103,106 @@ func (c *commandable) Del(keys ...string) *IntCmd { } func (c *commandable) Dump(key string) *StringCmd { - cmd := NewStringCmd("DUMP", key) + cmd := NewStringCmd("dump", key) c.Process(cmd) return cmd } func (c *commandable) Exists(key string) *BoolCmd { - cmd := NewBoolCmd("EXISTS", key) + cmd := NewBoolCmd("exists", key) c.Process(cmd) return cmd } func (c *commandable) Expire(key string, expiration time.Duration) *BoolCmd { - cmd := NewBoolCmd("EXPIRE", key, formatSec(expiration)) + cmd := NewBoolCmd("expire", key, formatSec(expiration)) c.Process(cmd) return cmd } func (c *commandable) ExpireAt(key string, tm time.Time) *BoolCmd { - cmd := NewBoolCmd("EXPIREAT", key, tm.Unix()) + cmd := NewBoolCmd("expireat", key, tm.Unix()) c.Process(cmd) return cmd } func (c *commandable) Keys(pattern string) *StringSliceCmd { - cmd := NewStringSliceCmd("KEYS", pattern) + cmd := NewStringSliceCmd("keys", pattern) c.Process(cmd) return cmd } func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd { cmd := NewStatusCmd( - "MIGRATE", + "migrate", host, port, key, db, formatMs(timeout), ) - cmd._clusterKeyPos = 3 cmd.setReadTimeout(readTimeout(timeout)) c.Process(cmd) return cmd } func (c *commandable) Move(key string, db int64) *BoolCmd { - cmd := NewBoolCmd("MOVE", key, db) + cmd := NewBoolCmd("move", key, db) c.Process(cmd) return cmd } func (c *commandable) ObjectRefCount(keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "OBJECT" - args[1] = "REFCOUNT" + args[0] = "object" + args[1] = "refcount" for i, key := range keys { args[2+i] = key } cmd := NewIntCmd(args...) - cmd._clusterKeyPos = 2 c.Process(cmd) return cmd } func (c *commandable) ObjectEncoding(keys ...string) *StringCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "OBJECT" - args[1] = "ENCODING" + args[0] = "object" + args[1] = "encoding" for i, key := range keys { args[2+i] = key } cmd := NewStringCmd(args...) - cmd._clusterKeyPos = 2 c.Process(cmd) return cmd } func (c *commandable) ObjectIdleTime(keys ...string) *DurationCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "OBJECT" - args[1] = "IDLETIME" + args[0] = "object" + args[1] = "idletime" for i, key := range keys { args[2+i] = key } cmd := NewDurationCmd(time.Second, args...) - cmd._clusterKeyPos = 2 c.Process(cmd) return cmd } func (c *commandable) Persist(key string) *BoolCmd { - cmd := NewBoolCmd("PERSIST", key) + cmd := NewBoolCmd("persist", key) c.Process(cmd) return cmd } func (c *commandable) PExpire(key string, expiration time.Duration) *BoolCmd { - cmd := NewBoolCmd("PEXPIRE", key, formatMs(expiration)) + cmd := NewBoolCmd("pexpire", key, formatMs(expiration)) c.Process(cmd) return cmd } func (c *commandable) PExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd( - "PEXPIREAT", + "pexpireat", key, tm.UnixNano()/int64(time.Millisecond), ) @@ -216,32 +211,32 @@ func (c *commandable) PExpireAt(key string, tm time.Time) *BoolCmd { } func (c *commandable) PTTL(key string) *DurationCmd { - cmd := NewDurationCmd(time.Millisecond, "PTTL", key) + cmd := NewDurationCmd(time.Millisecond, "pttl", key) c.Process(cmd) return cmd } func (c *commandable) RandomKey() *StringCmd { - cmd := NewStringCmd("RANDOMKEY") + cmd := NewStringCmd("randomkey") c.Process(cmd) return cmd } func (c *commandable) Rename(key, newkey string) *StatusCmd { - cmd := NewStatusCmd("RENAME", key, newkey) + cmd := NewStatusCmd("rename", key, newkey) c.Process(cmd) return cmd } func (c *commandable) RenameNX(key, newkey string) *BoolCmd { - cmd := NewBoolCmd("RENAMENX", key, newkey) + cmd := NewBoolCmd("renamenx", key, newkey) c.Process(cmd) return cmd } func (c *commandable) Restore(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( - "RESTORE", + "restore", key, formatMs(ttl), value, @@ -252,11 +247,11 @@ func (c *commandable) Restore(key string, ttl time.Duration, value string) *Stat func (c *commandable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( - "RESTORE", + "restore", key, formatMs(ttl), value, - "REPLACE", + "replace", ) c.Process(cmd) return cmd @@ -272,24 +267,24 @@ type Sort struct { } func (sort *Sort) args(key string) []interface{} { - args := []interface{}{"SORT", key} + args := []interface{}{"sort", key} if sort.By != "" { - args = append(args, "BY", sort.By) + args = append(args, "by", sort.By) } if sort.Offset != 0 || sort.Count != 0 { - args = append(args, "LIMIT", sort.Offset, sort.Count) + args = append(args, "limit", sort.Offset, sort.Count) } for _, get := range sort.Get { - args = append(args, "GET", get) + args = append(args, "get", get) } if sort.Order != "" { args = append(args, sort.Order) } if sort.IsAlpha { - args = append(args, "ALPHA") + args = append(args, "alpha") } if sort.Store != "" { - args = append(args, "STORE", sort.Store) + args = append(args, "store", sort.Store) } return args } @@ -307,24 +302,24 @@ func (c *commandable) SortInterfaces(key string, sort Sort) *SliceCmd { } func (c *commandable) TTL(key string) *DurationCmd { - cmd := NewDurationCmd(time.Second, "TTL", key) + cmd := NewDurationCmd(time.Second, "ttl", key) c.Process(cmd) return cmd } func (c *commandable) Type(key string) *StatusCmd { - cmd := NewStatusCmd("TYPE", key) + cmd := NewStatusCmd("type", key) c.Process(cmd) return cmd } func (c *commandable) Scan(cursor uint64, match string, count int64) Scanner { - args := []interface{}{"SCAN", cursor} + args := []interface{}{"scan", cursor} if match != "" { - args = append(args, "MATCH", match) + args = append(args, "match", match) } if count > 0 { - args = append(args, "COUNT", count) + args = append(args, "count", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -335,12 +330,12 @@ func (c *commandable) Scan(cursor uint64, match string, count int64) Scanner { } func (c *commandable) SScan(key string, cursor uint64, match string, count int64) Scanner { - args := []interface{}{"SSCAN", key, cursor} + args := []interface{}{"sscan", key, cursor} if match != "" { - args = append(args, "MATCH", match) + args = append(args, "match", match) } if count > 0 { - args = append(args, "COUNT", count) + args = append(args, "count", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -351,12 +346,12 @@ func (c *commandable) SScan(key string, cursor uint64, match string, count int64 } func (c *commandable) HScan(key string, cursor uint64, match string, count int64) Scanner { - args := []interface{}{"HSCAN", key, cursor} + args := []interface{}{"hscan", key, cursor} if match != "" { - args = append(args, "MATCH", match) + args = append(args, "match", match) } if count > 0 { - args = append(args, "COUNT", count) + args = append(args, "count", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -367,12 +362,12 @@ func (c *commandable) HScan(key string, cursor uint64, match string, count int64 } func (c *commandable) ZScan(key string, cursor uint64, match string, count int64) Scanner { - args := []interface{}{"ZSCAN", key, cursor} + args := []interface{}{"zscan", key, cursor} if match != "" { - args = append(args, "MATCH", match) + args = append(args, "match", match) } if count > 0 { - args = append(args, "COUNT", count) + args = append(args, "count", count) } cmd := NewScanCmd(args...) c.Process(cmd) @@ -385,7 +380,7 @@ func (c *commandable) ZScan(key string, cursor uint64, match string, count int64 //------------------------------------------------------------------------------ func (c *commandable) Append(key, value string) *IntCmd { - cmd := NewIntCmd("APPEND", key, value) + cmd := NewIntCmd("append", key, value) c.Process(cmd) return cmd } @@ -395,7 +390,7 @@ type BitCount struct { } func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { - args := []interface{}{"BITCOUNT", key} + args := []interface{}{"bitcount", key} if bitCount != nil { args = append( args, @@ -410,7 +405,7 @@ func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { func (c *commandable) bitOp(op, destKey string, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) - args[0] = "BITOP" + args[0] = "bitop" args[1] = op args[2] = destKey for i, key := range keys { @@ -422,24 +417,24 @@ func (c *commandable) bitOp(op, destKey string, keys ...string) *IntCmd { } func (c *commandable) BitOpAnd(destKey string, keys ...string) *IntCmd { - return c.bitOp("AND", destKey, keys...) + return c.bitOp("and", destKey, keys...) } func (c *commandable) BitOpOr(destKey string, keys ...string) *IntCmd { - return c.bitOp("OR", destKey, keys...) + return c.bitOp("or", destKey, keys...) } func (c *commandable) BitOpXor(destKey string, keys ...string) *IntCmd { - return c.bitOp("XOR", destKey, keys...) + return c.bitOp("xor", destKey, keys...) } func (c *commandable) BitOpNot(destKey string, key string) *IntCmd { - return c.bitOp("NOT", destKey, key) + return c.bitOp("not", destKey, key) } func (c *commandable) BitPos(key string, bit int64, pos ...int64) *IntCmd { args := make([]interface{}, 3+len(pos)) - args[0] = "BITPOS" + args[0] = "bitpos" args[1] = key args[2] = bit switch len(pos) { @@ -458,62 +453,62 @@ func (c *commandable) BitPos(key string, bit int64, pos ...int64) *IntCmd { } func (c *commandable) Decr(key string) *IntCmd { - cmd := NewIntCmd("DECR", key) + cmd := NewIntCmd("decr", key) c.Process(cmd) return cmd } func (c *commandable) DecrBy(key string, decrement int64) *IntCmd { - cmd := NewIntCmd("DECRBY", key, decrement) + cmd := NewIntCmd("decrby", key, decrement) c.Process(cmd) return cmd } func (c *commandable) Get(key string) *StringCmd { - cmd := NewStringCmd("GET", key) + cmd := NewStringCmd("get", key) c.Process(cmd) return cmd } func (c *commandable) GetBit(key string, offset int64) *IntCmd { - cmd := NewIntCmd("GETBIT", key, offset) + cmd := NewIntCmd("getbit", key, offset) c.Process(cmd) return cmd } func (c *commandable) GetRange(key string, start, end int64) *StringCmd { - cmd := NewStringCmd("GETRANGE", key, start, end) + cmd := NewStringCmd("getrange", key, start, end) c.Process(cmd) return cmd } func (c *commandable) GetSet(key string, value interface{}) *StringCmd { - cmd := NewStringCmd("GETSET", key, value) + cmd := NewStringCmd("getset", key, value) c.Process(cmd) return cmd } func (c *commandable) Incr(key string) *IntCmd { - cmd := NewIntCmd("INCR", key) + cmd := NewIntCmd("incr", key) c.Process(cmd) return cmd } func (c *commandable) IncrBy(key string, value int64) *IntCmd { - cmd := NewIntCmd("INCRBY", key, value) + cmd := NewIntCmd("incrby", key, value) c.Process(cmd) return cmd } func (c *commandable) IncrByFloat(key string, value float64) *FloatCmd { - cmd := NewFloatCmd("INCRBYFLOAT", key, value) + cmd := NewFloatCmd("incrbyfloat", key, value) c.Process(cmd) return cmd } func (c *commandable) MGet(keys ...string) *SliceCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "MGET" + args[0] = "mget" for i, key := range keys { args[1+i] = key } @@ -524,7 +519,7 @@ func (c *commandable) MGet(keys ...string) *SliceCmd { func (c *commandable) MSet(pairs ...string) *StatusCmd { args := make([]interface{}, 1+len(pairs)) - args[0] = "MSET" + args[0] = "mset" for i, pair := range pairs { args[1+i] = pair } @@ -535,7 +530,7 @@ func (c *commandable) MSet(pairs ...string) *StatusCmd { func (c *commandable) MSetNX(pairs ...string) *BoolCmd { args := make([]interface{}, 1+len(pairs)) - args[0] = "MSETNX" + args[0] = "msetnx" for i, pair := range pairs { args[1+i] = pair } @@ -548,15 +543,15 @@ func (c *commandable) MSetNX(pairs ...string) *BoolCmd { // // Zero expiration means the key has no expiration time. func (c *commandable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { - args := make([]interface{}, 3, 5) - args[0] = "SET" + args := make([]interface{}, 3, 4) + args[0] = "set" args[1] = key args[2] = value if expiration > 0 { if usePrecise(expiration) { - args = append(args, "PX", formatMs(expiration)) + args = append(args, "px", formatMs(expiration)) } else { - args = append(args, "EX", formatSec(expiration)) + args = append(args, "ex", formatSec(expiration)) } } cmd := NewStatusCmd(args...) @@ -582,12 +577,12 @@ func (c *commandable) SetNX(key string, value interface{}, expiration time.Durat var cmd *BoolCmd if expiration == 0 { // Use old `SETNX` to support old Redis versions. - cmd = NewBoolCmd("SETNX", key, value) + cmd = NewBoolCmd("setnx", key, value) } else { if usePrecise(expiration) { - cmd = NewBoolCmd("SET", key, value, "PX", formatMs(expiration), "NX") + cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "nx") } else { - cmd = NewBoolCmd("SET", key, value, "EX", formatSec(expiration), "NX") + cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "nx") } } c.Process(cmd) @@ -600,22 +595,22 @@ func (c *commandable) SetNX(key string, value interface{}, expiration time.Durat func (c *commandable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if usePrecise(expiration) { - cmd = NewBoolCmd("SET", key, value, "PX", formatMs(expiration), "XX") + cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "xx") } else { - cmd = NewBoolCmd("SET", key, value, "EX", formatSec(expiration), "XX") + cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "xx") } c.Process(cmd) return cmd } func (c *commandable) SetRange(key string, offset int64, value string) *IntCmd { - cmd := NewIntCmd("SETRANGE", key, offset, value) + cmd := NewIntCmd("setrange", key, offset, value) c.Process(cmd) return cmd } func (c *commandable) StrLen(key string) *IntCmd { - cmd := NewIntCmd("STRLEN", key) + cmd := NewIntCmd("strlen", key) c.Process(cmd) return cmd } @@ -624,7 +619,7 @@ func (c *commandable) StrLen(key string) *IntCmd { func (c *commandable) HDel(key string, fields ...string) *IntCmd { args := make([]interface{}, 2+len(fields)) - args[0] = "HDEL" + args[0] = "hdel" args[1] = key for i, field := range fields { args[2+i] = field @@ -635,50 +630,50 @@ func (c *commandable) HDel(key string, fields ...string) *IntCmd { } func (c *commandable) HExists(key, field string) *BoolCmd { - cmd := NewBoolCmd("HEXISTS", key, field) + cmd := NewBoolCmd("hexists", key, field) c.Process(cmd) return cmd } func (c *commandable) HGet(key, field string) *StringCmd { - cmd := NewStringCmd("HGET", key, field) + cmd := NewStringCmd("hget", key, field) c.Process(cmd) return cmd } func (c *commandable) HGetAll(key string) *StringStringMapCmd { - cmd := NewStringStringMapCmd("HGETALL", key) + cmd := NewStringStringMapCmd("hgetall", key) c.Process(cmd) return cmd } func (c *commandable) HIncrBy(key, field string, incr int64) *IntCmd { - cmd := NewIntCmd("HINCRBY", key, field, incr) + cmd := NewIntCmd("hincrby", key, field, incr) c.Process(cmd) return cmd } func (c *commandable) HIncrByFloat(key, field string, incr float64) *FloatCmd { - cmd := NewFloatCmd("HINCRBYFLOAT", key, field, incr) + cmd := NewFloatCmd("hincrbyfloat", key, field, incr) c.Process(cmd) return cmd } func (c *commandable) HKeys(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("HKEYS", key) + cmd := NewStringSliceCmd("hkeys", key) c.Process(cmd) return cmd } func (c *commandable) HLen(key string) *IntCmd { - cmd := NewIntCmd("HLEN", key) + cmd := NewIntCmd("hlen", key) c.Process(cmd) return cmd } func (c *commandable) HMGet(key string, fields ...string) *SliceCmd { args := make([]interface{}, 2+len(fields)) - args[0] = "HMGET" + args[0] = "hmget" args[1] = key for i, field := range fields { args[2+i] = field @@ -690,7 +685,7 @@ func (c *commandable) HMGet(key string, fields ...string) *SliceCmd { func (c *commandable) HMSet(key string, fields map[string]string) *StatusCmd { args := make([]interface{}, 2+len(fields)*2) - args[0] = "HMSET" + args[0] = "hmset" args[1] = key i := 2 for k, v := range fields { @@ -704,19 +699,19 @@ func (c *commandable) HMSet(key string, fields map[string]string) *StatusCmd { } func (c *commandable) HSet(key, field, value string) *BoolCmd { - cmd := NewBoolCmd("HSET", key, field, value) + cmd := NewBoolCmd("hset", key, field, value) c.Process(cmd) return cmd } func (c *commandable) HSetNX(key, field, value string) *BoolCmd { - cmd := NewBoolCmd("HSETNX", key, field, value) + cmd := NewBoolCmd("hsetnx", key, field, value) c.Process(cmd) return cmd } func (c *commandable) HVals(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("HVALS", key) + cmd := NewStringSliceCmd("hvals", key) c.Process(cmd) return cmd } @@ -724,8 +719,8 @@ func (c *commandable) HVals(key string) *StringSliceCmd { //------------------------------------------------------------------------------ func (c *commandable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "BLPOP" + args := make([]interface{}, 1+len(keys)+1) + args[0] = "blpop" for i, key := range keys { args[1+i] = key } @@ -737,12 +732,12 @@ func (c *commandable) BLPop(timeout time.Duration, keys ...string) *StringSliceC } func (c *commandable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "BRPOP" + args := make([]interface{}, 1+len(keys)+1) + args[0] = "brpop" for i, key := range keys { args[1+i] = key } - args[len(args)-1] = formatSec(timeout) + args[len(keys)+1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) cmd.setReadTimeout(readTimeout(timeout)) c.Process(cmd) @@ -751,7 +746,7 @@ func (c *commandable) BRPop(timeout time.Duration, keys ...string) *StringSliceC func (c *commandable) BRPopLPush(source, destination string, timeout time.Duration) *StringCmd { cmd := NewStringCmd( - "BRPOPLPUSH", + "brpoplpush", source, destination, formatSec(timeout), @@ -762,32 +757,32 @@ func (c *commandable) BRPopLPush(source, destination string, timeout time.Durati } func (c *commandable) LIndex(key string, index int64) *StringCmd { - cmd := NewStringCmd("LINDEX", key, index) + cmd := NewStringCmd("lindex", key, index) c.Process(cmd) return cmd } func (c *commandable) LInsert(key, op, pivot, value string) *IntCmd { - cmd := NewIntCmd("LINSERT", key, op, pivot, value) + cmd := NewIntCmd("linsert", key, op, pivot, value) c.Process(cmd) return cmd } func (c *commandable) LLen(key string) *IntCmd { - cmd := NewIntCmd("LLEN", key) + cmd := NewIntCmd("llen", key) c.Process(cmd) return cmd } func (c *commandable) LPop(key string) *StringCmd { - cmd := NewStringCmd("LPOP", key) + cmd := NewStringCmd("lpop", key) c.Process(cmd) return cmd } func (c *commandable) LPush(key string, values ...string) *IntCmd { args := make([]interface{}, 2+len(values)) - args[0] = "LPUSH" + args[0] = "lpush" args[1] = key for i, value := range values { args[2+i] = value @@ -798,14 +793,14 @@ func (c *commandable) LPush(key string, values ...string) *IntCmd { } func (c *commandable) LPushX(key, value interface{}) *IntCmd { - cmd := NewIntCmd("LPUSHX", key, value) + cmd := NewIntCmd("lpushx", key, value) c.Process(cmd) return cmd } func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd( - "LRANGE", + "lrange", key, start, stop, @@ -815,20 +810,20 @@ func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { } func (c *commandable) LRem(key string, count int64, value interface{}) *IntCmd { - cmd := NewIntCmd("LREM", key, count, value) + cmd := NewIntCmd("lrem", key, count, value) c.Process(cmd) return cmd } func (c *commandable) LSet(key string, index int64, value interface{}) *StatusCmd { - cmd := NewStatusCmd("LSET", key, index, value) + cmd := NewStatusCmd("lset", key, index, value) c.Process(cmd) return cmd } func (c *commandable) LTrim(key string, start, stop int64) *StatusCmd { cmd := NewStatusCmd( - "LTRIM", + "ltrim", key, start, stop, @@ -838,20 +833,20 @@ func (c *commandable) LTrim(key string, start, stop int64) *StatusCmd { } func (c *commandable) RPop(key string) *StringCmd { - cmd := NewStringCmd("RPOP", key) + cmd := NewStringCmd("rpop", key) c.Process(cmd) return cmd } func (c *commandable) RPopLPush(source, destination string) *StringCmd { - cmd := NewStringCmd("RPOPLPUSH", source, destination) + cmd := NewStringCmd("rpoplpush", source, destination) c.Process(cmd) return cmd } func (c *commandable) RPush(key string, values ...string) *IntCmd { args := make([]interface{}, 2+len(values)) - args[0] = "RPUSH" + args[0] = "rpush" args[1] = key for i, value := range values { args[2+i] = value @@ -862,7 +857,7 @@ func (c *commandable) RPush(key string, values ...string) *IntCmd { } func (c *commandable) RPushX(key string, value interface{}) *IntCmd { - cmd := NewIntCmd("RPUSHX", key, value) + cmd := NewIntCmd("rpushx", key, value) c.Process(cmd) return cmd } @@ -871,7 +866,7 @@ func (c *commandable) RPushX(key string, value interface{}) *IntCmd { func (c *commandable) SAdd(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) - args[0] = "SADD" + args[0] = "sadd" args[1] = key for i, member := range members { args[2+i] = member @@ -882,14 +877,14 @@ func (c *commandable) SAdd(key string, members ...string) *IntCmd { } func (c *commandable) SCard(key string) *IntCmd { - cmd := NewIntCmd("SCARD", key) + cmd := NewIntCmd("scard", key) c.Process(cmd) return cmd } func (c *commandable) SDiff(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "SDIFF" + args[0] = "sdiff" for i, key := range keys { args[1+i] = key } @@ -900,7 +895,7 @@ func (c *commandable) SDiff(keys ...string) *StringSliceCmd { func (c *commandable) SDiffStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "SDIFFSTORE" + args[0] = "sdiffstore" args[1] = destination for i, key := range keys { args[2+i] = key @@ -912,7 +907,7 @@ func (c *commandable) SDiffStore(destination string, keys ...string) *IntCmd { func (c *commandable) SInter(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "SINTER" + args[0] = "sinter" for i, key := range keys { args[1+i] = key } @@ -923,7 +918,7 @@ func (c *commandable) SInter(keys ...string) *StringSliceCmd { func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "SINTERSTORE" + args[0] = "sinterstore" args[1] = destination for i, key := range keys { args[2+i] = key @@ -934,46 +929,46 @@ func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { } func (c *commandable) SIsMember(key string, member interface{}) *BoolCmd { - cmd := NewBoolCmd("SISMEMBER", key, member) + cmd := NewBoolCmd("sismember", key, member) c.Process(cmd) return cmd } func (c *commandable) SMembers(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("SMEMBERS", key) + cmd := NewStringSliceCmd("smembers", key) c.Process(cmd) return cmd } func (c *commandable) SMove(source, destination string, member interface{}) *BoolCmd { - cmd := NewBoolCmd("SMOVE", source, destination, member) + cmd := NewBoolCmd("smove", source, destination, member) c.Process(cmd) return cmd } func (c *commandable) SPop(key string) *StringCmd { - cmd := NewStringCmd("SPOP", key) + cmd := NewStringCmd("spop", key) c.Process(cmd) return cmd } // Redis `SRANDMEMBER key` command. func (c *commandable) SRandMember(key string) *StringCmd { - cmd := NewStringCmd("SRANDMEMBER", key) + cmd := NewStringCmd("srandmember", key) c.Process(cmd) return cmd } // Redis `SRANDMEMBER key count` command. func (c *commandable) SRandMemberN(key string, count int64) *StringSliceCmd { - cmd := NewStringSliceCmd("SRANDMEMBER", key, count) + cmd := NewStringSliceCmd("srandmember", key, count) c.Process(cmd) return cmd } func (c *commandable) SRem(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) - args[0] = "SREM" + args[0] = "srem" args[1] = key for i, member := range members { args[2+i] = member @@ -985,7 +980,7 @@ func (c *commandable) SRem(key string, members ...string) *IntCmd { func (c *commandable) SUnion(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "SUNION" + args[0] = "sunion" for i, key := range keys { args[1+i] = key } @@ -996,7 +991,7 @@ func (c *commandable) SUnion(keys ...string) *StringSliceCmd { func (c *commandable) SUnionStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "SUNIONSTORE" + args[0] = "sunionstore" args[1] = destination for i, key := range keys { args[2+i] = key @@ -1035,7 +1030,7 @@ func (c *commandable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { const n = 2 a := make([]interface{}, n+2*len(members)) - a[0], a[1] = "ZADD", key + a[0], a[1] = "zadd", key return c.zAdd(a, n, members...) } @@ -1043,7 +1038,7 @@ func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { func (c *commandable) ZAddNX(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2] = "ZADD", key, "NX" + a[0], a[1], a[2] = "zadd", key, "nx" return c.zAdd(a, n, members...) } @@ -1051,7 +1046,7 @@ func (c *commandable) ZAddNX(key string, members ...Z) *IntCmd { func (c *commandable) ZAddXX(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2] = "ZADD", key, "XX" + a[0], a[1], a[2] = "zadd", key, "xx" return c.zAdd(a, n, members...) } @@ -1059,7 +1054,7 @@ func (c *commandable) ZAddXX(key string, members ...Z) *IntCmd { func (c *commandable) ZAddCh(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2] = "ZADD", key, "CH" + a[0], a[1], a[2] = "zadd", key, "ch" return c.zAdd(a, n, members...) } @@ -1067,7 +1062,7 @@ func (c *commandable) ZAddCh(key string, members ...Z) *IntCmd { func (c *commandable) ZAddNXCh(key string, members ...Z) *IntCmd { const n = 4 a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2], a[3] = "ZADD", key, "NX", "CH" + a[0], a[1], a[2], a[3] = "zadd", key, "nx", "ch" return c.zAdd(a, n, members...) } @@ -1075,7 +1070,7 @@ func (c *commandable) ZAddNXCh(key string, members ...Z) *IntCmd { func (c *commandable) ZAddXXCh(key string, members ...Z) *IntCmd { const n = 4 a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2], a[3] = "ZADD", key, "XX", "CH" + a[0], a[1], a[2], a[3] = "zadd", key, "xx", "ch" return c.zAdd(a, n, members...) } @@ -1093,7 +1088,7 @@ func (c *commandable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { func (c *commandable) ZIncr(key string, member Z) *FloatCmd { const n = 3 a := make([]interface{}, n+2) - a[0], a[1], a[2] = "ZADD", key, "INCR" + a[0], a[1], a[2] = "zadd", key, "incr" return c.zIncr(a, n, member) } @@ -1101,7 +1096,7 @@ func (c *commandable) ZIncr(key string, member Z) *FloatCmd { func (c *commandable) ZIncrNX(key string, member Z) *FloatCmd { const n = 4 a := make([]interface{}, n+2) - a[0], a[1], a[2], a[3] = "ZADD", key, "INCR", "NX" + a[0], a[1], a[2], a[3] = "zadd", key, "incr", "nx" return c.zIncr(a, n, member) } @@ -1109,44 +1104,44 @@ func (c *commandable) ZIncrNX(key string, member Z) *FloatCmd { func (c *commandable) ZIncrXX(key string, member Z) *FloatCmd { const n = 4 a := make([]interface{}, n+2) - a[0], a[1], a[2], a[3] = "ZADD", key, "INCR", "XX" + a[0], a[1], a[2], a[3] = "zadd", key, "incr", "xx" return c.zIncr(a, n, member) } func (c *commandable) ZCard(key string) *IntCmd { - cmd := NewIntCmd("ZCARD", key) + cmd := NewIntCmd("zcard", key) c.Process(cmd) return cmd } func (c *commandable) ZCount(key, min, max string) *IntCmd { - cmd := NewIntCmd("ZCOUNT", key, min, max) + cmd := NewIntCmd("zcount", key, min, max) c.Process(cmd) return cmd } func (c *commandable) ZIncrBy(key string, increment float64, member string) *FloatCmd { - cmd := NewFloatCmd("ZINCRBY", key, increment, member) + cmd := NewFloatCmd("zincrby", key, increment, member) c.Process(cmd) return cmd } func (c *commandable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) - args[0] = "ZINTERSTORE" + args[0] = "zinterstore" args[1] = destination args[2] = strconv.Itoa(len(keys)) for i, key := range keys { args[3+i] = key } if len(store.Weights) > 0 { - args = append(args, "WEIGHTS") + args = append(args, "weights") for _, weight := range store.Weights { args = append(args, weight) } } if store.Aggregate != "" { - args = append(args, "AGGREGATE", store.Aggregate) + args = append(args, "aggregate", store.Aggregate) } cmd := NewIntCmd(args...) c.Process(cmd) @@ -1155,13 +1150,13 @@ func (c *commandable) ZInterStore(destination string, store ZStore, keys ...stri func (c *commandable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { args := []interface{}{ - "ZRANGE", + "zrange", key, start, stop, } if withScores { - args = append(args, "WITHSCORES") + args = append(args, "withscores") } cmd := NewStringSliceCmd(args...) c.Process(cmd) @@ -1173,7 +1168,7 @@ func (c *commandable) ZRange(key string, start, stop int64) *StringSliceCmd { } func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { - cmd := NewZSliceCmd("ZRANGE", key, start, stop, "WITHSCORES") + cmd := NewZSliceCmd("zrange", key, start, stop, "withscores") c.Process(cmd) return cmd } @@ -1186,12 +1181,12 @@ type ZRangeBy struct { func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Min, opt.Max} if withScores { - args = append(args, "WITHSCORES") + args = append(args, "withscores") } if opt.Offset != 0 || opt.Count != 0 { args = append( args, - "LIMIT", + "limit", opt.Offset, opt.Count, ) @@ -1202,19 +1197,19 @@ func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) } func (c *commandable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRangeBy("ZRANGEBYSCORE", key, opt, false) + return c.zRangeBy("zrangebyscore", key, opt, false) } func (c *commandable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRangeBy("ZRANGEBYLEX", key, opt, false) + return c.zRangeBy("zrangebylex", key, opt, false) } func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { - args := []interface{}{"ZRANGEBYSCORE", key, opt.Min, opt.Max, "WITHSCORES"} + args := []interface{}{"zrangebyscore", key, opt.Min, opt.Max, "withscores"} if opt.Offset != 0 || opt.Count != 0 { args = append( args, - "LIMIT", + "limit", opt.Offset, opt.Count, ) @@ -1225,14 +1220,14 @@ func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceC } func (c *commandable) ZRank(key, member string) *IntCmd { - cmd := NewIntCmd("ZRANK", key, member) + cmd := NewIntCmd("zrank", key, member) c.Process(cmd) return cmd } func (c *commandable) ZRem(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) - args[0] = "ZREM" + args[0] = "zrem" args[1] = key for i, member := range members { args[2+i] = member @@ -1244,7 +1239,7 @@ func (c *commandable) ZRem(key string, members ...string) *IntCmd { func (c *commandable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { cmd := NewIntCmd( - "ZREMRANGEBYRANK", + "zremrangebyrank", key, start, stop, @@ -1254,19 +1249,19 @@ func (c *commandable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { } func (c *commandable) ZRemRangeByScore(key, min, max string) *IntCmd { - cmd := NewIntCmd("ZREMRANGEBYSCORE", key, min, max) + cmd := NewIntCmd("zremrangebyscore", key, min, max) c.Process(cmd) return cmd } func (c *commandable) ZRevRange(key string, start, stop int64) *StringSliceCmd { - cmd := NewStringSliceCmd("ZREVRANGE", key, start, stop) + cmd := NewStringSliceCmd("zrevrange", key, start, stop) c.Process(cmd) return cmd } func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { - cmd := NewZSliceCmd("ZREVRANGE", key, start, stop, "WITHSCORES") + cmd := NewZSliceCmd("zrevrange", key, start, stop, "withscores") c.Process(cmd) return cmd } @@ -1276,7 +1271,7 @@ func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCm if opt.Offset != 0 || opt.Count != 0 { args = append( args, - "LIMIT", + "limit", opt.Offset, opt.Count, ) @@ -1287,19 +1282,19 @@ func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCm } func (c *commandable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRevRangeBy("ZREVRANGEBYSCORE", key, opt) + return c.zRevRangeBy("zrevrangebyscore", key, opt) } func (c *commandable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRevRangeBy("ZREVRANGEBYLEX", key, opt) + return c.zRevRangeBy("zrevrangebylex", key, opt) } func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { - args := []interface{}{"ZREVRANGEBYSCORE", key, opt.Max, opt.Min, "WITHSCORES"} + args := []interface{}{"zrevrangebyscore", key, opt.Max, opt.Min, "withscores"} if opt.Offset != 0 || opt.Count != 0 { args = append( args, - "LIMIT", + "limit", opt.Offset, opt.Count, ) @@ -1310,33 +1305,33 @@ func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSli } func (c *commandable) ZRevRank(key, member string) *IntCmd { - cmd := NewIntCmd("ZREVRANK", key, member) + cmd := NewIntCmd("zrevrank", key, member) c.Process(cmd) return cmd } func (c *commandable) ZScore(key, member string) *FloatCmd { - cmd := NewFloatCmd("ZSCORE", key, member) + cmd := NewFloatCmd("zscore", key, member) c.Process(cmd) return cmd } func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) - args[0] = "ZUNIONSTORE" + args[0] = "zunionstore" args[1] = dest args[2] = strconv.Itoa(len(keys)) for i, key := range keys { args[3+i] = key } if len(store.Weights) > 0 { - args = append(args, "WEIGHTS") + args = append(args, "weights") for _, weight := range store.Weights { args = append(args, weight) } } if store.Aggregate != "" { - args = append(args, "AGGREGATE", store.Aggregate) + args = append(args, "aggregate", store.Aggregate) } cmd := NewIntCmd(args...) c.Process(cmd) @@ -1347,7 +1342,7 @@ func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *In func (c *commandable) PFAdd(key string, fields ...string) *IntCmd { args := make([]interface{}, 2+len(fields)) - args[0] = "PFADD" + args[0] = "pfadd" args[1] = key for i, field := range fields { args[2+i] = field @@ -1359,7 +1354,7 @@ func (c *commandable) PFAdd(key string, fields ...string) *IntCmd { func (c *commandable) PFCount(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "PFCOUNT" + args[0] = "pfcount" for i, key := range keys { args[1+i] = key } @@ -1370,7 +1365,7 @@ func (c *commandable) PFCount(keys ...string) *IntCmd { func (c *commandable) PFMerge(dest string, keys ...string) *StatusCmd { args := make([]interface{}, 2+len(keys)) - args[0] = "PFMERGE" + args[0] = "pfmerge" args[1] = dest for i, key := range keys { args[2+i] = key @@ -1383,96 +1378,87 @@ func (c *commandable) PFMerge(dest string, keys ...string) *StatusCmd { //------------------------------------------------------------------------------ func (c *commandable) BgRewriteAOF() *StatusCmd { - cmd := NewStatusCmd("BGREWRITEAOF") - cmd._clusterKeyPos = 0 + cmd := NewStatusCmd("bgrewriteaof") c.Process(cmd) return cmd } func (c *commandable) BgSave() *StatusCmd { - cmd := NewStatusCmd("BGSAVE") - cmd._clusterKeyPos = 0 + cmd := NewStatusCmd("bgsave") c.Process(cmd) return cmd } func (c *commandable) ClientKill(ipPort string) *StatusCmd { - cmd := NewStatusCmd("CLIENT", "KILL", ipPort) - cmd._clusterKeyPos = 0 + cmd := NewStatusCmd("client", "kill", ipPort) c.Process(cmd) return cmd } func (c *commandable) ClientList() *StringCmd { - cmd := NewStringCmd("CLIENT", "LIST") - cmd._clusterKeyPos = 0 + cmd := NewStringCmd("client", "list") c.Process(cmd) return cmd } func (c *commandable) ClientPause(dur time.Duration) *BoolCmd { - cmd := NewBoolCmd("CLIENT", "PAUSE", formatMs(dur)) - cmd._clusterKeyPos = 0 + cmd := NewBoolCmd("client", "pause", formatMs(dur)) c.Process(cmd) return cmd } // ClientSetName assigns a name to the one of many connections in the pool. func (c *commandable) ClientSetName(name string) *BoolCmd { - cmd := NewBoolCmd("CLIENT", "SETNAME", name) + cmd := NewBoolCmd("client", "setname", name) c.Process(cmd) return cmd } // ClientGetName returns the name of the one of many connections in the pool. func (c *Client) ClientGetName() *StringCmd { - cmd := NewStringCmd("CLIENT", "GETNAME") + cmd := NewStringCmd("client", "getname") c.Process(cmd) return cmd } func (c *commandable) ConfigGet(parameter string) *SliceCmd { - cmd := NewSliceCmd("CONFIG", "GET", parameter) - cmd._clusterKeyPos = 0 + cmd := NewSliceCmd("config", "get", parameter) c.Process(cmd) return cmd } func (c *commandable) ConfigResetStat() *StatusCmd { - cmd := NewStatusCmd("CONFIG", "RESETSTAT") - cmd._clusterKeyPos = 0 + cmd := NewStatusCmd("config", "resetstat") c.Process(cmd) return cmd } func (c *commandable) ConfigSet(parameter, value string) *StatusCmd { - cmd := NewStatusCmd("CONFIG", "SET", parameter, value) - cmd._clusterKeyPos = 0 + cmd := NewStatusCmd("config", "set", parameter, value) c.Process(cmd) return cmd } func (c *commandable) DbSize() *IntCmd { - cmd := NewIntCmd("DBSIZE") - cmd._clusterKeyPos = 0 + cmd := NewIntCmd("dbsize") c.Process(cmd) return cmd } func (c *commandable) FlushAll() *StatusCmd { - cmd := newKeylessStatusCmd("FLUSHALL") + cmd := NewStatusCmd("flushall") c.Process(cmd) return cmd } func (c *commandable) FlushDb() *StatusCmd { - cmd := newKeylessStatusCmd("FLUSHDB") + cmd := NewStatusCmd("flushdb") c.Process(cmd) return cmd } func (c *commandable) Info(section ...string) *StringCmd { - args := []interface{}{"INFO"} + args := []interface{}{"info"} if len(section) > 0 { args = append(args, section[0]) } @@ -1482,14 +1468,13 @@ func (c *commandable) Info(section ...string) *StringCmd { } func (c *commandable) LastSave() *IntCmd { - cmd := NewIntCmd("LASTSAVE") - cmd._clusterKeyPos = 0 + cmd := NewIntCmd("lastsave") c.Process(cmd) return cmd } func (c *commandable) Save() *StatusCmd { - cmd := newKeylessStatusCmd("SAVE") + cmd := NewStatusCmd("save") c.Process(cmd) return cmd } @@ -1497,11 +1482,11 @@ func (c *commandable) Save() *StatusCmd { func (c *commandable) shutdown(modifier string) *StatusCmd { var args []interface{} if modifier == "" { - args = []interface{}{"SHUTDOWN"} + args = []interface{}{"shutdown"} } else { - args = []interface{}{"SHUTDOWN", modifier} + args = []interface{}{"shutdown", modifier} } - cmd := newKeylessStatusCmd(args...) + cmd := NewStatusCmd(args...) c.Process(cmd) if err := cmd.Err(); err != nil { if err == io.EOF { @@ -1521,15 +1506,15 @@ func (c *commandable) Shutdown() *StatusCmd { } func (c *commandable) ShutdownSave() *StatusCmd { - return c.shutdown("SAVE") + return c.shutdown("save") } func (c *commandable) ShutdownNoSave() *StatusCmd { - return c.shutdown("NOSAVE") + return c.shutdown("nosave") } func (c *commandable) SlaveOf(host, port string) *StatusCmd { - cmd := newKeylessStatusCmd("SLAVEOF", host, port) + cmd := NewStatusCmd("slaveof", host, port) c.Process(cmd) return cmd } @@ -1543,8 +1528,7 @@ func (c *commandable) Sync() { } func (c *commandable) Time() *StringSliceCmd { - cmd := NewStringSliceCmd("TIME") - cmd._clusterKeyPos = 0 + cmd := NewStringSliceCmd("time") c.Process(cmd) return cmd } @@ -1553,7 +1537,7 @@ func (c *commandable) Time() *StringSliceCmd { func (c *commandable) Eval(script string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) - cmdArgs[0] = "EVAL" + cmdArgs[0] = "eval" cmdArgs[1] = script cmdArgs[2] = strconv.Itoa(len(keys)) for i, key := range keys { @@ -1564,16 +1548,13 @@ func (c *commandable) Eval(script string, keys []string, args ...interface{}) *C cmdArgs[pos+i] = arg } cmd := NewCmd(cmdArgs...) - if len(keys) > 0 { - cmd._clusterKeyPos = 3 - } c.Process(cmd) return cmd } func (c *commandable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) - cmdArgs[0] = "EVALSHA" + cmdArgs[0] = "evalsha" cmdArgs[1] = sha1 cmdArgs[2] = strconv.Itoa(len(keys)) for i, key := range keys { @@ -1584,41 +1565,36 @@ func (c *commandable) EvalSha(sha1 string, keys []string, args ...interface{}) * cmdArgs[pos+i] = arg } cmd := NewCmd(cmdArgs...) - if len(keys) > 0 { - cmd._clusterKeyPos = 3 - } c.Process(cmd) return cmd } func (c *commandable) ScriptExists(scripts ...string) *BoolSliceCmd { args := make([]interface{}, 2+len(scripts)) - args[0] = "SCRIPT" - args[1] = "EXISTS" + args[0] = "script" + args[1] = "exists" for i, script := range scripts { args[2+i] = script } cmd := NewBoolSliceCmd(args...) - cmd._clusterKeyPos = 0 c.Process(cmd) return cmd } func (c *commandable) ScriptFlush() *StatusCmd { - cmd := newKeylessStatusCmd("SCRIPT", "FLUSH") + cmd := NewStatusCmd("script", "flush") c.Process(cmd) return cmd } func (c *commandable) ScriptKill() *StatusCmd { - cmd := newKeylessStatusCmd("SCRIPT", "KILL") + cmd := NewStatusCmd("script", "kill") c.Process(cmd) return cmd } func (c *commandable) ScriptLoad(script string) *StringCmd { - cmd := NewStringCmd("SCRIPT", "LOAD", script) - cmd._clusterKeyPos = 0 + cmd := NewStringCmd("script", "load", script) c.Process(cmd) return cmd } @@ -1626,8 +1602,7 @@ func (c *commandable) ScriptLoad(script string) *StringCmd { //------------------------------------------------------------------------------ func (c *commandable) DebugObject(key string) *StringCmd { - cmd := NewStringCmd("DEBUG", "OBJECT", key) - cmd._clusterKeyPos = 2 + cmd := NewStringCmd("debug", "object", key) c.Process(cmd) return cmd } @@ -1635,32 +1610,29 @@ func (c *commandable) DebugObject(key string) *StringCmd { //------------------------------------------------------------------------------ func (c *commandable) PubSubChannels(pattern string) *StringSliceCmd { - args := []interface{}{"PUBSUB", "CHANNELS"} + args := []interface{}{"pubsub", "channels"} if pattern != "*" { args = append(args, pattern) } cmd := NewStringSliceCmd(args...) - cmd._clusterKeyPos = 0 c.Process(cmd) return cmd } func (c *commandable) PubSubNumSub(channels ...string) *StringIntMapCmd { args := make([]interface{}, 2+len(channels)) - args[0] = "PUBSUB" - args[1] = "NUMSUB" + args[0] = "pubsub" + args[1] = "numsub" for i, channel := range channels { args[2+i] = channel } cmd := NewStringIntMapCmd(args...) - cmd._clusterKeyPos = 0 c.Process(cmd) return cmd } func (c *commandable) PubSubNumPat() *IntCmd { - cmd := NewIntCmd("PUBSUB", "NUMPAT") - cmd._clusterKeyPos = 0 + cmd := NewIntCmd("pubsub", "numpat") c.Process(cmd) return cmd } @@ -1668,85 +1640,79 @@ func (c *commandable) PubSubNumPat() *IntCmd { //------------------------------------------------------------------------------ func (c *commandable) ClusterSlots() *ClusterSlotsCmd { - cmd := NewClusterSlotsCmd("CLUSTER", "slots") - cmd._clusterKeyPos = 0 + cmd := NewClusterSlotsCmd("cluster", "slots") c.Process(cmd) return cmd } func (c *commandable) ClusterNodes() *StringCmd { - cmd := NewStringCmd("CLUSTER", "nodes") - cmd._clusterKeyPos = 0 + cmd := NewStringCmd("cluster", "nodes") c.Process(cmd) return cmd } func (c *commandable) ClusterMeet(host, port string) *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "meet", host, port) + cmd := NewStatusCmd("cluster", "meet", host, port) c.Process(cmd) return cmd } func (c *commandable) ClusterForget(nodeID string) *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "forget", nodeID) + cmd := NewStatusCmd("cluster", "forget", nodeID) c.Process(cmd) return cmd } func (c *commandable) ClusterReplicate(nodeID string) *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "replicate", nodeID) + cmd := NewStatusCmd("cluster", "replicate", nodeID) c.Process(cmd) return cmd } func (c *commandable) ClusterResetSoft() *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "reset", "soft") + cmd := NewStatusCmd("cluster", "reset", "soft") c.Process(cmd) return cmd } func (c *commandable) ClusterResetHard() *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "reset", "hard") + cmd := NewStatusCmd("cluster", "reset", "hard") c.Process(cmd) return cmd } func (c *commandable) ClusterInfo() *StringCmd { - cmd := NewStringCmd("CLUSTER", "info") - cmd._clusterKeyPos = 0 + cmd := NewStringCmd("cluster", "info") c.Process(cmd) return cmd } func (c *commandable) ClusterKeySlot(key string) *IntCmd { - cmd := NewIntCmd("CLUSTER", "keyslot", key) - cmd._clusterKeyPos = 2 + cmd := NewIntCmd("cluster", "keyslot", key) c.Process(cmd) return cmd } func (c *commandable) ClusterCountFailureReports(nodeID string) *IntCmd { - cmd := NewIntCmd("CLUSTER", "count-failure-reports", nodeID) - cmd._clusterKeyPos = 2 + cmd := NewIntCmd("cluster", "count-failure-reports", nodeID) c.Process(cmd) return cmd } func (c *commandable) ClusterCountKeysInSlot(slot int) *IntCmd { - cmd := NewIntCmd("CLUSTER", "countkeysinslot", slot) - cmd._clusterKeyPos = 2 + cmd := NewIntCmd("cluster", "countkeysinslot", slot) c.Process(cmd) return cmd } func (c *commandable) ClusterDelSlots(slots ...int) *StatusCmd { args := make([]interface{}, 2+len(slots)) - args[0] = "CLUSTER" - args[1] = "DELSLOTS" + args[0] = "cluster" + args[1] = "delslots" for i, slot := range slots { args[2+i] = slot } - cmd := newKeylessStatusCmd(args...) + cmd := NewStatusCmd(args...) c.Process(cmd) return cmd } @@ -1761,46 +1727,43 @@ func (c *commandable) ClusterDelSlotsRange(min, max int) *StatusCmd { } func (c *commandable) ClusterSaveConfig() *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "saveconfig") + cmd := NewStatusCmd("cluster", "saveconfig") c.Process(cmd) return cmd } func (c *commandable) ClusterSlaves(nodeID string) *StringSliceCmd { - cmd := NewStringSliceCmd("CLUSTER", "SLAVES", nodeID) - cmd._clusterKeyPos = 2 + cmd := NewStringSliceCmd("cluster", "slaves", nodeID) c.Process(cmd) return cmd } -func (c *commandable) Readonly() *StatusCmd { - cmd := newKeylessStatusCmd("READONLY") - cmd._clusterKeyPos = 0 +func (c *commandable) ReadOnly() *StatusCmd { + cmd := NewStatusCmd("readonly") c.Process(cmd) return cmd } func (c *commandable) ReadWrite() *StatusCmd { - cmd := newKeylessStatusCmd("READWRITE") - cmd._clusterKeyPos = 0 + cmd := NewStatusCmd("readwrite") c.Process(cmd) return cmd } func (c *commandable) ClusterFailover() *StatusCmd { - cmd := newKeylessStatusCmd("CLUSTER", "failover") + cmd := NewStatusCmd("cluster", "failover") c.Process(cmd) return cmd } func (c *commandable) ClusterAddSlots(slots ...int) *StatusCmd { args := make([]interface{}, 2+len(slots)) - args[0] = "CLUSTER" - args[1] = "ADDSLOTS" + args[0] = "cluster" + args[1] = "addslots" for i, num := range slots { args[2+i] = strconv.Itoa(num) } - cmd := newKeylessStatusCmd(args...) + cmd := NewStatusCmd(args...) c.Process(cmd) return cmd } @@ -1818,7 +1781,7 @@ func (c *commandable) ClusterAddSlotsRange(min, max int) *StatusCmd { func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { args := make([]interface{}, 2+3*len(geoLocation)) - args[0] = "GEOADD" + args[0] = "geoadd" args[1] = key for i, eachLoc := range geoLocation { args[2+3*i] = eachLoc.Longitude @@ -1831,13 +1794,13 @@ func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { } func (c *commandable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { - cmd := NewGeoLocationCmd(query, "GEORADIUS", key, longitude, latitude) + cmd := NewGeoLocationCmd(query, "georadius", key, longitude, latitude) c.Process(cmd) return cmd } func (c *commandable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { - cmd := NewGeoLocationCmd(query, "GEORADIUSBYMEMBER", key, member) + cmd := NewGeoLocationCmd(query, "georadiusbymember", key, member) c.Process(cmd) return cmd } @@ -1846,14 +1809,14 @@ func (c *commandable) GeoDist(key string, member1, member2, unit string) *FloatC if unit == "" { unit = "km" } - cmd := NewFloatCmd("GEODIST", key, member1, member2, unit) + cmd := NewFloatCmd("geodist", key, member1, member2, unit) c.Process(cmd) return cmd } func (c *commandable) GeoHash(key string, members ...string) *StringSliceCmd { args := make([]interface{}, 2+len(members)) - args[0] = "GEOHASH" + args[0] = "geohash" args[1] = key for i, member := range members { args[2+i] = member @@ -1862,3 +1825,11 @@ func (c *commandable) GeoHash(key string, members ...string) *StringSliceCmd { c.Process(cmd) return cmd } + +//------------------------------------------------------------------------------ + +func (c *commandable) Command() *CommandsInfoCmd { + cmd := NewCommandsInfoCmd("command") + c.Process(cmd) + return cmd +} diff --git a/commands_test.go b/commands_test.go index d4f25ff863..9d65f96b2b 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2729,6 +2729,7 @@ var _ = Describe("Commands", func() { }) Describe("json marshaling/unmarshaling", func() { + BeforeEach(func() { value := &numberStruct{Number: 42} err := client.Set("key", value, 0).Err() @@ -2744,12 +2745,30 @@ var _ = Describe("Commands", func() { It("should scan custom values using json", func() { value := &numberStruct{} err := client.Get("key").Scan(value) - Expect(err).To(BeNil()) + Expect(err).NotTo(HaveOccurred()) Expect(value.Number).To(Equal(42)) }) }) + Describe("Command", func() { + + It("returns map of commands", func() { + cmds, err := client.Command().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(len(cmds)).To(BeNumerically("~", 173, 5)) + + cmd := cmds["mget"] + Expect(cmd.Name).To(Equal("mget")) + Expect(cmd.Arity).To(Equal(int8(-2))) + Expect(cmd.Flags).To(Equal([]string{"readonly"})) + Expect(cmd.FirstKeyPos).To(Equal(int8(1))) + Expect(cmd.LastKeyPos).To(Equal(int8(-1))) + Expect(cmd.StepCount).To(Equal(int8(1))) + }) + + }) + }) type numberStruct struct { diff --git a/options.go b/options.go index dea2784c42..e1c04030be 100644 --- a/options.go +++ b/options.go @@ -53,6 +53,9 @@ type Options struct { // The frequency of idle checks. // Default is 1 minute. IdleCheckFrequency time.Duration + + // Enables read queries for a connection to a Redis Cluster slave node. + ReadOnly bool } func (opt *Options) getNetwork() string { diff --git a/parser.go b/parser.go index 7a7b6f3a63..a58d3b1ccd 100644 --- a/parser.go +++ b/parser.go @@ -627,8 +627,8 @@ func clusterSlotsParser(cn *pool.Conn, slotNum int64) (interface{}, error) { func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { return func(cn *pool.Conn, n int64) (interface{}, error) { var loc GeoLocation - var err error + loc.Name, err = readStringReply(cn) if err != nil { return nil, err @@ -690,3 +690,70 @@ func newGeoLocationSliceParser(q *GeoRadiusQuery) multiBulkParser { return locs, nil } } + +func commandInfoParser(cn *pool.Conn, n int64) (interface{}, error) { + var cmd CommandInfo + var err error + + if n != 6 { + return nil, fmt.Errorf("redis: got %d elements in COMMAND reply, wanted 6") + } + + cmd.Name, err = readStringReply(cn) + if err != nil { + return nil, err + } + + arity, err := readIntReply(cn) + if err != nil { + return nil, err + } + cmd.Arity = int8(arity) + + flags, err := readReply(cn, stringSliceParser) + if err != nil { + return nil, err + } + cmd.Flags = flags.([]string) + + firstKeyPos, err := readIntReply(cn) + if err != nil { + return nil, err + } + cmd.FirstKeyPos = int8(firstKeyPos) + + lastKeyPos, err := readIntReply(cn) + if err != nil { + return nil, err + } + cmd.LastKeyPos = int8(lastKeyPos) + + stepCount, err := readIntReply(cn) + if err != nil { + return nil, err + } + cmd.StepCount = int8(stepCount) + + for _, flag := range cmd.Flags { + if flag == "readonly" { + cmd.ReadOnly = true + break + } + } + + return &cmd, nil +} + +func commandInfoSliceParser(cn *pool.Conn, n int64) (interface{}, error) { + m := make(map[string]*CommandInfo, n) + for i := int64(0); i < n; i++ { + v, err := readReply(cn, commandInfoParser) + if err != nil { + return nil, err + } + vv := v.(*CommandInfo) + m[vv.Name] = vv + + } + return m, nil +} diff --git a/redis.go b/redis.go index 9222db9105..e2c173b315 100644 --- a/redis.go +++ b/redis.go @@ -52,7 +52,7 @@ func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { func (c *baseClient) initConn(cn *pool.Conn) error { cn.Inited = true - if c.opt.Password == "" && c.opt.DB == 0 { + if c.opt.Password == "" && c.opt.DB == 0 && !c.opt.ReadOnly { return nil } @@ -71,6 +71,12 @@ func (c *baseClient) initConn(cn *pool.Conn) error { } } + if c.opt.ReadOnly { + if err := client.ReadOnly().Err(); err != nil { + return err + } + } + return nil } diff --git a/ring.go b/ring.go index cac25d4e46..31559e61d7 100644 --- a/ring.go +++ b/ring.go @@ -115,10 +115,13 @@ type Ring struct { opt *RingOptions nreplicas int - mx sync.RWMutex + mu sync.RWMutex hash *consistenthash.Map shards map[string]*ringShard + cmdsInfo map[string]*CommandInfo + cmdsInfoOnce *sync.Once + closed bool } @@ -130,6 +133,8 @@ func NewRing(opt *RingOptions) *Ring { hash: consistenthash.New(nreplicas, nil), shards: make(map[string]*ringShard), + + cmdsInfoOnce: new(sync.Once), } ring.commandable.process = ring.process for name, addr := range opt.Addrs { @@ -141,15 +146,40 @@ func NewRing(opt *RingOptions) *Ring { return ring } +func (ring *Ring) cmdInfo(name string) *CommandInfo { + ring.cmdsInfoOnce.Do(func() { + for _, shard := range ring.shards { + cmdsInfo, err := shard.Client.Command().Result() + if err == nil { + ring.cmdsInfo = cmdsInfo + return + } + } + ring.cmdsInfoOnce = &sync.Once{} + }) + if ring.cmdsInfo == nil { + return nil + } + return ring.cmdsInfo[name] +} + +func (ring *Ring) cmdFirstKey(cmd Cmder) string { + cmdInfo := ring.cmdInfo(cmd.arg(0)) + if cmdInfo == nil { + return "" + } + return cmd.arg(int(cmdInfo.FirstKeyPos)) +} + func (ring *Ring) addClient(name string, cl *Client) { - ring.mx.Lock() + ring.mu.Lock() ring.hash.Add(name) ring.shards[name] = &ringShard{Client: cl} - ring.mx.Unlock() + ring.mu.Unlock() } func (ring *Ring) getClient(key string) (*Client, error) { - ring.mx.RLock() + ring.mu.RLock() if ring.closed { return nil, pool.ErrClosed @@ -157,17 +187,17 @@ func (ring *Ring) getClient(key string) (*Client, error) { name := ring.hash.Get(hashtag.Key(key)) if name == "" { - ring.mx.RUnlock() + ring.mu.RUnlock() return nil, errRingShardsDown } cl := ring.shards[name].Client - ring.mx.RUnlock() + ring.mu.RUnlock() return cl, nil } func (ring *Ring) process(cmd Cmder) { - cl, err := ring.getClient(cmd.clusterKey()) + cl, err := ring.getClient(ring.cmdFirstKey(cmd)) if err != nil { cmd.setErr(err) return @@ -177,8 +207,8 @@ func (ring *Ring) process(cmd Cmder) { // rebalance removes dead shards from the ring. func (ring *Ring) rebalance() { - defer ring.mx.Unlock() - ring.mx.Lock() + defer ring.mu.Unlock() + ring.mu.Lock() ring.hash = consistenthash.New(ring.nreplicas, nil) for name, shard := range ring.shards { @@ -195,10 +225,10 @@ func (ring *Ring) heartbeat() { for _ = range ticker.C { var rebalance bool - ring.mx.RLock() + ring.mu.RLock() if ring.closed { - ring.mx.RUnlock() + ring.mu.RUnlock() break } @@ -210,7 +240,7 @@ func (ring *Ring) heartbeat() { } } - ring.mx.RUnlock() + ring.mu.RUnlock() if rebalance { ring.rebalance() @@ -223,8 +253,8 @@ func (ring *Ring) heartbeat() { // It is rare to Close a Ring, as the Ring is meant to be long-lived // and shared between many goroutines. func (ring *Ring) Close() (retErr error) { - defer ring.mx.Unlock() - ring.mx.Lock() + defer ring.mu.Unlock() + ring.mu.Lock() if ring.closed { return nil @@ -259,7 +289,7 @@ func (ring *Ring) pipelineExec(cmds []Cmder) error { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - name := ring.hash.Get(hashtag.Key(cmd.clusterKey())) + name := ring.hash.Get(hashtag.Key(ring.cmdFirstKey(cmd))) if name == "" { cmd.setErr(errRingShardsDown) if retErr == nil { From ae8483982963a7f1744670836bde553cad7c6b06 Mon Sep 17 00:00:00 2001 From: Tux Date: Wed, 25 May 2016 11:32:13 -0600 Subject: [PATCH 0176/1746] Update README.md to use redis.v4 documentation This changes the links from redis.v3 documentation to redis.v4 documentation. --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1d05c40108..0b85a11b73 100644 --- a/README.md +++ b/README.md @@ -3,26 +3,26 @@ Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- [Pub/Sub](http://godoc.org/gopkg.in/redis.v3#PubSub). -- [Transactions](http://godoc.org/gopkg.in/redis.v3#Multi). -- [Pipelining](http://godoc.org/gopkg.in/redis.v3#Client.Pipeline). -- [Scripting](http://godoc.org/gopkg.in/redis.v3#Script). -- [Timeouts](http://godoc.org/gopkg.in/redis.v3#Options). -- [Redis Sentinel](http://godoc.org/gopkg.in/redis.v3#NewFailoverClient). -- [Redis Cluster](http://godoc.org/gopkg.in/redis.v3#NewClusterClient). -- [Ring](http://godoc.org/gopkg.in/redis.v3#NewRing). +- [Pub/Sub](http://godoc.org/gopkg.in/redis.v4#PubSub). +- [Transactions](http://godoc.org/gopkg.in/redis.v4#Multi). +- [Pipelining](http://godoc.org/gopkg.in/redis.v4#Client.Pipeline). +- [Scripting](http://godoc.org/gopkg.in/redis.v4#Script). +- [Timeouts](http://godoc.org/gopkg.in/redis.v4#Options). +- [Redis Sentinel](http://godoc.org/gopkg.in/redis.v4#NewFailoverClient). +- [Redis Cluster](http://godoc.org/gopkg.in/redis.v4#NewClusterClient). +- [Ring](http://godoc.org/gopkg.in/redis.v4#NewRing). - [Cache friendly](https://github.com/go-redis/cache). - [Rate limiting](https://github.com/go-redis/rate). - [Distributed Locks](https://github.com/bsm/redis-lock). -API docs: http://godoc.org/gopkg.in/redis.v3. -Examples: http://godoc.org/gopkg.in/redis.v3#pkg-examples. +API docs: http://godoc.org/gopkg.in/redis.v4. +Examples: http://godoc.org/gopkg.in/redis.v4#pkg-examples. ## Installation Install: - go get gopkg.in/redis.v3 + go get gopkg.in/redis.v4 ## Quickstart @@ -66,7 +66,7 @@ func ExampleClient() { ## Howto -Please go through [examples](http://godoc.org/gopkg.in/redis.v3#pkg-examples) to get an idea how to use this package. +Please go through [examples](http://godoc.org/gopkg.in/redis.v4#pkg-examples) to get an idea how to use this package. ## Look and feel From e9233d8d17b43869f175103b038f9fa642518384 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 5 Jun 2016 08:05:50 +0000 Subject: [PATCH 0177/1746] Cleanup loggers. --- internal/log.go | 13 +++---------- redis.go | 2 -- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/internal/log.go b/internal/log.go index c1cdbf4ed9..fd14222eee 100644 --- a/internal/log.go +++ b/internal/log.go @@ -2,21 +2,14 @@ package internal import ( "fmt" - "io/ioutil" "log" ) -var Debug bool +var Logger *log.Logger -var Logger = log.New(ioutil.Discard, "redis: ", log.LstdFlags) - -func Debugf(s string, args ...interface{}) { - if !Debug { +func Logf(s string, args ...interface{}) { + if Logger == nil { return } Logger.Output(2, fmt.Sprintf(s, args...)) } - -func Logf(s string, args ...interface{}) { - Logger.Output(2, fmt.Sprintf(s, args...)) -} diff --git a/redis.go b/redis.go index e2c173b315..bb9fcd0a13 100644 --- a/redis.go +++ b/redis.go @@ -8,8 +8,6 @@ import ( "gopkg.in/redis.v4/internal/pool" ) -var Logger *log.Logger - func SetLogger(logger *log.Logger) { internal.Logger = logger } From ac162eb84325637ee86fe3aa74a5146ced9cae89 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 5 Jun 2016 09:45:39 +0000 Subject: [PATCH 0178/1746] Move Select to stateful commands and make it available only via Pipeline and Tx. --- cluster.go | 13 +- cluster_test.go | 22 +- commands.go | 760 +++++++++++++++++++++++------------------------ commands_test.go | 33 +- iterator.go | 4 +- options.go | 2 +- pipeline.go | 5 +- pubsub.go | 4 +- race_test.go | 4 +- redis.go | 39 ++- ring.go | 17 +- sentinel.go | 42 ++- tx.go | 25 +- 13 files changed, 483 insertions(+), 487 deletions(-) diff --git a/cluster.go b/cluster.go index cd433e7b63..e29d6ef720 100644 --- a/cluster.go +++ b/cluster.go @@ -21,7 +21,7 @@ type clusterNode struct { // or more underlying connections. It's safe for concurrent use by // multiple goroutines. type ClusterClient struct { - commandable + cmdable opt *ClusterOptions @@ -51,7 +51,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { cmdsInfoOnce: new(sync.Once), } - client.commandable.process = client.process + client.cmdable.process = client.Process for _, addr := range opt.Addrs { _ = client.nodeByAddr(addr) @@ -242,7 +242,7 @@ func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode) { return slot, c.slotMasterNode(slot) } -func (c *ClusterClient) process(cmd Cmder) { +func (c *ClusterClient) Process(cmd Cmder) { var ask bool slot, node := c.cmdSlotAndNode(cmd) @@ -398,11 +398,12 @@ func (c *ClusterClient) reaper(frequency time.Duration) { } func (c *ClusterClient) Pipeline() *Pipeline { - pipe := &Pipeline{ + pipe := Pipeline{ exec: c.pipelineExec, } - pipe.commandable.process = pipe.process - return pipe + pipe.cmdable.process = pipe.Process + pipe.statefulCmdable.process = pipe.Process + return &pipe } func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { diff --git a/cluster_test.go b/cluster_test.go index 6d081dfdc0..9cdbc00431 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -300,17 +300,17 @@ var _ = Describe("Cluster", func() { Expect(nodesList).Should(HaveLen(1)) }) - It("should CLUSTER READONLY", func() { - res, err := cluster.primary().ReadOnly().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal("OK")) - }) - - It("should CLUSTER READWRITE", func() { - res, err := cluster.primary().ReadWrite().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal("OK")) - }) + // It("should CLUSTER READONLY", func() { + // res, err := cluster.primary().ReadOnly().Result() + // Expect(err).NotTo(HaveOccurred()) + // Expect(res).To(Equal("OK")) + // }) + + // It("should CLUSTER READWRITE", func() { + // res, err := cluster.primary().ReadWrite().Result() + // Expect(err).NotTo(HaveOccurred()) + // Expect(res).To(Equal("OK")) + // }) }) Describe("Client", func() { diff --git a/commands.go b/commands.go index 00b047f084..5107b7b353 100644 --- a/commands.go +++ b/commands.go @@ -51,88 +51,88 @@ func formatSec(dur time.Duration) string { return formatInt(int64(dur / time.Second)) } -type commandable struct { +type cmdable struct { process func(cmd Cmder) } -func (c *commandable) Process(cmd Cmder) { - c.process(cmd) +type statefulCmdable struct { + process func(cmd Cmder) } //------------------------------------------------------------------------------ -func (c *commandable) Auth(password string) *StatusCmd { +func (c statefulCmdable) Auth(password string) *StatusCmd { cmd := NewStatusCmd("auth", password) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Echo(message string) *StringCmd { +func (c *cmdable) Echo(message string) *StringCmd { cmd := NewStringCmd("echo", message) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Ping() *StatusCmd { +func (c *cmdable) Ping() *StatusCmd { cmd := NewStatusCmd("ping") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Quit() *StatusCmd { +func (c cmdable) Quit() *StatusCmd { panic("not implemented") } -func (c *commandable) Select(index int64) *StatusCmd { +func (c statefulCmdable) Select(index int) *StatusCmd { cmd := NewStatusCmd("select", index) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) Del(keys ...string) *IntCmd { +func (c cmdable) Del(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) args[0] = "DEL" for i, key := range keys { args[1+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Dump(key string) *StringCmd { +func (c cmdable) Dump(key string) *StringCmd { cmd := NewStringCmd("dump", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Exists(key string) *BoolCmd { +func (c cmdable) Exists(key string) *BoolCmd { cmd := NewBoolCmd("exists", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Expire(key string, expiration time.Duration) *BoolCmd { +func (c cmdable) Expire(key string, expiration time.Duration) *BoolCmd { cmd := NewBoolCmd("expire", key, formatSec(expiration)) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ExpireAt(key string, tm time.Time) *BoolCmd { +func (c cmdable) ExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd("expireat", key, tm.Unix()) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Keys(pattern string) *StringSliceCmd { +func (c cmdable) Keys(pattern string) *StringSliceCmd { cmd := NewStringSliceCmd("keys", pattern) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd { +func (c cmdable) Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd { cmd := NewStatusCmd( "migrate", host, @@ -142,17 +142,17 @@ func (c *commandable) Migrate(host, port, key string, db int64, timeout time.Dur formatMs(timeout), ) cmd.setReadTimeout(readTimeout(timeout)) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Move(key string, db int64) *BoolCmd { +func (c cmdable) Move(key string, db int64) *BoolCmd { cmd := NewBoolCmd("move", key, db) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ObjectRefCount(keys ...string) *IntCmd { +func (c cmdable) ObjectRefCount(keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "object" args[1] = "refcount" @@ -160,11 +160,11 @@ func (c *commandable) ObjectRefCount(keys ...string) *IntCmd { args[2+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ObjectEncoding(keys ...string) *StringCmd { +func (c cmdable) ObjectEncoding(keys ...string) *StringCmd { args := make([]interface{}, 2+len(keys)) args[0] = "object" args[1] = "encoding" @@ -172,11 +172,11 @@ func (c *commandable) ObjectEncoding(keys ...string) *StringCmd { args[2+i] = key } cmd := NewStringCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ObjectIdleTime(keys ...string) *DurationCmd { +func (c cmdable) ObjectIdleTime(keys ...string) *DurationCmd { args := make([]interface{}, 2+len(keys)) args[0] = "object" args[1] = "idletime" @@ -184,68 +184,68 @@ func (c *commandable) ObjectIdleTime(keys ...string) *DurationCmd { args[2+i] = key } cmd := NewDurationCmd(time.Second, args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Persist(key string) *BoolCmd { +func (c cmdable) Persist(key string) *BoolCmd { cmd := NewBoolCmd("persist", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PExpire(key string, expiration time.Duration) *BoolCmd { +func (c cmdable) PExpire(key string, expiration time.Duration) *BoolCmd { cmd := NewBoolCmd("pexpire", key, formatMs(expiration)) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PExpireAt(key string, tm time.Time) *BoolCmd { +func (c cmdable) PExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd( "pexpireat", key, tm.UnixNano()/int64(time.Millisecond), ) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PTTL(key string) *DurationCmd { +func (c cmdable) PTTL(key string) *DurationCmd { cmd := NewDurationCmd(time.Millisecond, "pttl", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RandomKey() *StringCmd { +func (c cmdable) RandomKey() *StringCmd { cmd := NewStringCmd("randomkey") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Rename(key, newkey string) *StatusCmd { +func (c cmdable) Rename(key, newkey string) *StatusCmd { cmd := NewStatusCmd("rename", key, newkey) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RenameNX(key, newkey string) *BoolCmd { +func (c cmdable) RenameNX(key, newkey string) *BoolCmd { cmd := NewBoolCmd("renamenx", key, newkey) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Restore(key string, ttl time.Duration, value string) *StatusCmd { +func (c cmdable) Restore(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( "restore", key, formatMs(ttl), value, ) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { +func (c cmdable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( "restore", key, @@ -253,7 +253,7 @@ func (c *commandable) RestoreReplace(key string, ttl time.Duration, value string value, "replace", ) - c.Process(cmd) + c.process(cmd) return cmd } @@ -289,31 +289,31 @@ func (sort *Sort) args(key string) []interface{} { return args } -func (c *commandable) Sort(key string, sort Sort) *StringSliceCmd { +func (c cmdable) Sort(key string, sort Sort) *StringSliceCmd { cmd := NewStringSliceCmd(sort.args(key)...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SortInterfaces(key string, sort Sort) *SliceCmd { +func (c cmdable) SortInterfaces(key string, sort Sort) *SliceCmd { cmd := NewSliceCmd(sort.args(key)...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) TTL(key string) *DurationCmd { +func (c cmdable) TTL(key string) *DurationCmd { cmd := NewDurationCmd(time.Second, "ttl", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Type(key string) *StatusCmd { +func (c cmdable) Type(key string) *StatusCmd { cmd := NewStatusCmd("type", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Scan(cursor uint64, match string, count int64) Scanner { +func (c cmdable) Scan(cursor uint64, match string, count int64) Scanner { args := []interface{}{"scan", cursor} if match != "" { args = append(args, "match", match) @@ -322,14 +322,14 @@ func (c *commandable) Scan(cursor uint64, match string, count int64) Scanner { args = append(args, "count", count) } cmd := NewScanCmd(args...) - c.Process(cmd) + c.process(cmd) return Scanner{ client: c, ScanCmd: cmd, } } -func (c *commandable) SScan(key string, cursor uint64, match string, count int64) Scanner { +func (c cmdable) SScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"sscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -338,14 +338,14 @@ func (c *commandable) SScan(key string, cursor uint64, match string, count int64 args = append(args, "count", count) } cmd := NewScanCmd(args...) - c.Process(cmd) + c.process(cmd) return Scanner{ client: c, ScanCmd: cmd, } } -func (c *commandable) HScan(key string, cursor uint64, match string, count int64) Scanner { +func (c cmdable) HScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"hscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -354,14 +354,14 @@ func (c *commandable) HScan(key string, cursor uint64, match string, count int64 args = append(args, "count", count) } cmd := NewScanCmd(args...) - c.Process(cmd) + c.process(cmd) return Scanner{ client: c, ScanCmd: cmd, } } -func (c *commandable) ZScan(key string, cursor uint64, match string, count int64) Scanner { +func (c cmdable) ZScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"zscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -370,7 +370,7 @@ func (c *commandable) ZScan(key string, cursor uint64, match string, count int64 args = append(args, "count", count) } cmd := NewScanCmd(args...) - c.Process(cmd) + c.process(cmd) return Scanner{ client: c, ScanCmd: cmd, @@ -379,9 +379,9 @@ func (c *commandable) ZScan(key string, cursor uint64, match string, count int64 //------------------------------------------------------------------------------ -func (c *commandable) Append(key, value string) *IntCmd { +func (c cmdable) Append(key, value string) *IntCmd { cmd := NewIntCmd("append", key, value) - c.Process(cmd) + c.process(cmd) return cmd } @@ -389,7 +389,7 @@ type BitCount struct { Start, End int64 } -func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { +func (c cmdable) BitCount(key string, bitCount *BitCount) *IntCmd { args := []interface{}{"bitcount", key} if bitCount != nil { args = append( @@ -399,11 +399,11 @@ func (c *commandable) BitCount(key string, bitCount *BitCount) *IntCmd { ) } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) bitOp(op, destKey string, keys ...string) *IntCmd { +func (c cmdable) bitOp(op, destKey string, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "bitop" args[1] = op @@ -412,27 +412,27 @@ func (c *commandable) bitOp(op, destKey string, keys ...string) *IntCmd { args[3+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) BitOpAnd(destKey string, keys ...string) *IntCmd { +func (c cmdable) BitOpAnd(destKey string, keys ...string) *IntCmd { return c.bitOp("and", destKey, keys...) } -func (c *commandable) BitOpOr(destKey string, keys ...string) *IntCmd { +func (c cmdable) BitOpOr(destKey string, keys ...string) *IntCmd { return c.bitOp("or", destKey, keys...) } -func (c *commandable) BitOpXor(destKey string, keys ...string) *IntCmd { +func (c cmdable) BitOpXor(destKey string, keys ...string) *IntCmd { return c.bitOp("xor", destKey, keys...) } -func (c *commandable) BitOpNot(destKey string, key string) *IntCmd { +func (c cmdable) BitOpNot(destKey string, key string) *IntCmd { return c.bitOp("not", destKey, key) } -func (c *commandable) BitPos(key string, bit int64, pos ...int64) *IntCmd { +func (c cmdable) BitPos(key string, bit int64, pos ...int64) *IntCmd { args := make([]interface{}, 3+len(pos)) args[0] = "bitpos" args[1] = key @@ -448,101 +448,101 @@ func (c *commandable) BitPos(key string, bit int64, pos ...int64) *IntCmd { panic("too many arguments") } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Decr(key string) *IntCmd { +func (c cmdable) Decr(key string) *IntCmd { cmd := NewIntCmd("decr", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) DecrBy(key string, decrement int64) *IntCmd { +func (c cmdable) DecrBy(key string, decrement int64) *IntCmd { cmd := NewIntCmd("decrby", key, decrement) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Get(key string) *StringCmd { +func (c cmdable) Get(key string) *StringCmd { cmd := NewStringCmd("get", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GetBit(key string, offset int64) *IntCmd { +func (c cmdable) GetBit(key string, offset int64) *IntCmd { cmd := NewIntCmd("getbit", key, offset) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GetRange(key string, start, end int64) *StringCmd { +func (c cmdable) GetRange(key string, start, end int64) *StringCmd { cmd := NewStringCmd("getrange", key, start, end) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GetSet(key string, value interface{}) *StringCmd { +func (c cmdable) GetSet(key string, value interface{}) *StringCmd { cmd := NewStringCmd("getset", key, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Incr(key string) *IntCmd { +func (c cmdable) Incr(key string) *IntCmd { cmd := NewIntCmd("incr", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) IncrBy(key string, value int64) *IntCmd { +func (c cmdable) IncrBy(key string, value int64) *IntCmd { cmd := NewIntCmd("incrby", key, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) IncrByFloat(key string, value float64) *FloatCmd { +func (c cmdable) IncrByFloat(key string, value float64) *FloatCmd { cmd := NewFloatCmd("incrbyfloat", key, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) MGet(keys ...string) *SliceCmd { +func (c cmdable) MGet(keys ...string) *SliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "mget" for i, key := range keys { args[1+i] = key } cmd := NewSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) MSet(pairs ...string) *StatusCmd { +func (c cmdable) MSet(pairs ...string) *StatusCmd { args := make([]interface{}, 1+len(pairs)) args[0] = "mset" for i, pair := range pairs { args[1+i] = pair } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) MSetNX(pairs ...string) *BoolCmd { +func (c cmdable) MSetNX(pairs ...string) *BoolCmd { args := make([]interface{}, 1+len(pairs)) args[0] = "msetnx" for i, pair := range pairs { args[1+i] = pair } cmd := NewBoolCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } // Redis `SET key value [expiration]` command. // // Zero expiration means the key has no expiration time. -func (c *commandable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { +func (c cmdable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { args := make([]interface{}, 3, 4) args[0] = "set" args[1] = key @@ -555,25 +555,25 @@ func (c *commandable) Set(key string, value interface{}, expiration time.Duratio } } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SetBit(key string, offset int64, value int) *IntCmd { +func (c cmdable) SetBit(key string, offset int64, value int) *IntCmd { cmd := NewIntCmd( "SETBIT", key, offset, value, ) - c.Process(cmd) + c.process(cmd) return cmd } // Redis `SET key value [expiration] NX` command. // // Zero expiration means the key has no expiration time. -func (c *commandable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { +func (c cmdable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if expiration == 0 { // Use old `SETNX` to support old Redis versions. @@ -585,39 +585,39 @@ func (c *commandable) SetNX(key string, value interface{}, expiration time.Durat cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "nx") } } - c.Process(cmd) + c.process(cmd) return cmd } // Redis `SET key value [expiration] XX` command. // // Zero expiration means the key has no expiration time. -func (c *commandable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { +func (c cmdable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if usePrecise(expiration) { cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "xx") } else { cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "xx") } - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SetRange(key string, offset int64, value string) *IntCmd { +func (c cmdable) SetRange(key string, offset int64, value string) *IntCmd { cmd := NewIntCmd("setrange", key, offset, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) StrLen(key string) *IntCmd { +func (c cmdable) StrLen(key string) *IntCmd { cmd := NewIntCmd("strlen", key) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) HDel(key string, fields ...string) *IntCmd { +func (c cmdable) HDel(key string, fields ...string) *IntCmd { args := make([]interface{}, 2+len(fields)) args[0] = "hdel" args[1] = key @@ -625,53 +625,53 @@ func (c *commandable) HDel(key string, fields ...string) *IntCmd { args[2+i] = field } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HExists(key, field string) *BoolCmd { +func (c cmdable) HExists(key, field string) *BoolCmd { cmd := NewBoolCmd("hexists", key, field) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HGet(key, field string) *StringCmd { +func (c cmdable) HGet(key, field string) *StringCmd { cmd := NewStringCmd("hget", key, field) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HGetAll(key string) *StringStringMapCmd { +func (c cmdable) HGetAll(key string) *StringStringMapCmd { cmd := NewStringStringMapCmd("hgetall", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HIncrBy(key, field string, incr int64) *IntCmd { +func (c cmdable) HIncrBy(key, field string, incr int64) *IntCmd { cmd := NewIntCmd("hincrby", key, field, incr) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HIncrByFloat(key, field string, incr float64) *FloatCmd { +func (c cmdable) HIncrByFloat(key, field string, incr float64) *FloatCmd { cmd := NewFloatCmd("hincrbyfloat", key, field, incr) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HKeys(key string) *StringSliceCmd { +func (c cmdable) HKeys(key string) *StringSliceCmd { cmd := NewStringSliceCmd("hkeys", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HLen(key string) *IntCmd { +func (c cmdable) HLen(key string) *IntCmd { cmd := NewIntCmd("hlen", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HMGet(key string, fields ...string) *SliceCmd { +func (c cmdable) HMGet(key string, fields ...string) *SliceCmd { args := make([]interface{}, 2+len(fields)) args[0] = "hmget" args[1] = key @@ -679,11 +679,11 @@ func (c *commandable) HMGet(key string, fields ...string) *SliceCmd { args[2+i] = field } cmd := NewSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HMSet(key string, fields map[string]string) *StatusCmd { +func (c cmdable) HMSet(key string, fields map[string]string) *StatusCmd { args := make([]interface{}, 2+len(fields)*2) args[0] = "hmset" args[1] = key @@ -694,31 +694,31 @@ func (c *commandable) HMSet(key string, fields map[string]string) *StatusCmd { i += 2 } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HSet(key, field, value string) *BoolCmd { +func (c cmdable) HSet(key, field, value string) *BoolCmd { cmd := NewBoolCmd("hset", key, field, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HSetNX(key, field, value string) *BoolCmd { +func (c cmdable) HSetNX(key, field, value string) *BoolCmd { cmd := NewBoolCmd("hsetnx", key, field, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) HVals(key string) *StringSliceCmd { +func (c cmdable) HVals(key string) *StringSliceCmd { cmd := NewStringSliceCmd("hvals", key) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { +func (c cmdable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)+1) args[0] = "blpop" for i, key := range keys { @@ -727,11 +727,11 @@ func (c *commandable) BLPop(timeout time.Duration, keys ...string) *StringSliceC args[len(args)-1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) cmd.setReadTimeout(readTimeout(timeout)) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { +func (c cmdable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)+1) args[0] = "brpop" for i, key := range keys { @@ -740,11 +740,11 @@ func (c *commandable) BRPop(timeout time.Duration, keys ...string) *StringSliceC args[len(keys)+1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) cmd.setReadTimeout(readTimeout(timeout)) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) BRPopLPush(source, destination string, timeout time.Duration) *StringCmd { +func (c cmdable) BRPopLPush(source, destination string, timeout time.Duration) *StringCmd { cmd := NewStringCmd( "brpoplpush", source, @@ -752,35 +752,35 @@ func (c *commandable) BRPopLPush(source, destination string, timeout time.Durati formatSec(timeout), ) cmd.setReadTimeout(readTimeout(timeout)) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LIndex(key string, index int64) *StringCmd { +func (c cmdable) LIndex(key string, index int64) *StringCmd { cmd := NewStringCmd("lindex", key, index) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LInsert(key, op, pivot, value string) *IntCmd { +func (c cmdable) LInsert(key, op, pivot, value string) *IntCmd { cmd := NewIntCmd("linsert", key, op, pivot, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LLen(key string) *IntCmd { +func (c cmdable) LLen(key string) *IntCmd { cmd := NewIntCmd("llen", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LPop(key string) *StringCmd { +func (c cmdable) LPop(key string) *StringCmd { cmd := NewStringCmd("lpop", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LPush(key string, values ...string) *IntCmd { +func (c cmdable) LPush(key string, values ...string) *IntCmd { args := make([]interface{}, 2+len(values)) args[0] = "lpush" args[1] = key @@ -788,63 +788,63 @@ func (c *commandable) LPush(key string, values ...string) *IntCmd { args[2+i] = value } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LPushX(key, value interface{}) *IntCmd { +func (c cmdable) LPushX(key, value interface{}) *IntCmd { cmd := NewIntCmd("lpushx", key, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LRange(key string, start, stop int64) *StringSliceCmd { +func (c cmdable) LRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd( "lrange", key, start, stop, ) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LRem(key string, count int64, value interface{}) *IntCmd { +func (c cmdable) LRem(key string, count int64, value interface{}) *IntCmd { cmd := NewIntCmd("lrem", key, count, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LSet(key string, index int64, value interface{}) *StatusCmd { +func (c cmdable) LSet(key string, index int64, value interface{}) *StatusCmd { cmd := NewStatusCmd("lset", key, index, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LTrim(key string, start, stop int64) *StatusCmd { +func (c cmdable) LTrim(key string, start, stop int64) *StatusCmd { cmd := NewStatusCmd( "ltrim", key, start, stop, ) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RPop(key string) *StringCmd { +func (c cmdable) RPop(key string) *StringCmd { cmd := NewStringCmd("rpop", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RPopLPush(source, destination string) *StringCmd { +func (c cmdable) RPopLPush(source, destination string) *StringCmd { cmd := NewStringCmd("rpoplpush", source, destination) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RPush(key string, values ...string) *IntCmd { +func (c cmdable) RPush(key string, values ...string) *IntCmd { args := make([]interface{}, 2+len(values)) args[0] = "rpush" args[1] = key @@ -852,19 +852,19 @@ func (c *commandable) RPush(key string, values ...string) *IntCmd { args[2+i] = value } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) RPushX(key string, value interface{}) *IntCmd { +func (c cmdable) RPushX(key string, value interface{}) *IntCmd { cmd := NewIntCmd("rpushx", key, value) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) SAdd(key string, members ...string) *IntCmd { +func (c cmdable) SAdd(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "sadd" args[1] = key @@ -872,28 +872,28 @@ func (c *commandable) SAdd(key string, members ...string) *IntCmd { args[2+i] = member } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SCard(key string) *IntCmd { +func (c cmdable) SCard(key string) *IntCmd { cmd := NewIntCmd("scard", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SDiff(keys ...string) *StringSliceCmd { +func (c cmdable) SDiff(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "sdiff" for i, key := range keys { args[1+i] = key } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SDiffStore(destination string, keys ...string) *IntCmd { +func (c cmdable) SDiffStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "sdiffstore" args[1] = destination @@ -901,22 +901,22 @@ func (c *commandable) SDiffStore(destination string, keys ...string) *IntCmd { args[2+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SInter(keys ...string) *StringSliceCmd { +func (c cmdable) SInter(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "sinter" for i, key := range keys { args[1+i] = key } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { +func (c cmdable) SInterStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "sinterstore" args[1] = destination @@ -924,49 +924,49 @@ func (c *commandable) SInterStore(destination string, keys ...string) *IntCmd { args[2+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SIsMember(key string, member interface{}) *BoolCmd { +func (c cmdable) SIsMember(key string, member interface{}) *BoolCmd { cmd := NewBoolCmd("sismember", key, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SMembers(key string) *StringSliceCmd { +func (c cmdable) SMembers(key string) *StringSliceCmd { cmd := NewStringSliceCmd("smembers", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SMove(source, destination string, member interface{}) *BoolCmd { +func (c cmdable) SMove(source, destination string, member interface{}) *BoolCmd { cmd := NewBoolCmd("smove", source, destination, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SPop(key string) *StringCmd { +func (c cmdable) SPop(key string) *StringCmd { cmd := NewStringCmd("spop", key) - c.Process(cmd) + c.process(cmd) return cmd } // Redis `SRANDMEMBER key` command. -func (c *commandable) SRandMember(key string) *StringCmd { +func (c cmdable) SRandMember(key string) *StringCmd { cmd := NewStringCmd("srandmember", key) - c.Process(cmd) + c.process(cmd) return cmd } // Redis `SRANDMEMBER key count` command. -func (c *commandable) SRandMemberN(key string, count int64) *StringSliceCmd { +func (c cmdable) SRandMemberN(key string, count int64) *StringSliceCmd { cmd := NewStringSliceCmd("srandmember", key, count) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SRem(key string, members ...string) *IntCmd { +func (c cmdable) SRem(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "srem" args[1] = key @@ -974,22 +974,22 @@ func (c *commandable) SRem(key string, members ...string) *IntCmd { args[2+i] = member } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SUnion(keys ...string) *StringSliceCmd { +func (c cmdable) SUnion(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "sunion" for i, key := range keys { args[1+i] = key } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SUnionStore(destination string, keys ...string) *IntCmd { +func (c cmdable) SUnionStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "sunionstore" args[1] = destination @@ -997,7 +997,7 @@ func (c *commandable) SUnionStore(destination string, keys ...string) *IntCmd { args[2+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } @@ -1016,18 +1016,18 @@ type ZStore struct { Aggregate string } -func (c *commandable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { +func (c cmdable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { for i, m := range members { a[n+2*i] = m.Score a[n+2*i+1] = m.Member } cmd := NewIntCmd(a...) - c.Process(cmd) + c.process(cmd) return cmd } // Redis `ZADD key score member [score member ...]` command. -func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { +func (c cmdable) ZAdd(key string, members ...Z) *IntCmd { const n = 2 a := make([]interface{}, n+2*len(members)) a[0], a[1] = "zadd", key @@ -1035,7 +1035,7 @@ func (c *commandable) ZAdd(key string, members ...Z) *IntCmd { } // Redis `ZADD key NX score member [score member ...]` command. -func (c *commandable) ZAddNX(key string, members ...Z) *IntCmd { +func (c cmdable) ZAddNX(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2] = "zadd", key, "nx" @@ -1043,7 +1043,7 @@ func (c *commandable) ZAddNX(key string, members ...Z) *IntCmd { } // Redis `ZADD key XX score member [score member ...]` command. -func (c *commandable) ZAddXX(key string, members ...Z) *IntCmd { +func (c cmdable) ZAddXX(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2] = "zadd", key, "xx" @@ -1051,7 +1051,7 @@ func (c *commandable) ZAddXX(key string, members ...Z) *IntCmd { } // Redis `ZADD key CH score member [score member ...]` command. -func (c *commandable) ZAddCh(key string, members ...Z) *IntCmd { +func (c cmdable) ZAddCh(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2] = "zadd", key, "ch" @@ -1059,7 +1059,7 @@ func (c *commandable) ZAddCh(key string, members ...Z) *IntCmd { } // Redis `ZADD key NX CH score member [score member ...]` command. -func (c *commandable) ZAddNXCh(key string, members ...Z) *IntCmd { +func (c cmdable) ZAddNXCh(key string, members ...Z) *IntCmd { const n = 4 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2], a[3] = "zadd", key, "nx", "ch" @@ -1067,25 +1067,25 @@ func (c *commandable) ZAddNXCh(key string, members ...Z) *IntCmd { } // Redis `ZADD key XX CH score member [score member ...]` command. -func (c *commandable) ZAddXXCh(key string, members ...Z) *IntCmd { +func (c cmdable) ZAddXXCh(key string, members ...Z) *IntCmd { const n = 4 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2], a[3] = "zadd", key, "xx", "ch" return c.zAdd(a, n, members...) } -func (c *commandable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { +func (c cmdable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { for i, m := range members { a[n+2*i] = m.Score a[n+2*i+1] = m.Member } cmd := NewFloatCmd(a...) - c.Process(cmd) + c.process(cmd) return cmd } // Redis `ZADD key INCR score member` command. -func (c *commandable) ZIncr(key string, member Z) *FloatCmd { +func (c cmdable) ZIncr(key string, member Z) *FloatCmd { const n = 3 a := make([]interface{}, n+2) a[0], a[1], a[2] = "zadd", key, "incr" @@ -1093,7 +1093,7 @@ func (c *commandable) ZIncr(key string, member Z) *FloatCmd { } // Redis `ZADD key NX INCR score member` command. -func (c *commandable) ZIncrNX(key string, member Z) *FloatCmd { +func (c cmdable) ZIncrNX(key string, member Z) *FloatCmd { const n = 4 a := make([]interface{}, n+2) a[0], a[1], a[2], a[3] = "zadd", key, "incr", "nx" @@ -1101,32 +1101,32 @@ func (c *commandable) ZIncrNX(key string, member Z) *FloatCmd { } // Redis `ZADD key XX INCR score member` command. -func (c *commandable) ZIncrXX(key string, member Z) *FloatCmd { +func (c cmdable) ZIncrXX(key string, member Z) *FloatCmd { const n = 4 a := make([]interface{}, n+2) a[0], a[1], a[2], a[3] = "zadd", key, "incr", "xx" return c.zIncr(a, n, member) } -func (c *commandable) ZCard(key string) *IntCmd { +func (c cmdable) ZCard(key string) *IntCmd { cmd := NewIntCmd("zcard", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZCount(key, min, max string) *IntCmd { +func (c cmdable) ZCount(key, min, max string) *IntCmd { cmd := NewIntCmd("zcount", key, min, max) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZIncrBy(key string, increment float64, member string) *FloatCmd { +func (c cmdable) ZIncrBy(key string, increment float64, member string) *FloatCmd { cmd := NewFloatCmd("zincrby", key, increment, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { +func (c cmdable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "zinterstore" args[1] = destination @@ -1144,11 +1144,11 @@ func (c *commandable) ZInterStore(destination string, store ZStore, keys ...stri args = append(args, "aggregate", store.Aggregate) } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { +func (c cmdable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { args := []interface{}{ "zrange", key, @@ -1159,17 +1159,17 @@ func (c *commandable) zRange(key string, start, stop int64, withScores bool) *St args = append(args, "withscores") } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRange(key string, start, stop int64) *StringSliceCmd { +func (c cmdable) ZRange(key string, start, stop int64) *StringSliceCmd { return c.zRange(key, start, stop, false) } -func (c *commandable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { +func (c cmdable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { cmd := NewZSliceCmd("zrange", key, start, stop, "withscores") - c.Process(cmd) + c.process(cmd) return cmd } @@ -1178,7 +1178,7 @@ type ZRangeBy struct { Offset, Count int64 } -func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { +func (c cmdable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Min, opt.Max} if withScores { args = append(args, "withscores") @@ -1192,19 +1192,19 @@ func (c *commandable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) ) } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { +func (c cmdable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { return c.zRangeBy("zrangebyscore", key, opt, false) } -func (c *commandable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { +func (c cmdable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { return c.zRangeBy("zrangebylex", key, opt, false) } -func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { +func (c cmdable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"zrangebyscore", key, opt.Min, opt.Max, "withscores"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1215,17 +1215,17 @@ func (c *commandable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceC ) } cmd := NewZSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRank(key, member string) *IntCmd { +func (c cmdable) ZRank(key, member string) *IntCmd { cmd := NewIntCmd("zrank", key, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRem(key string, members ...string) *IntCmd { +func (c cmdable) ZRem(key string, members ...string) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "zrem" args[1] = key @@ -1233,40 +1233,40 @@ func (c *commandable) ZRem(key string, members ...string) *IntCmd { args[2+i] = member } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { +func (c cmdable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { cmd := NewIntCmd( "zremrangebyrank", key, start, stop, ) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRemRangeByScore(key, min, max string) *IntCmd { +func (c cmdable) ZRemRangeByScore(key, min, max string) *IntCmd { cmd := NewIntCmd("zremrangebyscore", key, min, max) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRevRange(key string, start, stop int64) *StringSliceCmd { +func (c cmdable) ZRevRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd("zrevrange", key, start, stop) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { +func (c cmdable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { cmd := NewZSliceCmd("zrevrange", key, start, stop, "withscores") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { +func (c cmdable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Max, opt.Min} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1277,19 +1277,19 @@ func (c *commandable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCm ) } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { +func (c cmdable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { return c.zRevRangeBy("zrevrangebyscore", key, opt) } -func (c *commandable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { +func (c cmdable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { return c.zRevRangeBy("zrevrangebylex", key, opt) } -func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { +func (c cmdable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"zrevrangebyscore", key, opt.Max, opt.Min, "withscores"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1300,23 +1300,23 @@ func (c *commandable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSli ) } cmd := NewZSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZRevRank(key, member string) *IntCmd { +func (c cmdable) ZRevRank(key, member string) *IntCmd { cmd := NewIntCmd("zrevrank", key, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZScore(key, member string) *FloatCmd { +func (c cmdable) ZScore(key, member string) *FloatCmd { cmd := NewFloatCmd("zscore", key, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { +func (c cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "zunionstore" args[1] = dest @@ -1334,13 +1334,13 @@ func (c *commandable) ZUnionStore(dest string, store ZStore, keys ...string) *In args = append(args, "aggregate", store.Aggregate) } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) PFAdd(key string, fields ...string) *IntCmd { +func (c cmdable) PFAdd(key string, fields ...string) *IntCmd { args := make([]interface{}, 2+len(fields)) args[0] = "pfadd" args[1] = key @@ -1348,22 +1348,22 @@ func (c *commandable) PFAdd(key string, fields ...string) *IntCmd { args[2+i] = field } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PFCount(keys ...string) *IntCmd { +func (c cmdable) PFCount(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) args[0] = "pfcount" for i, key := range keys { args[1+i] = key } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PFMerge(dest string, keys ...string) *StatusCmd { +func (c cmdable) PFMerge(dest string, keys ...string) *StatusCmd { args := make([]interface{}, 2+len(keys)) args[0] = "pfmerge" args[1] = dest @@ -1371,115 +1371,115 @@ func (c *commandable) PFMerge(dest string, keys ...string) *StatusCmd { args[2+i] = key } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) BgRewriteAOF() *StatusCmd { +func (c cmdable) BgRewriteAOF() *StatusCmd { cmd := NewStatusCmd("bgrewriteaof") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) BgSave() *StatusCmd { +func (c cmdable) BgSave() *StatusCmd { cmd := NewStatusCmd("bgsave") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClientKill(ipPort string) *StatusCmd { +func (c cmdable) ClientKill(ipPort string) *StatusCmd { cmd := NewStatusCmd("client", "kill", ipPort) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClientList() *StringCmd { +func (c cmdable) ClientList() *StringCmd { cmd := NewStringCmd("client", "list") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClientPause(dur time.Duration) *BoolCmd { +func (c cmdable) ClientPause(dur time.Duration) *BoolCmd { cmd := NewBoolCmd("client", "pause", formatMs(dur)) - c.Process(cmd) + c.process(cmd) return cmd } // ClientSetName assigns a name to the one of many connections in the pool. -func (c *commandable) ClientSetName(name string) *BoolCmd { +func (c cmdable) ClientSetName(name string) *BoolCmd { cmd := NewBoolCmd("client", "setname", name) - c.Process(cmd) + c.process(cmd) return cmd } // ClientGetName returns the name of the one of many connections in the pool. func (c *Client) ClientGetName() *StringCmd { cmd := NewStringCmd("client", "getname") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ConfigGet(parameter string) *SliceCmd { +func (c cmdable) ConfigGet(parameter string) *SliceCmd { cmd := NewSliceCmd("config", "get", parameter) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ConfigResetStat() *StatusCmd { +func (c cmdable) ConfigResetStat() *StatusCmd { cmd := NewStatusCmd("config", "resetstat") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ConfigSet(parameter, value string) *StatusCmd { +func (c cmdable) ConfigSet(parameter, value string) *StatusCmd { cmd := NewStatusCmd("config", "set", parameter, value) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) DbSize() *IntCmd { +func (c cmdable) DbSize() *IntCmd { cmd := NewIntCmd("dbsize") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) FlushAll() *StatusCmd { +func (c cmdable) FlushAll() *StatusCmd { cmd := NewStatusCmd("flushall") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) FlushDb() *StatusCmd { +func (c cmdable) FlushDb() *StatusCmd { cmd := NewStatusCmd("flushdb") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Info(section ...string) *StringCmd { +func (c cmdable) Info(section ...string) *StringCmd { args := []interface{}{"info"} if len(section) > 0 { args = append(args, section[0]) } cmd := NewStringCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) LastSave() *IntCmd { +func (c cmdable) LastSave() *IntCmd { cmd := NewIntCmd("lastsave") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) Save() *StatusCmd { +func (c cmdable) Save() *StatusCmd { cmd := NewStatusCmd("save") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) shutdown(modifier string) *StatusCmd { +func (c cmdable) shutdown(modifier string) *StatusCmd { var args []interface{} if modifier == "" { args = []interface{}{"shutdown"} @@ -1487,7 +1487,7 @@ func (c *commandable) shutdown(modifier string) *StatusCmd { args = []interface{}{"shutdown", modifier} } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) if err := cmd.Err(); err != nil { if err == io.EOF { // Server quit as expected. @@ -1501,41 +1501,41 @@ func (c *commandable) shutdown(modifier string) *StatusCmd { return cmd } -func (c *commandable) Shutdown() *StatusCmd { +func (c cmdable) Shutdown() *StatusCmd { return c.shutdown("") } -func (c *commandable) ShutdownSave() *StatusCmd { +func (c cmdable) ShutdownSave() *StatusCmd { return c.shutdown("save") } -func (c *commandable) ShutdownNoSave() *StatusCmd { +func (c cmdable) ShutdownNoSave() *StatusCmd { return c.shutdown("nosave") } -func (c *commandable) SlaveOf(host, port string) *StatusCmd { +func (c cmdable) SlaveOf(host, port string) *StatusCmd { cmd := NewStatusCmd("slaveof", host, port) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) SlowLog() { +func (c cmdable) SlowLog() { panic("not implemented") } -func (c *commandable) Sync() { +func (c cmdable) Sync() { panic("not implemented") } -func (c *commandable) Time() *StringSliceCmd { +func (c cmdable) Time() *StringSliceCmd { cmd := NewStringSliceCmd("time") - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) Eval(script string, keys []string, args ...interface{}) *Cmd { +func (c cmdable) Eval(script string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "eval" cmdArgs[1] = script @@ -1548,11 +1548,11 @@ func (c *commandable) Eval(script string, keys []string, args ...interface{}) *C cmdArgs[pos+i] = arg } cmd := NewCmd(cmdArgs...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { +func (c cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "evalsha" cmdArgs[1] = sha1 @@ -1565,11 +1565,11 @@ func (c *commandable) EvalSha(sha1 string, keys []string, args ...interface{}) * cmdArgs[pos+i] = arg } cmd := NewCmd(cmdArgs...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ScriptExists(scripts ...string) *BoolSliceCmd { +func (c cmdable) ScriptExists(scripts ...string) *BoolSliceCmd { args := make([]interface{}, 2+len(scripts)) args[0] = "script" args[1] = "exists" @@ -1577,49 +1577,49 @@ func (c *commandable) ScriptExists(scripts ...string) *BoolSliceCmd { args[2+i] = script } cmd := NewBoolSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ScriptFlush() *StatusCmd { +func (c cmdable) ScriptFlush() *StatusCmd { cmd := NewStatusCmd("script", "flush") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ScriptKill() *StatusCmd { +func (c cmdable) ScriptKill() *StatusCmd { cmd := NewStatusCmd("script", "kill") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ScriptLoad(script string) *StringCmd { +func (c cmdable) ScriptLoad(script string) *StringCmd { cmd := NewStringCmd("script", "load", script) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) DebugObject(key string) *StringCmd { +func (c cmdable) DebugObject(key string) *StringCmd { cmd := NewStringCmd("debug", "object", key) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) PubSubChannels(pattern string) *StringSliceCmd { +func (c cmdable) PubSubChannels(pattern string) *StringSliceCmd { args := []interface{}{"pubsub", "channels"} if pattern != "*" { args = append(args, pattern) } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PubSubNumSub(channels ...string) *StringIntMapCmd { +func (c cmdable) PubSubNumSub(channels ...string) *StringIntMapCmd { args := make([]interface{}, 2+len(channels)) args[0] = "pubsub" args[1] = "numsub" @@ -1627,85 +1627,85 @@ func (c *commandable) PubSubNumSub(channels ...string) *StringIntMapCmd { args[2+i] = channel } cmd := NewStringIntMapCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) PubSubNumPat() *IntCmd { +func (c cmdable) PubSubNumPat() *IntCmd { cmd := NewIntCmd("pubsub", "numpat") - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) ClusterSlots() *ClusterSlotsCmd { +func (c cmdable) ClusterSlots() *ClusterSlotsCmd { cmd := NewClusterSlotsCmd("cluster", "slots") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterNodes() *StringCmd { +func (c cmdable) ClusterNodes() *StringCmd { cmd := NewStringCmd("cluster", "nodes") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterMeet(host, port string) *StatusCmd { +func (c cmdable) ClusterMeet(host, port string) *StatusCmd { cmd := NewStatusCmd("cluster", "meet", host, port) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterForget(nodeID string) *StatusCmd { +func (c cmdable) ClusterForget(nodeID string) *StatusCmd { cmd := NewStatusCmd("cluster", "forget", nodeID) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterReplicate(nodeID string) *StatusCmd { +func (c cmdable) ClusterReplicate(nodeID string) *StatusCmd { cmd := NewStatusCmd("cluster", "replicate", nodeID) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterResetSoft() *StatusCmd { +func (c cmdable) ClusterResetSoft() *StatusCmd { cmd := NewStatusCmd("cluster", "reset", "soft") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterResetHard() *StatusCmd { +func (c cmdable) ClusterResetHard() *StatusCmd { cmd := NewStatusCmd("cluster", "reset", "hard") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterInfo() *StringCmd { +func (c cmdable) ClusterInfo() *StringCmd { cmd := NewStringCmd("cluster", "info") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterKeySlot(key string) *IntCmd { +func (c cmdable) ClusterKeySlot(key string) *IntCmd { cmd := NewIntCmd("cluster", "keyslot", key) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterCountFailureReports(nodeID string) *IntCmd { +func (c cmdable) ClusterCountFailureReports(nodeID string) *IntCmd { cmd := NewIntCmd("cluster", "count-failure-reports", nodeID) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterCountKeysInSlot(slot int) *IntCmd { +func (c cmdable) ClusterCountKeysInSlot(slot int) *IntCmd { cmd := NewIntCmd("cluster", "countkeysinslot", slot) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterDelSlots(slots ...int) *StatusCmd { +func (c cmdable) ClusterDelSlots(slots ...int) *StatusCmd { args := make([]interface{}, 2+len(slots)) args[0] = "cluster" args[1] = "delslots" @@ -1713,11 +1713,11 @@ func (c *commandable) ClusterDelSlots(slots ...int) *StatusCmd { args[2+i] = slot } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterDelSlotsRange(min, max int) *StatusCmd { +func (c cmdable) ClusterDelSlotsRange(min, max int) *StatusCmd { size := max - min + 1 slots := make([]int, size) for i := 0; i < size; i++ { @@ -1726,37 +1726,37 @@ func (c *commandable) ClusterDelSlotsRange(min, max int) *StatusCmd { return c.ClusterDelSlots(slots...) } -func (c *commandable) ClusterSaveConfig() *StatusCmd { +func (c cmdable) ClusterSaveConfig() *StatusCmd { cmd := NewStatusCmd("cluster", "saveconfig") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterSlaves(nodeID string) *StringSliceCmd { +func (c cmdable) ClusterSlaves(nodeID string) *StringSliceCmd { cmd := NewStringSliceCmd("cluster", "slaves", nodeID) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ReadOnly() *StatusCmd { +func (c statefulCmdable) ReadOnly() *StatusCmd { cmd := NewStatusCmd("readonly") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ReadWrite() *StatusCmd { +func (c statefulCmdable) ReadWrite() *StatusCmd { cmd := NewStatusCmd("readwrite") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterFailover() *StatusCmd { +func (c cmdable) ClusterFailover() *StatusCmd { cmd := NewStatusCmd("cluster", "failover") - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterAddSlots(slots ...int) *StatusCmd { +func (c cmdable) ClusterAddSlots(slots ...int) *StatusCmd { args := make([]interface{}, 2+len(slots)) args[0] = "cluster" args[1] = "addslots" @@ -1764,11 +1764,11 @@ func (c *commandable) ClusterAddSlots(slots ...int) *StatusCmd { args[2+i] = strconv.Itoa(num) } cmd := NewStatusCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) ClusterAddSlotsRange(min, max int) *StatusCmd { +func (c cmdable) ClusterAddSlotsRange(min, max int) *StatusCmd { size := max - min + 1 slots := make([]int, size) for i := 0; i < size; i++ { @@ -1779,7 +1779,7 @@ func (c *commandable) ClusterAddSlotsRange(min, max int) *StatusCmd { //------------------------------------------------------------------------------ -func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { +func (c cmdable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { args := make([]interface{}, 2+3*len(geoLocation)) args[0] = "geoadd" args[1] = key @@ -1789,32 +1789,32 @@ func (c *commandable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { args[2+3*i+2] = eachLoc.Name } cmd := NewIntCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { +func (c cmdable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { cmd := NewGeoLocationCmd(query, "georadius", key, longitude, latitude) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { +func (c cmdable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { cmd := NewGeoLocationCmd(query, "georadiusbymember", key, member) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GeoDist(key string, member1, member2, unit string) *FloatCmd { +func (c cmdable) GeoDist(key string, member1, member2, unit string) *FloatCmd { if unit == "" { unit = "km" } cmd := NewFloatCmd("geodist", key, member1, member2, unit) - c.Process(cmd) + c.process(cmd) return cmd } -func (c *commandable) GeoHash(key string, members ...string) *StringSliceCmd { +func (c cmdable) GeoHash(key string, members ...string) *StringSliceCmd { args := make([]interface{}, 2+len(members)) args[0] = "geohash" args[1] = key @@ -1822,14 +1822,14 @@ func (c *commandable) GeoHash(key string, members ...string) *StringSliceCmd { args[2+i] = member } cmd := NewStringSliceCmd(args...) - c.Process(cmd) + c.process(cmd) return cmd } //------------------------------------------------------------------------------ -func (c *commandable) Command() *CommandsInfoCmd { +func (c cmdable) Command() *CommandsInfoCmd { cmd := NewCommandsInfoCmd("command") - c.Process(cmd) + c.process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index 9d65f96b2b..466f991fe9 100644 --- a/commands_test.go +++ b/commands_test.go @@ -26,11 +26,11 @@ var _ = Describe("Commands", func() { Describe("server", func() { - It("should Auth", func() { - auth := client.Auth("password") - Expect(auth.Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) - Expect(auth.Val()).To(Equal("")) - }) + // It("should Auth", func() { + // auth := client.Auth("password") + // Expect(auth.Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) + // Expect(auth.Val()).To(Equal("")) + // }) It("should Echo", func() { echo := client.Echo("hello") @@ -44,11 +44,11 @@ var _ = Describe("Commands", func() { Expect(ping.Val()).To(Equal("PONG")) }) - It("should Select", func() { - sel := client.Select(1) - Expect(sel.Err()).NotTo(HaveOccurred()) - Expect(sel.Val()).To(Equal("OK")) - }) + // It("should Select", func() { + // sel := client.Select(1) + // Expect(sel.Err()).NotTo(HaveOccurred()) + // Expect(sel.Val()).To(Equal("OK")) + // }) It("should BgRewriteAOF", func() { Skip("flaky test") @@ -309,15 +309,14 @@ var _ = Describe("Commands", func() { Expect(get.Err()).To(Equal(redis.Nil)) Expect(get.Val()).To(Equal("")) - sel := client.Select(2) - Expect(sel.Err()).NotTo(HaveOccurred()) - Expect(sel.Val()).To(Equal("OK")) + pipe := client.Pipeline() + pipe.Select(2) + get = pipe.Get("key") + pipe.FlushDb() - get = client.Get("key") - Expect(get.Err()).NotTo(HaveOccurred()) + _, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) Expect(get.Val()).To(Equal("hello")) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - Expect(client.Select(1).Err()).NotTo(HaveOccurred()) }) It("should Object", func() { diff --git a/iterator.go b/iterator.go index c57c5afdef..0219ab4555 100644 --- a/iterator.go +++ b/iterator.go @@ -3,7 +3,7 @@ package redis import "sync" type Scanner struct { - client *commandable + client cmdable *ScanCmd } @@ -54,7 +54,7 @@ func (it *ScanIterator) Next() bool { // Fetch next page. it.ScanCmd._args[1] = it.ScanCmd.cursor it.ScanCmd.reset() - it.client.Process(it.ScanCmd) + it.client.process(it.ScanCmd) if it.ScanCmd.Err() != nil { return false } diff --git a/options.go b/options.go index e1c04030be..e3db4ad1b5 100644 --- a/options.go +++ b/options.go @@ -22,7 +22,7 @@ type Options struct { // requirepass server configuration option. Password string // A database to be selected after connecting to server. - DB int64 + DB int // The maximum number of retries before giving up. // Default is to not retry failed commands. diff --git a/pipeline.go b/pipeline.go index fcddc61823..8d2d884a2c 100644 --- a/pipeline.go +++ b/pipeline.go @@ -11,7 +11,8 @@ import ( // http://redis.io/topics/pipelining. It's safe for concurrent use // by multiple goroutines. type Pipeline struct { - commandable + cmdable + statefulCmdable exec func([]Cmder) error @@ -21,7 +22,7 @@ type Pipeline struct { closed int32 } -func (pipe *Pipeline) process(cmd Cmder) { +func (pipe *Pipeline) Process(cmd Cmder) { pipe.mu.Lock() pipe.cmds = append(pipe.cmds, cmd) pipe.mu.Unlock() diff --git a/pubsub.go b/pubsub.go index aef4a76b08..5bbda5966a 100644 --- a/pubsub.go +++ b/pubsub.go @@ -20,7 +20,7 @@ func (c *Client) Publish(channel, message string) *IntCmd { // http://redis.io/topics/pubsub. It's NOT safe for concurrent use by // multiple goroutines. type PubSub struct { - base *baseClient + base baseClient channels []string patterns []string @@ -31,7 +31,7 @@ type PubSub struct { // Deprecated. Use Subscribe/PSubscribe instead. func (c *Client) PubSub() *PubSub { return &PubSub{ - base: &baseClient{ + base: baseClient{ opt: c.opt, connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), }, diff --git a/race_test.go b/race_test.go index f654f87ab2..75ade87cab 100644 --- a/race_test.go +++ b/race_test.go @@ -172,7 +172,7 @@ var _ = Describe("races", func() { perform(C, func(id int) { opt := redisOptions() - opt.DB = int64(id) + opt.DB = id client := redis.NewClient(opt) for i := 0; i < N; i++ { err := client.Set("db", id, 0).Err() @@ -194,7 +194,7 @@ var _ = Describe("races", func() { It("should select DB with read timeout", func() { perform(C, func(id int) { opt := redisOptions() - opt.DB = int64(id) + opt.DB = id opt.ReadTimeout = time.Nanosecond client := redis.NewClient(opt) diff --git a/redis.go b/redis.go index bb9fcd0a13..d44e86e7b1 100644 --- a/redis.go +++ b/redis.go @@ -56,29 +56,25 @@ func (c *baseClient) initConn(cn *pool.Conn) error { // Temp client for Auth and Select. client := newClient(c.opt, pool.NewSingleConnPool(cn)) - - if c.opt.Password != "" { - if err := client.Auth(c.opt.Password).Err(); err != nil { - return err + _, err := client.Pipelined(func(pipe *Pipeline) error { + if c.opt.Password != "" { + pipe.Auth(c.opt.Password) } - } - if c.opt.DB > 0 { - if err := client.Select(c.opt.DB).Err(); err != nil { - return err + if c.opt.DB > 0 { + pipe.Select(c.opt.DB) } - } - if c.opt.ReadOnly { - if err := client.ReadOnly().Err(); err != nil { - return err + if c.opt.ReadOnly { + pipe.ReadOnly() } - } - return nil + return nil + }) + return err } -func (c *baseClient) process(cmd Cmder) { +func (c *baseClient) Process(cmd Cmder) { for i := 0; i <= c.opt.MaxRetries; i++ { if i > 0 { cmd.reset() @@ -145,16 +141,14 @@ func (c *baseClient) Close() error { // goroutines. type Client struct { baseClient - commandable + cmdable } func newClient(opt *Options, pool pool.Pooler) *Client { base := baseClient{opt: opt, connPool: pool} client := &Client{ baseClient: base, - commandable: commandable{ - process: base.process, - }, + cmdable: cmdable{base.Process}, } return client } @@ -178,11 +172,12 @@ func (c *Client) PoolStats() *PoolStats { } func (c *Client) Pipeline() *Pipeline { - pipe := &Pipeline{ + pipe := Pipeline{ exec: c.pipelineExec, } - pipe.commandable.process = pipe.process - return pipe + pipe.cmdable.process = pipe.Process + pipe.statefulCmdable.process = pipe.Process + return &pipe } func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { diff --git a/ring.go b/ring.go index 31559e61d7..17f53056f4 100644 --- a/ring.go +++ b/ring.go @@ -24,7 +24,7 @@ type RingOptions struct { // Following options are copied from Options struct. - DB int64 + DB int Password string MaxRetries int @@ -110,7 +110,7 @@ func (shard *ringShard) Vote(up bool) bool { // and can tolerate losing data when one of the servers dies. // Otherwise you should use Redis Cluster. type Ring struct { - commandable + cmdable opt *RingOptions nreplicas int @@ -136,7 +136,7 @@ func NewRing(opt *RingOptions) *Ring { cmdsInfoOnce: new(sync.Once), } - ring.commandable.process = ring.process + ring.cmdable.process = ring.Process for name, addr := range opt.Addrs { clopt := opt.clientOptions() clopt.Addr = addr @@ -196,13 +196,13 @@ func (ring *Ring) getClient(key string) (*Client, error) { return cl, nil } -func (ring *Ring) process(cmd Cmder) { +func (ring *Ring) Process(cmd Cmder) { cl, err := ring.getClient(ring.cmdFirstKey(cmd)) if err != nil { cmd.setErr(err) return } - cl.baseClient.process(cmd) + cl.baseClient.Process(cmd) } // rebalance removes dead shards from the ring. @@ -273,11 +273,12 @@ func (ring *Ring) Close() (retErr error) { } func (ring *Ring) Pipeline() *Pipeline { - pipe := &Pipeline{ + pipe := Pipeline{ exec: ring.pipelineExec, } - pipe.commandable.process = pipe.process - return pipe + pipe.cmdable.process = pipe.Process + pipe.statefulCmdable.process = pipe.Process + return &pipe } func (ring *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { diff --git a/sentinel.go b/sentinel.go index 53c40a9aa2..6cd3986355 100644 --- a/sentinel.go +++ b/sentinel.go @@ -25,7 +25,7 @@ type FailoverOptions struct { // Following options are copied from Options struct. Password string - DB int64 + DB int MaxRetries int @@ -70,43 +70,41 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt: opt, } - base := baseClient{ - opt: opt, - connPool: failover.Pool(), - - onClose: func() error { - return failover.Close() - }, - } - return &Client{ - baseClient: base, - commandable: commandable{ - process: base.process, + client := Client{ + baseClient: baseClient{ + opt: opt, + connPool: failover.Pool(), + + onClose: func() error { + return failover.Close() + }, }, } + client.cmdable.process = client.Process + return &client } //------------------------------------------------------------------------------ type sentinelClient struct { + cmdable baseClient - commandable } func newSentinel(opt *Options) *sentinelClient { - base := baseClient{ - opt: opt, - connPool: newConnPool(opt), - } - return &sentinelClient{ - baseClient: base, - commandable: commandable{process: base.process}, + client := sentinelClient{ + baseClient: baseClient{ + opt: opt, + connPool: newConnPool(opt), + }, } + client.cmdable = cmdable{client.Process} + return &client } func (c *sentinelClient) PubSub() *PubSub { return &PubSub{ - base: &baseClient{ + base: baseClient{ opt: c.opt, connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), }, diff --git a/tx.go b/tx.go index 34e828bc57..43271faafb 100644 --- a/tx.go +++ b/tx.go @@ -15,23 +15,24 @@ var errDiscard = errors.New("redis: Discard can be used only inside Exec") // by multiple goroutines, because Exec resets list of watched keys. // If you don't need WATCH it is better to use Pipeline. type Tx struct { - commandable - - base *baseClient + cmdable + statefulCmdable + baseClient cmds []Cmder closed bool } func (c *Client) newTx() *Tx { - tx := &Tx{ - base: &baseClient{ + tx := Tx{ + baseClient: baseClient{ opt: c.opt, connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true), }, } - tx.commandable.process = tx.process - return tx + tx.cmdable.process = tx.Process + tx.statefulCmdable.process = tx.Process + return &tx } func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { @@ -49,9 +50,9 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return retErr } -func (tx *Tx) process(cmd Cmder) { +func (tx *Tx) Process(cmd Cmder) { if tx.cmds == nil { - tx.base.process(cmd) + tx.baseClient.Process(cmd) } else { tx.cmds = append(tx.cmds, cmd) } @@ -66,7 +67,7 @@ func (tx *Tx) close() error { if err := tx.Unwatch().Err(); err != nil { internal.Logf("Unwatch failed: %s", err) } - return tx.base.Close() + return tx.baseClient.Close() } // Watch marks the keys to be watched for conditional execution @@ -133,14 +134,14 @@ func (tx *Tx) MultiExec(fn func() error) ([]Cmder, error) { // Strip MULTI and EXEC commands. retCmds := cmds[1 : len(cmds)-1] - cn, err := tx.base.conn() + cn, err := tx.conn() if err != nil { setCmdsErr(retCmds, err) return retCmds, err } err = tx.execCmds(cn, cmds) - tx.base.putConn(cn, err, false) + tx.putConn(cn, err, false) return retCmds, err } From 079b7ce393bf8d80fab5bd23de463fff611d7b28 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 5 Jun 2016 11:10:30 +0000 Subject: [PATCH 0179/1746] Rework Options initialisation. --- cluster.go | 21 ++++++++++---------- options.go | 55 +++++++++++++++-------------------------------------- redis.go | 1 + ring.go | 3 +++ sentinel.go | 5 +++++ 5 files changed, 34 insertions(+), 51 deletions(-) diff --git a/cluster.go b/cluster.go index e29d6ef720..983480b58a 100644 --- a/cluster.go +++ b/cluster.go @@ -41,10 +41,7 @@ type ClusterClient struct { // NewClusterClient returns a Redis Cluster client as described in // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { - if opt.RouteByLatency { - opt.ReadOnly = true - } - + opt.init() client := &ClusterClient{ opt: opt, nodes: make(map[string]*clusterNode), @@ -246,7 +243,7 @@ func (c *ClusterClient) Process(cmd Cmder) { var ask bool slot, node := c.cmdSlotAndNode(cmd) - for attempt := 0; attempt <= c.opt.getMaxRedirects(); attempt++ { + for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { if attempt > 0 { cmd.reset() } @@ -419,7 +416,7 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { cmdsMap[node] = append(cmdsMap[node], cmd) } - for attempt := 0; attempt <= c.opt.getMaxRedirects(); attempt++ { + for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { @@ -516,14 +513,16 @@ type ClusterOptions struct { IdleCheckFrequency time.Duration } -func (opt *ClusterOptions) getMaxRedirects() int { +func (opt *ClusterOptions) init() { if opt.MaxRedirects == -1 { - return 0 + opt.MaxRedirects = 0 + } else if opt.MaxRedirects == 0 { + opt.MaxRedirects = 16 } - if opt.MaxRedirects == 0 { - return 16 + + if opt.RouteByLatency { + opt.ReadOnly = true } - return opt.MaxRedirects } func (opt *ClusterOptions) clientOptions() *Options { diff --git a/options.go b/options.go index e3db4ad1b5..4cd62f9316 100644 --- a/options.go +++ b/options.go @@ -58,61 +58,36 @@ type Options struct { ReadOnly bool } -func (opt *Options) getNetwork() string { +func (opt *Options) init() { if opt.Network == "" { - return "tcp" + opt.Network = "tcp" } - return opt.Network -} - -func (opt *Options) getDialer() func() (net.Conn, error) { - if opt.Dialer != nil { - return opt.Dialer + if opt.Dialer == nil { + opt.Dialer = func() (net.Conn, error) { + return net.DialTimeout(opt.Network, opt.Addr, opt.DialTimeout) + } } - return func() (net.Conn, error) { - return net.DialTimeout(opt.getNetwork(), opt.Addr, opt.getDialTimeout()) - } -} - -func (opt *Options) getPoolSize() int { if opt.PoolSize == 0 { - return 10 + opt.PoolSize = 10 } - return opt.PoolSize -} - -func (opt *Options) getDialTimeout() time.Duration { if opt.DialTimeout == 0 { - return 5 * time.Second + opt.DialTimeout = 5 * time.Second } - return opt.DialTimeout -} - -func (opt *Options) getPoolTimeout() time.Duration { if opt.PoolTimeout == 0 { - return 1 * time.Second + opt.PoolTimeout = 1 * time.Second } - return opt.PoolTimeout -} - -func (opt *Options) getIdleTimeout() time.Duration { - return opt.IdleTimeout -} - -func (opt *Options) getIdleCheckFrequency() time.Duration { if opt.IdleCheckFrequency == 0 { - return time.Minute + opt.IdleCheckFrequency = time.Minute } - return opt.IdleCheckFrequency } func newConnPool(opt *Options) *pool.ConnPool { return pool.NewConnPool( - opt.getDialer(), - opt.getPoolSize(), - opt.getPoolTimeout(), - opt.getIdleTimeout(), - opt.getIdleCheckFrequency(), + opt.Dialer, + opt.PoolSize, + opt.PoolTimeout, + opt.IdleTimeout, + opt.IdleCheckFrequency, ) } diff --git a/redis.go b/redis.go index d44e86e7b1..121c5b24a9 100644 --- a/redis.go +++ b/redis.go @@ -155,6 +155,7 @@ func newClient(opt *Options, pool pool.Pooler) *Client { // NewClient returns a client to the Redis Server specified by Options. func NewClient(opt *Options) *Client { + opt.init() return newClient(opt, newConnPool(opt)) } diff --git a/ring.go b/ring.go index 17f53056f4..bc749c1d04 100644 --- a/ring.go +++ b/ring.go @@ -39,6 +39,8 @@ type RingOptions struct { IdleCheckFrequency time.Duration } +func (opt *RingOptions) init() {} + func (opt *RingOptions) clientOptions() *Options { return &Options{ DB: opt.DB, @@ -127,6 +129,7 @@ type Ring struct { func NewRing(opt *RingOptions) *Ring { const nreplicas = 100 + opt.init() ring := &Ring{ opt: opt, nreplicas: nreplicas, diff --git a/sentinel.go b/sentinel.go index 6cd3986355..66f0974ce6 100644 --- a/sentinel.go +++ b/sentinel.go @@ -64,12 +64,15 @@ func (opt *FailoverOptions) options() *Options { // goroutines. func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt := failoverOpt.options() + opt.init() + failover := &sentinelFailover{ masterName: failoverOpt.MasterName, sentinelAddrs: failoverOpt.SentinelAddrs, opt: opt, } + client := Client{ baseClient: baseClient{ opt: opt, @@ -81,6 +84,7 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { }, } client.cmdable.process = client.Process + return &client } @@ -92,6 +96,7 @@ type sentinelClient struct { } func newSentinel(opt *Options) *sentinelClient { + opt.init() client := sentinelClient{ baseClient: baseClient{ opt: opt, From 4e64d5aa6e42c1a6545476feadb6edaa7f8cd4f9 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 5 Jun 2016 11:30:56 +0000 Subject: [PATCH 0180/1746] Run cluster tests on Client with RouteByLatency option. --- cluster.go | 164 ++++++++++++++++++++++++++++++++---------------- cluster_test.go | 99 ++++++++++++++++------------- 2 files changed, 164 insertions(+), 99 deletions(-) diff --git a/cluster.go b/cluster.go index 983480b58a..b0c4659b94 100644 --- a/cluster.go +++ b/cluster.go @@ -51,7 +51,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { client.cmdable.process = client.Process for _, addr := range opt.Addrs { - _ = client.nodeByAddr(addr) + _, _ = client.nodeByAddr(addr) } client.reloadSlots() @@ -86,7 +86,10 @@ func (c *ClusterClient) getNodes() map[string]*clusterNode { } func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { - node := c.slotMasterNode(hashtag.Slot(keys[0])) + node, err := c.slotMasterNode(hashtag.Slot(keys[0])) + if err != nil { + return err + } return node.Client.Watch(fn, keys...) } @@ -122,26 +125,29 @@ func (c *ClusterClient) Close() error { return nil } -func (c *ClusterClient) nodeByAddr(addr string) *clusterNode { +func (c *ClusterClient) nodeByAddr(addr string) (*clusterNode, error) { c.mu.RLock() node, ok := c.nodes[addr] c.mu.RUnlock() if ok { - return node + return node, nil } + defer c.mu.Unlock() c.mu.Lock() - if !c.closed { - node, ok = c.nodes[addr] - if !ok { - node = c.newNode(addr) - c.nodes[addr] = node - c.addrs = append(c.addrs, node.Addr) - } + + if c.closed { + return nil, pool.ErrClosed + } + + node, ok = c.nodes[addr] + if !ok { + node = c.newNode(addr) + c.nodes[addr] = node + c.addrs = append(c.addrs, node.Addr) } - c.mu.Unlock() - return node + return node, nil } func (c *ClusterClient) newNode(addr string) *clusterNode { @@ -161,70 +167,81 @@ func (c *ClusterClient) slotNodes(slot int) []*clusterNode { } // randomNode returns random live node. -func (c *ClusterClient) randomNode() *clusterNode { +func (c *ClusterClient) randomNode() (*clusterNode, error) { var node *clusterNode + var err error for i := 0; i < 10; i++ { c.mu.RLock() + closed := c.closed addrs := c.addrs c.mu.RUnlock() - if len(addrs) == 0 { - return nil + if closed { + return nil, pool.ErrClosed } n := rand.Intn(len(addrs)) - node = c.nodeByAddr(addrs[n]) + node, err = c.nodeByAddr(addrs[n]) + if err != nil { + return nil, err + } if node.Client.ClusterInfo().Err() == nil { - return node + break } } - return node + return node, nil } -func (c *ClusterClient) slotMasterNode(slot int) *clusterNode { +func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) if len(nodes) == 0 { return c.randomNode() } - return nodes[0] + return nodes[0], nil } -func (c *ClusterClient) slotSlaveNode(slot int) *clusterNode { +func (c *ClusterClient) slotSlaveNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) switch len(nodes) { case 0: return c.randomNode() case 1: - return nodes[0] + return nodes[0], nil case 2: - return nodes[1] + return nodes[1], nil default: n := rand.Intn(len(nodes)-1) + 1 - return nodes[n] + return nodes[n], nil } } -func (c *ClusterClient) slotClosestNode(slot int) *clusterNode { +func (c *ClusterClient) slotClosestNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) + if len(nodes) == 0 { + return c.randomNode() + } + var node *clusterNode for _, n := range nodes { if node == nil || n.Latency < node.Latency { node = n } } - return node + return node, nil } -func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode) { +func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmdInfo(cmd.arg(0)) if cmdInfo == nil { - return 0, c.randomNode() + node, err := c.randomNode() + return 0, node, err } if cmdInfo.FirstKeyPos == -1 { - return 0, c.randomNode() + node, err := c.randomNode() + return 0, node, err } firstKey := cmd.arg(int(cmdInfo.FirstKeyPos)) @@ -232,23 +249,28 @@ func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode) { if cmdInfo.ReadOnly && c.opt.ReadOnly { if c.opt.RouteByLatency { - return slot, c.slotClosestNode(slot) + node, err := c.slotClosestNode(slot) + return slot, node, err } - return slot, c.slotSlaveNode(slot) + + node, err := c.slotSlaveNode(slot) + return slot, node, err } - return slot, c.slotMasterNode(slot) + + node, err := c.slotMasterNode(slot) + return slot, node, err } func (c *ClusterClient) Process(cmd Cmder) { var ask bool - slot, node := c.cmdSlotAndNode(cmd) - + slot, node, err := c.cmdSlotAndNode(cmd) for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { if attempt > 0 { cmd.reset() } - if node == nil { - cmd.setErr(pool.ErrClosed) + + if err != nil { + cmd.setErr(err) return } @@ -271,7 +293,7 @@ func (c *ClusterClient) Process(cmd Cmder) { // On network errors try random node. if shouldRetry(err) { - node = c.randomNode() + node, err = c.randomNode() continue } @@ -279,11 +301,12 @@ func (c *ClusterClient) Process(cmd Cmder) { var addr string moved, ask, addr = isMovedError(err) if moved || ask { - if moved && c.slotMasterNode(slot).Addr != addr { + master, _ := c.slotMasterNode(slot) + if moved && (master == nil || master.Addr != addr) { c.lazyReloadSlots() } - node = c.nodeByAddr(addr) + node, err = c.nodeByAddr(addr) continue } @@ -310,7 +333,10 @@ func (c *ClusterClient) setSlots(cs []ClusterSlot) { for _, s := range cs { var nodes []*clusterNode for _, n := range s.Nodes { - nodes = append(nodes, c.nodeByAddr(n.Addr)) + node, err := c.nodeByAddr(n.Addr) + if err == nil { + nodes = append(nodes, node) + } } for i := s.Start; i <= s.End; i++ { @@ -341,8 +367,8 @@ func (c *ClusterClient) setNodesLatency() { func (c *ClusterClient) reloadSlots() { defer atomic.StoreUint32(&c.reloading, 0) - node := c.randomNode() - if node == nil { + node, err := c.randomNode() + if err != nil { return } @@ -409,10 +435,20 @@ func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { func (c *ClusterClient) pipelineExec(cmds []Cmder) error { var retErr error + returnError := func(err error) { + if retErr == nil { + retErr = err + } + } cmdsMap := make(map[*clusterNode][]Cmder) for _, cmd := range cmds { - _, node := c.cmdSlotAndNode(cmd) + _, node, err := c.cmdSlotAndNode(cmd) + if err != nil { + cmd.setErr(err) + returnError(err) + continue + } cmdsMap[node] = append(cmdsMap[node], cmd) } @@ -421,19 +457,25 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { for node, cmds := range cmdsMap { if node == nil { - node = c.randomNode() + var err error + node, err = c.randomNode() + if err != nil { + setCmdsErr(cmds, err) + returnError(err) + continue + } } cn, err := node.Client.conn() if err != nil { - setCmdsErr(cmds, err) - retErr = err + setCmdsErr(cmds, retErr) + returnError(err) continue } failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) if err != nil { - retErr = err + returnError(err) } node.Client.putConn(cn, err, false) } @@ -452,7 +494,13 @@ func (c *ClusterClient) execClusterCmds( return failedCmds, err } - var firstCmdErr error + var retErr error + returnError := func(err error) { + if retErr == nil { + retErr = err + } + } + for i, cmd := range cmds { err := cmd.readReply(cn) if err == nil { @@ -465,18 +513,26 @@ func (c *ClusterClient) execClusterCmds( } else if moved, ask, addr := isMovedError(err); moved { c.lazyReloadSlots() cmd.reset() - node := c.nodeByAddr(addr) + node, err := c.nodeByAddr(addr) + if err != nil { + returnError(err) + continue + } failedCmds[node] = append(failedCmds[node], cmd) } else if ask { cmd.reset() - node := c.nodeByAddr(addr) + node, err := c.nodeByAddr(addr) + if err != nil { + returnError(err) + continue + } failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) - } else if firstCmdErr == nil { - firstCmdErr = err + } else { + returnError(err) } } - return failedCmds, firstCmdErr + return failedCmds, retErr } //------------------------------------------------------------------------------ diff --git a/cluster_test.go b/cluster_test.go index 9cdbc00431..446e20df9e 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -312,21 +312,12 @@ var _ = Describe("Cluster", func() { // Expect(res).To(Equal("OK")) // }) }) +}) - Describe("Client", func() { - var client *redis.ClusterClient - - BeforeEach(func() { - client = cluster.clusterClient(nil) - }) - - AfterEach(func() { - for _, client := range cluster.masters() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - } - Expect(client.Close()).NotTo(HaveOccurred()) - }) +var _ = Describe("ClusterClient", func() { + var client *redis.ClusterClient + describeClusterClient := func() { It("should GET/SET/DEL", func() { val, err := client.Get("A").Result() Expect(err).To(Equal(redis.Nil)) @@ -358,15 +349,10 @@ var _ = Describe("Cluster", func() { val, err := client.Get("A").Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("VALUE")) - - Eventually(func() []string { - return client.SlotAddrs(slot) - }, "10s").Should(Equal([]string{"127.0.0.1:8221", "127.0.0.1:8224"})) }) It("should return error when there are no attempts left", func() { - Expect(client.Close()).NotTo(HaveOccurred()) - client = cluster.clusterClient(&redis.ClusterOptions{ + client := cluster.clusterClient(&redis.ClusterOptions{ MaxRedirects: -1, }) @@ -376,6 +362,8 @@ var _ = Describe("Cluster", func() { err := client.Get("A").Err() Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("MOVED")) + + Expect(client.Close()).NotTo(HaveOccurred()) }) It("should Watch", func() { @@ -417,23 +405,8 @@ var _ = Describe("Cluster", func() { Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(100))) }) - }) - - Describe("pipeline", func() { - var client *redis.ClusterClient - - BeforeEach(func() { - client = cluster.clusterClient(nil) - }) - - AfterEach(func() { - for _, client := range cluster.masters() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - } - Expect(client.Close()).NotTo(HaveOccurred()) - }) - It("performs multi-pipelines", func() { + It("supports pipeline", func() { slot := hashtag.Slot("A") Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) @@ -441,27 +414,31 @@ var _ = Describe("Cluster", func() { defer pipe.Close() keys := []string{"A", "B", "C", "D", "E", "F", "G"} + for i, key := range keys { pipe.Set(key, key+"_value", 0) pipe.Expire(key, time.Duration(i+1)*time.Hour) } + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(14)) + for _, key := range keys { pipe.Get(key) pipe.TTL(key) } - - cmds, err := pipe.Exec() + cmds, err = pipe.Exec() Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(28)) - Expect(cmds[14].(*redis.StringCmd).Val()).To(Equal("A_value")) - Expect(cmds[15].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) - Expect(cmds[20].(*redis.StringCmd).Val()).To(Equal("D_value")) - Expect(cmds[21].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) - Expect(cmds[26].(*redis.StringCmd).Val()).To(Equal("G_value")) - Expect(cmds[27].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) + Expect(cmds).To(HaveLen(14)) + Expect(cmds[0].(*redis.StringCmd).Val()).To(Equal("A_value")) + Expect(cmds[1].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) + Expect(cmds[6].(*redis.StringCmd).Val()).To(Equal("D_value")) + Expect(cmds[7].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) + Expect(cmds[12].(*redis.StringCmd).Val()).To(Equal("G_value")) + Expect(cmds[13].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) }) - It("works with missing keys", func() { + It("supports pipeline with missing keys", func() { Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) @@ -484,6 +461,38 @@ var _ = Describe("Cluster", func() { Expect(c.Err()).NotTo(HaveOccurred()) Expect(c.Val()).To(Equal("C_value")) }) + } + + Describe("default ClusterClient", func() { + BeforeEach(func() { + client = cluster.clusterClient(nil) + }) + + AfterEach(func() { + for _, client := range cluster.masters() { + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + } + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + describeClusterClient() + }) + + Describe("ClusterClient with RouteByLatency", func() { + BeforeEach(func() { + client = cluster.clusterClient(&redis.ClusterOptions{ + RouteByLatency: true, + }) + }) + + AfterEach(func() { + for _, client := range cluster.masters() { + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + } + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + describeClusterClient() }) }) From cd582ed5769b18f2aff68916ef3807e8a10669c4 Mon Sep 17 00:00:00 2001 From: Mattias Lundell Date: Thu, 9 Jun 2016 17:39:51 +0200 Subject: [PATCH 0181/1746] add support for SPOP with a count argument (available in REDIS 3.2) --- commands.go | 8 ++++++++ commands_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/commands.go b/commands.go index 5107b7b353..f6994f04c4 100644 --- a/commands.go +++ b/commands.go @@ -946,12 +946,20 @@ func (c cmdable) SMove(source, destination string, member interface{}) *BoolCmd return cmd } +// Redis `SPOP key` command. func (c cmdable) SPop(key string) *StringCmd { cmd := NewStringCmd("spop", key) c.process(cmd) return cmd } +// Redis `SPOP key count` command. +func (c cmdable) SPopN(key string, count int64) *StringSliceCmd { + cmd := NewStringSliceCmd("spop", key, count) + c.process(cmd) + return cmd +} + // Redis `SRANDMEMBER key` command. func (c cmdable) SRandMember(key string) *StringCmd { cmd := NewStringCmd("srandmember", key) diff --git a/commands_test.go b/commands_test.go index 466f991fe9..b4d0ff3b6b 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1790,6 +1790,34 @@ var _ = Describe("Commands", func() { sMembers := client.SMembers("set") Expect(sMembers.Err()).NotTo(HaveOccurred()) Expect(sMembers.Val()).To(HaveLen(2)) + + }) + + It("should SPopN", func() { + sAdd := client.SAdd("set", "one") + Expect(sAdd.Err()).NotTo(HaveOccurred()) + sAdd = client.SAdd("set", "two") + Expect(sAdd.Err()).NotTo(HaveOccurred()) + sAdd = client.SAdd("set", "three") + Expect(sAdd.Err()).NotTo(HaveOccurred()) + sAdd = client.SAdd("set", "four") + Expect(sAdd.Err()).NotTo(HaveOccurred()) + + sPopN := client.SPopN("set", 1) + Expect(sPopN.Err()).NotTo(HaveOccurred()) + Expect(sPopN.Val()).NotTo(Equal([]string{""})) + + sMembers := client.SMembers("set") + Expect(sMembers.Err()).NotTo(HaveOccurred()) + Expect(sMembers.Val()).To(HaveLen(3)) + + sPopN = client.SPopN("set", 4) + Expect(sPopN.Err()).NotTo(HaveOccurred()) + Expect(sPopN.Val()).To(HaveLen(3)) + + sMembers = client.SMembers("set") + Expect(sMembers.Err()).NotTo(HaveOccurred()) + Expect(sMembers.Val()).To(HaveLen(0)) }) It("should SRandMember and SRandMemberN", func() { From b1f6610fc63ad24fdf6969e06d108c777eae1c24 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 14 Jun 2016 10:22:16 +0000 Subject: [PATCH 0182/1746] Support more interface values. --- commands.go | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/commands.go b/commands.go index f6994f04c4..b9c7d25d10 100644 --- a/commands.go +++ b/commands.go @@ -67,7 +67,7 @@ func (c statefulCmdable) Auth(password string) *StatusCmd { return cmd } -func (c *cmdable) Echo(message string) *StringCmd { +func (c *cmdable) Echo(message interface{}) *StringCmd { cmd := NewStringCmd("echo", message) c.process(cmd) return cmd @@ -517,7 +517,7 @@ func (c cmdable) MGet(keys ...string) *SliceCmd { return cmd } -func (c cmdable) MSet(pairs ...string) *StatusCmd { +func (c cmdable) MSet(pairs ...interface{}) *StatusCmd { args := make([]interface{}, 1+len(pairs)) args[0] = "mset" for i, pair := range pairs { @@ -528,7 +528,7 @@ func (c cmdable) MSet(pairs ...string) *StatusCmd { return cmd } -func (c cmdable) MSetNX(pairs ...string) *BoolCmd { +func (c cmdable) MSetNX(pairs ...interface{}) *BoolCmd { args := make([]interface{}, 1+len(pairs)) args[0] = "msetnx" for i, pair := range pairs { @@ -762,12 +762,24 @@ func (c cmdable) LIndex(key string, index int64) *StringCmd { return cmd } -func (c cmdable) LInsert(key, op, pivot, value string) *IntCmd { +func (c cmdable) LInsert(key, op string, pivot, value interface{}) *IntCmd { cmd := NewIntCmd("linsert", key, op, pivot, value) c.process(cmd) return cmd } +func (c cmdable) LInsertBefore(key string, pivot, value interface{}) *IntCmd { + cmd := NewIntCmd("linsert", key, "before", pivot, value) + c.process(cmd) + return cmd +} + +func (c cmdable) LInsertAfter(key string, pivot, value interface{}) *IntCmd { + cmd := NewIntCmd("linsert", key, "after", pivot, value) + c.process(cmd) + return cmd +} + func (c cmdable) LLen(key string) *IntCmd { cmd := NewIntCmd("llen", key) c.process(cmd) @@ -780,7 +792,7 @@ func (c cmdable) LPop(key string) *StringCmd { return cmd } -func (c cmdable) LPush(key string, values ...string) *IntCmd { +func (c cmdable) LPush(key string, values ...interface{}) *IntCmd { args := make([]interface{}, 2+len(values)) args[0] = "lpush" args[1] = key @@ -792,7 +804,7 @@ func (c cmdable) LPush(key string, values ...string) *IntCmd { return cmd } -func (c cmdable) LPushX(key, value interface{}) *IntCmd { +func (c cmdable) LPushX(key string, value interface{}) *IntCmd { cmd := NewIntCmd("lpushx", key, value) c.process(cmd) return cmd @@ -844,7 +856,7 @@ func (c cmdable) RPopLPush(source, destination string) *StringCmd { return cmd } -func (c cmdable) RPush(key string, values ...string) *IntCmd { +func (c cmdable) RPush(key string, values ...interface{}) *IntCmd { args := make([]interface{}, 2+len(values)) args[0] = "rpush" args[1] = key @@ -864,7 +876,7 @@ func (c cmdable) RPushX(key string, value interface{}) *IntCmd { //------------------------------------------------------------------------------ -func (c cmdable) SAdd(key string, members ...string) *IntCmd { +func (c cmdable) SAdd(key string, members ...interface{}) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "sadd" args[1] = key @@ -974,7 +986,7 @@ func (c cmdable) SRandMemberN(key string, count int64) *StringSliceCmd { return cmd } -func (c cmdable) SRem(key string, members ...string) *IntCmd { +func (c cmdable) SRem(key string, members ...interface{}) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "srem" args[1] = key @@ -1233,7 +1245,7 @@ func (c cmdable) ZRank(key, member string) *IntCmd { return cmd } -func (c cmdable) ZRem(key string, members ...string) *IntCmd { +func (c cmdable) ZRem(key string, members ...interface{}) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "zrem" args[1] = key @@ -1348,12 +1360,12 @@ func (c cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd //------------------------------------------------------------------------------ -func (c cmdable) PFAdd(key string, fields ...string) *IntCmd { - args := make([]interface{}, 2+len(fields)) +func (c cmdable) PFAdd(key string, els ...interface{}) *IntCmd { + args := make([]interface{}, 2+len(els)) args[0] = "pfadd" args[1] = key - for i, field := range fields { - args[2+i] = field + for i, el := range els { + args[2+i] = el } cmd := NewIntCmd(args...) c.process(cmd) From 4761c242181d8e133e7abf2a2261d79565ddbb86 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 17 Jun 2016 12:09:38 +0000 Subject: [PATCH 0183/1746] Add ForEachMaster API. --- cluster.go | 121 ++++++++++++++++++++++++++++++++---------------- cluster_test.go | 40 ++++++++++++---- command.go | 5 +- commands.go | 4 +- pipeline.go | 3 +- redis.go | 10 ++-- ring.go | 6 +-- tx.go | 8 ++-- 8 files changed, 133 insertions(+), 64 deletions(-) diff --git a/cluster.go b/cluster.go index b0c4659b94..1ae6082b61 100644 --- a/cluster.go +++ b/cluster.go @@ -13,8 +13,8 @@ import ( type clusterNode struct { Addr string - Latency int Client *Client + Latency time.Duration } // ClusterClient is a Redis Cluster client representing a pool of zero @@ -73,8 +73,8 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo { } func (c *ClusterClient) getNodes() map[string]*clusterNode { - c.mu.RLock() var nodes map[string]*clusterNode + c.mu.RLock() if !c.closed { nodes = make(map[string]*clusterNode, len(c.nodes)) for addr, node := range c.nodes { @@ -95,7 +95,7 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { // PoolStats returns accumulated connection pool stats. func (c *ClusterClient) PoolStats() *PoolStats { - acc := PoolStats{} + var acc PoolStats for _, node := range c.getNodes() { s := node.Client.connPool.Stats() acc.Requests += s.Requests @@ -214,7 +214,6 @@ func (c *ClusterClient) slotSlaveNode(slot int) (*clusterNode, error) { n := rand.Intn(len(nodes)-1) + 1 return nodes[n], nil } - } func (c *ClusterClient) slotClosestNode(slot int) (*clusterNode, error) { @@ -261,19 +260,19 @@ func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { return slot, node, err } -func (c *ClusterClient) Process(cmd Cmder) { - var ask bool +func (c *ClusterClient) Process(cmd Cmder) error { slot, node, err := c.cmdSlotAndNode(cmd) + if err != nil { + cmd.setErr(err) + return err + } + + var ask bool for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { if attempt > 0 { cmd.reset() } - if err != nil { - cmd.setErr(err) - return - } - if ask { pipe := node.Client.Pipeline() pipe.Process(NewCmd("ASKING")) @@ -288,7 +287,7 @@ func (c *ClusterClient) Process(cmd Cmder) { // If there is no (real) error, we are done! err := cmd.Err() if err == nil { - return + return nil } // On network errors try random node. @@ -307,11 +306,58 @@ func (c *ClusterClient) Process(cmd Cmder) { } node, err = c.nodeByAddr(addr) + if err != nil { + cmd.setErr(err) + return err + } continue } break } + + return cmd.Err() +} + +// ForEachMaster concurrently calls the fn on each master node in the cluster. +// It returns the first error if any. +func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { + c.mu.RLock() + slots := c.slots + c.mu.RUnlock() + + var retErr error + var mu sync.Mutex + + var wg sync.WaitGroup + visited := make(map[*clusterNode]struct{}) + for _, nodes := range slots { + if len(nodes) == 0 { + continue + } + + master := nodes[0] + if _, ok := visited[master]; ok { + continue + } + visited[master] = struct{}{} + + wg.Add(1) + go func(node *clusterNode) { + err := fn(node.Client) + if err != nil { + mu.Lock() + if retErr == nil { + retErr = err + } + mu.Unlock() + } + wg.Done() + }(master) + } + wg.Wait() + + return retErr } // closeClients closes all clients and returns the first error if there are any. @@ -327,9 +373,6 @@ func (c *ClusterClient) closeClients() error { func (c *ClusterClient) setSlots(cs []ClusterSlot) { slots := make([][]*clusterNode, hashtag.SlotNumber) - for i := 0; i < hashtag.SlotNumber; i++ { - slots[i] = nil - } for _, s := range cs { var nodes []*clusterNode for _, n := range s.Nodes { @@ -351,17 +394,11 @@ func (c *ClusterClient) setSlots(cs []ClusterSlot) { c.mu.Unlock() } -func (c *ClusterClient) setNodesLatency() { - nodes := c.getNodes() - for _, node := range nodes { - var latency int - for i := 0; i < 10; i++ { - t1 := time.Now() - node.Client.Ping() - latency += int(time.Since(t1) / time.Millisecond) - } - node.Latency = latency +func (c *ClusterClient) lazyReloadSlots() { + if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { + return } + go c.reloadSlots() } func (c *ClusterClient) reloadSlots() { @@ -384,11 +421,17 @@ func (c *ClusterClient) reloadSlots() { } } -func (c *ClusterClient) lazyReloadSlots() { - if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { - return +func (c *ClusterClient) setNodesLatency() { + const n = 10 + for _, node := range c.getNodes() { + var latency time.Duration + for i := 0; i < n; i++ { + t1 := time.Now() + node.Client.Ping() + latency += time.Since(t1) + } + node.Latency = latency / n } - go c.reloadSlots() } // reaper closes idle connections to the cluster. @@ -435,7 +478,7 @@ func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { func (c *ClusterClient) pipelineExec(cmds []Cmder) error { var retErr error - returnError := func(err error) { + setRetErr := func(err error) { if retErr == nil { retErr = err } @@ -446,7 +489,7 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { _, node, err := c.cmdSlotAndNode(cmd) if err != nil { cmd.setErr(err) - returnError(err) + setRetErr(err) continue } cmdsMap[node] = append(cmdsMap[node], cmd) @@ -461,21 +504,21 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { node, err = c.randomNode() if err != nil { setCmdsErr(cmds, err) - returnError(err) + setRetErr(err) continue } } cn, err := node.Client.conn() if err != nil { - setCmdsErr(cmds, retErr) - returnError(err) + setCmdsErr(cmds, err) + setRetErr(err) continue } failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) if err != nil { - returnError(err) + setRetErr(err) } node.Client.putConn(cn, err, false) } @@ -495,7 +538,7 @@ func (c *ClusterClient) execClusterCmds( } var retErr error - returnError := func(err error) { + setRetErr := func(err error) { if retErr == nil { retErr = err } @@ -515,7 +558,7 @@ func (c *ClusterClient) execClusterCmds( cmd.reset() node, err := c.nodeByAddr(addr) if err != nil { - returnError(err) + setRetErr(err) continue } failedCmds[node] = append(failedCmds[node], cmd) @@ -523,12 +566,12 @@ func (c *ClusterClient) execClusterCmds( cmd.reset() node, err := c.nodeByAddr(addr) if err != nil { - returnError(err) + setRetErr(err) continue } failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) } else { - returnError(err) + setRetErr(err) } } diff --git a/cluster_test.go b/cluster_test.go index 446e20df9e..6889d4e09f 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -336,11 +336,11 @@ var _ = Describe("ClusterClient", func() { Expect(cnt).To(Equal(int64(1))) }) - It("should return pool stats", func() { + It("returns pool stats", func() { Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) }) - It("should follow redirects", func() { + It("follows redirects", func() { Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) slot := hashtag.Slot("A") @@ -351,7 +351,7 @@ var _ = Describe("ClusterClient", func() { Expect(val).To(Equal("VALUE")) }) - It("should return error when there are no attempts left", func() { + It("returns an error when there are no attempts left", func() { client := cluster.clusterClient(&redis.ClusterOptions{ MaxRedirects: -1, }) @@ -366,7 +366,7 @@ var _ = Describe("ClusterClient", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("should Watch", func() { + It("supports Watch", func() { var incr func(string) error // Transactionally increments key using GET and SET commands. @@ -461,17 +461,35 @@ var _ = Describe("ClusterClient", func() { Expect(c.Err()).NotTo(HaveOccurred()) Expect(c.Val()).To(Equal("C_value")) }) + + It("calls fn for every master node", func() { + for i := 0; i < 10; i++ { + Expect(client.Set(strconv.Itoa(i), "", 0).Err()).NotTo(HaveOccurred()) + } + + err := client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDb().Err() + }) + Expect(err).NotTo(HaveOccurred()) + + for _, client := range cluster.masters() { + keys, err := client.Keys("*").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(keys).To(HaveLen(0)) + } + }) } Describe("default ClusterClient", func() { BeforeEach(func() { client = cluster.clusterClient(nil) + + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDb().Err() + }) }) AfterEach(func() { - for _, client := range cluster.masters() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - } Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -483,12 +501,14 @@ var _ = Describe("ClusterClient", func() { client = cluster.clusterClient(&redis.ClusterOptions{ RouteByLatency: true, }) + + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDb().Err() + }) }) AfterEach(func() { - for _, client := range cluster.masters() { - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - } + client.FlushDb() Expect(client.Close()).NotTo(HaveOccurred()) }) diff --git a/command.go b/command.go index 2db8fa2f5d..c56258284a 100644 --- a/command.go +++ b/command.go @@ -31,6 +31,7 @@ var ( type Cmder interface { args() []interface{} arg(int) string + readReply(*pool.Conn) error setErr(error) reset() @@ -142,7 +143,9 @@ type Cmd struct { } func NewCmd(args ...interface{}) *Cmd { - return &Cmd{baseCmd: newBaseCmd(args)} + return &Cmd{ + baseCmd: newBaseCmd(args), + } } func (cmd *Cmd) reset() { diff --git a/commands.go b/commands.go index b9c7d25d10..b21a8eec57 100644 --- a/commands.go +++ b/commands.go @@ -52,11 +52,11 @@ func formatSec(dur time.Duration) string { } type cmdable struct { - process func(cmd Cmder) + process func(cmd Cmder) error } type statefulCmdable struct { - process func(cmd Cmder) + process func(cmd Cmder) error } //------------------------------------------------------------------------------ diff --git a/pipeline.go b/pipeline.go index 8d2d884a2c..e946b9e9e0 100644 --- a/pipeline.go +++ b/pipeline.go @@ -22,10 +22,11 @@ type Pipeline struct { closed int32 } -func (pipe *Pipeline) Process(cmd Cmder) { +func (pipe *Pipeline) Process(cmd Cmder) error { pipe.mu.Lock() pipe.cmds = append(pipe.cmds, cmd) pipe.mu.Unlock() + return nil } // Close closes the pipeline, releasing any open resources. diff --git a/redis.go b/redis.go index 121c5b24a9..5c93db0010 100644 --- a/redis.go +++ b/redis.go @@ -74,7 +74,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { return err } -func (c *baseClient) Process(cmd Cmder) { +func (c *baseClient) Process(cmd Cmder) error { for i := 0; i <= c.opt.MaxRetries; i++ { if i > 0 { cmd.reset() @@ -83,7 +83,7 @@ func (c *baseClient) Process(cmd Cmder) { cn, err := c.conn() if err != nil { cmd.setErr(err) - return + return err } readTimeout := cmd.readTimeout() @@ -100,7 +100,7 @@ func (c *baseClient) Process(cmd Cmder) { if err != nil && shouldRetry(err) { continue } - return + return err } err = cmd.readReply(cn) @@ -109,8 +109,10 @@ func (c *baseClient) Process(cmd Cmder) { continue } - return + return err } + + return cmd.Err() } func (c *baseClient) closed() bool { diff --git a/ring.go b/ring.go index bc749c1d04..50476fa894 100644 --- a/ring.go +++ b/ring.go @@ -199,13 +199,13 @@ func (ring *Ring) getClient(key string) (*Client, error) { return cl, nil } -func (ring *Ring) Process(cmd Cmder) { +func (ring *Ring) Process(cmd Cmder) error { cl, err := ring.getClient(ring.cmdFirstKey(cmd)) if err != nil { cmd.setErr(err) - return + return err } - cl.baseClient.Process(cmd) + return cl.baseClient.Process(cmd) } // rebalance removes dead shards from the ring. diff --git a/tx.go b/tx.go index 43271faafb..4f85cc8d90 100644 --- a/tx.go +++ b/tx.go @@ -50,12 +50,12 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return retErr } -func (tx *Tx) Process(cmd Cmder) { +func (tx *Tx) Process(cmd Cmder) error { if tx.cmds == nil { - tx.baseClient.Process(cmd) - } else { - tx.cmds = append(tx.cmds, cmd) + return tx.baseClient.Process(cmd) } + tx.cmds = append(tx.cmds, cmd) + return nil } // close closes the transaction, releasing any open resources. From 5d0dda688fb64cb5283944bcbd1e730c4e9176c5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 27 Jun 2016 09:54:22 +0000 Subject: [PATCH 0184/1746] Make readLine more strict. --- parser.go | 12 +++++++----- race_test.go | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/parser.go b/parser.go index a58d3b1ccd..f6e5ca62bf 100644 --- a/parser.go +++ b/parser.go @@ -1,6 +1,7 @@ package redis import ( + "bufio" "errors" "fmt" "net" @@ -19,9 +20,7 @@ const ( type multiBulkParser func(cn *pool.Conn, n int64) (interface{}, error) -var ( - errReaderTooSmall = errors.New("redis: reader is too small") -) +var errEmptyReply = errors.New("redis: reply is empty") //------------------------------------------------------------------------------ @@ -227,10 +226,13 @@ func scan(b []byte, val interface{}) error { func readLine(cn *pool.Conn) ([]byte, error) { line, isPrefix, err := cn.Rd.ReadLine() if err != nil { - return line, err + return nil, err } if isPrefix { - return line, errReaderTooSmall + return nil, bufio.ErrBufferFull + } + if len(line) == 0 { + return nil, errEmptyReply } if isNilReply(line) { return nil, Nil diff --git a/race_test.go b/race_test.go index 75ade87cab..ca4fb7f297 100644 --- a/race_test.go +++ b/race_test.go @@ -120,7 +120,6 @@ var _ = Describe("races", func() { Expect(got).To(Equal(bigVal)) } }) - }) It("should handle big vals in Set", func() { From 1c4c05e970b928908e03dce3b7677cc0d9a09079 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Fri, 1 Jul 2016 13:25:28 +0100 Subject: [PATCH 0185/1746] Ensure to use pointer methods where appropriate. Tidy up godoc. --- commands.go | 404 ++++++++++++++++++++++++++-------------------------- iterator.go | 2 +- pipeline.go | 54 +++---- ring.go | 110 +++++++------- tx.go | 54 +++---- 5 files changed, 312 insertions(+), 312 deletions(-) diff --git a/commands.go b/commands.go index b21a8eec57..f84311c51f 100644 --- a/commands.go +++ b/commands.go @@ -61,7 +61,7 @@ type statefulCmdable struct { //------------------------------------------------------------------------------ -func (c statefulCmdable) Auth(password string) *StatusCmd { +func (c *statefulCmdable) Auth(password string) *StatusCmd { cmd := NewStatusCmd("auth", password) c.process(cmd) return cmd @@ -79,11 +79,11 @@ func (c *cmdable) Ping() *StatusCmd { return cmd } -func (c cmdable) Quit() *StatusCmd { +func (c *cmdable) Quit() *StatusCmd { panic("not implemented") } -func (c statefulCmdable) Select(index int) *StatusCmd { +func (c *statefulCmdable) Select(index int) *StatusCmd { cmd := NewStatusCmd("select", index) c.process(cmd) return cmd @@ -91,7 +91,7 @@ func (c statefulCmdable) Select(index int) *StatusCmd { //------------------------------------------------------------------------------ -func (c cmdable) Del(keys ...string) *IntCmd { +func (c *cmdable) Del(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) args[0] = "DEL" for i, key := range keys { @@ -102,37 +102,37 @@ func (c cmdable) Del(keys ...string) *IntCmd { return cmd } -func (c cmdable) Dump(key string) *StringCmd { +func (c *cmdable) Dump(key string) *StringCmd { cmd := NewStringCmd("dump", key) c.process(cmd) return cmd } -func (c cmdable) Exists(key string) *BoolCmd { +func (c *cmdable) Exists(key string) *BoolCmd { cmd := NewBoolCmd("exists", key) c.process(cmd) return cmd } -func (c cmdable) Expire(key string, expiration time.Duration) *BoolCmd { +func (c *cmdable) Expire(key string, expiration time.Duration) *BoolCmd { cmd := NewBoolCmd("expire", key, formatSec(expiration)) c.process(cmd) return cmd } -func (c cmdable) ExpireAt(key string, tm time.Time) *BoolCmd { +func (c *cmdable) ExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd("expireat", key, tm.Unix()) c.process(cmd) return cmd } -func (c cmdable) Keys(pattern string) *StringSliceCmd { +func (c *cmdable) Keys(pattern string) *StringSliceCmd { cmd := NewStringSliceCmd("keys", pattern) c.process(cmd) return cmd } -func (c cmdable) Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd { +func (c *cmdable) Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd { cmd := NewStatusCmd( "migrate", host, @@ -146,13 +146,13 @@ func (c cmdable) Migrate(host, port, key string, db int64, timeout time.Duration return cmd } -func (c cmdable) Move(key string, db int64) *BoolCmd { +func (c *cmdable) Move(key string, db int64) *BoolCmd { cmd := NewBoolCmd("move", key, db) c.process(cmd) return cmd } -func (c cmdable) ObjectRefCount(keys ...string) *IntCmd { +func (c *cmdable) ObjectRefCount(keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "object" args[1] = "refcount" @@ -164,7 +164,7 @@ func (c cmdable) ObjectRefCount(keys ...string) *IntCmd { return cmd } -func (c cmdable) ObjectEncoding(keys ...string) *StringCmd { +func (c *cmdable) ObjectEncoding(keys ...string) *StringCmd { args := make([]interface{}, 2+len(keys)) args[0] = "object" args[1] = "encoding" @@ -176,7 +176,7 @@ func (c cmdable) ObjectEncoding(keys ...string) *StringCmd { return cmd } -func (c cmdable) ObjectIdleTime(keys ...string) *DurationCmd { +func (c *cmdable) ObjectIdleTime(keys ...string) *DurationCmd { args := make([]interface{}, 2+len(keys)) args[0] = "object" args[1] = "idletime" @@ -188,19 +188,19 @@ func (c cmdable) ObjectIdleTime(keys ...string) *DurationCmd { return cmd } -func (c cmdable) Persist(key string) *BoolCmd { +func (c *cmdable) Persist(key string) *BoolCmd { cmd := NewBoolCmd("persist", key) c.process(cmd) return cmd } -func (c cmdable) PExpire(key string, expiration time.Duration) *BoolCmd { +func (c *cmdable) PExpire(key string, expiration time.Duration) *BoolCmd { cmd := NewBoolCmd("pexpire", key, formatMs(expiration)) c.process(cmd) return cmd } -func (c cmdable) PExpireAt(key string, tm time.Time) *BoolCmd { +func (c *cmdable) PExpireAt(key string, tm time.Time) *BoolCmd { cmd := NewBoolCmd( "pexpireat", key, @@ -210,31 +210,31 @@ func (c cmdable) PExpireAt(key string, tm time.Time) *BoolCmd { return cmd } -func (c cmdable) PTTL(key string) *DurationCmd { +func (c *cmdable) PTTL(key string) *DurationCmd { cmd := NewDurationCmd(time.Millisecond, "pttl", key) c.process(cmd) return cmd } -func (c cmdable) RandomKey() *StringCmd { +func (c *cmdable) RandomKey() *StringCmd { cmd := NewStringCmd("randomkey") c.process(cmd) return cmd } -func (c cmdable) Rename(key, newkey string) *StatusCmd { +func (c *cmdable) Rename(key, newkey string) *StatusCmd { cmd := NewStatusCmd("rename", key, newkey) c.process(cmd) return cmd } -func (c cmdable) RenameNX(key, newkey string) *BoolCmd { +func (c *cmdable) RenameNX(key, newkey string) *BoolCmd { cmd := NewBoolCmd("renamenx", key, newkey) c.process(cmd) return cmd } -func (c cmdable) Restore(key string, ttl time.Duration, value string) *StatusCmd { +func (c *cmdable) Restore(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( "restore", key, @@ -245,7 +245,7 @@ func (c cmdable) Restore(key string, ttl time.Duration, value string) *StatusCmd return cmd } -func (c cmdable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { +func (c *cmdable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { cmd := NewStatusCmd( "restore", key, @@ -289,31 +289,31 @@ func (sort *Sort) args(key string) []interface{} { return args } -func (c cmdable) Sort(key string, sort Sort) *StringSliceCmd { +func (c *cmdable) Sort(key string, sort Sort) *StringSliceCmd { cmd := NewStringSliceCmd(sort.args(key)...) c.process(cmd) return cmd } -func (c cmdable) SortInterfaces(key string, sort Sort) *SliceCmd { +func (c *cmdable) SortInterfaces(key string, sort Sort) *SliceCmd { cmd := NewSliceCmd(sort.args(key)...) c.process(cmd) return cmd } -func (c cmdable) TTL(key string) *DurationCmd { +func (c *cmdable) TTL(key string) *DurationCmd { cmd := NewDurationCmd(time.Second, "ttl", key) c.process(cmd) return cmd } -func (c cmdable) Type(key string) *StatusCmd { +func (c *cmdable) Type(key string) *StatusCmd { cmd := NewStatusCmd("type", key) c.process(cmd) return cmd } -func (c cmdable) Scan(cursor uint64, match string, count int64) Scanner { +func (c *cmdable) Scan(cursor uint64, match string, count int64) Scanner { args := []interface{}{"scan", cursor} if match != "" { args = append(args, "match", match) @@ -329,7 +329,7 @@ func (c cmdable) Scan(cursor uint64, match string, count int64) Scanner { } } -func (c cmdable) SScan(key string, cursor uint64, match string, count int64) Scanner { +func (c *cmdable) SScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"sscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -345,7 +345,7 @@ func (c cmdable) SScan(key string, cursor uint64, match string, count int64) Sca } } -func (c cmdable) HScan(key string, cursor uint64, match string, count int64) Scanner { +func (c *cmdable) HScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"hscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -361,7 +361,7 @@ func (c cmdable) HScan(key string, cursor uint64, match string, count int64) Sca } } -func (c cmdable) ZScan(key string, cursor uint64, match string, count int64) Scanner { +func (c *cmdable) ZScan(key string, cursor uint64, match string, count int64) Scanner { args := []interface{}{"zscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -379,7 +379,7 @@ func (c cmdable) ZScan(key string, cursor uint64, match string, count int64) Sca //------------------------------------------------------------------------------ -func (c cmdable) Append(key, value string) *IntCmd { +func (c *cmdable) Append(key, value string) *IntCmd { cmd := NewIntCmd("append", key, value) c.process(cmd) return cmd @@ -389,7 +389,7 @@ type BitCount struct { Start, End int64 } -func (c cmdable) BitCount(key string, bitCount *BitCount) *IntCmd { +func (c *cmdable) BitCount(key string, bitCount *BitCount) *IntCmd { args := []interface{}{"bitcount", key} if bitCount != nil { args = append( @@ -403,7 +403,7 @@ func (c cmdable) BitCount(key string, bitCount *BitCount) *IntCmd { return cmd } -func (c cmdable) bitOp(op, destKey string, keys ...string) *IntCmd { +func (c *cmdable) bitOp(op, destKey string, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "bitop" args[1] = op @@ -416,23 +416,23 @@ func (c cmdable) bitOp(op, destKey string, keys ...string) *IntCmd { return cmd } -func (c cmdable) BitOpAnd(destKey string, keys ...string) *IntCmd { +func (c *cmdable) BitOpAnd(destKey string, keys ...string) *IntCmd { return c.bitOp("and", destKey, keys...) } -func (c cmdable) BitOpOr(destKey string, keys ...string) *IntCmd { +func (c *cmdable) BitOpOr(destKey string, keys ...string) *IntCmd { return c.bitOp("or", destKey, keys...) } -func (c cmdable) BitOpXor(destKey string, keys ...string) *IntCmd { +func (c *cmdable) BitOpXor(destKey string, keys ...string) *IntCmd { return c.bitOp("xor", destKey, keys...) } -func (c cmdable) BitOpNot(destKey string, key string) *IntCmd { +func (c *cmdable) BitOpNot(destKey string, key string) *IntCmd { return c.bitOp("not", destKey, key) } -func (c cmdable) BitPos(key string, bit int64, pos ...int64) *IntCmd { +func (c *cmdable) BitPos(key string, bit int64, pos ...int64) *IntCmd { args := make([]interface{}, 3+len(pos)) args[0] = "bitpos" args[1] = key @@ -452,61 +452,61 @@ func (c cmdable) BitPos(key string, bit int64, pos ...int64) *IntCmd { return cmd } -func (c cmdable) Decr(key string) *IntCmd { +func (c *cmdable) Decr(key string) *IntCmd { cmd := NewIntCmd("decr", key) c.process(cmd) return cmd } -func (c cmdable) DecrBy(key string, decrement int64) *IntCmd { +func (c *cmdable) DecrBy(key string, decrement int64) *IntCmd { cmd := NewIntCmd("decrby", key, decrement) c.process(cmd) return cmd } -func (c cmdable) Get(key string) *StringCmd { +func (c *cmdable) Get(key string) *StringCmd { cmd := NewStringCmd("get", key) c.process(cmd) return cmd } -func (c cmdable) GetBit(key string, offset int64) *IntCmd { +func (c *cmdable) GetBit(key string, offset int64) *IntCmd { cmd := NewIntCmd("getbit", key, offset) c.process(cmd) return cmd } -func (c cmdable) GetRange(key string, start, end int64) *StringCmd { +func (c *cmdable) GetRange(key string, start, end int64) *StringCmd { cmd := NewStringCmd("getrange", key, start, end) c.process(cmd) return cmd } -func (c cmdable) GetSet(key string, value interface{}) *StringCmd { +func (c *cmdable) GetSet(key string, value interface{}) *StringCmd { cmd := NewStringCmd("getset", key, value) c.process(cmd) return cmd } -func (c cmdable) Incr(key string) *IntCmd { +func (c *cmdable) Incr(key string) *IntCmd { cmd := NewIntCmd("incr", key) c.process(cmd) return cmd } -func (c cmdable) IncrBy(key string, value int64) *IntCmd { +func (c *cmdable) IncrBy(key string, value int64) *IntCmd { cmd := NewIntCmd("incrby", key, value) c.process(cmd) return cmd } -func (c cmdable) IncrByFloat(key string, value float64) *FloatCmd { +func (c *cmdable) IncrByFloat(key string, value float64) *FloatCmd { cmd := NewFloatCmd("incrbyfloat", key, value) c.process(cmd) return cmd } -func (c cmdable) MGet(keys ...string) *SliceCmd { +func (c *cmdable) MGet(keys ...string) *SliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "mget" for i, key := range keys { @@ -517,7 +517,7 @@ func (c cmdable) MGet(keys ...string) *SliceCmd { return cmd } -func (c cmdable) MSet(pairs ...interface{}) *StatusCmd { +func (c *cmdable) MSet(pairs ...interface{}) *StatusCmd { args := make([]interface{}, 1+len(pairs)) args[0] = "mset" for i, pair := range pairs { @@ -528,7 +528,7 @@ func (c cmdable) MSet(pairs ...interface{}) *StatusCmd { return cmd } -func (c cmdable) MSetNX(pairs ...interface{}) *BoolCmd { +func (c *cmdable) MSetNX(pairs ...interface{}) *BoolCmd { args := make([]interface{}, 1+len(pairs)) args[0] = "msetnx" for i, pair := range pairs { @@ -542,7 +542,7 @@ func (c cmdable) MSetNX(pairs ...interface{}) *BoolCmd { // Redis `SET key value [expiration]` command. // // Zero expiration means the key has no expiration time. -func (c cmdable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { +func (c *cmdable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { args := make([]interface{}, 3, 4) args[0] = "set" args[1] = key @@ -559,7 +559,7 @@ func (c cmdable) Set(key string, value interface{}, expiration time.Duration) *S return cmd } -func (c cmdable) SetBit(key string, offset int64, value int) *IntCmd { +func (c *cmdable) SetBit(key string, offset int64, value int) *IntCmd { cmd := NewIntCmd( "SETBIT", key, @@ -573,7 +573,7 @@ func (c cmdable) SetBit(key string, offset int64, value int) *IntCmd { // Redis `SET key value [expiration] NX` command. // // Zero expiration means the key has no expiration time. -func (c cmdable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { +func (c *cmdable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if expiration == 0 { // Use old `SETNX` to support old Redis versions. @@ -592,7 +592,7 @@ func (c cmdable) SetNX(key string, value interface{}, expiration time.Duration) // Redis `SET key value [expiration] XX` command. // // Zero expiration means the key has no expiration time. -func (c cmdable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { +func (c *cmdable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd if usePrecise(expiration) { cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "xx") @@ -603,13 +603,13 @@ func (c cmdable) SetXX(key string, value interface{}, expiration time.Duration) return cmd } -func (c cmdable) SetRange(key string, offset int64, value string) *IntCmd { +func (c *cmdable) SetRange(key string, offset int64, value string) *IntCmd { cmd := NewIntCmd("setrange", key, offset, value) c.process(cmd) return cmd } -func (c cmdable) StrLen(key string) *IntCmd { +func (c *cmdable) StrLen(key string) *IntCmd { cmd := NewIntCmd("strlen", key) c.process(cmd) return cmd @@ -617,7 +617,7 @@ func (c cmdable) StrLen(key string) *IntCmd { //------------------------------------------------------------------------------ -func (c cmdable) HDel(key string, fields ...string) *IntCmd { +func (c *cmdable) HDel(key string, fields ...string) *IntCmd { args := make([]interface{}, 2+len(fields)) args[0] = "hdel" args[1] = key @@ -629,49 +629,49 @@ func (c cmdable) HDel(key string, fields ...string) *IntCmd { return cmd } -func (c cmdable) HExists(key, field string) *BoolCmd { +func (c *cmdable) HExists(key, field string) *BoolCmd { cmd := NewBoolCmd("hexists", key, field) c.process(cmd) return cmd } -func (c cmdable) HGet(key, field string) *StringCmd { +func (c *cmdable) HGet(key, field string) *StringCmd { cmd := NewStringCmd("hget", key, field) c.process(cmd) return cmd } -func (c cmdable) HGetAll(key string) *StringStringMapCmd { +func (c *cmdable) HGetAll(key string) *StringStringMapCmd { cmd := NewStringStringMapCmd("hgetall", key) c.process(cmd) return cmd } -func (c cmdable) HIncrBy(key, field string, incr int64) *IntCmd { +func (c *cmdable) HIncrBy(key, field string, incr int64) *IntCmd { cmd := NewIntCmd("hincrby", key, field, incr) c.process(cmd) return cmd } -func (c cmdable) HIncrByFloat(key, field string, incr float64) *FloatCmd { +func (c *cmdable) HIncrByFloat(key, field string, incr float64) *FloatCmd { cmd := NewFloatCmd("hincrbyfloat", key, field, incr) c.process(cmd) return cmd } -func (c cmdable) HKeys(key string) *StringSliceCmd { +func (c *cmdable) HKeys(key string) *StringSliceCmd { cmd := NewStringSliceCmd("hkeys", key) c.process(cmd) return cmd } -func (c cmdable) HLen(key string) *IntCmd { +func (c *cmdable) HLen(key string) *IntCmd { cmd := NewIntCmd("hlen", key) c.process(cmd) return cmd } -func (c cmdable) HMGet(key string, fields ...string) *SliceCmd { +func (c *cmdable) HMGet(key string, fields ...string) *SliceCmd { args := make([]interface{}, 2+len(fields)) args[0] = "hmget" args[1] = key @@ -683,7 +683,7 @@ func (c cmdable) HMGet(key string, fields ...string) *SliceCmd { return cmd } -func (c cmdable) HMSet(key string, fields map[string]string) *StatusCmd { +func (c *cmdable) HMSet(key string, fields map[string]string) *StatusCmd { args := make([]interface{}, 2+len(fields)*2) args[0] = "hmset" args[1] = key @@ -698,19 +698,19 @@ func (c cmdable) HMSet(key string, fields map[string]string) *StatusCmd { return cmd } -func (c cmdable) HSet(key, field, value string) *BoolCmd { +func (c *cmdable) HSet(key, field, value string) *BoolCmd { cmd := NewBoolCmd("hset", key, field, value) c.process(cmd) return cmd } -func (c cmdable) HSetNX(key, field, value string) *BoolCmd { +func (c *cmdable) HSetNX(key, field, value string) *BoolCmd { cmd := NewBoolCmd("hsetnx", key, field, value) c.process(cmd) return cmd } -func (c cmdable) HVals(key string) *StringSliceCmd { +func (c *cmdable) HVals(key string) *StringSliceCmd { cmd := NewStringSliceCmd("hvals", key) c.process(cmd) return cmd @@ -718,7 +718,7 @@ func (c cmdable) HVals(key string) *StringSliceCmd { //------------------------------------------------------------------------------ -func (c cmdable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { +func (c *cmdable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)+1) args[0] = "blpop" for i, key := range keys { @@ -731,7 +731,7 @@ func (c cmdable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { return cmd } -func (c cmdable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { +func (c *cmdable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)+1) args[0] = "brpop" for i, key := range keys { @@ -744,7 +744,7 @@ func (c cmdable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { return cmd } -func (c cmdable) BRPopLPush(source, destination string, timeout time.Duration) *StringCmd { +func (c *cmdable) BRPopLPush(source, destination string, timeout time.Duration) *StringCmd { cmd := NewStringCmd( "brpoplpush", source, @@ -756,43 +756,43 @@ func (c cmdable) BRPopLPush(source, destination string, timeout time.Duration) * return cmd } -func (c cmdable) LIndex(key string, index int64) *StringCmd { +func (c *cmdable) LIndex(key string, index int64) *StringCmd { cmd := NewStringCmd("lindex", key, index) c.process(cmd) return cmd } -func (c cmdable) LInsert(key, op string, pivot, value interface{}) *IntCmd { +func (c *cmdable) LInsert(key, op string, pivot, value interface{}) *IntCmd { cmd := NewIntCmd("linsert", key, op, pivot, value) c.process(cmd) return cmd } -func (c cmdable) LInsertBefore(key string, pivot, value interface{}) *IntCmd { +func (c *cmdable) LInsertBefore(key string, pivot, value interface{}) *IntCmd { cmd := NewIntCmd("linsert", key, "before", pivot, value) c.process(cmd) return cmd } -func (c cmdable) LInsertAfter(key string, pivot, value interface{}) *IntCmd { +func (c *cmdable) LInsertAfter(key string, pivot, value interface{}) *IntCmd { cmd := NewIntCmd("linsert", key, "after", pivot, value) c.process(cmd) return cmd } -func (c cmdable) LLen(key string) *IntCmd { +func (c *cmdable) LLen(key string) *IntCmd { cmd := NewIntCmd("llen", key) c.process(cmd) return cmd } -func (c cmdable) LPop(key string) *StringCmd { +func (c *cmdable) LPop(key string) *StringCmd { cmd := NewStringCmd("lpop", key) c.process(cmd) return cmd } -func (c cmdable) LPush(key string, values ...interface{}) *IntCmd { +func (c *cmdable) LPush(key string, values ...interface{}) *IntCmd { args := make([]interface{}, 2+len(values)) args[0] = "lpush" args[1] = key @@ -804,13 +804,13 @@ func (c cmdable) LPush(key string, values ...interface{}) *IntCmd { return cmd } -func (c cmdable) LPushX(key string, value interface{}) *IntCmd { +func (c *cmdable) LPushX(key string, value interface{}) *IntCmd { cmd := NewIntCmd("lpushx", key, value) c.process(cmd) return cmd } -func (c cmdable) LRange(key string, start, stop int64) *StringSliceCmd { +func (c *cmdable) LRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd( "lrange", key, @@ -821,19 +821,19 @@ func (c cmdable) LRange(key string, start, stop int64) *StringSliceCmd { return cmd } -func (c cmdable) LRem(key string, count int64, value interface{}) *IntCmd { +func (c *cmdable) LRem(key string, count int64, value interface{}) *IntCmd { cmd := NewIntCmd("lrem", key, count, value) c.process(cmd) return cmd } -func (c cmdable) LSet(key string, index int64, value interface{}) *StatusCmd { +func (c *cmdable) LSet(key string, index int64, value interface{}) *StatusCmd { cmd := NewStatusCmd("lset", key, index, value) c.process(cmd) return cmd } -func (c cmdable) LTrim(key string, start, stop int64) *StatusCmd { +func (c *cmdable) LTrim(key string, start, stop int64) *StatusCmd { cmd := NewStatusCmd( "ltrim", key, @@ -844,19 +844,19 @@ func (c cmdable) LTrim(key string, start, stop int64) *StatusCmd { return cmd } -func (c cmdable) RPop(key string) *StringCmd { +func (c *cmdable) RPop(key string) *StringCmd { cmd := NewStringCmd("rpop", key) c.process(cmd) return cmd } -func (c cmdable) RPopLPush(source, destination string) *StringCmd { +func (c *cmdable) RPopLPush(source, destination string) *StringCmd { cmd := NewStringCmd("rpoplpush", source, destination) c.process(cmd) return cmd } -func (c cmdable) RPush(key string, values ...interface{}) *IntCmd { +func (c *cmdable) RPush(key string, values ...interface{}) *IntCmd { args := make([]interface{}, 2+len(values)) args[0] = "rpush" args[1] = key @@ -868,7 +868,7 @@ func (c cmdable) RPush(key string, values ...interface{}) *IntCmd { return cmd } -func (c cmdable) RPushX(key string, value interface{}) *IntCmd { +func (c *cmdable) RPushX(key string, value interface{}) *IntCmd { cmd := NewIntCmd("rpushx", key, value) c.process(cmd) return cmd @@ -876,7 +876,7 @@ func (c cmdable) RPushX(key string, value interface{}) *IntCmd { //------------------------------------------------------------------------------ -func (c cmdable) SAdd(key string, members ...interface{}) *IntCmd { +func (c *cmdable) SAdd(key string, members ...interface{}) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "sadd" args[1] = key @@ -888,13 +888,13 @@ func (c cmdable) SAdd(key string, members ...interface{}) *IntCmd { return cmd } -func (c cmdable) SCard(key string) *IntCmd { +func (c *cmdable) SCard(key string) *IntCmd { cmd := NewIntCmd("scard", key) c.process(cmd) return cmd } -func (c cmdable) SDiff(keys ...string) *StringSliceCmd { +func (c *cmdable) SDiff(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "sdiff" for i, key := range keys { @@ -905,7 +905,7 @@ func (c cmdable) SDiff(keys ...string) *StringSliceCmd { return cmd } -func (c cmdable) SDiffStore(destination string, keys ...string) *IntCmd { +func (c *cmdable) SDiffStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "sdiffstore" args[1] = destination @@ -917,7 +917,7 @@ func (c cmdable) SDiffStore(destination string, keys ...string) *IntCmd { return cmd } -func (c cmdable) SInter(keys ...string) *StringSliceCmd { +func (c *cmdable) SInter(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "sinter" for i, key := range keys { @@ -928,7 +928,7 @@ func (c cmdable) SInter(keys ...string) *StringSliceCmd { return cmd } -func (c cmdable) SInterStore(destination string, keys ...string) *IntCmd { +func (c *cmdable) SInterStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "sinterstore" args[1] = destination @@ -940,53 +940,53 @@ func (c cmdable) SInterStore(destination string, keys ...string) *IntCmd { return cmd } -func (c cmdable) SIsMember(key string, member interface{}) *BoolCmd { +func (c *cmdable) SIsMember(key string, member interface{}) *BoolCmd { cmd := NewBoolCmd("sismember", key, member) c.process(cmd) return cmd } -func (c cmdable) SMembers(key string) *StringSliceCmd { +func (c *cmdable) SMembers(key string) *StringSliceCmd { cmd := NewStringSliceCmd("smembers", key) c.process(cmd) return cmd } -func (c cmdable) SMove(source, destination string, member interface{}) *BoolCmd { +func (c *cmdable) SMove(source, destination string, member interface{}) *BoolCmd { cmd := NewBoolCmd("smove", source, destination, member) c.process(cmd) return cmd } // Redis `SPOP key` command. -func (c cmdable) SPop(key string) *StringCmd { +func (c *cmdable) SPop(key string) *StringCmd { cmd := NewStringCmd("spop", key) c.process(cmd) return cmd } // Redis `SPOP key count` command. -func (c cmdable) SPopN(key string, count int64) *StringSliceCmd { +func (c *cmdable) SPopN(key string, count int64) *StringSliceCmd { cmd := NewStringSliceCmd("spop", key, count) c.process(cmd) return cmd } // Redis `SRANDMEMBER key` command. -func (c cmdable) SRandMember(key string) *StringCmd { +func (c *cmdable) SRandMember(key string) *StringCmd { cmd := NewStringCmd("srandmember", key) c.process(cmd) return cmd } // Redis `SRANDMEMBER key count` command. -func (c cmdable) SRandMemberN(key string, count int64) *StringSliceCmd { +func (c *cmdable) SRandMemberN(key string, count int64) *StringSliceCmd { cmd := NewStringSliceCmd("srandmember", key, count) c.process(cmd) return cmd } -func (c cmdable) SRem(key string, members ...interface{}) *IntCmd { +func (c *cmdable) SRem(key string, members ...interface{}) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "srem" args[1] = key @@ -998,7 +998,7 @@ func (c cmdable) SRem(key string, members ...interface{}) *IntCmd { return cmd } -func (c cmdable) SUnion(keys ...string) *StringSliceCmd { +func (c *cmdable) SUnion(keys ...string) *StringSliceCmd { args := make([]interface{}, 1+len(keys)) args[0] = "sunion" for i, key := range keys { @@ -1009,7 +1009,7 @@ func (c cmdable) SUnion(keys ...string) *StringSliceCmd { return cmd } -func (c cmdable) SUnionStore(destination string, keys ...string) *IntCmd { +func (c *cmdable) SUnionStore(destination string, keys ...string) *IntCmd { args := make([]interface{}, 2+len(keys)) args[0] = "sunionstore" args[1] = destination @@ -1036,7 +1036,7 @@ type ZStore struct { Aggregate string } -func (c cmdable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { +func (c *cmdable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { for i, m := range members { a[n+2*i] = m.Score a[n+2*i+1] = m.Member @@ -1047,7 +1047,7 @@ func (c cmdable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { } // Redis `ZADD key score member [score member ...]` command. -func (c cmdable) ZAdd(key string, members ...Z) *IntCmd { +func (c *cmdable) ZAdd(key string, members ...Z) *IntCmd { const n = 2 a := make([]interface{}, n+2*len(members)) a[0], a[1] = "zadd", key @@ -1055,7 +1055,7 @@ func (c cmdable) ZAdd(key string, members ...Z) *IntCmd { } // Redis `ZADD key NX score member [score member ...]` command. -func (c cmdable) ZAddNX(key string, members ...Z) *IntCmd { +func (c *cmdable) ZAddNX(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2] = "zadd", key, "nx" @@ -1063,7 +1063,7 @@ func (c cmdable) ZAddNX(key string, members ...Z) *IntCmd { } // Redis `ZADD key XX score member [score member ...]` command. -func (c cmdable) ZAddXX(key string, members ...Z) *IntCmd { +func (c *cmdable) ZAddXX(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2] = "zadd", key, "xx" @@ -1071,7 +1071,7 @@ func (c cmdable) ZAddXX(key string, members ...Z) *IntCmd { } // Redis `ZADD key CH score member [score member ...]` command. -func (c cmdable) ZAddCh(key string, members ...Z) *IntCmd { +func (c *cmdable) ZAddCh(key string, members ...Z) *IntCmd { const n = 3 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2] = "zadd", key, "ch" @@ -1079,7 +1079,7 @@ func (c cmdable) ZAddCh(key string, members ...Z) *IntCmd { } // Redis `ZADD key NX CH score member [score member ...]` command. -func (c cmdable) ZAddNXCh(key string, members ...Z) *IntCmd { +func (c *cmdable) ZAddNXCh(key string, members ...Z) *IntCmd { const n = 4 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2], a[3] = "zadd", key, "nx", "ch" @@ -1087,14 +1087,14 @@ func (c cmdable) ZAddNXCh(key string, members ...Z) *IntCmd { } // Redis `ZADD key XX CH score member [score member ...]` command. -func (c cmdable) ZAddXXCh(key string, members ...Z) *IntCmd { +func (c *cmdable) ZAddXXCh(key string, members ...Z) *IntCmd { const n = 4 a := make([]interface{}, n+2*len(members)) a[0], a[1], a[2], a[3] = "zadd", key, "xx", "ch" return c.zAdd(a, n, members...) } -func (c cmdable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { +func (c *cmdable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { for i, m := range members { a[n+2*i] = m.Score a[n+2*i+1] = m.Member @@ -1105,7 +1105,7 @@ func (c cmdable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { } // Redis `ZADD key INCR score member` command. -func (c cmdable) ZIncr(key string, member Z) *FloatCmd { +func (c *cmdable) ZIncr(key string, member Z) *FloatCmd { const n = 3 a := make([]interface{}, n+2) a[0], a[1], a[2] = "zadd", key, "incr" @@ -1113,7 +1113,7 @@ func (c cmdable) ZIncr(key string, member Z) *FloatCmd { } // Redis `ZADD key NX INCR score member` command. -func (c cmdable) ZIncrNX(key string, member Z) *FloatCmd { +func (c *cmdable) ZIncrNX(key string, member Z) *FloatCmd { const n = 4 a := make([]interface{}, n+2) a[0], a[1], a[2], a[3] = "zadd", key, "incr", "nx" @@ -1121,32 +1121,32 @@ func (c cmdable) ZIncrNX(key string, member Z) *FloatCmd { } // Redis `ZADD key XX INCR score member` command. -func (c cmdable) ZIncrXX(key string, member Z) *FloatCmd { +func (c *cmdable) ZIncrXX(key string, member Z) *FloatCmd { const n = 4 a := make([]interface{}, n+2) a[0], a[1], a[2], a[3] = "zadd", key, "incr", "xx" return c.zIncr(a, n, member) } -func (c cmdable) ZCard(key string) *IntCmd { +func (c *cmdable) ZCard(key string) *IntCmd { cmd := NewIntCmd("zcard", key) c.process(cmd) return cmd } -func (c cmdable) ZCount(key, min, max string) *IntCmd { +func (c *cmdable) ZCount(key, min, max string) *IntCmd { cmd := NewIntCmd("zcount", key, min, max) c.process(cmd) return cmd } -func (c cmdable) ZIncrBy(key string, increment float64, member string) *FloatCmd { +func (c *cmdable) ZIncrBy(key string, increment float64, member string) *FloatCmd { cmd := NewFloatCmd("zincrby", key, increment, member) c.process(cmd) return cmd } -func (c cmdable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { +func (c *cmdable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "zinterstore" args[1] = destination @@ -1168,7 +1168,7 @@ func (c cmdable) ZInterStore(destination string, store ZStore, keys ...string) * return cmd } -func (c cmdable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { +func (c *cmdable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { args := []interface{}{ "zrange", key, @@ -1183,11 +1183,11 @@ func (c cmdable) zRange(key string, start, stop int64, withScores bool) *StringS return cmd } -func (c cmdable) ZRange(key string, start, stop int64) *StringSliceCmd { +func (c *cmdable) ZRange(key string, start, stop int64) *StringSliceCmd { return c.zRange(key, start, stop, false) } -func (c cmdable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { +func (c *cmdable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { cmd := NewZSliceCmd("zrange", key, start, stop, "withscores") c.process(cmd) return cmd @@ -1198,7 +1198,7 @@ type ZRangeBy struct { Offset, Count int64 } -func (c cmdable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { +func (c *cmdable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Min, opt.Max} if withScores { args = append(args, "withscores") @@ -1216,15 +1216,15 @@ func (c cmdable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *Stri return cmd } -func (c cmdable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { +func (c *cmdable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { return c.zRangeBy("zrangebyscore", key, opt, false) } -func (c cmdable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { +func (c *cmdable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { return c.zRangeBy("zrangebylex", key, opt, false) } -func (c cmdable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { +func (c *cmdable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"zrangebyscore", key, opt.Min, opt.Max, "withscores"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1239,13 +1239,13 @@ func (c cmdable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { return cmd } -func (c cmdable) ZRank(key, member string) *IntCmd { +func (c *cmdable) ZRank(key, member string) *IntCmd { cmd := NewIntCmd("zrank", key, member) c.process(cmd) return cmd } -func (c cmdable) ZRem(key string, members ...interface{}) *IntCmd { +func (c *cmdable) ZRem(key string, members ...interface{}) *IntCmd { args := make([]interface{}, 2+len(members)) args[0] = "zrem" args[1] = key @@ -1257,7 +1257,7 @@ func (c cmdable) ZRem(key string, members ...interface{}) *IntCmd { return cmd } -func (c cmdable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { +func (c *cmdable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { cmd := NewIntCmd( "zremrangebyrank", key, @@ -1268,25 +1268,25 @@ func (c cmdable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { return cmd } -func (c cmdable) ZRemRangeByScore(key, min, max string) *IntCmd { +func (c *cmdable) ZRemRangeByScore(key, min, max string) *IntCmd { cmd := NewIntCmd("zremrangebyscore", key, min, max) c.process(cmd) return cmd } -func (c cmdable) ZRevRange(key string, start, stop int64) *StringSliceCmd { +func (c *cmdable) ZRevRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd("zrevrange", key, start, stop) c.process(cmd) return cmd } -func (c cmdable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { +func (c *cmdable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { cmd := NewZSliceCmd("zrevrange", key, start, stop, "withscores") c.process(cmd) return cmd } -func (c cmdable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { +func (c *cmdable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { args := []interface{}{zcmd, key, opt.Max, opt.Min} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1301,15 +1301,15 @@ func (c cmdable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { return cmd } -func (c cmdable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { +func (c *cmdable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { return c.zRevRangeBy("zrevrangebyscore", key, opt) } -func (c cmdable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { +func (c *cmdable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { return c.zRevRangeBy("zrevrangebylex", key, opt) } -func (c cmdable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { +func (c *cmdable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { args := []interface{}{"zrevrangebyscore", key, opt.Max, opt.Min, "withscores"} if opt.Offset != 0 || opt.Count != 0 { args = append( @@ -1324,19 +1324,19 @@ func (c cmdable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd return cmd } -func (c cmdable) ZRevRank(key, member string) *IntCmd { +func (c *cmdable) ZRevRank(key, member string) *IntCmd { cmd := NewIntCmd("zrevrank", key, member) c.process(cmd) return cmd } -func (c cmdable) ZScore(key, member string) *FloatCmd { +func (c *cmdable) ZScore(key, member string) *FloatCmd { cmd := NewFloatCmd("zscore", key, member) c.process(cmd) return cmd } -func (c cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { +func (c *cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { args := make([]interface{}, 3+len(keys)) args[0] = "zunionstore" args[1] = dest @@ -1360,7 +1360,7 @@ func (c cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd //------------------------------------------------------------------------------ -func (c cmdable) PFAdd(key string, els ...interface{}) *IntCmd { +func (c *cmdable) PFAdd(key string, els ...interface{}) *IntCmd { args := make([]interface{}, 2+len(els)) args[0] = "pfadd" args[1] = key @@ -1372,7 +1372,7 @@ func (c cmdable) PFAdd(key string, els ...interface{}) *IntCmd { return cmd } -func (c cmdable) PFCount(keys ...string) *IntCmd { +func (c *cmdable) PFCount(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) args[0] = "pfcount" for i, key := range keys { @@ -1383,7 +1383,7 @@ func (c cmdable) PFCount(keys ...string) *IntCmd { return cmd } -func (c cmdable) PFMerge(dest string, keys ...string) *StatusCmd { +func (c *cmdable) PFMerge(dest string, keys ...string) *StatusCmd { args := make([]interface{}, 2+len(keys)) args[0] = "pfmerge" args[1] = dest @@ -1397,38 +1397,38 @@ func (c cmdable) PFMerge(dest string, keys ...string) *StatusCmd { //------------------------------------------------------------------------------ -func (c cmdable) BgRewriteAOF() *StatusCmd { +func (c *cmdable) BgRewriteAOF() *StatusCmd { cmd := NewStatusCmd("bgrewriteaof") c.process(cmd) return cmd } -func (c cmdable) BgSave() *StatusCmd { +func (c *cmdable) BgSave() *StatusCmd { cmd := NewStatusCmd("bgsave") c.process(cmd) return cmd } -func (c cmdable) ClientKill(ipPort string) *StatusCmd { +func (c *cmdable) ClientKill(ipPort string) *StatusCmd { cmd := NewStatusCmd("client", "kill", ipPort) c.process(cmd) return cmd } -func (c cmdable) ClientList() *StringCmd { +func (c *cmdable) ClientList() *StringCmd { cmd := NewStringCmd("client", "list") c.process(cmd) return cmd } -func (c cmdable) ClientPause(dur time.Duration) *BoolCmd { +func (c *cmdable) ClientPause(dur time.Duration) *BoolCmd { cmd := NewBoolCmd("client", "pause", formatMs(dur)) c.process(cmd) return cmd } // ClientSetName assigns a name to the one of many connections in the pool. -func (c cmdable) ClientSetName(name string) *BoolCmd { +func (c *cmdable) ClientSetName(name string) *BoolCmd { cmd := NewBoolCmd("client", "setname", name) c.process(cmd) return cmd @@ -1441,43 +1441,43 @@ func (c *Client) ClientGetName() *StringCmd { return cmd } -func (c cmdable) ConfigGet(parameter string) *SliceCmd { +func (c *cmdable) ConfigGet(parameter string) *SliceCmd { cmd := NewSliceCmd("config", "get", parameter) c.process(cmd) return cmd } -func (c cmdable) ConfigResetStat() *StatusCmd { +func (c *cmdable) ConfigResetStat() *StatusCmd { cmd := NewStatusCmd("config", "resetstat") c.process(cmd) return cmd } -func (c cmdable) ConfigSet(parameter, value string) *StatusCmd { +func (c *cmdable) ConfigSet(parameter, value string) *StatusCmd { cmd := NewStatusCmd("config", "set", parameter, value) c.process(cmd) return cmd } -func (c cmdable) DbSize() *IntCmd { +func (c *cmdable) DbSize() *IntCmd { cmd := NewIntCmd("dbsize") c.process(cmd) return cmd } -func (c cmdable) FlushAll() *StatusCmd { +func (c *cmdable) FlushAll() *StatusCmd { cmd := NewStatusCmd("flushall") c.process(cmd) return cmd } -func (c cmdable) FlushDb() *StatusCmd { +func (c *cmdable) FlushDb() *StatusCmd { cmd := NewStatusCmd("flushdb") c.process(cmd) return cmd } -func (c cmdable) Info(section ...string) *StringCmd { +func (c *cmdable) Info(section ...string) *StringCmd { args := []interface{}{"info"} if len(section) > 0 { args = append(args, section[0]) @@ -1487,19 +1487,19 @@ func (c cmdable) Info(section ...string) *StringCmd { return cmd } -func (c cmdable) LastSave() *IntCmd { +func (c *cmdable) LastSave() *IntCmd { cmd := NewIntCmd("lastsave") c.process(cmd) return cmd } -func (c cmdable) Save() *StatusCmd { +func (c *cmdable) Save() *StatusCmd { cmd := NewStatusCmd("save") c.process(cmd) return cmd } -func (c cmdable) shutdown(modifier string) *StatusCmd { +func (c *cmdable) shutdown(modifier string) *StatusCmd { var args []interface{} if modifier == "" { args = []interface{}{"shutdown"} @@ -1521,33 +1521,33 @@ func (c cmdable) shutdown(modifier string) *StatusCmd { return cmd } -func (c cmdable) Shutdown() *StatusCmd { +func (c *cmdable) Shutdown() *StatusCmd { return c.shutdown("") } -func (c cmdable) ShutdownSave() *StatusCmd { +func (c *cmdable) ShutdownSave() *StatusCmd { return c.shutdown("save") } -func (c cmdable) ShutdownNoSave() *StatusCmd { +func (c *cmdable) ShutdownNoSave() *StatusCmd { return c.shutdown("nosave") } -func (c cmdable) SlaveOf(host, port string) *StatusCmd { +func (c *cmdable) SlaveOf(host, port string) *StatusCmd { cmd := NewStatusCmd("slaveof", host, port) c.process(cmd) return cmd } -func (c cmdable) SlowLog() { +func (c *cmdable) SlowLog() { panic("not implemented") } -func (c cmdable) Sync() { +func (c *cmdable) Sync() { panic("not implemented") } -func (c cmdable) Time() *StringSliceCmd { +func (c *cmdable) Time() *StringSliceCmd { cmd := NewStringSliceCmd("time") c.process(cmd) return cmd @@ -1555,7 +1555,7 @@ func (c cmdable) Time() *StringSliceCmd { //------------------------------------------------------------------------------ -func (c cmdable) Eval(script string, keys []string, args ...interface{}) *Cmd { +func (c *cmdable) Eval(script string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "eval" cmdArgs[1] = script @@ -1572,7 +1572,7 @@ func (c cmdable) Eval(script string, keys []string, args ...interface{}) *Cmd { return cmd } -func (c cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { +func (c *cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "evalsha" cmdArgs[1] = sha1 @@ -1589,7 +1589,7 @@ func (c cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { return cmd } -func (c cmdable) ScriptExists(scripts ...string) *BoolSliceCmd { +func (c *cmdable) ScriptExists(scripts ...string) *BoolSliceCmd { args := make([]interface{}, 2+len(scripts)) args[0] = "script" args[1] = "exists" @@ -1601,19 +1601,19 @@ func (c cmdable) ScriptExists(scripts ...string) *BoolSliceCmd { return cmd } -func (c cmdable) ScriptFlush() *StatusCmd { +func (c *cmdable) ScriptFlush() *StatusCmd { cmd := NewStatusCmd("script", "flush") c.process(cmd) return cmd } -func (c cmdable) ScriptKill() *StatusCmd { +func (c *cmdable) ScriptKill() *StatusCmd { cmd := NewStatusCmd("script", "kill") c.process(cmd) return cmd } -func (c cmdable) ScriptLoad(script string) *StringCmd { +func (c *cmdable) ScriptLoad(script string) *StringCmd { cmd := NewStringCmd("script", "load", script) c.process(cmd) return cmd @@ -1621,7 +1621,7 @@ func (c cmdable) ScriptLoad(script string) *StringCmd { //------------------------------------------------------------------------------ -func (c cmdable) DebugObject(key string) *StringCmd { +func (c *cmdable) DebugObject(key string) *StringCmd { cmd := NewStringCmd("debug", "object", key) c.process(cmd) return cmd @@ -1629,7 +1629,7 @@ func (c cmdable) DebugObject(key string) *StringCmd { //------------------------------------------------------------------------------ -func (c cmdable) PubSubChannels(pattern string) *StringSliceCmd { +func (c *cmdable) PubSubChannels(pattern string) *StringSliceCmd { args := []interface{}{"pubsub", "channels"} if pattern != "*" { args = append(args, pattern) @@ -1639,7 +1639,7 @@ func (c cmdable) PubSubChannels(pattern string) *StringSliceCmd { return cmd } -func (c cmdable) PubSubNumSub(channels ...string) *StringIntMapCmd { +func (c *cmdable) PubSubNumSub(channels ...string) *StringIntMapCmd { args := make([]interface{}, 2+len(channels)) args[0] = "pubsub" args[1] = "numsub" @@ -1651,7 +1651,7 @@ func (c cmdable) PubSubNumSub(channels ...string) *StringIntMapCmd { return cmd } -func (c cmdable) PubSubNumPat() *IntCmd { +func (c *cmdable) PubSubNumPat() *IntCmd { cmd := NewIntCmd("pubsub", "numpat") c.process(cmd) return cmd @@ -1659,73 +1659,73 @@ func (c cmdable) PubSubNumPat() *IntCmd { //------------------------------------------------------------------------------ -func (c cmdable) ClusterSlots() *ClusterSlotsCmd { +func (c *cmdable) ClusterSlots() *ClusterSlotsCmd { cmd := NewClusterSlotsCmd("cluster", "slots") c.process(cmd) return cmd } -func (c cmdable) ClusterNodes() *StringCmd { +func (c *cmdable) ClusterNodes() *StringCmd { cmd := NewStringCmd("cluster", "nodes") c.process(cmd) return cmd } -func (c cmdable) ClusterMeet(host, port string) *StatusCmd { +func (c *cmdable) ClusterMeet(host, port string) *StatusCmd { cmd := NewStatusCmd("cluster", "meet", host, port) c.process(cmd) return cmd } -func (c cmdable) ClusterForget(nodeID string) *StatusCmd { +func (c *cmdable) ClusterForget(nodeID string) *StatusCmd { cmd := NewStatusCmd("cluster", "forget", nodeID) c.process(cmd) return cmd } -func (c cmdable) ClusterReplicate(nodeID string) *StatusCmd { +func (c *cmdable) ClusterReplicate(nodeID string) *StatusCmd { cmd := NewStatusCmd("cluster", "replicate", nodeID) c.process(cmd) return cmd } -func (c cmdable) ClusterResetSoft() *StatusCmd { +func (c *cmdable) ClusterResetSoft() *StatusCmd { cmd := NewStatusCmd("cluster", "reset", "soft") c.process(cmd) return cmd } -func (c cmdable) ClusterResetHard() *StatusCmd { +func (c *cmdable) ClusterResetHard() *StatusCmd { cmd := NewStatusCmd("cluster", "reset", "hard") c.process(cmd) return cmd } -func (c cmdable) ClusterInfo() *StringCmd { +func (c *cmdable) ClusterInfo() *StringCmd { cmd := NewStringCmd("cluster", "info") c.process(cmd) return cmd } -func (c cmdable) ClusterKeySlot(key string) *IntCmd { +func (c *cmdable) ClusterKeySlot(key string) *IntCmd { cmd := NewIntCmd("cluster", "keyslot", key) c.process(cmd) return cmd } -func (c cmdable) ClusterCountFailureReports(nodeID string) *IntCmd { +func (c *cmdable) ClusterCountFailureReports(nodeID string) *IntCmd { cmd := NewIntCmd("cluster", "count-failure-reports", nodeID) c.process(cmd) return cmd } -func (c cmdable) ClusterCountKeysInSlot(slot int) *IntCmd { +func (c *cmdable) ClusterCountKeysInSlot(slot int) *IntCmd { cmd := NewIntCmd("cluster", "countkeysinslot", slot) c.process(cmd) return cmd } -func (c cmdable) ClusterDelSlots(slots ...int) *StatusCmd { +func (c *cmdable) ClusterDelSlots(slots ...int) *StatusCmd { args := make([]interface{}, 2+len(slots)) args[0] = "cluster" args[1] = "delslots" @@ -1737,7 +1737,7 @@ func (c cmdable) ClusterDelSlots(slots ...int) *StatusCmd { return cmd } -func (c cmdable) ClusterDelSlotsRange(min, max int) *StatusCmd { +func (c *cmdable) ClusterDelSlotsRange(min, max int) *StatusCmd { size := max - min + 1 slots := make([]int, size) for i := 0; i < size; i++ { @@ -1746,37 +1746,37 @@ func (c cmdable) ClusterDelSlotsRange(min, max int) *StatusCmd { return c.ClusterDelSlots(slots...) } -func (c cmdable) ClusterSaveConfig() *StatusCmd { +func (c *cmdable) ClusterSaveConfig() *StatusCmd { cmd := NewStatusCmd("cluster", "saveconfig") c.process(cmd) return cmd } -func (c cmdable) ClusterSlaves(nodeID string) *StringSliceCmd { +func (c *cmdable) ClusterSlaves(nodeID string) *StringSliceCmd { cmd := NewStringSliceCmd("cluster", "slaves", nodeID) c.process(cmd) return cmd } -func (c statefulCmdable) ReadOnly() *StatusCmd { +func (c *statefulCmdable) ReadOnly() *StatusCmd { cmd := NewStatusCmd("readonly") c.process(cmd) return cmd } -func (c statefulCmdable) ReadWrite() *StatusCmd { +func (c *statefulCmdable) ReadWrite() *StatusCmd { cmd := NewStatusCmd("readwrite") c.process(cmd) return cmd } -func (c cmdable) ClusterFailover() *StatusCmd { +func (c *cmdable) ClusterFailover() *StatusCmd { cmd := NewStatusCmd("cluster", "failover") c.process(cmd) return cmd } -func (c cmdable) ClusterAddSlots(slots ...int) *StatusCmd { +func (c *cmdable) ClusterAddSlots(slots ...int) *StatusCmd { args := make([]interface{}, 2+len(slots)) args[0] = "cluster" args[1] = "addslots" @@ -1788,7 +1788,7 @@ func (c cmdable) ClusterAddSlots(slots ...int) *StatusCmd { return cmd } -func (c cmdable) ClusterAddSlotsRange(min, max int) *StatusCmd { +func (c *cmdable) ClusterAddSlotsRange(min, max int) *StatusCmd { size := max - min + 1 slots := make([]int, size) for i := 0; i < size; i++ { @@ -1799,7 +1799,7 @@ func (c cmdable) ClusterAddSlotsRange(min, max int) *StatusCmd { //------------------------------------------------------------------------------ -func (c cmdable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { +func (c *cmdable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { args := make([]interface{}, 2+3*len(geoLocation)) args[0] = "geoadd" args[1] = key @@ -1813,19 +1813,19 @@ func (c cmdable) GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd { return cmd } -func (c cmdable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { +func (c *cmdable) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { cmd := NewGeoLocationCmd(query, "georadius", key, longitude, latitude) c.process(cmd) return cmd } -func (c cmdable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { +func (c *cmdable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { cmd := NewGeoLocationCmd(query, "georadiusbymember", key, member) c.process(cmd) return cmd } -func (c cmdable) GeoDist(key string, member1, member2, unit string) *FloatCmd { +func (c *cmdable) GeoDist(key string, member1, member2, unit string) *FloatCmd { if unit == "" { unit = "km" } @@ -1834,7 +1834,7 @@ func (c cmdable) GeoDist(key string, member1, member2, unit string) *FloatCmd { return cmd } -func (c cmdable) GeoHash(key string, members ...string) *StringSliceCmd { +func (c *cmdable) GeoHash(key string, members ...string) *StringSliceCmd { args := make([]interface{}, 2+len(members)) args[0] = "geohash" args[1] = key @@ -1848,7 +1848,7 @@ func (c cmdable) GeoHash(key string, members ...string) *StringSliceCmd { //------------------------------------------------------------------------------ -func (c cmdable) Command() *CommandsInfoCmd { +func (c *cmdable) Command() *CommandsInfoCmd { cmd := NewCommandsInfoCmd("command") c.process(cmd) return cmd diff --git a/iterator.go b/iterator.go index 0219ab4555..e9dfbad938 100644 --- a/iterator.go +++ b/iterator.go @@ -3,7 +3,7 @@ package redis import "sync" type Scanner struct { - client cmdable + client *cmdable *ScanCmd } diff --git a/pipeline.go b/pipeline.go index e946b9e9e0..fed6e0c1a0 100644 --- a/pipeline.go +++ b/pipeline.go @@ -22,32 +22,32 @@ type Pipeline struct { closed int32 } -func (pipe *Pipeline) Process(cmd Cmder) error { - pipe.mu.Lock() - pipe.cmds = append(pipe.cmds, cmd) - pipe.mu.Unlock() +func (c *Pipeline) Process(cmd Cmder) error { + c.mu.Lock() + c.cmds = append(c.cmds, cmd) + c.mu.Unlock() return nil } // Close closes the pipeline, releasing any open resources. -func (pipe *Pipeline) Close() error { - atomic.StoreInt32(&pipe.closed, 1) - pipe.Discard() +func (c *Pipeline) Close() error { + atomic.StoreInt32(&c.closed, 1) + c.Discard() return nil } -func (pipe *Pipeline) isClosed() bool { - return atomic.LoadInt32(&pipe.closed) == 1 +func (c *Pipeline) isClosed() bool { + return atomic.LoadInt32(&c.closed) == 1 } // Discard resets the pipeline and discards queued commands. -func (pipe *Pipeline) Discard() error { - defer pipe.mu.Unlock() - pipe.mu.Lock() - if pipe.isClosed() { +func (c *Pipeline) Discard() error { + defer c.mu.Unlock() + c.mu.Lock() + if c.isClosed() { return pool.ErrClosed } - pipe.cmds = pipe.cmds[:0] + c.cmds = c.cmds[:0] return nil } @@ -56,30 +56,30 @@ func (pipe *Pipeline) Discard() error { // // Exec always returns list of commands and error of the first failed // command if any. -func (pipe *Pipeline) Exec() ([]Cmder, error) { - if pipe.isClosed() { +func (c *Pipeline) Exec() ([]Cmder, error) { + if c.isClosed() { return nil, pool.ErrClosed } - defer pipe.mu.Unlock() - pipe.mu.Lock() + defer c.mu.Unlock() + c.mu.Lock() - if len(pipe.cmds) == 0 { - return pipe.cmds, nil + if len(c.cmds) == 0 { + return c.cmds, nil } - cmds := pipe.cmds - pipe.cmds = nil + cmds := c.cmds + c.cmds = nil - return cmds, pipe.exec(cmds) + return cmds, c.exec(cmds) } -func (pipe *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { - if err := fn(pipe); err != nil { +func (c *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + if err := fn(c); err != nil { return nil, err } - cmds, err := pipe.Exec() - _ = pipe.Close() + cmds, err := c.Exec() + _ = c.Close() return cmds, err } diff --git a/ring.go b/ring.go index 50476fa894..8aba07e29e 100644 --- a/ring.go +++ b/ring.go @@ -102,7 +102,7 @@ func (shard *ringShard) Vote(up bool) bool { // concurrent use by multiple goroutines. // // Ring monitors the state of each shard and removes dead shards from -// the ring. When shard comes online it is added back to the ring. This +// the c. When shard comes online it is added back to the c. This // gives you maximum availability and partition tolerance, but no // consistency between different shards or even clients. Each client // uses shards that are available to the client and does not do any @@ -149,58 +149,58 @@ func NewRing(opt *RingOptions) *Ring { return ring } -func (ring *Ring) cmdInfo(name string) *CommandInfo { - ring.cmdsInfoOnce.Do(func() { - for _, shard := range ring.shards { +func (c *Ring) cmdInfo(name string) *CommandInfo { + c.cmdsInfoOnce.Do(func() { + for _, shard := range c.shards { cmdsInfo, err := shard.Client.Command().Result() if err == nil { - ring.cmdsInfo = cmdsInfo + c.cmdsInfo = cmdsInfo return } } - ring.cmdsInfoOnce = &sync.Once{} + c.cmdsInfoOnce = &sync.Once{} }) - if ring.cmdsInfo == nil { + if c.cmdsInfo == nil { return nil } - return ring.cmdsInfo[name] + return c.cmdsInfo[name] } -func (ring *Ring) cmdFirstKey(cmd Cmder) string { - cmdInfo := ring.cmdInfo(cmd.arg(0)) +func (c *Ring) cmdFirstKey(cmd Cmder) string { + cmdInfo := c.cmdInfo(cmd.arg(0)) if cmdInfo == nil { return "" } return cmd.arg(int(cmdInfo.FirstKeyPos)) } -func (ring *Ring) addClient(name string, cl *Client) { - ring.mu.Lock() - ring.hash.Add(name) - ring.shards[name] = &ringShard{Client: cl} - ring.mu.Unlock() +func (c *Ring) addClient(name string, cl *Client) { + c.mu.Lock() + c.hash.Add(name) + c.shards[name] = &ringShard{Client: cl} + c.mu.Unlock() } -func (ring *Ring) getClient(key string) (*Client, error) { - ring.mu.RLock() +func (c *Ring) getClient(key string) (*Client, error) { + c.mu.RLock() - if ring.closed { + if c.closed { return nil, pool.ErrClosed } - name := ring.hash.Get(hashtag.Key(key)) + name := c.hash.Get(hashtag.Key(key)) if name == "" { - ring.mu.RUnlock() + c.mu.RUnlock() return nil, errRingShardsDown } - cl := ring.shards[name].Client - ring.mu.RUnlock() + cl := c.shards[name].Client + c.mu.RUnlock() return cl, nil } -func (ring *Ring) Process(cmd Cmder) error { - cl, err := ring.getClient(ring.cmdFirstKey(cmd)) +func (c *Ring) Process(cmd Cmder) error { + cl, err := c.getClient(c.cmdFirstKey(cmd)) if err != nil { cmd.setErr(err) return err @@ -208,34 +208,34 @@ func (ring *Ring) Process(cmd Cmder) error { return cl.baseClient.Process(cmd) } -// rebalance removes dead shards from the ring. -func (ring *Ring) rebalance() { - defer ring.mu.Unlock() - ring.mu.Lock() +// rebalance removes dead shards from the c. +func (c *Ring) rebalance() { + defer c.mu.Unlock() + c.mu.Lock() - ring.hash = consistenthash.New(ring.nreplicas, nil) - for name, shard := range ring.shards { + c.hash = consistenthash.New(c.nreplicas, nil) + for name, shard := range c.shards { if shard.IsUp() { - ring.hash.Add(name) + c.hash.Add(name) } } } -// heartbeat monitors state of each shard in the ring. -func (ring *Ring) heartbeat() { +// heartbeat monitors state of each shard in the c. +func (c *Ring) heartbeat() { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for _ = range ticker.C { var rebalance bool - ring.mu.RLock() + c.mu.RLock() - if ring.closed { - ring.mu.RUnlock() + if c.closed { + c.mu.RUnlock() break } - for _, shard := range ring.shards { + for _, shard := range c.shards { err := shard.Client.Ping().Err() if shard.Vote(err == nil || err == pool.ErrPoolTimeout) { internal.Logf("ring shard state changed: %s", shard) @@ -243,10 +243,10 @@ func (ring *Ring) heartbeat() { } } - ring.mu.RUnlock() + c.mu.RUnlock() if rebalance { - ring.rebalance() + c.rebalance() } } } @@ -255,45 +255,45 @@ func (ring *Ring) heartbeat() { // // It is rare to Close a Ring, as the Ring is meant to be long-lived // and shared between many goroutines. -func (ring *Ring) Close() (retErr error) { - defer ring.mu.Unlock() - ring.mu.Lock() +func (c *Ring) Close() (retErr error) { + defer c.mu.Unlock() + c.mu.Lock() - if ring.closed { + if c.closed { return nil } - ring.closed = true + c.closed = true - for _, shard := range ring.shards { + for _, shard := range c.shards { if err := shard.Client.Close(); err != nil { retErr = err } } - ring.hash = nil - ring.shards = nil + c.hash = nil + c.shards = nil return retErr } -func (ring *Ring) Pipeline() *Pipeline { +func (c *Ring) Pipeline() *Pipeline { pipe := Pipeline{ - exec: ring.pipelineExec, + exec: c.pipelineExec, } pipe.cmdable.process = pipe.Process pipe.statefulCmdable.process = pipe.Process return &pipe } -func (ring *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { - return ring.Pipeline().pipelined(fn) +func (c *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) } -func (ring *Ring) pipelineExec(cmds []Cmder) error { +func (c *Ring) pipelineExec(cmds []Cmder) error { var retErr error cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - name := ring.hash.Get(hashtag.Key(ring.cmdFirstKey(cmd))) + name := c.hash.Get(hashtag.Key(c.cmdFirstKey(cmd))) if name == "" { cmd.setErr(errRingShardsDown) if retErr == nil { @@ -304,11 +304,11 @@ func (ring *Ring) pipelineExec(cmds []Cmder) error { cmdsMap[name] = append(cmdsMap[name], cmd) } - for i := 0; i <= ring.opt.MaxRetries; i++ { + for i := 0; i <= c.opt.MaxRetries; i++ { failedCmdsMap := make(map[string][]Cmder) for name, cmds := range cmdsMap { - client := ring.shards[name].Client + client := c.shards[name].Client cn, err := client.conn() if err != nil { setCmdsErr(cmds, err) diff --git a/tx.go b/tx.go index 4f85cc8d90..5ed223c043 100644 --- a/tx.go +++ b/tx.go @@ -39,7 +39,7 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { tx := c.newTx() if len(keys) > 0 { if err := tx.Watch(keys...).Err(); err != nil { - tx.close() + _ = tx.close() return err } } @@ -50,57 +50,57 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return retErr } -func (tx *Tx) Process(cmd Cmder) error { - if tx.cmds == nil { - return tx.baseClient.Process(cmd) +func (c *Tx) Process(cmd Cmder) error { + if c.cmds == nil { + return c.baseClient.Process(cmd) } - tx.cmds = append(tx.cmds, cmd) + c.cmds = append(c.cmds, cmd) return nil } // close closes the transaction, releasing any open resources. -func (tx *Tx) close() error { - if tx.closed { +func (c *Tx) close() error { + if c.closed { return nil } - tx.closed = true - if err := tx.Unwatch().Err(); err != nil { + c.closed = true + if err := c.Unwatch().Err(); err != nil { internal.Logf("Unwatch failed: %s", err) } - return tx.baseClient.Close() + return c.baseClient.Close() } // Watch marks the keys to be watched for conditional execution // of a transaction. -func (tx *Tx) Watch(keys ...string) *StatusCmd { +func (c *Tx) Watch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) args[0] = "WATCH" for i, key := range keys { args[1+i] = key } cmd := NewStatusCmd(args...) - tx.Process(cmd) + c.Process(cmd) return cmd } // Unwatch flushes all the previously watched keys for a transaction. -func (tx *Tx) Unwatch(keys ...string) *StatusCmd { +func (c *Tx) Unwatch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) args[0] = "UNWATCH" for i, key := range keys { args[1+i] = key } cmd := NewStatusCmd(args...) - tx.Process(cmd) + c.Process(cmd) return cmd } // Discard discards queued commands. -func (tx *Tx) Discard() error { - if tx.cmds == nil { +func (c *Tx) Discard() error { + if c.cmds == nil { return errDiscard } - tx.cmds = tx.cmds[:1] + c.cmds = c.cmds[:1] return nil } @@ -113,19 +113,19 @@ func (tx *Tx) Discard() error { // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. -func (tx *Tx) MultiExec(fn func() error) ([]Cmder, error) { - if tx.closed { +func (c *Tx) MultiExec(fn func() error) ([]Cmder, error) { + if c.closed { return nil, pool.ErrClosed } - tx.cmds = []Cmder{NewStatusCmd("MULTI")} + c.cmds = []Cmder{NewStatusCmd("MULTI")} if err := fn(); err != nil { return nil, err } - tx.cmds = append(tx.cmds, NewSliceCmd("EXEC")) + c.cmds = append(c.cmds, NewSliceCmd("EXEC")) - cmds := tx.cmds - tx.cmds = nil + cmds := c.cmds + c.cmds = nil if len(cmds) == 2 { return []Cmder{}, nil @@ -134,18 +134,18 @@ func (tx *Tx) MultiExec(fn func() error) ([]Cmder, error) { // Strip MULTI and EXEC commands. retCmds := cmds[1 : len(cmds)-1] - cn, err := tx.conn() + cn, err := c.conn() if err != nil { setCmdsErr(retCmds, err) return retCmds, err } - err = tx.execCmds(cn, cmds) - tx.putConn(cn, err, false) + err = c.execCmds(cn, cmds) + c.putConn(cn, err, false) return retCmds, err } -func (tx *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { +func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { err := writeCmd(cn, cmds...) if err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) From aa063fe0a2e718b1c123ad1f10abb296fc2c6f4c Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Sat, 2 Jul 2016 09:07:27 +0100 Subject: [PATCH 0186/1746] Fix ring comments --- ring.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ring.go b/ring.go index 8aba07e29e..8c998a8395 100644 --- a/ring.go +++ b/ring.go @@ -102,7 +102,7 @@ func (shard *ringShard) Vote(up bool) bool { // concurrent use by multiple goroutines. // // Ring monitors the state of each shard and removes dead shards from -// the c. When shard comes online it is added back to the c. This +// the ring. When shard comes online it is added back to the ring. This // gives you maximum availability and partition tolerance, but no // consistency between different shards or even clients. Each client // uses shards that are available to the client and does not do any @@ -221,7 +221,7 @@ func (c *Ring) rebalance() { } } -// heartbeat monitors state of each shard in the c. +// heartbeat monitors state of each shard in the ring. func (c *Ring) heartbeat() { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() From 7d856c5595af82b702b05f14116793a6563f82da Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Sat, 2 Jul 2016 13:52:10 +0100 Subject: [PATCH 0187/1746] Make proto/parser an internal package --- cluster.go | 9 +- cluster_test.go | 1 + command.go | 47 ++- commands.go | 19 +- error.go | 81 ----- internal/errors/errors.go | 67 ++++ internal/pool/conn.go | 23 +- internal/pool/pool.go | 5 +- internal/proto/proto.go | 122 +++++++ internal/proto/proto_test.go | 13 + internal/proto/reader.go | 264 ++++++++++++++ internal/proto/reader_test.go | 86 +++++ internal/proto/writebuffer.go | 101 ++++++ internal/proto/writebuffer_test.go | 62 ++++ main_test.go | 2 + parser.go | 546 ++++------------------------- parser_test.go | 57 --- pipeline.go | 3 +- pipeline_test.go | 2 + pubsub.go | 3 +- redis.go | 10 +- safe.go | 7 - tx.go | 9 +- unsafe.go | 14 - 24 files changed, 841 insertions(+), 712 deletions(-) delete mode 100644 error.go create mode 100644 internal/errors/errors.go create mode 100644 internal/proto/proto.go create mode 100644 internal/proto/proto_test.go create mode 100644 internal/proto/reader.go create mode 100644 internal/proto/reader_test.go create mode 100644 internal/proto/writebuffer.go create mode 100644 internal/proto/writebuffer_test.go delete mode 100644 parser_test.go delete mode 100644 safe.go delete mode 100644 unsafe.go diff --git a/cluster.go b/cluster.go index 1ae6082b61..217cadd942 100644 --- a/cluster.go +++ b/cluster.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/redis.v4/internal" + "gopkg.in/redis.v4/internal/errors" "gopkg.in/redis.v4/internal/hashtag" "gopkg.in/redis.v4/internal/pool" ) @@ -291,14 +292,14 @@ func (c *ClusterClient) Process(cmd Cmder) error { } // On network errors try random node. - if shouldRetry(err) { + if errors.IsRetryable(err) { node, err = c.randomNode() continue } var moved bool var addr string - moved, ask, addr = isMovedError(err) + moved, ask, addr = errors.IsMoved(err) if moved || ask { master, _ := c.slotMasterNode(slot) if moved && (master == nil || master.Addr != addr) { @@ -549,11 +550,11 @@ func (c *ClusterClient) execClusterCmds( if err == nil { continue } - if isNetworkError(err) { + if errors.IsNetwork(err) { cmd.reset() failedCmds[nil] = append(failedCmds[nil], cmds[i:]...) break - } else if moved, ask, addr := isMovedError(err); moved { + } else if moved, ask, addr := errors.IsMoved(err); moved { c.lazyReloadSlots() cmd.reset() node, err := c.nodeByAddr(addr) diff --git a/cluster_test.go b/cluster_test.go index 6889d4e09f..17d5113318 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -393,6 +393,7 @@ var _ = Describe("ClusterClient", func() { for i := 0; i < 100; i++ { wg.Add(1) go func() { + defer GinkgoRecover() defer wg.Done() err := incr("key") diff --git a/command.go b/command.go index c56258284a..9cb705879a 100644 --- a/command.go +++ b/command.go @@ -8,6 +8,7 @@ import ( "time" "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v4/internal/proto" ) var ( @@ -55,16 +56,14 @@ func resetCmds(cmds []Cmder) { } func writeCmd(cn *pool.Conn, cmds ...Cmder) error { - cn.Buf = cn.Buf[:0] + cn.Wb.Reset() for _, cmd := range cmds { - var err error - cn.Buf, err = appendArgs(cn.Buf, cmd.args()) - if err != nil { + if err := cn.Wb.Append(cmd.args()); err != nil { return err } } - _, err := cn.Write(cn.Buf) + _, err := cn.Write(cn.Wb.Bytes()) return err } @@ -166,7 +165,7 @@ func (cmd *Cmd) String() string { } func (cmd *Cmd) readReply(cn *pool.Conn) error { - val, err := readReply(cn, sliceParser) + val, err := cn.Rd.ReadReply(sliceParser) if err != nil { cmd.err = err return cmd.err @@ -211,7 +210,7 @@ func (cmd *SliceCmd) String() string { } func (cmd *SliceCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, sliceParser) + v, err := cn.Rd.ReadArrayReply(sliceParser) if err != nil { cmd.err = err return err @@ -251,7 +250,7 @@ func (cmd *StatusCmd) String() string { } func (cmd *StatusCmd) readReply(cn *pool.Conn) error { - cmd.val, cmd.err = readStringReply(cn) + cmd.val, cmd.err = cn.Rd.ReadStringReply() return cmd.err } @@ -286,7 +285,7 @@ func (cmd *IntCmd) String() string { } func (cmd *IntCmd) readReply(cn *pool.Conn) error { - cmd.val, cmd.err = readIntReply(cn) + cmd.val, cmd.err = cn.Rd.ReadIntReply() return cmd.err } @@ -325,7 +324,7 @@ func (cmd *DurationCmd) String() string { } func (cmd *DurationCmd) readReply(cn *pool.Conn) error { - n, err := readIntReply(cn) + n, err := cn.Rd.ReadIntReply() if err != nil { cmd.err = err return err @@ -367,7 +366,7 @@ func (cmd *BoolCmd) String() string { var ok = []byte("OK") func (cmd *BoolCmd) readReply(cn *pool.Conn) error { - v, err := readReply(cn, nil) + v, err := cn.Rd.ReadReply(nil) // `SET key value NX` returns nil when key already exists. But // `SETNX key value` returns bool (0/1). So convert nil to bool. // TODO: is this okay? @@ -410,7 +409,7 @@ func (cmd *StringCmd) reset() { } func (cmd *StringCmd) Val() string { - return bytesToString(cmd.val) + return string(cmd.val) } func (cmd *StringCmd) Result() (string, error) { @@ -446,7 +445,7 @@ func (cmd *StringCmd) Scan(val interface{}) error { if cmd.err != nil { return cmd.err } - return scan(cmd.val, val) + return proto.Scan(cmd.val, val) } func (cmd *StringCmd) String() string { @@ -454,7 +453,7 @@ func (cmd *StringCmd) String() string { } func (cmd *StringCmd) readReply(cn *pool.Conn) error { - b, err := readBytesReply(cn) + b, err := cn.Rd.ReadBytesReply() if err != nil { cmd.err = err return err @@ -498,7 +497,7 @@ func (cmd *FloatCmd) String() string { } func (cmd *FloatCmd) readReply(cn *pool.Conn) error { - cmd.val, cmd.err = readFloatReply(cn) + cmd.val, cmd.err = cn.Rd.ReadFloatReply() return cmd.err } @@ -533,7 +532,7 @@ func (cmd *StringSliceCmd) String() string { } func (cmd *StringSliceCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, stringSliceParser) + v, err := cn.Rd.ReadArrayReply(stringSliceParser) if err != nil { cmd.err = err return err @@ -573,7 +572,7 @@ func (cmd *BoolSliceCmd) String() string { } func (cmd *BoolSliceCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, boolSliceParser) + v, err := cn.Rd.ReadArrayReply(boolSliceParser) if err != nil { cmd.err = err return err @@ -613,7 +612,7 @@ func (cmd *StringStringMapCmd) String() string { } func (cmd *StringStringMapCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, stringStringMapParser) + v, err := cn.Rd.ReadArrayReply(stringStringMapParser) if err != nil { cmd.err = err return err @@ -653,7 +652,7 @@ func (cmd *StringIntMapCmd) reset() { } func (cmd *StringIntMapCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, stringIntMapParser) + v, err := cn.Rd.ReadArrayReply(stringIntMapParser) if err != nil { cmd.err = err return err @@ -693,7 +692,7 @@ func (cmd *ZSliceCmd) String() string { } func (cmd *ZSliceCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, zSliceParser) + v, err := cn.Rd.ReadArrayReply(zSliceParser) if err != nil { cmd.err = err return err @@ -737,7 +736,7 @@ func (cmd *ScanCmd) String() string { } func (cmd *ScanCmd) readReply(cn *pool.Conn) error { - page, cursor, err := readScanReply(cn) + page, cursor, err := cn.Rd.ReadScanReply() if err != nil { cmd.err = err return cmd.err @@ -789,7 +788,7 @@ func (cmd *ClusterSlotsCmd) reset() { } func (cmd *ClusterSlotsCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, clusterSlotsParser) + v, err := cn.Rd.ReadArrayReply(clusterSlotsParser) if err != nil { cmd.err = err return err @@ -874,7 +873,7 @@ func (cmd *GeoLocationCmd) String() string { } func (cmd *GeoLocationCmd) readReply(cn *pool.Conn) error { - reply, err := readArrayReply(cn, newGeoLocationSliceParser(cmd.q)) + reply, err := cn.Rd.ReadArrayReply(newGeoLocationSliceParser(cmd.q)) if err != nil { cmd.err = err return err @@ -924,7 +923,7 @@ func (cmd *CommandsInfoCmd) reset() { } func (cmd *CommandsInfoCmd) readReply(cn *pool.Conn) error { - v, err := readArrayReply(cn, commandInfoSliceParser) + v, err := cn.Rd.ReadArrayReply(commandInfoSliceParser) if err != nil { cmd.err = err return err diff --git a/commands.go b/commands.go index f84311c51f..412adaec0c 100644 --- a/commands.go +++ b/commands.go @@ -6,20 +6,9 @@ import ( "time" "gopkg.in/redis.v4/internal" + "gopkg.in/redis.v4/internal/errors" ) -func formatInt(i int64) string { - return strconv.FormatInt(i, 10) -} - -func formatUint(i uint64) string { - return strconv.FormatUint(i, 10) -} - -func formatFloat(f float64) string { - return strconv.FormatFloat(f, 'f', -1, 64) -} - func readTimeout(timeout time.Duration) time.Duration { if timeout == 0 { return 0 @@ -38,7 +27,7 @@ func formatMs(dur time.Duration) string { dur, time.Millisecond, ) } - return formatInt(int64(dur / time.Millisecond)) + return strconv.FormatInt(int64(dur/time.Millisecond), 10) } func formatSec(dur time.Duration) string { @@ -48,7 +37,7 @@ func formatSec(dur time.Duration) string { dur, time.Second, ) } - return formatInt(int64(dur / time.Second)) + return strconv.FormatInt(int64(dur/time.Second), 10) } type cmdable struct { @@ -1515,7 +1504,7 @@ func (c *cmdable) shutdown(modifier string) *StatusCmd { } } else { // Server did not quit. String reply contains the reason. - cmd.err = errorf(cmd.val) + cmd.err = errors.RedisError(cmd.val) cmd.val = "" } return cmd diff --git a/error.go b/error.go deleted file mode 100644 index f0b27a8466..0000000000 --- a/error.go +++ /dev/null @@ -1,81 +0,0 @@ -package redis - -import ( - "fmt" - "io" - "net" - "strings" -) - -// Redis nil reply, .e.g. when key does not exist. -var Nil = errorf("redis: nil") - -// Redis transaction failed. -var TxFailedErr = errorf("redis: transaction failed") - -type redisError struct { - s string -} - -func errorf(s string, args ...interface{}) redisError { - return redisError{s: fmt.Sprintf(s, args...)} -} - -func (err redisError) Error() string { - return err.s -} - -func isInternalError(err error) bool { - _, ok := err.(redisError) - return ok -} - -func isNetworkError(err error) bool { - if err == io.EOF { - return true - } - _, ok := err.(net.Error) - return ok -} - -func isBadConn(err error, allowTimeout bool) bool { - if err == nil { - return false - } - if isInternalError(err) { - return false - } - if allowTimeout { - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return false - } - } - return true -} - -func isMovedError(err error) (moved bool, ask bool, addr string) { - if _, ok := err.(redisError); !ok { - return - } - - s := err.Error() - if strings.HasPrefix(s, "MOVED ") { - moved = true - } else if strings.HasPrefix(s, "ASK ") { - ask = true - } else { - return - } - - ind := strings.LastIndexByte(s, ' ') - if ind == -1 { - return false, false, "" - } - addr = s[ind+1:] - return -} - -// shouldRetry reports whether failed command should be retried. -func shouldRetry(err error) bool { - return isNetworkError(err) -} diff --git a/internal/errors/errors.go b/internal/errors/errors.go new file mode 100644 index 0000000000..6d664ddfe1 --- /dev/null +++ b/internal/errors/errors.go @@ -0,0 +1,67 @@ +package errors + +import ( + "io" + "net" + "strings" +) + +const Nil = RedisError("redis: nil") + +type RedisError string + +func (e RedisError) Error() string { return string(e) } + +func IsRetryable(err error) bool { + return IsNetwork(err) +} + +func IsInternal(err error) bool { + _, ok := err.(RedisError) + return ok +} + +func IsNetwork(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func IsBadConn(err error, allowTimeout bool) bool { + if err == nil { + return false + } + if IsInternal(err) { + return false + } + if allowTimeout { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return false + } + } + return true +} + +func IsMoved(err error) (moved bool, ask bool, addr string) { + if !IsInternal(err) { + return + } + + s := err.Error() + if strings.HasPrefix(s, "MOVED ") { + moved = true + } else if strings.HasPrefix(s, "ASK ") { + ask = true + } else { + return + } + + ind := strings.LastIndexByte(s, ' ') + if ind == -1 { + return false, false, "" + } + addr = s[ind+1:] + return +} diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 497fd4e46c..bb0922f384 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -1,10 +1,10 @@ package pool import ( - "bufio" - "io" "net" "time" + + "gopkg.in/redis.v4/internal/proto" ) const defaultBufSize = 4096 @@ -13,8 +13,8 @@ var noDeadline = time.Time{} type Conn struct { NetConn net.Conn - Rd *bufio.Reader - Buf []byte + Rd *proto.Reader + Wb *proto.WriteBuffer Inited bool UsedAt time.Time @@ -26,11 +26,11 @@ type Conn struct { func NewConn(netConn net.Conn) *Conn { cn := &Conn{ NetConn: netConn, - Buf: make([]byte, defaultBufSize), + Wb: proto.NewWriteBuffer(), UsedAt: time.Now(), } - cn.Rd = bufio.NewReader(cn) + cn.Rd = proto.NewReader(cn) return cn } @@ -62,17 +62,6 @@ func (cn *Conn) RemoteAddr() net.Addr { return cn.NetConn.RemoteAddr() } -func (cn *Conn) ReadN(n int) ([]byte, error) { - if d := n - cap(cn.Buf); d > 0 { - cn.Buf = cn.Buf[:cap(cn.Buf)] - cn.Buf = append(cn.Buf, make([]byte, d)...) - } else { - cn.Buf = cn.Buf[:n] - } - _, err := io.ReadFull(cn.Rd, cn.Buf) - return cn.Buf, err -} - func (cn *Conn) Close() error { return cn.NetConn.Close() } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 330767c989..da1e381fd3 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -205,9 +205,8 @@ func (p *ConnPool) Get() (*Conn, error) { } func (p *ConnPool) Put(cn *Conn) error { - if cn.Rd.Buffered() != 0 { - b, _ := cn.Rd.Peek(cn.Rd.Buffered()) - err := fmt.Errorf("connection has unread data: %q", b) + if data := cn.Rd.PeekBuffered(); data != nil { + err := fmt.Errorf("connection has unread data: %q", data) internal.Logf(err.Error()) return p.Remove(cn, err) } diff --git a/internal/proto/proto.go b/internal/proto/proto.go new file mode 100644 index 0000000000..c63caaac18 --- /dev/null +++ b/internal/proto/proto.go @@ -0,0 +1,122 @@ +package proto + +import ( + "encoding" + "fmt" + "strconv" + + "gopkg.in/redis.v4/internal/errors" +) + +const ( + ErrorReply = '-' + StatusReply = '+' + IntReply = ':' + StringReply = '$' + ArrayReply = '*' +) + +const defaultBufSize = 4096 + +var errScanNil = errors.RedisError("redis: Scan(nil)") + +func Scan(b []byte, val interface{}) error { + switch v := val.(type) { + case nil: + return errScanNil + case *string: + *v = string(b) + return nil + case *[]byte: + *v = b + return nil + case *int: + var err error + *v, err = strconv.Atoi(string(b)) + return err + case *int8: + n, err := strconv.ParseInt(string(b), 10, 8) + if err != nil { + return err + } + *v = int8(n) + return nil + case *int16: + n, err := strconv.ParseInt(string(b), 10, 16) + if err != nil { + return err + } + *v = int16(n) + return nil + case *int32: + n, err := strconv.ParseInt(string(b), 10, 32) + if err != nil { + return err + } + *v = int32(n) + return nil + case *int64: + n, err := strconv.ParseInt(string(b), 10, 64) + if err != nil { + return err + } + *v = n + return nil + case *uint: + n, err := strconv.ParseUint(string(b), 10, 64) + if err != nil { + return err + } + *v = uint(n) + return nil + case *uint8: + n, err := strconv.ParseUint(string(b), 10, 8) + if err != nil { + return err + } + *v = uint8(n) + return nil + case *uint16: + n, err := strconv.ParseUint(string(b), 10, 16) + if err != nil { + return err + } + *v = uint16(n) + return nil + case *uint32: + n, err := strconv.ParseUint(string(b), 10, 32) + if err != nil { + return err + } + *v = uint32(n) + return nil + case *uint64: + n, err := strconv.ParseUint(string(b), 10, 64) + if err != nil { + return err + } + *v = n + return nil + case *float32: + n, err := strconv.ParseFloat(string(b), 32) + if err != nil { + return err + } + *v = float32(n) + return err + case *float64: + var err error + *v, err = strconv.ParseFloat(string(b), 64) + return err + case *bool: + *v = len(b) == 1 && b[0] == '1' + return nil + default: + if bu, ok := val.(encoding.BinaryUnmarshaler); ok { + return bu.UnmarshalBinary(b) + } + err := fmt.Errorf( + "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", val) + return err + } +} diff --git a/internal/proto/proto_test.go b/internal/proto/proto_test.go new file mode 100644 index 0000000000..c9a820eb11 --- /dev/null +++ b/internal/proto/proto_test.go @@ -0,0 +1,13 @@ +package proto_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestGinkgoSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "proto") +} diff --git a/internal/proto/reader.go b/internal/proto/reader.go new file mode 100644 index 0000000000..9d2f51ae3d --- /dev/null +++ b/internal/proto/reader.go @@ -0,0 +1,264 @@ +package proto + +import ( + "bufio" + "errors" + "fmt" + "io" + "strconv" + + ierrors "gopkg.in/redis.v4/internal/errors" +) + +type MultiBulkParse func(*Reader, int64) (interface{}, error) + +var errEmptyReply = errors.New("redis: reply is empty") + +type Reader struct { + src *bufio.Reader + buf []byte +} + +func NewReader(rd io.Reader) *Reader { + return &Reader{ + src: bufio.NewReader(rd), + buf: make([]byte, 0, defaultBufSize), + } +} + +func (p *Reader) PeekBuffered() []byte { + if n := p.src.Buffered(); n != 0 { + b, _ := p.src.Peek(n) + return b + } + return nil +} + +func (p *Reader) ReadN(n int) ([]byte, error) { + // grow internal buffer, if necessary + if d := n - cap(p.buf); d > 0 { + p.buf = p.buf[:cap(p.buf)] + p.buf = append(p.buf, make([]byte, d)...) + } else { + p.buf = p.buf[:n] + } + + _, err := io.ReadFull(p.src, p.buf) + return p.buf, err +} + +func (p *Reader) ReadLine() ([]byte, error) { + line, isPrefix, err := p.src.ReadLine() + if err != nil { + return nil, err + } + if isPrefix { + return nil, bufio.ErrBufferFull + } + if len(line) == 0 { + return nil, errEmptyReply + } + if isNilReply(line) { + return nil, ierrors.Nil + } + return line, nil +} + +func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { + line, err := p.ReadLine() + if err != nil { + return nil, err + } + + switch line[0] { + case ErrorReply: + return nil, parseErrorValue(line) + case StatusReply: + return parseStatusValue(line) + case IntReply: + return parseIntValue(line) + case StringReply: + return p.parseBytesValue(line) + case ArrayReply: + n, err := parseArrayLen(line) + if err != nil { + return nil, err + } + return m(p, n) + } + return nil, fmt.Errorf("redis: can't parse %.100q", line) +} + +func (p *Reader) ReadIntReply() (int64, error) { + line, err := p.ReadLine() + if err != nil { + return 0, err + } + switch line[0] { + case ErrorReply: + return 0, parseErrorValue(line) + case IntReply: + return parseIntValue(line) + default: + return 0, fmt.Errorf("redis: can't parse int reply: %.100q", line) + } +} + +func (p *Reader) ReadBytesReply() ([]byte, error) { + line, err := p.ReadLine() + if err != nil { + return nil, err + } + switch line[0] { + case ErrorReply: + return nil, parseErrorValue(line) + case StringReply: + return p.parseBytesValue(line) + case StatusReply: + return parseStatusValue(line) + default: + return nil, fmt.Errorf("redis: can't parse string reply: %.100q", line) + } +} + +func (p *Reader) ReadStringReply() (string, error) { + b, err := p.ReadBytesReply() + if err != nil { + return "", err + } + return string(b), nil +} + +func (p *Reader) ReadFloatReply() (float64, error) { + s, err := p.ReadStringReply() + if err != nil { + return 0, err + } + return strconv.ParseFloat(s, 64) +} + +func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { + line, err := p.ReadLine() + if err != nil { + return nil, err + } + switch line[0] { + case ErrorReply: + return nil, parseErrorValue(line) + case ArrayReply: + n, err := parseArrayLen(line) + if err != nil { + return nil, err + } + return m(p, n) + default: + return nil, fmt.Errorf("redis: can't parse array reply: %.100q", line) + } +} + +func (p *Reader) ReadArrayLen() (int64, error) { + line, err := p.ReadLine() + if err != nil { + return 0, err + } + switch line[0] { + case ErrorReply: + return 0, parseErrorValue(line) + case ArrayReply: + return parseArrayLen(line) + default: + return 0, fmt.Errorf("redis: can't parse array reply: %.100q", line) + } +} + +func (p *Reader) ReadScanReply() ([]string, uint64, error) { + n, err := p.ReadArrayLen() + if err != nil { + return nil, 0, err + } + if n != 2 { + return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n) + } + + s, err := p.ReadStringReply() + if err != nil { + return nil, 0, err + } + + cursor, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, 0, err + } + + n, err = p.ReadArrayLen() + if err != nil { + return nil, 0, err + } + + keys := make([]string, n) + for i := int64(0); i < n; i++ { + key, err := p.ReadStringReply() + if err != nil { + return nil, 0, err + } + keys[i] = key + } + + return keys, cursor, err +} + +func (p *Reader) parseBytesValue(line []byte) ([]byte, error) { + if isNilReply(line) { + return nil, ierrors.Nil + } + + replyLen, err := strconv.Atoi(string(line[1:])) + if err != nil { + return nil, err + } + + b, err := p.ReadN(replyLen + 2) + if err != nil { + return nil, err + } + return b[:replyLen], nil +} + +// -------------------------------------------------------------------- + +func formatInt(n int64) string { + return strconv.FormatInt(n, 10) +} + +func formatUint(u uint64) string { + return strconv.FormatUint(u, 10) +} + +func formatFloat(f float64) string { + return strconv.FormatFloat(f, 'f', -1, 64) +} + +func isNilReply(b []byte) bool { + return len(b) == 3 && + (b[0] == StringReply || b[0] == ArrayReply) && + b[1] == '-' && b[2] == '1' +} + +func parseErrorValue(line []byte) error { + return ierrors.RedisError(string(line[1:])) +} + +func parseStatusValue(line []byte) ([]byte, error) { + return line[1:], nil +} + +func parseIntValue(line []byte) (int64, error) { + return strconv.ParseInt(string(line[1:]), 10, 64) +} + +func parseArrayLen(line []byte) (int64, error) { + if isNilReply(line) { + return 0, ierrors.Nil + } + return parseIntValue(line) +} diff --git a/internal/proto/reader_test.go b/internal/proto/reader_test.go new file mode 100644 index 0000000000..5169262661 --- /dev/null +++ b/internal/proto/reader_test.go @@ -0,0 +1,86 @@ +package proto_test + +import ( + "bytes" + "strings" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "gopkg.in/redis.v4/internal/proto" +) + +var _ = Describe("Reader", func() { + + It("should read n bytes", func() { + data, err := proto.NewReader(strings.NewReader("ABCDEFGHIJKLMNO")).ReadN(10) + Expect(err).NotTo(HaveOccurred()) + Expect(len(data)).To(Equal(10)) + Expect(string(data)).To(Equal("ABCDEFGHIJ")) + + data, err = proto.NewReader(strings.NewReader(strings.Repeat("x", 8192))).ReadN(6000) + Expect(err).NotTo(HaveOccurred()) + Expect(len(data)).To(Equal(6000)) + }) + + It("should read lines", func() { + p := proto.NewReader(strings.NewReader("$5\r\nhello\r\n")) + + data, err := p.ReadLine() + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal("$5")) + + data, err = p.ReadLine() + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal("hello")) + }) + +}) + +func BenchmarkReader_ParseReply_Status(b *testing.B) { + benchmarkParseReply(b, "+OK\r\n", nil, false) +} + +func BenchmarkReader_ParseReply_Int(b *testing.B) { + benchmarkParseReply(b, ":1\r\n", nil, false) +} + +func BenchmarkReader_ParseReply_Error(b *testing.B) { + benchmarkParseReply(b, "-Error message\r\n", nil, true) +} + +func BenchmarkReader_ParseReply_String(b *testing.B) { + benchmarkParseReply(b, "$5\r\nhello\r\n", nil, false) +} + +func BenchmarkReader_ParseReply_Slice(b *testing.B) { + benchmarkParseReply(b, "*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n", multiBulkParse, false) +} + +func benchmarkParseReply(b *testing.B, reply string, m proto.MultiBulkParse, wanterr bool) { + buf := &bytes.Buffer{} + for i := 0; i < b.N; i++ { + buf.WriteString(reply) + } + p := proto.NewReader(buf) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := p.ReadReply(m) + if !wanterr && err != nil { + b.Fatal(err) + } + } +} + +func multiBulkParse(p *proto.Reader, n int64) (interface{}, error) { + vv := make([]interface{}, 0, n) + for i := int64(0); i < n; i++ { + v, err := p.ReadReply(multiBulkParse) + if err != nil { + return nil, err + } + vv = append(vv, v) + } + return vv, nil +} diff --git a/internal/proto/writebuffer.go b/internal/proto/writebuffer.go new file mode 100644 index 0000000000..8164f7f20e --- /dev/null +++ b/internal/proto/writebuffer.go @@ -0,0 +1,101 @@ +package proto + +import ( + "encoding" + "fmt" + "strconv" +) + +type WriteBuffer struct{ b []byte } + +func NewWriteBuffer() *WriteBuffer { + return &WriteBuffer{ + b: make([]byte, 0, defaultBufSize), + } +} + +func (w *WriteBuffer) Len() int { return len(w.b) } +func (w *WriteBuffer) Bytes() []byte { return w.b } +func (w *WriteBuffer) Reset() { w.b = w.b[:0] } + +func (w *WriteBuffer) Append(args []interface{}) error { + w.b = append(w.b, ArrayReply) + w.b = strconv.AppendUint(w.b, uint64(len(args)), 10) + w.b = append(w.b, '\r', '\n') + + for _, arg := range args { + if err := w.append(arg); err != nil { + return err + } + } + return nil +} + +func (w *WriteBuffer) append(val interface{}) error { + switch v := val.(type) { + case nil: + w.AppendString("") + case string: + w.AppendString(v) + case []byte: + w.AppendBytes(v) + case int: + w.AppendString(formatInt(int64(v))) + case int8: + w.AppendString(formatInt(int64(v))) + case int16: + w.AppendString(formatInt(int64(v))) + case int32: + w.AppendString(formatInt(int64(v))) + case int64: + w.AppendString(formatInt(v)) + case uint: + w.AppendString(formatUint(uint64(v))) + case uint8: + w.AppendString(formatUint(uint64(v))) + case uint16: + w.AppendString(formatUint(uint64(v))) + case uint32: + w.AppendString(formatUint(uint64(v))) + case uint64: + w.AppendString(formatUint(v)) + case float32: + w.AppendString(formatFloat(float64(v))) + case float64: + w.AppendString(formatFloat(v)) + case bool: + if v { + w.AppendString("1") + } else { + w.AppendString("0") + } + default: + if bm, ok := val.(encoding.BinaryMarshaler); ok { + bb, err := bm.MarshalBinary() + if err != nil { + return err + } + w.AppendBytes(bb) + } else { + return fmt.Errorf( + "redis: can't marshal %T (consider implementing encoding.BinaryMarshaler)", val) + } + } + return nil +} + +func (w *WriteBuffer) AppendString(s string) { + w.b = append(w.b, StringReply) + w.b = strconv.AppendUint(w.b, uint64(len(s)), 10) + w.b = append(w.b, '\r', '\n') + w.b = append(w.b, s...) + w.b = append(w.b, '\r', '\n') +} + +func (w *WriteBuffer) AppendBytes(p []byte) { + w.b = append(w.b, StringReply) + w.b = strconv.AppendUint(w.b, uint64(len(p)), 10) + w.b = append(w.b, '\r', '\n') + w.b = append(w.b, p...) + w.b = append(w.b, '\r', '\n') +} diff --git a/internal/proto/writebuffer_test.go b/internal/proto/writebuffer_test.go new file mode 100644 index 0000000000..6316ded78d --- /dev/null +++ b/internal/proto/writebuffer_test.go @@ -0,0 +1,62 @@ +package proto_test + +import ( + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "gopkg.in/redis.v4/internal/proto" +) + +var _ = Describe("WriteBuffer", func() { + var buf *proto.WriteBuffer + + BeforeEach(func() { + buf = proto.NewWriteBuffer() + }) + + It("should reset", func() { + buf.AppendString("string") + Expect(buf.Len()).To(Equal(12)) + buf.Reset() + Expect(buf.Len()).To(Equal(0)) + }) + + It("should append args", func() { + err := buf.Append([]interface{}{ + "string", + 12, + 34.56, + []byte{'b', 'y', 't', 'e', 's'}, + true, + nil, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte("*6\r\n" + + "$6\r\nstring\r\n" + + "$2\r\n12\r\n" + + "$5\r\n34.56\r\n" + + "$5\r\nbytes\r\n" + + "$1\r\n1\r\n" + + "$0\r\n" + + "\r\n"))) + }) + + It("should append marshalable args", func() { + err := buf.Append([]interface{}{time.Unix(1414141414, 0)}) + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Len()).To(Equal(26)) + }) + +}) + +func BenchmarkWriteBuffer_Append(b *testing.B) { + buf := proto.NewWriteBuffer() + args := []interface{}{"hello", "world", "foo", "bar"} + + for i := 0; i < b.N; i++ { + buf.Append(args) + buf.Reset() + } +} diff --git a/main_test.go b/main_test.go index b0277e868c..5208903fb6 100644 --- a/main_test.go +++ b/main_test.go @@ -136,6 +136,8 @@ func eventually(fn func() error, timeout time.Duration) error { done := make(chan struct{}) go func() { + defer GinkgoRecover() + for atomic.LoadInt32(&exit) == 0 { err := fn() if err == nil { diff --git a/parser.go b/parser.go index f6e5ca62bf..95a4297bab 100644 --- a/parser.go +++ b/parser.go @@ -1,446 +1,18 @@ package redis import ( - "bufio" - "errors" "fmt" "net" "strconv" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v4/internal/proto" ) -const ( - errorReply = '-' - statusReply = '+' - intReply = ':' - stringReply = '$' - arrayReply = '*' -) - -type multiBulkParser func(cn *pool.Conn, n int64) (interface{}, error) - -var errEmptyReply = errors.New("redis: reply is empty") - -//------------------------------------------------------------------------------ - -// Copy of encoding.BinaryMarshaler. -type binaryMarshaler interface { - MarshalBinary() (data []byte, err error) -} - -// Copy of encoding.BinaryUnmarshaler. -type binaryUnmarshaler interface { - UnmarshalBinary(data []byte) error -} - -func appendString(b []byte, s string) []byte { - b = append(b, '$') - b = strconv.AppendUint(b, uint64(len(s)), 10) - b = append(b, '\r', '\n') - b = append(b, s...) - b = append(b, '\r', '\n') - return b -} - -func appendBytes(b, bb []byte) []byte { - b = append(b, '$') - b = strconv.AppendUint(b, uint64(len(bb)), 10) - b = append(b, '\r', '\n') - b = append(b, bb...) - b = append(b, '\r', '\n') - return b -} - -func appendArg(b []byte, val interface{}) ([]byte, error) { - switch v := val.(type) { - case nil: - b = appendString(b, "") - case string: - b = appendString(b, v) - case []byte: - b = appendBytes(b, v) - case int: - b = appendString(b, formatInt(int64(v))) - case int8: - b = appendString(b, formatInt(int64(v))) - case int16: - b = appendString(b, formatInt(int64(v))) - case int32: - b = appendString(b, formatInt(int64(v))) - case int64: - b = appendString(b, formatInt(v)) - case uint: - b = appendString(b, formatUint(uint64(v))) - case uint8: - b = appendString(b, formatUint(uint64(v))) - case uint16: - b = appendString(b, formatUint(uint64(v))) - case uint32: - b = appendString(b, formatUint(uint64(v))) - case uint64: - b = appendString(b, formatUint(v)) - case float32: - b = appendString(b, formatFloat(float64(v))) - case float64: - b = appendString(b, formatFloat(v)) - case bool: - if v { - b = appendString(b, "1") - } else { - b = appendString(b, "0") - } - default: - if bm, ok := val.(binaryMarshaler); ok { - bb, err := bm.MarshalBinary() - if err != nil { - return nil, err - } - b = appendBytes(b, bb) - } else { - err := fmt.Errorf( - "redis: can't marshal %T (consider implementing BinaryMarshaler)", val) - return nil, err - } - } - return b, nil -} - -func appendArgs(b []byte, args []interface{}) ([]byte, error) { - b = append(b, arrayReply) - b = strconv.AppendUint(b, uint64(len(args)), 10) - b = append(b, '\r', '\n') - for _, arg := range args { - var err error - b, err = appendArg(b, arg) - if err != nil { - return nil, err - } - } - return b, nil -} - -func scan(b []byte, val interface{}) error { - switch v := val.(type) { - case nil: - return errorf("redis: Scan(nil)") - case *string: - *v = bytesToString(b) - return nil - case *[]byte: - *v = b - return nil - case *int: - var err error - *v, err = strconv.Atoi(bytesToString(b)) - return err - case *int8: - n, err := strconv.ParseInt(bytesToString(b), 10, 8) - if err != nil { - return err - } - *v = int8(n) - return nil - case *int16: - n, err := strconv.ParseInt(bytesToString(b), 10, 16) - if err != nil { - return err - } - *v = int16(n) - return nil - case *int32: - n, err := strconv.ParseInt(bytesToString(b), 10, 16) - if err != nil { - return err - } - *v = int32(n) - return nil - case *int64: - n, err := strconv.ParseInt(bytesToString(b), 10, 64) - if err != nil { - return err - } - *v = n - return nil - case *uint: - n, err := strconv.ParseUint(bytesToString(b), 10, 64) - if err != nil { - return err - } - *v = uint(n) - return nil - case *uint8: - n, err := strconv.ParseUint(bytesToString(b), 10, 8) - if err != nil { - return err - } - *v = uint8(n) - return nil - case *uint16: - n, err := strconv.ParseUint(bytesToString(b), 10, 16) - if err != nil { - return err - } - *v = uint16(n) - return nil - case *uint32: - n, err := strconv.ParseUint(bytesToString(b), 10, 32) - if err != nil { - return err - } - *v = uint32(n) - return nil - case *uint64: - n, err := strconv.ParseUint(bytesToString(b), 10, 64) - if err != nil { - return err - } - *v = n - return nil - case *float32: - n, err := strconv.ParseFloat(bytesToString(b), 32) - if err != nil { - return err - } - *v = float32(n) - return err - case *float64: - var err error - *v, err = strconv.ParseFloat(bytesToString(b), 64) - return err - case *bool: - *v = len(b) == 1 && b[0] == '1' - return nil - default: - if bu, ok := val.(binaryUnmarshaler); ok { - return bu.UnmarshalBinary(b) - } - err := fmt.Errorf( - "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", val) - return err - } -} - -//------------------------------------------------------------------------------ - -func readLine(cn *pool.Conn) ([]byte, error) { - line, isPrefix, err := cn.Rd.ReadLine() - if err != nil { - return nil, err - } - if isPrefix { - return nil, bufio.ErrBufferFull - } - if len(line) == 0 { - return nil, errEmptyReply - } - if isNilReply(line) { - return nil, Nil - } - return line, nil -} - -func isNilReply(b []byte) bool { - return len(b) == 3 && - (b[0] == stringReply || b[0] == arrayReply) && - b[1] == '-' && b[2] == '1' -} - -//------------------------------------------------------------------------------ - -func parseErrorReply(cn *pool.Conn, line []byte) error { - return errorf(string(line[1:])) -} - -func parseStatusReply(cn *pool.Conn, line []byte) ([]byte, error) { - return line[1:], nil -} - -func parseIntReply(cn *pool.Conn, line []byte) (int64, error) { - n, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) - if err != nil { - return 0, err - } - return n, nil -} - -func readIntReply(cn *pool.Conn) (int64, error) { - line, err := readLine(cn) - if err != nil { - return 0, err - } - switch line[0] { - case errorReply: - return 0, parseErrorReply(cn, line) - case intReply: - return parseIntReply(cn, line) - default: - return 0, fmt.Errorf("redis: can't parse int reply: %.100q", line) - } -} - -func parseBytesReply(cn *pool.Conn, line []byte) ([]byte, error) { - if isNilReply(line) { - return nil, Nil - } - - replyLen, err := strconv.Atoi(bytesToString(line[1:])) - if err != nil { - return nil, err - } - - b, err := cn.ReadN(replyLen + 2) - if err != nil { - return nil, err - } - - return b[:replyLen], nil -} - -func readBytesReply(cn *pool.Conn) ([]byte, error) { - line, err := readLine(cn) - if err != nil { - return nil, err - } - switch line[0] { - case errorReply: - return nil, parseErrorReply(cn, line) - case stringReply: - return parseBytesReply(cn, line) - case statusReply: - return parseStatusReply(cn, line) - default: - return nil, fmt.Errorf("redis: can't parse string reply: %.100q", line) - } -} - -func readStringReply(cn *pool.Conn) (string, error) { - b, err := readBytesReply(cn) - if err != nil { - return "", err - } - return string(b), nil -} - -func readFloatReply(cn *pool.Conn) (float64, error) { - b, err := readBytesReply(cn) - if err != nil { - return 0, err - } - return strconv.ParseFloat(bytesToString(b), 64) -} - -func parseArrayHeader(cn *pool.Conn, line []byte) (int64, error) { - if isNilReply(line) { - return 0, Nil - } - - n, err := strconv.ParseInt(bytesToString(line[1:]), 10, 64) - if err != nil { - return 0, err - } - return n, nil -} - -func parseArrayReply(cn *pool.Conn, p multiBulkParser, line []byte) (interface{}, error) { - n, err := parseArrayHeader(cn, line) - if err != nil { - return nil, err - } - return p(cn, n) -} - -func readArrayHeader(cn *pool.Conn) (int64, error) { - line, err := readLine(cn) - if err != nil { - return 0, err - } - switch line[0] { - case errorReply: - return 0, parseErrorReply(cn, line) - case arrayReply: - return parseArrayHeader(cn, line) - default: - return 0, fmt.Errorf("redis: can't parse array reply: %.100q", line) - } -} - -func readArrayReply(cn *pool.Conn, p multiBulkParser) (interface{}, error) { - line, err := readLine(cn) - if err != nil { - return nil, err - } - switch line[0] { - case errorReply: - return nil, parseErrorReply(cn, line) - case arrayReply: - return parseArrayReply(cn, p, line) - default: - return nil, fmt.Errorf("redis: can't parse array reply: %.100q", line) - } -} - -func readReply(cn *pool.Conn, p multiBulkParser) (interface{}, error) { - line, err := readLine(cn) - if err != nil { - return nil, err - } - - switch line[0] { - case errorReply: - return nil, parseErrorReply(cn, line) - case statusReply: - return parseStatusReply(cn, line) - case intReply: - return parseIntReply(cn, line) - case stringReply: - return parseBytesReply(cn, line) - case arrayReply: - return parseArrayReply(cn, p, line) - } - return nil, fmt.Errorf("redis: can't parse %.100q", line) -} - -func readScanReply(cn *pool.Conn) ([]string, uint64, error) { - n, err := readArrayHeader(cn) - if err != nil { - return nil, 0, err - } - if n != 2 { - return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n) - } - - b, err := readBytesReply(cn) - if err != nil { - return nil, 0, err - } - - cursor, err := strconv.ParseUint(bytesToString(b), 10, 64) - if err != nil { - return nil, 0, err - } - - n, err = readArrayHeader(cn) - if err != nil { - return nil, 0, err - } - - keys := make([]string, n) - for i := int64(0); i < n; i++ { - key, err := readStringReply(cn) - if err != nil { - return nil, 0, err - } - keys[i] = key - } - - return keys, cursor, err -} - -func sliceParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func sliceParser(rd *proto.Reader, n int64) (interface{}, error) { vals := make([]interface{}, 0, n) for i := int64(0); i < n; i++ { - v, err := readReply(cn, sliceParser) + v, err := rd.ReadReply(sliceParser) if err == Nil { vals = append(vals, nil) } else if err != nil { @@ -457,10 +29,11 @@ func sliceParser(cn *pool.Conn, n int64) (interface{}, error) { return vals, nil } -func intSliceParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func intSliceParser(rd *proto.Reader, n int64) (interface{}, error) { ints := make([]int64, 0, n) for i := int64(0); i < n; i++ { - n, err := readIntReply(cn) + n, err := rd.ReadIntReply() if err != nil { return nil, err } @@ -469,10 +42,11 @@ func intSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return ints, nil } -func boolSliceParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func boolSliceParser(rd *proto.Reader, n int64) (interface{}, error) { bools := make([]bool, 0, n) for i := int64(0); i < n; i++ { - n, err := readIntReply(cn) + n, err := rd.ReadIntReply() if err != nil { return nil, err } @@ -481,10 +55,11 @@ func boolSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return bools, nil } -func stringSliceParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func stringSliceParser(rd *proto.Reader, n int64) (interface{}, error) { ss := make([]string, 0, n) for i := int64(0); i < n; i++ { - s, err := readStringReply(cn) + s, err := rd.ReadStringReply() if err == Nil { ss = append(ss, "") } else if err != nil { @@ -496,10 +71,11 @@ func stringSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return ss, nil } -func floatSliceParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func floatSliceParser(rd *proto.Reader, n int64) (interface{}, error) { nn := make([]float64, 0, n) for i := int64(0); i < n; i++ { - n, err := readFloatReply(cn) + n, err := rd.ReadFloatReply() if err != nil { return nil, err } @@ -508,15 +84,16 @@ func floatSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return nn, nil } -func stringStringMapParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func stringStringMapParser(rd *proto.Reader, n int64) (interface{}, error) { m := make(map[string]string, n/2) for i := int64(0); i < n; i += 2 { - key, err := readStringReply(cn) + key, err := rd.ReadStringReply() if err != nil { return nil, err } - value, err := readStringReply(cn) + value, err := rd.ReadStringReply() if err != nil { return nil, err } @@ -526,15 +103,16 @@ func stringStringMapParser(cn *pool.Conn, n int64) (interface{}, error) { return m, nil } -func stringIntMapParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func stringIntMapParser(rd *proto.Reader, n int64) (interface{}, error) { m := make(map[string]int64, n/2) for i := int64(0); i < n; i += 2 { - key, err := readStringReply(cn) + key, err := rd.ReadStringReply() if err != nil { return nil, err } - n, err := readIntReply(cn) + n, err := rd.ReadIntReply() if err != nil { return nil, err } @@ -544,19 +122,20 @@ func stringIntMapParser(cn *pool.Conn, n int64) (interface{}, error) { return m, nil } -func zSliceParser(cn *pool.Conn, n int64) (interface{}, error) { +// Implements proto.MultiBulkParse +func zSliceParser(rd *proto.Reader, n int64) (interface{}, error) { zz := make([]Z, n/2) for i := int64(0); i < n; i += 2 { var err error z := &zz[i/2] - z.Member, err = readStringReply(cn) + z.Member, err = rd.ReadStringReply() if err != nil { return nil, err } - z.Score, err = readFloatReply(cn) + z.Score, err = rd.ReadFloatReply() if err != nil { return nil, err } @@ -564,10 +143,11 @@ func zSliceParser(cn *pool.Conn, n int64) (interface{}, error) { return zz, nil } -func clusterSlotsParser(cn *pool.Conn, slotNum int64) (interface{}, error) { - slots := make([]ClusterSlot, slotNum) - for slotInd := 0; slotInd < len(slots); slotInd++ { - n, err := readArrayHeader(cn) +// Implements proto.MultiBulkParse +func clusterSlotsParser(rd *proto.Reader, n int64) (interface{}, error) { + slots := make([]ClusterSlot, n) + for i := 0; i < len(slots); i++ { + n, err := rd.ReadArrayLen() if err != nil { return nil, err } @@ -576,19 +156,19 @@ func clusterSlotsParser(cn *pool.Conn, slotNum int64) (interface{}, error) { return nil, err } - start, err := readIntReply(cn) + start, err := rd.ReadIntReply() if err != nil { return nil, err } - end, err := readIntReply(cn) + end, err := rd.ReadIntReply() if err != nil { return nil, err } nodes := make([]ClusterNode, n-2) - for nodeInd := 0; nodeInd < len(nodes); nodeInd++ { - n, err := readArrayHeader(cn) + for j := 0; j < len(nodes); j++ { + n, err := rd.ReadArrayLen() if err != nil { return nil, err } @@ -597,27 +177,27 @@ func clusterSlotsParser(cn *pool.Conn, slotNum int64) (interface{}, error) { return nil, err } - ip, err := readStringReply(cn) + ip, err := rd.ReadStringReply() if err != nil { return nil, err } - port, err := readIntReply(cn) + port, err := rd.ReadIntReply() if err != nil { return nil, err } - nodes[nodeInd].Addr = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) + nodes[j].Addr = net.JoinHostPort(ip, strconv.FormatInt(port, 10)) if n == 3 { - id, err := readStringReply(cn) + id, err := rd.ReadStringReply() if err != nil { return nil, err } - nodes[nodeInd].Id = id + nodes[j].Id = id } } - slots[slotInd] = ClusterSlot{ + slots[i] = ClusterSlot{ Start: int(start), End: int(end), Nodes: nodes, @@ -626,29 +206,29 @@ func clusterSlotsParser(cn *pool.Conn, slotNum int64) (interface{}, error) { return slots, nil } -func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { - return func(cn *pool.Conn, n int64) (interface{}, error) { +func newGeoLocationParser(q *GeoRadiusQuery) proto.MultiBulkParse { + return func(rd *proto.Reader, n int64) (interface{}, error) { var loc GeoLocation var err error - loc.Name, err = readStringReply(cn) + loc.Name, err = rd.ReadStringReply() if err != nil { return nil, err } if q.WithDist { - loc.Dist, err = readFloatReply(cn) + loc.Dist, err = rd.ReadFloatReply() if err != nil { return nil, err } } if q.WithGeoHash { - loc.GeoHash, err = readIntReply(cn) + loc.GeoHash, err = rd.ReadIntReply() if err != nil { return nil, err } } if q.WithCoord { - n, err := readArrayHeader(cn) + n, err := rd.ReadArrayLen() if err != nil { return nil, err } @@ -656,11 +236,11 @@ func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { return nil, fmt.Errorf("got %d coordinates, expected 2", n) } - loc.Longitude, err = readFloatReply(cn) + loc.Longitude, err = rd.ReadFloatReply() if err != nil { return nil, err } - loc.Latitude, err = readFloatReply(cn) + loc.Latitude, err = rd.ReadFloatReply() if err != nil { return nil, err } @@ -670,11 +250,11 @@ func newGeoLocationParser(q *GeoRadiusQuery) multiBulkParser { } } -func newGeoLocationSliceParser(q *GeoRadiusQuery) multiBulkParser { - return func(cn *pool.Conn, n int64) (interface{}, error) { +func newGeoLocationSliceParser(q *GeoRadiusQuery) proto.MultiBulkParse { + return func(rd *proto.Reader, n int64) (interface{}, error) { locs := make([]GeoLocation, 0, n) for i := int64(0); i < n; i++ { - v, err := readReply(cn, newGeoLocationParser(q)) + v, err := rd.ReadReply(newGeoLocationParser(q)) if err != nil { return nil, err } @@ -693,7 +273,7 @@ func newGeoLocationSliceParser(q *GeoRadiusQuery) multiBulkParser { } } -func commandInfoParser(cn *pool.Conn, n int64) (interface{}, error) { +func commandInfoParser(rd *proto.Reader, n int64) (interface{}, error) { var cmd CommandInfo var err error @@ -701,36 +281,36 @@ func commandInfoParser(cn *pool.Conn, n int64) (interface{}, error) { return nil, fmt.Errorf("redis: got %d elements in COMMAND reply, wanted 6") } - cmd.Name, err = readStringReply(cn) + cmd.Name, err = rd.ReadStringReply() if err != nil { return nil, err } - arity, err := readIntReply(cn) + arity, err := rd.ReadIntReply() if err != nil { return nil, err } cmd.Arity = int8(arity) - flags, err := readReply(cn, stringSliceParser) + flags, err := rd.ReadReply(stringSliceParser) if err != nil { return nil, err } cmd.Flags = flags.([]string) - firstKeyPos, err := readIntReply(cn) + firstKeyPos, err := rd.ReadIntReply() if err != nil { return nil, err } cmd.FirstKeyPos = int8(firstKeyPos) - lastKeyPos, err := readIntReply(cn) + lastKeyPos, err := rd.ReadIntReply() if err != nil { return nil, err } cmd.LastKeyPos = int8(lastKeyPos) - stepCount, err := readIntReply(cn) + stepCount, err := rd.ReadIntReply() if err != nil { return nil, err } @@ -746,10 +326,10 @@ func commandInfoParser(cn *pool.Conn, n int64) (interface{}, error) { return &cmd, nil } -func commandInfoSliceParser(cn *pool.Conn, n int64) (interface{}, error) { +func commandInfoSliceParser(rd *proto.Reader, n int64) (interface{}, error) { m := make(map[string]*CommandInfo, n) for i := int64(0); i < n; i++ { - v, err := readReply(cn, commandInfoParser) + v, err := rd.ReadReply(commandInfoParser) if err != nil { return nil, err } diff --git a/parser_test.go b/parser_test.go deleted file mode 100644 index d8fa360955..0000000000 --- a/parser_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package redis - -import ( - "bufio" - "bytes" - "testing" - - "gopkg.in/redis.v4/internal/pool" -) - -func BenchmarkParseReplyStatus(b *testing.B) { - benchmarkParseReply(b, "+OK\r\n", nil, false) -} - -func BenchmarkParseReplyInt(b *testing.B) { - benchmarkParseReply(b, ":1\r\n", nil, false) -} - -func BenchmarkParseReplyError(b *testing.B) { - benchmarkParseReply(b, "-Error message\r\n", nil, true) -} - -func BenchmarkParseReplyString(b *testing.B) { - benchmarkParseReply(b, "$5\r\nhello\r\n", nil, false) -} - -func BenchmarkParseReplySlice(b *testing.B) { - benchmarkParseReply(b, "*2\r\n$5\r\nhello\r\n$5\r\nworld\r\n", sliceParser, false) -} - -func benchmarkParseReply(b *testing.B, reply string, p multiBulkParser, wanterr bool) { - buf := &bytes.Buffer{} - for i := 0; i < b.N; i++ { - buf.WriteString(reply) - } - cn := &pool.Conn{ - Rd: bufio.NewReader(buf), - Buf: make([]byte, 4096), - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - _, err := readReply(cn, p) - if !wanterr && err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkAppendArgs(b *testing.B) { - buf := make([]byte, 0, 64) - args := []interface{}{"hello", "world", "foo", "bar"} - for i := 0; i < b.N; i++ { - appendArgs(buf, args) - } -} diff --git a/pipeline.go b/pipeline.go index fed6e0c1a0..bf80d408a8 100644 --- a/pipeline.go +++ b/pipeline.go @@ -4,6 +4,7 @@ import ( "sync" "sync/atomic" + "gopkg.in/redis.v4/internal/errors" "gopkg.in/redis.v4/internal/pool" ) @@ -99,7 +100,7 @@ func execCmds(cn *pool.Conn, cmds []Cmder) ([]Cmder, error) { if firstCmdErr == nil { firstCmdErr = err } - if shouldRetry(err) { + if errors.IsRetryable(err) { failedCmds = append(failedCmds, cmd) } } diff --git a/pipeline_test.go b/pipeline_test.go index 939c8b8499..c3a37f6651 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -156,6 +156,8 @@ var _ = Describe("Pipelining", func() { wg.Add(N) for i := 0; i < N; i++ { go func() { + defer GinkgoRecover() + pipeline.Ping() wg.Done() }() diff --git a/pubsub.go b/pubsub.go index 5bbda5966a..6c72b8f76c 100644 --- a/pubsub.go +++ b/pubsub.go @@ -6,6 +6,7 @@ import ( "time" "gopkg.in/redis.v4/internal" + "gopkg.in/redis.v4/internal/errors" "gopkg.in/redis.v4/internal/pool" ) @@ -248,7 +249,7 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { for { msgi, err := c.ReceiveTimeout(timeout) if err != nil { - if !isNetworkError(err) { + if !errors.IsNetwork(err) { return nil, err } diff --git a/redis.go b/redis.go index 5c93db0010..b67d422727 100644 --- a/redis.go +++ b/redis.go @@ -5,9 +5,13 @@ import ( "log" "gopkg.in/redis.v4/internal" + "gopkg.in/redis.v4/internal/errors" "gopkg.in/redis.v4/internal/pool" ) +// Redis nil reply, .e.g. when key does not exist. +const Nil = errors.Nil + func SetLogger(logger *log.Logger) { internal.Logger = logger } @@ -38,7 +42,7 @@ func (c *baseClient) conn() (*pool.Conn, error) { } func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { - if isBadConn(err, allowTimeout) { + if errors.IsBadConn(err, allowTimeout) { _ = c.connPool.Remove(cn, err) return false } @@ -97,7 +101,7 @@ func (c *baseClient) Process(cmd Cmder) error { if err := writeCmd(cn, cmd); err != nil { c.putConn(cn, err, false) cmd.setErr(err) - if err != nil && shouldRetry(err) { + if err != nil && errors.IsRetryable(err) { continue } return err @@ -105,7 +109,7 @@ func (c *baseClient) Process(cmd Cmder) error { err = cmd.readReply(cn) c.putConn(cn, err, readTimeout != nil) - if err != nil && shouldRetry(err) { + if err != nil && errors.IsRetryable(err) { continue } diff --git a/safe.go b/safe.go deleted file mode 100644 index d66dc56a09..0000000000 --- a/safe.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build appengine - -package redis - -func bytesToString(b []byte) string { - return string(b) -} diff --git a/tx.go b/tx.go index 5ed223c043..ca425ebba9 100644 --- a/tx.go +++ b/tx.go @@ -5,9 +5,14 @@ import ( "fmt" "gopkg.in/redis.v4/internal" + ierrors "gopkg.in/redis.v4/internal/errors" "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v4/internal/proto" ) +// Redis transaction failed. +const TxFailedErr = ierrors.RedisError("redis: transaction failed") + var errDiscard = errors.New("redis: Discard can be used only inside Exec") // Tx implements Redis transactions as described in @@ -166,7 +171,7 @@ func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { } // Parse number of replies. - line, err := readLine(cn) + line, err := cn.Rd.ReadLine() if err != nil { if err == Nil { err = TxFailedErr @@ -174,7 +179,7 @@ func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { setCmdsErr(cmds[1:len(cmds)-1], err) return err } - if line[0] != '*' { + if line[0] != proto.ArrayReply { err := fmt.Errorf("redis: expected '*', but got line %q", line) setCmdsErr(cmds[1:len(cmds)-1], err) return err diff --git a/unsafe.go b/unsafe.go deleted file mode 100644 index 3cd8d1c182..0000000000 --- a/unsafe.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !appengine - -package redis - -import ( - "reflect" - "unsafe" -) - -func bytesToString(b []byte) string { - bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - strHeader := reflect.StringHeader{bytesHeader.Data, bytesHeader.Len} - return *(*string)(unsafe.Pointer(&strHeader)) -} From 65a64fe7aac20b56d8da4a8fd40d7c4bcffcc432 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 8 Jul 2016 09:24:02 +0000 Subject: [PATCH 0188/1746] Fix Del command case. --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 412adaec0c..0d8e8b2db4 100644 --- a/commands.go +++ b/commands.go @@ -82,7 +82,7 @@ func (c *statefulCmdable) Select(index int) *StatusCmd { func (c *cmdable) Del(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "DEL" + args[0] = "del" for i, key := range keys { args[1+i] = key } From 261cf7ae704fd1a8ba71ab34c53629d547068d80 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 8 Jul 2016 09:24:02 +0000 Subject: [PATCH 0189/1746] Fix Del command case. --- cluster.go | 1 + commands.go | 4 ++-- ring.go | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 217cadd942..07de993274 100644 --- a/cluster.go +++ b/cluster.go @@ -235,6 +235,7 @@ func (c *ClusterClient) slotClosestNode(slot int) (*clusterNode, error) { func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmdInfo(cmd.arg(0)) if cmdInfo == nil { + internal.Logf("info for cmd=%s not found", cmd.arg(0)) node, err := c.randomNode() return 0, node, err } diff --git a/commands.go b/commands.go index 412adaec0c..db9d8061a5 100644 --- a/commands.go +++ b/commands.go @@ -82,7 +82,7 @@ func (c *statefulCmdable) Select(index int) *StatusCmd { func (c *cmdable) Del(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "DEL" + args[0] = "del" for i, key := range keys { args[1+i] = key } @@ -550,7 +550,7 @@ func (c *cmdable) Set(key string, value interface{}, expiration time.Duration) * func (c *cmdable) SetBit(key string, offset int64, value int) *IntCmd { cmd := NewIntCmd( - "SETBIT", + "setbit", key, offset, value, diff --git a/ring.go b/ring.go index 8c998a8395..b0c017930c 100644 --- a/ring.go +++ b/ring.go @@ -169,6 +169,7 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { func (c *Ring) cmdFirstKey(cmd Cmder) string { cmdInfo := c.cmdInfo(cmd.arg(0)) if cmdInfo == nil { + internal.Logf("info for cmd=%s not found", cmd.arg(0)) return "" } return cmd.arg(int(cmdInfo.FirstKeyPos)) From dcdf3fc9c3a226c0929c86e07a40b402eaafbea5 Mon Sep 17 00:00:00 2001 From: yzprofile Date: Thu, 21 Jul 2016 13:50:28 +0800 Subject: [PATCH 0190/1746] Feature: Export cmdable as an interface --- commands.go | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/commands.go b/commands.go index db9d8061a5..00b5fcec01 100644 --- a/commands.go +++ b/commands.go @@ -44,6 +44,209 @@ type cmdable struct { process func(cmd Cmder) error } +type Commander interface { + Echo(message interface{}) *StringCmd + Ping() *StatusCmd + Quit() *StatusCmd + Del(keys ...string) *IntCmd + Dump(key string) *StringCmd + Exists(key string) *BoolCmd + Expire(key string, expiration time.Duration) *BoolCmd + ExpireAt(key string, tm time.Time) *BoolCmd + Keys(pattern string) *StringSliceCmd + Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd + Move(key string, db int64) *BoolCmd + ObjectRefCount(keys ...string) *IntCmd + ObjectEncoding(keys ...string) *StringCmd + ObjectIdleTime(keys ...string) *DurationCmd + Persist(key string) *BoolCmd + PExpire(key string, expiration time.Duration) *BoolCmd + PExpireAt(key string, tm time.Time) *BoolCmd + PTTL(key string) *DurationCmd + RandomKey() *StringCmd + Rename(key, newkey string) *StatusCmd + RenameNX(key, newkey string) *BoolCmd + Restore(key string, ttl time.Duration, value string) *StatusCmd + RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd + Sort(key string, sort Sort) *StringSliceCmd + SortInterfaces(key string, sort Sort) *SliceCmd + TTL(key string) *DurationCmd + Type(key string) *StatusCmd + Scan(cursor uint64, match string, count int64) Scanner + SScan(key string, cursor uint64, match string, count int64) Scanner + HScan(key string, cursor uint64, match string, count int64) Scanner + ZScan(key string, cursor uint64, match string, count int64) Scanner + Append(key, value string) *IntCmd + BitCount(key string, bitCount *BitCount) *IntCmd + bitOp(op, destKey string, keys ...string) *IntCmd + BitOpAnd(destKey string, keys ...string) *IntCmd + BitOpOr(destKey string, keys ...string) *IntCmd + BitOpXor(destKey string, keys ...string) *IntCmd + BitOpNot(destKey string, key string) *IntCmd + BitPos(key string, bit int64, pos ...int64) *IntCmd + Decr(key string) *IntCmd + DecrBy(key string, decrement int64) *IntCmd + Get(key string) *StringCmd + GetBit(key string, offset int64) *IntCmd + GetRange(key string, start, end int64) *StringCmd + GetSet(key string, value interface{}) *StringCmd + Incr(key string) *IntCmd + IncrBy(key string, value int64) *IntCmd + IncrByFloat(key string, value float64) *FloatCmd + MGet(keys ...string) *SliceCmd + MSet(pairs ...interface{}) *StatusCmd + MSetNX(pairs ...interface{}) *BoolCmd + Set(key string, value interface{}, expiration time.Duration) *StatusCmd + SetBit(key string, offset int64, value int) *IntCmd + SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd + SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd + SetRange(key string, offset int64, value string) *IntCmd + StrLen(key string) *IntCmd + HDel(key string, fields ...string) *IntCmd + HExists(key, field string) *BoolCmd + HGet(key, field string) *StringCmd + HGetAll(key string) *StringStringMapCmd + HIncrBy(key, field string, incr int64) *IntCmd + HIncrByFloat(key, field string, incr float64) *FloatCmd + HKeys(key string) *StringSliceCmd + HLen(key string) *IntCmd + HMGet(key string, fields ...string) *SliceCmd + HMSet(key string, fields map[string]string) *StatusCmd + HSet(key, field, value string) *BoolCmd + HSetNX(key, field, value string) *BoolCmd + HVals(key string) *StringSliceCmd + BLPop(timeout time.Duration, keys ...string) *StringSliceCmd + BRPop(timeout time.Duration, keys ...string) *StringSliceCmd + BRPopLPush(source, destination string, timeout time.Duration) *StringCmd + LIndex(key string, index int64) *StringCmd + LInsert(key, op string, pivot, value interface{}) *IntCmd + LInsertBefore(key string, pivot, value interface{}) *IntCmd + LInsertAfter(key string, pivot, value interface{}) *IntCmd + LLen(key string) *IntCmd + LPop(key string) *StringCmd + LPush(key string, values ...interface{}) *IntCmd + LPushX(key string, value interface{}) *IntCmd + LRange(key string, start, stop int64) *StringSliceCmd + LRem(key string, count int64, value interface{}) *IntCmd + LSet(key string, index int64, value interface{}) *StatusCmd + LTrim(key string, start, stop int64) *StatusCmd + RPop(key string) *StringCmd + RPopLPush(source, destination string) *StringCmd + RPush(key string, values ...interface{}) *IntCmd + RPushX(key string, value interface{}) *IntCmd + SAdd(key string, members ...interface{}) *IntCmd + SCard(key string) *IntCmd + SDiff(keys ...string) *StringSliceCmd + SDiffStore(destination string, keys ...string) *IntCmd + SInter(keys ...string) *StringSliceCmd + SInterStore(destination string, keys ...string) *IntCmd + SIsMember(key string, member interface{}) *BoolCmd + SMembers(key string) *StringSliceCmd + SMove(source, destination string, member interface{}) *BoolCmd + SPop(key string) *StringCmd + SPopN(key string, count int64) *StringSliceCmd + SRandMember(key string) *StringCmd + SRandMemberN(key string, count int64) *StringSliceCmd + SRem(key string, members ...interface{}) *IntCmd + SUnion(keys ...string) *StringSliceCmd + SUnionStore(destination string, keys ...string) *IntCmd + zAdd(a []interface{}, n int, members ...Z) *IntCmd + ZAdd(key string, members ...Z) *IntCmd + ZAddNX(key string, members ...Z) *IntCmd + ZAddXX(key string, members ...Z) *IntCmd + ZAddCh(key string, members ...Z) *IntCmd + ZAddNXCh(key string, members ...Z) *IntCmd + ZAddXXCh(key string, members ...Z) *IntCmd + zIncr(a []interface{}, n int, members ...Z) *FloatCmd + ZIncr(key string, member Z) *FloatCmd + ZIncrNX(key string, member Z) *FloatCmd + ZIncrXX(key string, member Z) *FloatCmd + ZCard(key string) *IntCmd + ZCount(key, min, max string) *IntCmd + ZIncrBy(key string, increment float64, member string) *FloatCmd + ZInterStore(destination string, store ZStore, keys ...string) *IntCmd + zRange(key string, start, stop int64, withScores bool) *StringSliceCmd + ZRange(key string, start, stop int64) *StringSliceCmd + ZRangeWithScores(key string, start, stop int64) *ZSliceCmd + zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd + ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd + ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd + ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd + ZRank(key, member string) *IntCmd + ZRem(key string, members ...interface{}) *IntCmd + ZRemRangeByRank(key string, start, stop int64) *IntCmd + ZRemRangeByScore(key, min, max string) *IntCmd + ZRevRange(key string, start, stop int64) *StringSliceCmd + ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd + zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd + ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd + ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd + ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd + ZRevRank(key, member string) *IntCmd + ZScore(key, member string) *FloatCmd + ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd + PFAdd(key string, els ...interface{}) *IntCmd + PFCount(keys ...string) *IntCmd + PFMerge(dest string, keys ...string) *StatusCmd + BgRewriteAOF() *StatusCmd + BgSave() *StatusCmd + ClientKill(ipPort string) *StatusCmd + ClientList() *StringCmd + ClientPause(dur time.Duration) *BoolCmd + ClientSetName(name string) *BoolCmd + ConfigGet(parameter string) *SliceCmd + ConfigResetStat() *StatusCmd + ConfigSet(parameter, value string) *StatusCmd + DbSize() *IntCmd + FlushAll() *StatusCmd + FlushDb() *StatusCmd + Info(section ...string) *StringCmd + LastSave() *IntCmd + Save() *StatusCmd + shutdown(modifier string) *StatusCmd + Shutdown() *StatusCmd + ShutdownSave() *StatusCmd + ShutdownNoSave() *StatusCmd + SlaveOf(host, port string) *StatusCmd + SlowLog() + Sync() + Time() *StringSliceCmd + Eval(script string, keys []string, args ...interface{}) *Cmd + EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd + ScriptExists(scripts ...string) *BoolSliceCmd + ScriptFlush() *StatusCmd + ScriptKill() *StatusCmd + ScriptLoad(script string) *StringCmd + DebugObject(key string) *StringCmd + PubSubChannels(pattern string) *StringSliceCmd + PubSubNumSub(channels ...string) *StringIntMapCmd + PubSubNumPat() *IntCmd + ClusterSlots() *ClusterSlotsCmd + ClusterNodes() *StringCmd + ClusterMeet(host, port string) *StatusCmd + ClusterForget(nodeID string) *StatusCmd + ClusterReplicate(nodeID string) *StatusCmd + ClusterResetSoft() *StatusCmd + ClusterResetHard() *StatusCmd + ClusterInfo() *StringCmd + ClusterKeySlot(key string) *IntCmd + ClusterCountFailureReports(nodeID string) *IntCmd + ClusterCountKeysInSlot(slot int) *IntCmd + ClusterDelSlots(slots ...int) *StatusCmd + ClusterDelSlotsRange(min, max int) *StatusCmd + ClusterSaveConfig() *StatusCmd + ClusterSlaves(nodeID string) *StringSliceCmd + ClusterFailover() *StatusCmd + ClusterAddSlots(slots ...int) *StatusCmd + ClusterAddSlotsRange(min, max int) *StatusCmd + GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd + GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd + GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd + GeoDist(key string, member1, member2, unit string) *FloatCmd + GeoHash(key string, members ...string) *StringSliceCmd + Command() *CommandsInfoCmd +} + type statefulCmdable struct { process func(cmd Cmder) error } From 4210c090b14cdbf1553d6b1a8db7dac0c8cb9ca1 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 21 Jul 2016 13:04:40 +0000 Subject: [PATCH 0191/1746] Move Publish channel to cmdable. Remove method that was deprecated in v3. --- commands.go | 7 +++++++ pool_test.go | 4 ++-- pubsub.go | 29 ----------------------------- redis.go | 21 +++++++++++++++++++++ redis_test.go | 10 +++++++--- 5 files changed, 37 insertions(+), 34 deletions(-) diff --git a/commands.go b/commands.go index db9d8061a5..81a691e8a5 100644 --- a/commands.go +++ b/commands.go @@ -1618,6 +1618,13 @@ func (c *cmdable) DebugObject(key string) *StringCmd { //------------------------------------------------------------------------------ +// Publish posts the message to the channel. +func (c *cmdable) Publish(channel, message string) *IntCmd { + cmd := NewIntCmd("PUBLISH", channel, message) + c.process(cmd) + return cmd +} + func (c *cmdable) PubSubChannels(pattern string) *StringSliceCmd { args := []interface{}{"pubsub", "channels"} if pattern != "*" { diff --git a/pool_test.go b/pool_test.go index 31bf9687ab..c1d2f6810b 100644 --- a/pool_test.go +++ b/pool_test.go @@ -81,8 +81,8 @@ var _ = Describe("pool", func() { connPool.(*pool.ConnPool).DialLimiter = nil perform(1000, func(id int) { - pubsub := client.PubSub() - Expect(pubsub.Subscribe()).NotTo(HaveOccurred()) + pubsub, err := client.Subscribe() + Expect(err).NotTo(HaveOccurred()) Expect(pubsub.Close()).NotTo(HaveOccurred()) }) diff --git a/pubsub.go b/pubsub.go index 6c72b8f76c..1d1fc939b8 100644 --- a/pubsub.go +++ b/pubsub.go @@ -10,13 +10,6 @@ import ( "gopkg.in/redis.v4/internal/pool" ) -// Posts a message to the given channel. -func (c *Client) Publish(channel, message string) *IntCmd { - req := NewIntCmd("PUBLISH", channel, message) - c.Process(req) - return req -} - // PubSub implements Pub/Sub commands as described in // http://redis.io/topics/pubsub. It's NOT safe for concurrent use by // multiple goroutines. @@ -29,28 +22,6 @@ type PubSub struct { nsub int // number of active subscriptions } -// Deprecated. Use Subscribe/PSubscribe instead. -func (c *Client) PubSub() *PubSub { - return &PubSub{ - base: baseClient{ - opt: c.opt, - connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), - }, - } -} - -// Subscribes the client to the specified channels. -func (c *Client) Subscribe(channels ...string) (*PubSub, error) { - pubsub := c.PubSub() - return pubsub, pubsub.Subscribe(channels...) -} - -// Subscribes the client to the given patterns. -func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { - pubsub := c.PubSub() - return pubsub, pubsub.PSubscribe(channels...) -} - func (c *PubSub) subscribe(redisCmd string, channels ...string) error { cn, err := c.base.conn() if err != nil { diff --git a/redis.go b/redis.go index b67d422727..b8ceab550e 100644 --- a/redis.go +++ b/redis.go @@ -215,3 +215,24 @@ func (c *Client) pipelineExec(cmds []Cmder) error { } return retErr } + +func (c *Client) pubSub() *PubSub { + return &PubSub{ + base: baseClient{ + opt: c.opt, + connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), + }, + } +} + +// Subscribe subscribes the client to the specified channels. +func (c *Client) Subscribe(channels ...string) (*PubSub, error) { + pubsub := c.pubSub() + return pubsub, pubsub.Subscribe(channels...) +} + +// PSubscribe subscribes the client to the given patterns. +func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { + pubsub := c.pubSub() + return pubsub, pubsub.PSubscribe(channels...) +} diff --git a/redis_test.go b/redis_test.go index bbb00e0a31..4b62f3ed1a 100644 --- a/redis_test.go +++ b/redis_test.go @@ -57,10 +57,10 @@ var _ = Describe("Client", func() { }) It("should close pubsub without closing the client", func() { - pubsub := client.PubSub() + pubsub, err := client.Subscribe() Expect(pubsub.Close()).NotTo(HaveOccurred()) - _, err := pubsub.Receive() + _, err = pubsub.Receive() Expect(err).To(MatchError("redis: client is closed")) Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) @@ -90,8 +90,12 @@ var _ = Describe("Client", func() { }) It("should close pubsub when client is closed", func() { - pubsub := client.PubSub() + pubsub, err := client.Subscribe() Expect(client.Close()).NotTo(HaveOccurred()) + + _, err = pubsub.Receive() + Expect(err).To(HaveOccurred()) + Expect(pubsub.Close()).NotTo(HaveOccurred()) }) From ff8419f6430c9f57c7857703708684c8b3737d91 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 21 Jul 2016 13:24:58 +0000 Subject: [PATCH 0192/1746] Rename Commander to Cmdable to match internal name. --- commands.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/commands.go b/commands.go index 112a547796..769dcb1c0c 100644 --- a/commands.go +++ b/commands.go @@ -40,11 +40,7 @@ func formatSec(dur time.Duration) string { return strconv.FormatInt(int64(dur/time.Second), 10) } -type cmdable struct { - process func(cmd Cmder) error -} - -type Commander interface { +type Cmdable interface { Echo(message interface{}) *StringCmd Ping() *StatusCmd Quit() *StatusCmd @@ -208,8 +204,6 @@ type Commander interface { ShutdownSave() *StatusCmd ShutdownNoSave() *StatusCmd SlaveOf(host, port string) *StatusCmd - SlowLog() - Sync() Time() *StringSliceCmd Eval(script string, keys []string, args ...interface{}) *Cmd EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd @@ -247,6 +241,12 @@ type Commander interface { Command() *CommandsInfoCmd } +type cmdable struct { + process func(cmd Cmder) error +} + +var _ Cmdable = (*cmdable)(nil) + type statefulCmdable struct { process func(cmd Cmder) error } @@ -1739,6 +1739,7 @@ func (c *cmdable) Sync() { panic("not implemented") } +// TODO: add TimeCmd and use it here func (c *cmdable) Time() *StringSliceCmd { cmd := NewStringSliceCmd("time") c.process(cmd) From 91993128b0eade4284de442b3c4276e2838566d2 Mon Sep 17 00:00:00 2001 From: Yin Jifeng Date: Thu, 4 Aug 2016 12:52:00 +0800 Subject: [PATCH 0193/1746] fix iterator for hscan/sscan/zscan --- iterator.go | 6 +++++- iterator_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/iterator.go b/iterator.go index e9dfbad938..0a3df06326 100644 --- a/iterator.go +++ b/iterator.go @@ -52,7 +52,11 @@ func (it *ScanIterator) Next() bool { } // Fetch next page. - it.ScanCmd._args[1] = it.ScanCmd.cursor + if it.ScanCmd._args[0] == "scan" { + it.ScanCmd._args[1] = it.ScanCmd.cursor + } else { + it.ScanCmd._args[2] = it.ScanCmd.cursor + } it.ScanCmd.reset() it.client.process(it.ScanCmd) if it.ScanCmd.Err() != nil { diff --git a/iterator_test.go b/iterator_test.go index 327feeb5bd..f752f68c0f 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -21,6 +21,16 @@ var _ = Describe("ScanIterator", func() { return err } + var hashKey = "K_HASHTEST" + var hashSeed = func(n int) error { + pipe := client.Pipeline() + for i := 1; i <= n; i++ { + pipe.HSet(hashKey, fmt.Sprintf("K%02d", i), "x").Err() + } + _, err := pipe.Exec() + return err + } + BeforeEach(func() { client = redis.NewClient(redisOptions()) Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) @@ -62,6 +72,20 @@ var _ = Describe("ScanIterator", func() { Expect(vals).To(ContainElement("K71")) }) + It("should hscan across multiple pages", func() { + Expect(hashSeed(71)).NotTo(HaveOccurred()) + + var vals []string + iter := client.HScan(hashKey, 0, "", 10).Iterator() + for iter.Next() { + vals = append(vals, iter.Val()) + } + Expect(iter.Err()).NotTo(HaveOccurred()) + Expect(vals).To(HaveLen(71 * 2)) + Expect(vals).To(ContainElement("K01")) + Expect(vals).To(ContainElement("K71")) + }) + It("should scan to page borders", func() { Expect(seed(20)).NotTo(HaveOccurred()) From 66f2eb1584ac814fc47a771dd6c73ec70d6161bb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 9 Aug 2016 13:32:08 +0000 Subject: [PATCH 0194/1746] ring: reduce HeartbeatFrequency. --- options.go | 31 +++++++++++++++---------------- ring.go | 22 ++++++++++++++-------- ring_test.go | 7 ++++--- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/options.go b/options.go index 4cd62f9316..e1ad28adc3 100644 --- a/options.go +++ b/options.go @@ -18,43 +18,42 @@ type Options struct { // Network and Addr options. Dialer func() (net.Conn, error) - // An optional password. Must match the password specified in the + // Optional password. Must match the password specified in the // requirepass server configuration option. Password string - // A database to be selected after connecting to server. + // Database to be selected after connecting to the server. DB int - // The maximum number of retries before giving up. + // Maximum number of retries before giving up. // Default is to not retry failed commands. MaxRetries int - // Sets the deadline for establishing new connections. If reached, - // dial will fail with a timeout. + // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration - // Sets the deadline for socket reads. If reached, commands will - // fail with a timeout instead of blocking. + // Timeout for socket reads. If reached, commands will fail + // with a timeout instead of blocking. ReadTimeout time.Duration - // Sets the deadline for socket writes. If reached, commands will - // fail with a timeout instead of blocking. + // Timeout for socket writes. If reached, commands will fail + // with a timeout instead of blocking. WriteTimeout time.Duration - // The maximum number of socket connections. + // Maximum number of socket connections. // Default is 10 connections. PoolSize int - // Specifies amount of time client waits for connection if all - // connections are busy before returning an error. + // Amount of time client waits for connection if all connections + // are busy before returning an error. // Default is 1 second. PoolTimeout time.Duration - // Specifies amount of time after which client closes idle - // connections. Should be less than server's timeout. + // Amount of time after which client closes idle connections. + // Should be less than server's timeout. // Default is to not close idle connections. IdleTimeout time.Duration - // The frequency of idle checks. + // Frequency of idle checks. // Default is 1 minute. IdleCheckFrequency time.Duration - // Enables read queries for a connection to a Redis Cluster slave node. + // Enables read only queries on slave nodes. ReadOnly bool } diff --git a/ring.go b/ring.go index b0c017930c..39733e2a76 100644 --- a/ring.go +++ b/ring.go @@ -12,16 +12,18 @@ import ( "gopkg.in/redis.v4/internal/pool" ) -var ( - errRingShardsDown = errors.New("redis: all ring shards are down") -) +var errRingShardsDown = errors.New("redis: all ring shards are down") // RingOptions are used to configure a ring client and should be // passed to NewRing. type RingOptions struct { - // A map of name => host:port addresses of ring shards. + // Map of name => host:port addresses of ring shards. Addrs map[string]string + // Frequency of PING commands sent to check shards availability. + // Shard is considered down after 3 subsequent failed checks. + HeartbeatFrequency time.Duration + // Following options are copied from Options struct. DB int @@ -39,7 +41,11 @@ type RingOptions struct { IdleCheckFrequency time.Duration } -func (opt *RingOptions) init() {} +func (opt *RingOptions) init() { + if opt.HeartbeatFrequency == 0 { + opt.HeartbeatFrequency = 500 * time.Millisecond + } +} func (opt *RingOptions) clientOptions() *Options { return &Options{ @@ -73,7 +79,7 @@ func (shard *ringShard) String() string { } func (shard *ringShard) IsDown() bool { - const threshold = 5 + const threshold = 3 return shard.down >= threshold } @@ -108,7 +114,7 @@ func (shard *ringShard) Vote(up bool) bool { // uses shards that are available to the client and does not do any // coordination when shard state is changed. // -// Ring should be used when you use multiple Redis servers for caching +// Ring should be used when you need multiple Redis servers for caching // and can tolerate losing data when one of the servers dies. // Otherwise you should use Redis Cluster. type Ring struct { @@ -224,7 +230,7 @@ func (c *Ring) rebalance() { // heartbeat monitors state of each shard in the ring. func (c *Ring) heartbeat() { - ticker := time.NewTicker(100 * time.Millisecond) + ticker := time.NewTicker(c.opt.HeartbeatFrequency) defer ticker.Stop() for _ = range ticker.C { var rebalance bool diff --git a/ring_test.go b/ring_test.go index 9e1576b757..a63f5c8e27 100644 --- a/ring_test.go +++ b/ring_test.go @@ -12,6 +12,7 @@ import ( ) var _ = Describe("Redis ring", func() { + const heartbeat = 100 * time.Millisecond var ring *redis.Ring setRingKeys := func() { @@ -27,6 +28,7 @@ var _ = Describe("Redis ring", func() { "ringShardOne": ":" + ringShard1Port, "ringShardTwo": ":" + ringShard2Port, }, + HeartbeatFrequency: heartbeat, }) // Shards should not have any keys. @@ -53,10 +55,9 @@ var _ = Describe("Redis ring", func() { // Stop ringShard2. Expect(ringShard2.Close()).NotTo(HaveOccurred()) - // Ring needs 5 * heartbeat time to detect that node is down. + // Ring needs 3 * heartbeat time to detect that node is down. // Give it more to be sure. - heartbeat := 100 * time.Millisecond - time.Sleep(2 * 5 * heartbeat) + time.Sleep(2 * 3 * heartbeat) setRingKeys() From 5760a88db3a3160fd90fd39067d2c024edd23e08 Mon Sep 17 00:00:00 2001 From: Jamie Markle Date: Mon, 15 Aug 2016 15:22:50 -0400 Subject: [PATCH 0195/1746] add WrapProcess --- commands.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/commands.go b/commands.go index 769dcb1c0c..d7f22495c5 100644 --- a/commands.go +++ b/commands.go @@ -245,6 +245,14 @@ type cmdable struct { process func(cmd Cmder) error } +// WrapProcess replaces the process func. It takes a function createWrapper +// which is supplied by the user. createWrapper takes the old process func as +// an input and returns the new wrapper process func. createWrapper should +// use call the old process func within the new process func. +func (c *cmdable) WrapProcess(createWrapper func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { + c.process = createWrapper(c.process) +} + var _ Cmdable = (*cmdable)(nil) type statefulCmdable struct { From ac1c5e46f981d300ad0051ebabbf08d78cfd395b Mon Sep 17 00:00:00 2001 From: Sergey Shcherbina Date: Mon, 22 Aug 2016 02:32:06 +0500 Subject: [PATCH 0196/1746] support geopos command --- command.go | 42 ++++++++++++++++++++++++++++++++++++++++++ commands.go | 13 +++++++++++++ parser.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/command.go b/command.go index 9cb705879a..d89eab651c 100644 --- a/command.go +++ b/command.go @@ -806,6 +806,10 @@ type GeoLocation struct { GeoHash int64 } +type GeoPosition struct { + Longitude, Latitude float64 +} + // GeoRadiusQuery is used with GeoRadius to query geospatial index. type GeoRadiusQuery struct { Radius float64 @@ -882,6 +886,44 @@ func (cmd *GeoLocationCmd) readReply(cn *pool.Conn) error { return nil } +type GeoPosCmd struct { + baseCmd + + positions []*GeoPosition +} + +func NewGeoPosCmd(args ...interface{}) *GeoPosCmd { + cmd := newBaseCmd(args) + return &GeoPosCmd{baseCmd: cmd} +} + +func (cmd *GeoPosCmd) Val() []*GeoPosition { + return cmd.positions +} + +func (cmd *GeoPosCmd) Result() ([]*GeoPosition, error) { + return cmd.Val(), cmd.Err() +} + +func (cmd *GeoPosCmd) String() string { + return cmdString(cmd, cmd.positions) +} + +func (cmd *GeoPosCmd) reset() { + cmd.positions = nil + cmd.err = nil +} + +func (cmd *GeoPosCmd) readReply(cn *pool.Conn) error { + reply, err := cn.Rd.ReadArrayReply(newGeoPositionSliceParser()) + if err != nil { + cmd.err = err + return err + } + cmd.positions = reply.([]*GeoPosition) + return nil +} + //------------------------------------------------------------------------------ type CommandInfo struct { diff --git a/commands.go b/commands.go index d7f22495c5..bcaeadad4a 100644 --- a/commands.go +++ b/commands.go @@ -234,6 +234,7 @@ type Cmdable interface { ClusterAddSlots(slots ...int) *StatusCmd ClusterAddSlotsRange(min, max int) *StatusCmd GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd + GeoPos(key string, name ...string) *GeoPosCmd GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd GeoDist(key string, member1, member2, unit string) *FloatCmd @@ -2054,6 +2055,18 @@ func (c *cmdable) GeoHash(key string, members ...string) *StringSliceCmd { return cmd } +func (c *cmdable) GeoPos(key string, names ...string) *GeoPosCmd { + args := make([]interface{}, 2+len(names)) + args[0] = "geopos" + args[1] = key + for i, name := range names { + args[2+i] = name + } + cmd := NewGeoPosCmd(args...) + c.process(cmd) + return cmd +} + //------------------------------------------------------------------------------ func (c *cmdable) Command() *CommandsInfoCmd { diff --git a/parser.go b/parser.go index 95a4297bab..c1db8d0d00 100644 --- a/parser.go +++ b/parser.go @@ -273,6 +273,48 @@ func newGeoLocationSliceParser(q *GeoRadiusQuery) proto.MultiBulkParse { } } +func newGeoPositionParser() proto.MultiBulkParse { + return func(rd *proto.Reader, n int64) (interface{}, error) { + var pos GeoPosition + var err error + + pos.Longitude, err = rd.ReadFloatReply() + if err != nil { + return nil, err + } + pos.Latitude, err = rd.ReadFloatReply() + if err != nil { + return nil, err + } + + return &pos, nil + } +} + +func newGeoPositionSliceParser() proto.MultiBulkParse { + return func(rd *proto.Reader, n int64) (interface{}, error) { + positions := make([]*GeoPosition, 0, n) + for i := int64(0); i < n; i++ { + v, err := rd.ReadReply(newGeoPositionParser()) + if err != nil { + // response may contain nil results + if err == Nil { + positions = append(positions, nil) + continue + } + return nil, err + } + switch vv := v.(type) { + case *GeoPosition: + positions = append(positions, vv) + default: + return nil, fmt.Errorf("got %T, expected *GeoPosition", v) + } + } + return positions, nil + } +} + func commandInfoParser(rd *proto.Reader, n int64) (interface{}, error) { var cmd CommandInfo var err error From 235dc49d5f96fb6705962341a06f1a77f0fd8681 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 22 Aug 2016 09:39:22 +0000 Subject: [PATCH 0197/1746] Rename GeoPosition to GeoPos for consistency with Redis Server. Simplify code where possible. --- command.go | 20 ++++++++++-------- commands.go | 10 ++++----- parser.go | 60 +++++++++++++++++++++++++---------------------------- 3 files changed, 44 insertions(+), 46 deletions(-) diff --git a/command.go b/command.go index d89eab651c..8ac048acfe 100644 --- a/command.go +++ b/command.go @@ -806,10 +806,6 @@ type GeoLocation struct { GeoHash int64 } -type GeoPosition struct { - Longitude, Latitude float64 -} - // GeoRadiusQuery is used with GeoRadius to query geospatial index. type GeoRadiusQuery struct { Radius float64 @@ -886,10 +882,16 @@ func (cmd *GeoLocationCmd) readReply(cn *pool.Conn) error { return nil } +//------------------------------------------------------------------------------ + +type GeoPos struct { + Longitude, Latitude float64 +} + type GeoPosCmd struct { baseCmd - positions []*GeoPosition + positions []*GeoPos } func NewGeoPosCmd(args ...interface{}) *GeoPosCmd { @@ -897,11 +899,11 @@ func NewGeoPosCmd(args ...interface{}) *GeoPosCmd { return &GeoPosCmd{baseCmd: cmd} } -func (cmd *GeoPosCmd) Val() []*GeoPosition { +func (cmd *GeoPosCmd) Val() []*GeoPos { return cmd.positions } -func (cmd *GeoPosCmd) Result() ([]*GeoPosition, error) { +func (cmd *GeoPosCmd) Result() ([]*GeoPos, error) { return cmd.Val(), cmd.Err() } @@ -915,12 +917,12 @@ func (cmd *GeoPosCmd) reset() { } func (cmd *GeoPosCmd) readReply(cn *pool.Conn) error { - reply, err := cn.Rd.ReadArrayReply(newGeoPositionSliceParser()) + reply, err := cn.Rd.ReadArrayReply(geoPosSliceParser) if err != nil { cmd.err = err return err } - cmd.positions = reply.([]*GeoPosition) + cmd.positions = reply.([]*GeoPos) return nil } diff --git a/commands.go b/commands.go index bcaeadad4a..3d3fe2f60f 100644 --- a/commands.go +++ b/commands.go @@ -234,7 +234,7 @@ type Cmdable interface { ClusterAddSlots(slots ...int) *StatusCmd ClusterAddSlotsRange(min, max int) *StatusCmd GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd - GeoPos(key string, name ...string) *GeoPosCmd + GeoPos(key string, members ...string) *GeoPosCmd GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd GeoDist(key string, member1, member2, unit string) *FloatCmd @@ -2055,12 +2055,12 @@ func (c *cmdable) GeoHash(key string, members ...string) *StringSliceCmd { return cmd } -func (c *cmdable) GeoPos(key string, names ...string) *GeoPosCmd { - args := make([]interface{}, 2+len(names)) +func (c *cmdable) GeoPos(key string, members ...string) *GeoPosCmd { + args := make([]interface{}, 2+len(members)) args[0] = "geopos" args[1] = key - for i, name := range names { - args[2+i] = name + for i, member := range members { + args[2+i] = member } cmd := NewGeoPosCmd(args...) c.process(cmd) diff --git a/parser.go b/parser.go index c1db8d0d00..46220d4051 100644 --- a/parser.go +++ b/parser.go @@ -273,46 +273,42 @@ func newGeoLocationSliceParser(q *GeoRadiusQuery) proto.MultiBulkParse { } } -func newGeoPositionParser() proto.MultiBulkParse { - return func(rd *proto.Reader, n int64) (interface{}, error) { - var pos GeoPosition - var err error +func geoPosParser(rd *proto.Reader, n int64) (interface{}, error) { + var pos GeoPos + var err error - pos.Longitude, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - pos.Latitude, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } + pos.Longitude, err = rd.ReadFloatReply() + if err != nil { + return nil, err + } - return &pos, nil + pos.Latitude, err = rd.ReadFloatReply() + if err != nil { + return nil, err } + + return &pos, nil } -func newGeoPositionSliceParser() proto.MultiBulkParse { - return func(rd *proto.Reader, n int64) (interface{}, error) { - positions := make([]*GeoPosition, 0, n) - for i := int64(0); i < n; i++ { - v, err := rd.ReadReply(newGeoPositionParser()) - if err != nil { - // response may contain nil results - if err == Nil { - positions = append(positions, nil) - continue - } - return nil, err - } - switch vv := v.(type) { - case *GeoPosition: - positions = append(positions, vv) - default: - return nil, fmt.Errorf("got %T, expected *GeoPosition", v) +func geoPosSliceParser(rd *proto.Reader, n int64) (interface{}, error) { + positions := make([]*GeoPos, 0, n) + for i := int64(0); i < n; i++ { + v, err := rd.ReadReply(geoPosParser) + if err != nil { + if err == Nil { + positions = append(positions, nil) + continue } + return nil, err + } + switch v := v.(type) { + case *GeoPos: + positions = append(positions, v) + default: + return nil, fmt.Errorf("got %T, expected *GeoPos", v) } - return positions, nil } + return positions, nil } func commandInfoParser(rd *proto.Reader, n int64) (interface{}, error) { From 8b7922d185896d0df2436a92a278454e588139b7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 22 Aug 2016 09:46:42 +0000 Subject: [PATCH 0198/1746] Add test for GeoPos. --- commands_test.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/commands_test.go b/commands_test.go index b4d0ff3b6b..ea33e1b8f7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2689,20 +2689,35 @@ var _ = Describe("Commands", func() { // "166274.15156960033" // GEODIST Sicily Palermo Catania km // "166.27415156960032" - geoDist := client.GeoDist("Sicily", "Palermo", "Catania", "km") - Expect(geoDist.Err()).NotTo(HaveOccurred()) - Expect(geoDist.Val()).To(BeNumerically("~", 166.27, 0.01)) + dist, err := client.GeoDist("Sicily", "Palermo", "Catania", "km").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(dist).To(BeNumerically("~", 166.27, 0.01)) - geoDist = client.GeoDist("Sicily", "Palermo", "Catania", "m") - Expect(geoDist.Err()).NotTo(HaveOccurred()) - Expect(geoDist.Val()).To(BeNumerically("~", 166274.15, 0.01)) + dist, err = client.GeoDist("Sicily", "Palermo", "Catania", "m").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(dist).To(BeNumerically("~", 166274.15, 0.01)) }) It("should get geo hash in string representation", func() { - res, err := client.GeoHash("Sicily", "Palermo", "Catania").Result() + hashes, err := client.GeoHash("Sicily", "Palermo", "Catania").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(hashes).To(ConsistOf([]string{"sqc8b49rny0", "sqdtr74hyu0"})) + }) + + It("should return geo position", func() { + pos, err := client.GeoPos("Sicily", "Palermo", "Catania", "NonExisting").Result() Expect(err).NotTo(HaveOccurred()) - Expect(res[0]).To(Equal("sqc8b49rny0")) - Expect(res[1]).To(Equal("sqdtr74hyu0")) + Expect(pos).To(ConsistOf([]*redis.GeoPos{ + { + Longitude: 13.361389338970184, + Latitude: 38.1155563954963, + }, + { + Longitude: 15.087267458438873, + Latitude: 37.50266842333162, + }, + nil, + })) }) }) From 559e13782cb19009c8154b0a40f32455e0019827 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 22 Aug 2016 09:47:13 +0000 Subject: [PATCH 0199/1746] travis: test on Go 1.7. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index afbe1a48c9..25bce050ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ services: - redis-server go: - - 1.5 - 1.6 + - 1.7 - tip matrix: From d07be2cd0446af36e6dea6c9b552b98e1fd39ded Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 22 Aug 2016 21:20:49 +0100 Subject: [PATCH 0200/1746] Add Cmd constructors for testing/mock purposes. --- result.go | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 result.go diff --git a/result.go b/result.go new file mode 100644 index 0000000000..481673d234 --- /dev/null +++ b/result.go @@ -0,0 +1,140 @@ +package redis + +import "time" + +// NewCmdResult returns a Cmd initalised with val and err for testing +func NewCmdResult(val interface{}, err error) *Cmd { + var cmd Cmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewSliceResult returns a SliceCmd initalised with val and err for testing +func NewSliceResult(val []interface{}, err error) *SliceCmd { + var cmd SliceCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewStatusResult returns a StatusCmd initalised with val and err for testing +func NewStatusResult(val string, err error) *StatusCmd { + var cmd StatusCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewIntResult returns an IntCmd initalised with val and err for testing +func NewIntResult(val int64, err error) *IntCmd { + var cmd IntCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewDurationResult returns a DurationCmd initalised with val and err for testing +func NewDurationResult(val time.Duration, err error) *DurationCmd { + var cmd DurationCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewBoolResult returns a BoolCmd initalised with val and err for testing +func NewBoolResult(val bool, err error) *BoolCmd { + var cmd BoolCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewStringResult returns a StringCmd initalised with val and err for testing +func NewStringResult(val []byte, err error) *StringCmd { + var cmd StringCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewFloatResult returns a FloatCmd initalised with val and err for testing +func NewFloatResult(val float64, err error) *FloatCmd { + var cmd FloatCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewStringSliceResult returns a StringSliceCmd initalised with val and err for testing +func NewStringSliceResult(val []string, err error) *StringSliceCmd { + var cmd StringSliceCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewBoolSliceResult returns a BoolSliceCmd initalised with val and err for testing +func NewBoolSliceResult(val []bool, err error) *BoolSliceCmd { + var cmd BoolSliceCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewStringStringMapResult returns a StringStringMapCmd initalised with val and err for testing +func NewStringStringMapResult(val map[string]string, err error) *StringStringMapCmd { + var cmd StringStringMapCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewStringIntMapCmdResult returns a StringIntMapCmd initalised with val and err for testing +func NewStringIntMapCmdResult(val map[string]int64, err error) *StringIntMapCmd { + var cmd StringIntMapCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewZSliceCmdResult returns a ZSliceCmd initalised with val and err for testing +func NewZSliceCmdResult(val []Z, err error) *ZSliceCmd { + var cmd ZSliceCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewScanCmdResult returns a ScanCmd initalised with val and err for testing +func NewScanCmdResult(keys []string, cursor uint64, err error) *ScanCmd { + var cmd ScanCmd + cmd.page = keys + cmd.cursor = cursor + cmd.setErr(err) + return &cmd +} + +// NewClusterSlotsCmdResult returns a ClusterSlotsCmd initalised with val and err for testing +func NewClusterSlotsCmdResult(val []ClusterSlot, err error) *ClusterSlotsCmd { + var cmd ClusterSlotsCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} + +// NewGeoLocationCmdResult returns a GeoLocationCmd initalised with val and err for testing +func NewGeoLocationCmdResult(val []GeoLocation, err error) *GeoLocationCmd { + var cmd GeoLocationCmd + cmd.locations = val + cmd.setErr(err) + return &cmd +} + +// NewCommandsInfoCmdResult returns a CommandsInfoCmd initalised with val and err for testing +func NewCommandsInfoCmdResult(val map[string]*CommandInfo, err error) *CommandsInfoCmd { + var cmd CommandsInfoCmd + cmd.val = val + cmd.setErr(err) + return &cmd +} From 76c33da3ee407a5e83125627c6ff916432aeda15 Mon Sep 17 00:00:00 2001 From: vkd Date: Thu, 8 Sep 2016 16:07:49 +0300 Subject: [PATCH 0201/1746] fix iterator across empty pages --- iterator.go | 39 ++++++++++++++++++++++----------------- iterator_test.go | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/iterator.go b/iterator.go index 0a3df06326..e8872fb3ec 100644 --- a/iterator.go +++ b/iterator.go @@ -46,25 +46,30 @@ func (it *ScanIterator) Next() bool { return true } - // Return if there is more data to fetch. - if it.ScanCmd.cursor == 0 { - return false - } + for { + // Return if there is more data to fetch. + if it.ScanCmd.cursor == 0 { + return false + } - // Fetch next page. - if it.ScanCmd._args[0] == "scan" { - it.ScanCmd._args[1] = it.ScanCmd.cursor - } else { - it.ScanCmd._args[2] = it.ScanCmd.cursor - } - it.ScanCmd.reset() - it.client.process(it.ScanCmd) - if it.ScanCmd.Err() != nil { - return false - } + // Fetch next page. + if it.ScanCmd._args[0] == "scan" { + it.ScanCmd._args[1] = it.ScanCmd.cursor + } else { + it.ScanCmd._args[2] = it.ScanCmd.cursor + } + it.ScanCmd.reset() + it.client.process(it.ScanCmd) + if it.ScanCmd.Err() != nil { + return false + } - it.pos = 1 - return len(it.ScanCmd.page) > 0 + it.pos = 1 + if len(it.ScanCmd.page) > 0 { + return true + } + } + return false } // Val returns the key/field at the current cursor position. diff --git a/iterator_test.go b/iterator_test.go index f752f68c0f..7b301768f1 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -21,6 +21,18 @@ var _ = Describe("ScanIterator", func() { return err } + var extraSeed = func(n int, m int) error { + pipe := client.Pipeline() + for i := 1; i <= m; i++ { + pipe.Set(fmt.Sprintf("A%02d", i), "x", 0).Err() + } + for i := 1; i <= n; i++ { + pipe.Set(fmt.Sprintf("K%02d", i), "x", 0).Err() + } + _, err := pipe.Exec() + return err + } + var hashKey = "K_HASHTEST" var hashSeed = func(n int) error { pipe := client.Pipeline() @@ -110,4 +122,15 @@ var _ = Describe("ScanIterator", func() { Expect(vals).To(HaveLen(13)) }) + It("should scan with match across empty pages", func() { + Expect(extraSeed(2, 10)).NotTo(HaveOccurred()) + + var vals []string + iter := client.Scan(0, "K*", 1).Iterator() + for iter.Next() { + vals = append(vals, iter.Val()) + } + Expect(iter.Err()).NotTo(HaveOccurred()) + Expect(vals).To(HaveLen(2)) + }) }) From cbf8ff22f55038e0681e43be48c01b37847b507c Mon Sep 17 00:00:00 2001 From: vkd Date: Thu, 8 Sep 2016 16:40:45 +0300 Subject: [PATCH 0202/1746] fix comments --- iterator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iterator.go b/iterator.go index e8872fb3ec..d48ad7b895 100644 --- a/iterator.go +++ b/iterator.go @@ -47,7 +47,7 @@ func (it *ScanIterator) Next() bool { } for { - // Return if there is more data to fetch. + // Return if there is no more data to fetch. if it.ScanCmd.cursor == 0 { return false } @@ -65,6 +65,8 @@ func (it *ScanIterator) Next() bool { } it.pos = 1 + + // Redis can occasionally return empty page if len(it.ScanCmd.page) > 0 { return true } From aab4e040c6588d7656ee930903dc6128157a1456 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 10 Sep 2016 13:54:49 +0000 Subject: [PATCH 0203/1746] internal/pool: fix reaper to properly close connections. --- internal/pool/pool.go | 70 ++++++++++++++++++++++---------------- internal/pool/pool_test.go | 26 ++++++++++++-- 2 files changed, 64 insertions(+), 32 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index da1e381fd3..16e508c084 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -185,7 +185,7 @@ func (p *ConnPool) Get() (*Conn, error) { if !cn.IsStale(p.idleTimeout) { return cn, nil } - _ = cn.Close() + _ = p.closeConn(cn, errConnStale) } newcn, err := p.NewConn() @@ -196,7 +196,7 @@ func (p *ConnPool) Get() (*Conn, error) { p.connsMu.Lock() if cn != nil { - p.remove(cn, errConnStale) + p.removeConn(cn) } p.conns = append(p.conns, newcn) p.connsMu.Unlock() @@ -218,16 +218,20 @@ func (p *ConnPool) Put(cn *Conn) error { } func (p *ConnPool) Remove(cn *Conn, reason error) error { - _ = cn.Close() - p.connsMu.Lock() p.remove(cn, reason) - p.connsMu.Unlock() p.queue <- struct{}{} return nil } func (p *ConnPool) remove(cn *Conn, reason error) { - p.storeLastErr(reason.Error()) + _ = p.closeConn(cn, reason) + + p.connsMu.Lock() + p.removeConn(cn) + p.connsMu.Unlock() +} + +func (p *ConnPool) removeConn(cn *Conn) { for i, c := range p.conns { if c == cn { p.conns = append(p.conns[:i], p.conns[i+1:]...) @@ -272,13 +276,12 @@ func (p *ConnPool) Close() (retErr error) { } p.connsMu.Lock() - // Close all connections. for _, cn := range p.conns { if cn == nil { continue } - if err := p.closeConn(cn); err != nil && retErr == nil { + if err := p.closeConn(cn, ErrClosed); err != nil && retErr == nil { retErr = err } } @@ -292,41 +295,48 @@ func (p *ConnPool) Close() (retErr error) { return retErr } -func (p *ConnPool) closeConn(cn *Conn) error { +func (p *ConnPool) closeConn(cn *Conn, reason error) error { + p.storeLastErr(reason.Error()) if p.OnClose != nil { _ = p.OnClose(cn) } return cn.Close() } -func (p *ConnPool) ReapStaleConns() (n int, err error) { - <-p.queue - p.freeConnsMu.Lock() - +func (p *ConnPool) reapStaleConn() bool { if len(p.freeConns) == 0 { + return false + } + + cn := p.freeConns[0] + if !cn.IsStale(p.idleTimeout) { + return false + } + + p.remove(cn, errConnStale) + p.freeConns = append(p.freeConns[:0], p.freeConns[1:]...) + + return true +} + +func (p *ConnPool) ReapStaleConns() (int, error) { + var n int + for { + <-p.queue + p.freeConnsMu.Lock() + + reaped := p.reapStaleConn() + p.freeConnsMu.Unlock() p.queue <- struct{}{} - return - } - var idx int - var cn *Conn - for idx, cn = range p.freeConns { - if !cn.IsStale(p.idleTimeout) { + if reaped { + n++ + } else { break } - p.connsMu.Lock() - p.remove(cn, errConnStale) - p.connsMu.Unlock() - n++ - } - if idx > 0 { - p.freeConns = append(p.freeConns[:0], p.freeConns[idx:]...) } - - p.freeConnsMu.Unlock() - p.queue <- struct{}{} - return + return n, nil } func (p *ConnPool) reaper(frequency time.Duration) { diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index ceec428f82..2c8908feb3 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -94,20 +94,31 @@ var _ = Describe("ConnPool", func() { }) var _ = Describe("conns reaper", func() { + const idleTimeout = time.Minute + var connPool *pool.ConnPool + var idleConns, closedConns []*pool.Conn BeforeEach(func() { connPool = pool.NewConnPool( - dummyDialer, 10, time.Second, time.Millisecond, time.Hour) + dummyDialer, 10, time.Second, idleTimeout, time.Hour) + + closedConns = nil + connPool.OnClose = func(cn *pool.Conn) error { + closedConns = append(closedConns, cn) + return nil + } var cns []*pool.Conn // add stale connections + idleConns = nil for i := 0; i < 3; i++ { cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) - cn.UsedAt = time.Now().Add(-2 * time.Minute) + cn.UsedAt = time.Now().Add(-2 * idleTimeout) cns = append(cns, cn) + idleConns = append(idleConns, cn) } // add fresh connections @@ -139,6 +150,17 @@ var _ = Describe("conns reaper", func() { Expect(connPool.FreeLen()).To(Equal(3)) }) + It("does not reap fresh connections", func() { + n, err := connPool.ReapStaleConns() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(0)) + }) + + It("stale connections are closed", func() { + Expect(closedConns).To(HaveLen(3)) + Expect(closedConns).To(ConsistOf(idleConns)) + }) + It("pool is functional", func() { for j := 0; j < 3; j++ { var freeCns []*pool.Conn From 4b0862b5fd0a5ae4e63c76476a64655752d6031b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 10 Sep 2016 14:09:39 +0000 Subject: [PATCH 0204/1746] internal/pool: improve tests. --- internal/pool/pool_test.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 2c8908feb3..5fe7e8dbc5 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -2,7 +2,6 @@ package pool_test import ( "errors" - "net" "testing" "time" @@ -97,7 +96,7 @@ var _ = Describe("conns reaper", func() { const idleTimeout = time.Minute var connPool *pool.ConnPool - var idleConns, closedConns []*pool.Conn + var conns, idleConns, closedConns []*pool.Conn BeforeEach(func() { connPool = pool.NewConnPool( @@ -109,7 +108,7 @@ var _ = Describe("conns reaper", func() { return nil } - var cns []*pool.Conn + conns = nil // add stale connections idleConns = nil @@ -117,19 +116,18 @@ var _ = Describe("conns reaper", func() { cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) cn.UsedAt = time.Now().Add(-2 * idleTimeout) - cns = append(cns, cn) + conns = append(conns, cn) idleConns = append(idleConns, cn) } // add fresh connections for i := 0; i < 3; i++ { - cn := pool.NewConn(&net.TCPConn{}) cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) - cns = append(cns, cn) + conns = append(conns, cn) } - for _, cn := range cns { + for _, cn := range conns { Expect(connPool.Put(cn)).NotTo(HaveOccurred()) } @@ -142,7 +140,11 @@ var _ = Describe("conns reaper", func() { }) AfterEach(func() { - connPool.Close() + _ = connPool.Close() + Expect(connPool.Len()).To(Equal(0)) + Expect(connPool.FreeLen()).To(Equal(0)) + Expect(len(closedConns)).To(Equal(len(conns))) + Expect(closedConns).To(ConsistOf(conns)) }) It("reaps stale connections", func() { @@ -157,7 +159,7 @@ var _ = Describe("conns reaper", func() { }) It("stale connections are closed", func() { - Expect(closedConns).To(HaveLen(3)) + Expect(len(closedConns)).To(Equal(len(idleConns))) Expect(closedConns).To(ConsistOf(idleConns)) }) @@ -177,6 +179,7 @@ var _ = Describe("conns reaper", func() { cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) + conns = append(conns, cn) Expect(connPool.Len()).To(Equal(4)) Expect(connPool.FreeLen()).To(Equal(0)) From 455abb190609d01db0080252f58fc39ba469ae14 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 12 Sep 2016 06:15:59 +0000 Subject: [PATCH 0205/1746] internal/pool: more idiomatic work with channels. --- internal/pool/pool.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 16e508c084..5c3fa06559 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -84,9 +84,6 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout, idleCheckF conns: make([]*Conn, 0, poolSize), freeConns: make([]*Conn, 0, poolSize), } - for i := 0; i < poolSize; i++ { - p.queue <- struct{}{} - } if idleTimeout > 0 && idleCheckFrequency > 0 { go p.reaper(idleCheckFrequency) } @@ -125,7 +122,7 @@ func (p *ConnPool) PopFree() *Conn { } select { - case <-p.queue: + case p.queue <- struct{}{}: timers.Put(timer) case <-timer.C: timers.Put(timer) @@ -138,7 +135,7 @@ func (p *ConnPool) PopFree() *Conn { p.freeConnsMu.Unlock() if cn == nil { - p.queue <- struct{}{} + <-p.queue } return cn } @@ -168,7 +165,7 @@ func (p *ConnPool) Get() (*Conn, error) { } select { - case <-p.queue: + case p.queue <- struct{}{}: timers.Put(timer) case <-timer.C: timers.Put(timer) @@ -190,7 +187,7 @@ func (p *ConnPool) Get() (*Conn, error) { newcn, err := p.NewConn() if err != nil { - p.queue <- struct{}{} + <-p.queue return nil, err } @@ -213,13 +210,13 @@ func (p *ConnPool) Put(cn *Conn) error { p.freeConnsMu.Lock() p.freeConns = append(p.freeConns, cn) p.freeConnsMu.Unlock() - p.queue <- struct{}{} + <-p.queue return nil } func (p *ConnPool) Remove(cn *Conn, reason error) error { p.remove(cn, reason) - p.queue <- struct{}{} + <-p.queue return nil } @@ -322,13 +319,13 @@ func (p *ConnPool) reapStaleConn() bool { func (p *ConnPool) ReapStaleConns() (int, error) { var n int for { - <-p.queue + p.queue <- struct{}{} p.freeConnsMu.Lock() reaped := p.reapStaleConn() p.freeConnsMu.Unlock() - p.queue <- struct{}{} + <-p.queue if reaped { n++ From f69688538d5ec4a817732b2a521a4b0c7cd5c08e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 12 Sep 2016 12:00:28 +0000 Subject: [PATCH 0206/1746] Fix license. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 261f1ec375..298bed9bea 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2016 The github.com/go-redis/redis Contributors. +Copyright (c) 2013 The github.com/go-redis/redis Authors. All rights reserved. Redistribution and use in source and binary forms, with or without From cea5c239f518481612c46841c488de1874680501 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 13 Sep 2016 14:44:41 +0000 Subject: [PATCH 0207/1746] Set some sane default for every day usage. --- options.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index e1ad28adc3..3f9a46277b 100644 --- a/options.go +++ b/options.go @@ -33,9 +33,11 @@ type Options struct { DialTimeout time.Duration // Timeout for socket reads. If reached, commands will fail // with a timeout instead of blocking. + // Default is 3 seconds. ReadTimeout time.Duration // Timeout for socket writes. If reached, commands will fail // with a timeout instead of blocking. + // Default is 3 seconds. WriteTimeout time.Duration // Maximum number of socket connections. @@ -43,7 +45,7 @@ type Options struct { PoolSize int // Amount of time client waits for connection if all connections // are busy before returning an error. - // Default is 1 second. + // Default is ReadTimeout + 1 second. PoolTimeout time.Duration // Amount of time after which client closes idle connections. // Should be less than server's timeout. @@ -72,8 +74,17 @@ func (opt *Options) init() { if opt.DialTimeout == 0 { opt.DialTimeout = 5 * time.Second } + if opt.ReadTimeout == 0 { + opt.ReadTimeout = 3 * time.Second + } + if opt.WriteTimeout == 0 { + opt.WriteTimeout = opt.ReadTimeout + } if opt.PoolTimeout == 0 { - opt.PoolTimeout = 1 * time.Second + opt.PoolTimeout = opt.ReadTimeout + time.Second + } + if opt.IdleTimeout == 0 { + opt.IdleTimeout = 5 * time.Minute } if opt.IdleCheckFrequency == 0 { opt.IdleCheckFrequency = time.Minute From 73b28d0372e37e8f06d96539a57818d34d6c4e06 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 14 Sep 2016 10:01:56 +0000 Subject: [PATCH 0208/1746] Fix Cmdable interface. --- commands.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/commands.go b/commands.go index 3d3fe2f60f..8277418dc3 100644 --- a/commands.go +++ b/commands.go @@ -74,7 +74,6 @@ type Cmdable interface { ZScan(key string, cursor uint64, match string, count int64) Scanner Append(key, value string) *IntCmd BitCount(key string, bitCount *BitCount) *IntCmd - bitOp(op, destKey string, keys ...string) *IntCmd BitOpAnd(destKey string, keys ...string) *IntCmd BitOpOr(destKey string, keys ...string) *IntCmd BitOpXor(destKey string, keys ...string) *IntCmd @@ -153,7 +152,6 @@ type Cmdable interface { ZAddCh(key string, members ...Z) *IntCmd ZAddNXCh(key string, members ...Z) *IntCmd ZAddXXCh(key string, members ...Z) *IntCmd - zIncr(a []interface{}, n int, members ...Z) *FloatCmd ZIncr(key string, member Z) *FloatCmd ZIncrNX(key string, member Z) *FloatCmd ZIncrXX(key string, member Z) *FloatCmd From 39333495f9ef2fe77a2efdaf920d818912fceb79 Mon Sep 17 00:00:00 2001 From: Roger Clotet Date: Wed, 14 Sep 2016 12:47:46 +0200 Subject: [PATCH 0209/1746] Remove unexported functions from Cmdable interface --- commands.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/commands.go b/commands.go index 8277418dc3..99314b908b 100644 --- a/commands.go +++ b/commands.go @@ -145,7 +145,6 @@ type Cmdable interface { SRem(key string, members ...interface{}) *IntCmd SUnion(keys ...string) *StringSliceCmd SUnionStore(destination string, keys ...string) *IntCmd - zAdd(a []interface{}, n int, members ...Z) *IntCmd ZAdd(key string, members ...Z) *IntCmd ZAddNX(key string, members ...Z) *IntCmd ZAddXX(key string, members ...Z) *IntCmd @@ -159,10 +158,8 @@ type Cmdable interface { ZCount(key, min, max string) *IntCmd ZIncrBy(key string, increment float64, member string) *FloatCmd ZInterStore(destination string, store ZStore, keys ...string) *IntCmd - zRange(key string, start, stop int64, withScores bool) *StringSliceCmd ZRange(key string, start, stop int64) *StringSliceCmd ZRangeWithScores(key string, start, stop int64) *ZSliceCmd - zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd @@ -172,7 +169,6 @@ type Cmdable interface { ZRemRangeByScore(key, min, max string) *IntCmd ZRevRange(key string, start, stop int64) *StringSliceCmd ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd - zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd @@ -197,7 +193,6 @@ type Cmdable interface { Info(section ...string) *StringCmd LastSave() *IntCmd Save() *StatusCmd - shutdown(modifier string) *StatusCmd Shutdown() *StatusCmd ShutdownSave() *StatusCmd ShutdownNoSave() *StatusCmd From 04cdc41cc4dfc91ed91e7fa785778bf029861fdb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 14 Sep 2016 11:00:34 +0000 Subject: [PATCH 0210/1746] Try to fix Go 1.4. --- .travis.yml | 1 + internal/errors/errors.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 25bce050ea..1d042e4c35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ services: - redis-server go: + - 1.4 - 1.6 - 1.7 - tip diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 6d664ddfe1..a17a3d37b6 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -58,7 +58,7 @@ func IsMoved(err error) (moved bool, ask bool, addr string) { return } - ind := strings.LastIndexByte(s, ' ') + ind := strings.LastIndex(s, " ") if ind == -1 { return false, false, "" } From c0b792a0592d1a509f04c35f7dc3515739d78c94 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 23 Sep 2016 11:52:19 +0000 Subject: [PATCH 0211/1746] Don't panic when cluster does not have valid nodes. --- cluster.go | 19 +++++++++++-------- cluster_test.go | 13 +++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cluster.go b/cluster.go index 07de993274..8290db0c4e 100644 --- a/cluster.go +++ b/cluster.go @@ -160,17 +160,18 @@ func (c *ClusterClient) newNode(addr string) *clusterNode { } } -func (c *ClusterClient) slotNodes(slot int) []*clusterNode { +func (c *ClusterClient) slotNodes(slot int) (nodes []*clusterNode) { c.mu.RLock() - nodes := c.slots[slot] + if slot < len(c.slots) { + nodes = c.slots[slot] + } c.mu.RUnlock() return nodes } // randomNode returns random live node. func (c *ClusterClient) randomNode() (*clusterNode, error) { - var node *clusterNode - var err error + var nodeErr error for i := 0; i < 10; i++ { c.mu.RLock() closed := c.closed @@ -182,16 +183,18 @@ func (c *ClusterClient) randomNode() (*clusterNode, error) { } n := rand.Intn(len(addrs)) - node, err = c.nodeByAddr(addrs[n]) + + node, err := c.nodeByAddr(addrs[n]) if err != nil { return nil, err } - if node.Client.ClusterInfo().Err() == nil { - break + nodeErr = node.Client.ClusterInfo().Err() + if nodeErr == nil { + return node, nil } } - return node, nil + return nil, nodeErr } func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { diff --git a/cluster_test.go b/cluster_test.go index 17d5113318..659e134e49 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -515,6 +515,19 @@ var _ = Describe("ClusterClient", func() { describeClusterClient() }) + + Describe("ClusterClient without valid nodes", func() { + BeforeEach(func() { + client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{redisAddr}, + }) + }) + + It("returns an error", func() { + err := client.Ping().Err() + Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + }) + }) }) //------------------------------------------------------------------------------ From 8f5aeb70652fb9b537dea1c78d1bc34d6937550b Mon Sep 17 00:00:00 2001 From: evalphobia Date: Mon, 26 Sep 2016 15:22:39 +0900 Subject: [PATCH 0212/1746] Disable idle checks for cluster connection --- cluster.go | 3 +++ options.go | 1 + 2 files changed, 4 insertions(+) diff --git a/cluster.go b/cluster.go index 07de993274..6aacd03d35 100644 --- a/cluster.go +++ b/cluster.go @@ -627,6 +627,8 @@ func (opt *ClusterOptions) init() { } func (opt *ClusterOptions) clientOptions() *Options { + const disableIdleCheck = -1 + return &Options{ Password: opt.Password, ReadOnly: opt.ReadOnly, @@ -640,5 +642,6 @@ func (opt *ClusterOptions) clientOptions() *Options { IdleTimeout: opt.IdleTimeout, // IdleCheckFrequency is not copied to disable reaper + IdleCheckFrequency: disableIdleCheck, } } diff --git a/options.go b/options.go index 3f9a46277b..bfe2e8a2bf 100644 --- a/options.go +++ b/options.go @@ -53,6 +53,7 @@ type Options struct { IdleTimeout time.Duration // Frequency of idle checks. // Default is 1 minute. + // When minus value is set, then idle check is disabled. IdleCheckFrequency time.Duration // Enables read only queries on slave nodes. From 0b706418d98c97e8e9588f7a62760ea6680c65ea Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 27 Sep 2016 09:24:14 +0000 Subject: [PATCH 0213/1746] Add Pipeline to Cmdable. --- cluster.go | 2 ++ commands.go | 5 +++-- redis.go | 2 ++ ring.go | 2 ++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 6aacd03d35..a53b793cc2 100644 --- a/cluster.go +++ b/cluster.go @@ -39,6 +39,8 @@ type ClusterClient struct { reloading uint32 } +var _ Cmdable = (*ClusterClient)(nil) + // NewClusterClient returns a Redis Cluster client as described in // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { diff --git a/commands.go b/commands.go index 99314b908b..f1173296ca 100644 --- a/commands.go +++ b/commands.go @@ -41,6 +41,9 @@ func formatSec(dur time.Duration) string { } type Cmdable interface { + Pipeline() *Pipeline + Pipelined(fn func(*Pipeline) error) ([]Cmder, error) + Echo(message interface{}) *StringCmd Ping() *StatusCmd Quit() *StatusCmd @@ -247,8 +250,6 @@ func (c *cmdable) WrapProcess(createWrapper func(oldProcess func(cmd Cmder) erro c.process = createWrapper(c.process) } -var _ Cmdable = (*cmdable)(nil) - type statefulCmdable struct { process func(cmd Cmder) error } diff --git a/redis.go b/redis.go index b8ceab550e..ea0a2b4319 100644 --- a/redis.go +++ b/redis.go @@ -150,6 +150,8 @@ type Client struct { cmdable } +var _ Cmdable = (*Client)(nil) + func newClient(opt *Options, pool pool.Pooler) *Client { base := baseClient{opt: opt, connPool: pool} client := &Client{ diff --git a/ring.go b/ring.go index 39733e2a76..5b2fddeeda 100644 --- a/ring.go +++ b/ring.go @@ -133,6 +133,8 @@ type Ring struct { closed bool } +var _ Cmdable = (*Ring)(nil) + func NewRing(opt *RingOptions) *Ring { const nreplicas = 100 opt.init() From 850045d6a69756aafe383c7bf1067f449c9c84e3 Mon Sep 17 00:00:00 2001 From: Sergey Shcherbina Date: Thu, 29 Sep 2016 15:12:35 +0500 Subject: [PATCH 0214/1746] Use appendIfNotExists instead of append. Fixed bug when connection loss to server leads to exponential grow of channels and patterns array in PubSub in every reconnect --- pubsub.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pubsub.go b/pubsub.go index 1d1fc939b8..de8d35c4b6 100644 --- a/pubsub.go +++ b/pubsub.go @@ -43,7 +43,7 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { func (c *PubSub) Subscribe(channels ...string) error { err := c.subscribe("SUBSCRIBE", channels...) if err == nil { - c.channels = append(c.channels, channels...) + c.channels = appendIfNotExists(c.channels, channels...) c.nsub += len(channels) } return err @@ -53,7 +53,7 @@ func (c *PubSub) Subscribe(channels ...string) error { func (c *PubSub) PSubscribe(patterns ...string) error { err := c.subscribe("PSUBSCRIBE", patterns...) if err == nil { - c.patterns = append(c.patterns, patterns...) + c.patterns = appendIfNotExists(c.patterns, patterns...) c.nsub += len(patterns) } return err @@ -74,6 +74,23 @@ func remove(ss []string, es ...string) []string { return ss } +func appendIfNotExists(ss []string, es ...string) []string { + for _, e := range es { + found := false + for _, s := range ss { + if s == e { + found = true + break + } + } + + if !found { + ss = append(ss, e) + } + } + return ss +} + // Unsubscribes the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { From e57ac63b6e4cd1d370e3641b9c00674f1a865c9f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 29 Sep 2016 12:07:04 +0000 Subject: [PATCH 0215/1746] Simplify resubscribing in PubSub. --- cluster.go | 2 +- internal/pool/bench_test.go | 4 +- internal/pool/pool.go | 14 +++--- internal/pool/pool_single.go | 4 +- internal/pool/pool_sticky.go | 12 ++--- internal/pool/pool_test.go | 22 ++++----- pool_test.go | 2 +- pubsub.go | 93 +++++++++++++++++------------------- pubsub_test.go | 2 +- redis.go | 14 +++--- redis_test.go | 6 +-- ring.go | 2 +- tx.go | 2 +- tx_test.go | 4 +- 14 files changed, 90 insertions(+), 93 deletions(-) diff --git a/cluster.go b/cluster.go index 15b17aa770..402447ea7c 100644 --- a/cluster.go +++ b/cluster.go @@ -516,7 +516,7 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { } } - cn, err := node.Client.conn() + cn, _, err := node.Client.conn() if err != nil { setCmdsErr(cmds, err) setRetErr(err) diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index 878b20260b..663abc0bd2 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -16,7 +16,7 @@ func benchmarkPoolGetPut(b *testing.B, poolSize int) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - cn, err := connPool.Get() + cn, _, err := connPool.Get() if err != nil { b.Fatal(err) } @@ -48,7 +48,7 @@ func benchmarkPoolGetRemove(b *testing.B, poolSize int) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - cn, err := connPool.Get() + cn, _, err := connPool.Get() if err != nil { b.Fatal(err) } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 5c3fa06559..55c1f9fda9 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -36,7 +36,7 @@ type PoolStats struct { } type Pooler interface { - Get() (*Conn, error) + Get() (*Conn, bool, error) Put(*Conn) error Remove(*Conn, error) error Len() int @@ -152,9 +152,9 @@ func (p *ConnPool) popFree() *Conn { } // Get returns existed connection from the pool or creates a new one. -func (p *ConnPool) Get() (*Conn, error) { +func (p *ConnPool) Get() (*Conn, bool, error) { if p.Closed() { - return nil, ErrClosed + return nil, false, ErrClosed } atomic.AddUint32(&p.stats.Requests, 1) @@ -170,7 +170,7 @@ func (p *ConnPool) Get() (*Conn, error) { case <-timer.C: timers.Put(timer) atomic.AddUint32(&p.stats.Timeouts, 1) - return nil, ErrPoolTimeout + return nil, false, ErrPoolTimeout } p.freeConnsMu.Lock() @@ -180,7 +180,7 @@ func (p *ConnPool) Get() (*Conn, error) { if cn != nil { atomic.AddUint32(&p.stats.Hits, 1) if !cn.IsStale(p.idleTimeout) { - return cn, nil + return cn, false, nil } _ = p.closeConn(cn, errConnStale) } @@ -188,7 +188,7 @@ func (p *ConnPool) Get() (*Conn, error) { newcn, err := p.NewConn() if err != nil { <-p.queue - return nil, err + return nil, false, err } p.connsMu.Lock() @@ -198,7 +198,7 @@ func (p *ConnPool) Get() (*Conn, error) { p.conns = append(p.conns, newcn) p.connsMu.Unlock() - return newcn, nil + return newcn, true, nil } func (p *ConnPool) Put(cn *Conn) error { diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index cb1863eb13..0cf6c7c8a9 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -16,8 +16,8 @@ func (p *SingleConnPool) First() *Conn { return p.cn } -func (p *SingleConnPool) Get() (*Conn, error) { - return p.cn, nil +func (p *SingleConnPool) Get() (*Conn, bool, error) { + return p.cn, false, nil } func (p *SingleConnPool) Put(cn *Conn) error { diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 24a4f755ff..a2649e5d2e 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -30,23 +30,23 @@ func (p *StickyConnPool) First() *Conn { return cn } -func (p *StickyConnPool) Get() (*Conn, error) { +func (p *StickyConnPool) Get() (*Conn, bool, error) { defer p.mx.Unlock() p.mx.Lock() if p.closed { - return nil, ErrClosed + return nil, false, ErrClosed } if p.cn != nil { - return p.cn, nil + return p.cn, false, nil } - cn, err := p.pool.Get() + cn, _, err := p.pool.Get() if err != nil { - return nil, err + return nil, false, err } p.cn = cn - return cn, nil + return cn, true, nil } func (p *StickyConnPool) put() (err error) { diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 5fe7e8dbc5..425ce9280b 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -26,7 +26,7 @@ var _ = Describe("ConnPool", func() { It("rate limits dial", func() { var rateErr error for i := 0; i < 1000; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() if err != nil { rateErr = err break @@ -40,13 +40,13 @@ var _ = Describe("ConnPool", func() { It("should unblock client when conn is removed", func() { // Reserve one connection. - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) // Reserve all other connections. var cns []*pool.Conn for i := 0; i < 9; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) cns = append(cns, cn) } @@ -57,7 +57,7 @@ var _ = Describe("ConnPool", func() { defer GinkgoRecover() started <- true - _, err := connPool.Get() + _, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) done <- true @@ -113,7 +113,7 @@ var _ = Describe("conns reaper", func() { // add stale connections idleConns = nil for i := 0; i < 3; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) cn.UsedAt = time.Now().Add(-2 * idleTimeout) conns = append(conns, cn) @@ -122,7 +122,7 @@ var _ = Describe("conns reaper", func() { // add fresh connections for i := 0; i < 3; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) conns = append(conns, cn) } @@ -167,7 +167,7 @@ var _ = Describe("conns reaper", func() { for j := 0; j < 3; j++ { var freeCns []*pool.Conn for i := 0; i < 3; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) freeCns = append(freeCns, cn) @@ -176,7 +176,7 @@ var _ = Describe("conns reaper", func() { Expect(connPool.Len()).To(Equal(3)) Expect(connPool.FreeLen()).To(Equal(0)) - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) conns = append(conns, cn) @@ -224,7 +224,7 @@ var _ = Describe("race", func() { perform(C, func(id int) { for i := 0; i < N; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) if err == nil { Expect(connPool.Put(cn)).NotTo(HaveOccurred()) @@ -232,7 +232,7 @@ var _ = Describe("race", func() { } }, func(id int) { for i := 0; i < N; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) if err == nil { Expect(connPool.Remove(cn, errors.New("test"))).NotTo(HaveOccurred()) @@ -248,7 +248,7 @@ var _ = Describe("race", func() { perform(C, func(id int) { for i := 0; i < N; i++ { - cn, err := connPool.Get() + cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) if err == nil { Expect(connPool.Put(cn)).NotTo(HaveOccurred()) diff --git a/pool_test.go b/pool_test.go index c1d2f6810b..13edd700bf 100644 --- a/pool_test.go +++ b/pool_test.go @@ -91,7 +91,7 @@ var _ = Describe("pool", func() { }) It("should remove broken connections", func() { - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) diff --git a/pubsub.go b/pubsub.go index de8d35c4b6..796bd4a25d 100644 --- a/pubsub.go +++ b/pubsub.go @@ -18,12 +18,25 @@ type PubSub struct { channels []string patterns []string +} + +func (c *PubSub) conn() (*pool.Conn, bool, error) { + cn, isNew, err := c.base.conn() + if err != nil { + return nil, false, err + } + if isNew { + c.resubscribe() + } + return cn, isNew, nil +} - nsub int // number of active subscriptions +func (c *PubSub) putConn(cn *pool.Conn, err error) { + c.base.putConn(cn, err, true) } func (c *PubSub) subscribe(redisCmd string, channels ...string) error { - cn, err := c.base.conn() + cn, _, err := c.conn() if err != nil { return err } @@ -44,7 +57,6 @@ func (c *PubSub) Subscribe(channels ...string) error { err := c.subscribe("SUBSCRIBE", channels...) if err == nil { c.channels = appendIfNotExists(c.channels, channels...) - c.nsub += len(channels) } return err } @@ -54,43 +66,10 @@ func (c *PubSub) PSubscribe(patterns ...string) error { err := c.subscribe("PSUBSCRIBE", patterns...) if err == nil { c.patterns = appendIfNotExists(c.patterns, patterns...) - c.nsub += len(patterns) } return err } -func remove(ss []string, es ...string) []string { - if len(es) == 0 { - return ss[:0] - } - for _, e := range es { - for i, s := range ss { - if s == e { - ss = append(ss[:i], ss[i+1:]...) - break - } - } - } - return ss -} - -func appendIfNotExists(ss []string, es ...string) []string { - for _, e := range es { - found := false - for _, s := range ss { - if s == e { - found = true - break - } - } - - if !found { - ss = append(ss, e) - } - } - return ss -} - // Unsubscribes the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { @@ -116,7 +95,7 @@ func (c *PubSub) Close() error { } func (c *PubSub) Ping(payload string) error { - cn, err := c.base.conn() + cn, _, err := c.conn() if err != nil { return err } @@ -198,11 +177,7 @@ func (c *PubSub) newMessage(reply []interface{}) (interface{}, error) { // is not received in time. This is low-level API and most clients // should use ReceiveMessage. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { - if c.nsub == 0 { - c.resubscribe() - } - - cn, err := c.base.conn() + cn, _, err := c.conn() if err != nil { return nil, err } @@ -274,12 +249,6 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { } } -func (c *PubSub) putConn(cn *pool.Conn, err error) { - if !c.base.putConn(cn, err, true) { - c.nsub = 0 - } -} - func (c *PubSub) resubscribe() { if c.base.closed() { return @@ -295,3 +264,31 @@ func (c *PubSub) resubscribe() { } } } + +func remove(ss []string, es ...string) []string { + if len(es) == 0 { + return ss[:0] + } + for _, e := range es { + for i, s := range ss { + if s == e { + ss = append(ss[:i], ss[i+1:]...) + break + } + } + } + return ss +} + +func appendIfNotExists(ss []string, es ...string) []string { +loop: + for _, e := range es { + for _, s := range ss { + if s == e { + continue loop + } + } + ss = append(ss, e) + } + return ss +} diff --git a/pubsub_test.go b/pubsub_test.go index 116099b5a6..957d3031e9 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -288,7 +288,7 @@ var _ = Describe("PubSub", func() { }) expectReceiveMessageOnError := func(pubsub *redis.PubSub) { - cn1, err := pubsub.Pool().Get() + cn1, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn1.NetConn = &badConn{ readErr: io.EOF, diff --git a/redis.go b/redis.go index ea0a2b4319..4b535ff47d 100644 --- a/redis.go +++ b/redis.go @@ -27,18 +27,18 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.opt.Addr, c.opt.DB) } -func (c *baseClient) conn() (*pool.Conn, error) { - cn, err := c.connPool.Get() +func (c *baseClient) conn() (*pool.Conn, bool, error) { + cn, isNew, err := c.connPool.Get() if err != nil { - return nil, err + return nil, false, err } if !cn.Inited { if err := c.initConn(cn); err != nil { _ = c.connPool.Remove(cn, err) - return nil, err + return nil, false, err } } - return cn, err + return cn, isNew, nil } func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { @@ -84,7 +84,7 @@ func (c *baseClient) Process(cmd Cmder) error { cmd.reset() } - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { cmd.setErr(err) return err @@ -197,7 +197,7 @@ func (c *Client) pipelineExec(cmds []Cmder) error { var retErr error failedCmds := cmds for i := 0; i <= c.opt.MaxRetries; i++ { - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { setCmdsErr(failedCmds, err) return err diff --git a/redis_test.go b/redis_test.go index 4b62f3ed1a..0b59547145 100644 --- a/redis_test.go +++ b/redis_test.go @@ -144,7 +144,7 @@ var _ = Describe("Client", func() { }) // Put bad connection in the pool. - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} @@ -156,7 +156,7 @@ var _ = Describe("Client", func() { }) It("should update conn.UsedAt on read/write", func() { - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn.UsedAt).NotTo(BeZero()) createdAt := cn.UsedAt @@ -168,7 +168,7 @@ var _ = Describe("Client", func() { err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) - cn, err = client.Pool().Get() + cn, _, err = client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) Expect(cn.UsedAt.After(createdAt)).To(BeTrue()) diff --git a/ring.go b/ring.go index 5b2fddeeda..570c53573f 100644 --- a/ring.go +++ b/ring.go @@ -318,7 +318,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { for name, cmds := range cmdsMap { client := c.shards[name].Client - cn, err := client.conn() + cn, _, err := client.conn() if err != nil { setCmdsErr(cmds, err) if retErr == nil { diff --git a/tx.go b/tx.go index ca425ebba9..62e770183a 100644 --- a/tx.go +++ b/tx.go @@ -139,7 +139,7 @@ func (c *Tx) MultiExec(fn func() error) ([]Cmder, error) { // Strip MULTI and EXEC commands. retCmds := cmds[1 : len(cmds)-1] - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { setCmdsErr(retCmds, err) return retCmds, err diff --git a/tx_test.go b/tx_test.go index 7ff84dd09e..ddb9ddfa80 100644 --- a/tx_test.go +++ b/tx_test.go @@ -126,7 +126,7 @@ var _ = Describe("Tx", func() { It("should recover from bad connection", func() { // Put bad connection in the pool. - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} @@ -153,7 +153,7 @@ var _ = Describe("Tx", func() { It("should recover from bad connection when there are no commands", func() { // Put bad connection in the pool. - cn, err := client.Pool().Get() + cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} From 7cbee9d33726d49b95fda0cdd047ad4f761a1e91 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 2 Oct 2016 12:44:01 +0000 Subject: [PATCH 0216/1746] Enable reaper on ClusterClient and add tests. --- cluster.go | 39 ++++++---- cluster_test.go | 129 ++++++++----------------------- commands_test.go | 2 +- internal/hashtag/hashtag_test.go | 49 ++++++++++++ internal/pool/pool.go | 55 +++++++------ internal/pool/pool_single.go | 2 +- internal/pool/pool_sticky.go | 4 +- main_test.go | 32 +++++++- pool_test.go | 36 +++++++-- ring.go | 55 ++++++++++++- ring_test.go | 23 +++--- 11 files changed, 257 insertions(+), 169 deletions(-) diff --git a/cluster.go b/cluster.go index 402447ea7c..ad56af70ed 100644 --- a/cluster.go +++ b/cluster.go @@ -45,20 +45,25 @@ var _ Cmdable = (*ClusterClient)(nil) // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt.init() - client := &ClusterClient{ + + c := &ClusterClient{ opt: opt, nodes: make(map[string]*clusterNode), cmdsInfoOnce: new(sync.Once), } - client.cmdable.process = client.Process + c.cmdable.process = c.Process for _, addr := range opt.Addrs { - _, _ = client.nodeByAddr(addr) + _, _ = c.nodeByAddr(addr) + } + c.reloadSlots() + + if opt.IdleCheckFrequency > 0 { + go c.reaper(opt.IdleCheckFrequency) } - client.reloadSlots() - return client + return c } func (c *ClusterClient) cmdInfo(name string) *CommandInfo { @@ -333,11 +338,9 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { slots := c.slots c.mu.RUnlock() - var retErr error - var mu sync.Mutex - var wg sync.WaitGroup visited := make(map[*clusterNode]struct{}) + errCh := make(chan error, 1) for _, nodes := range slots { if len(nodes) == 0 { continue @@ -351,20 +354,24 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { wg.Add(1) go func(node *clusterNode) { + defer wg.Done() err := fn(node.Client) if err != nil { - mu.Lock() - if retErr == nil { - retErr = err + select { + case errCh <- err: + default: } - mu.Unlock() } - wg.Done() }(master) } wg.Wait() - return retErr + select { + case err := <-errCh: + return err + default: + return nil + } } // closeClients closes all clients and returns the first error if there are any. @@ -442,8 +449,8 @@ func (c *ClusterClient) setNodesLatency() { } // reaper closes idle connections to the cluster. -func (c *ClusterClient) reaper(frequency time.Duration) { - ticker := time.NewTicker(frequency) +func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { + ticker := time.NewTicker(idleCheckFrequency) defer ticker.Stop() for _ = range ticker.C { diff --git a/cluster_test.go b/cluster_test.go index 659e134e49..74b260f8a5 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -2,7 +2,6 @@ package redis_test import ( "fmt" - "math/rand" "net" "strconv" "strings" @@ -50,17 +49,6 @@ func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.Cluste for i, port := range s.ports { addrs[i] = net.JoinHostPort("127.0.0.1", port) } - if opt == nil { - opt = &redis.ClusterOptions{ - DialTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - PoolSize: 10, - PoolTimeout: 30 * time.Second, - IdleTimeout: time.Second, - IdleCheckFrequency: time.Second, - } - } opt.Addrs = addrs return redis.NewClusterClient(opt) } @@ -193,52 +181,12 @@ func stopCluster(scenario *clusterScenario) error { //------------------------------------------------------------------------------ -var _ = Describe("Cluster", func() { - Describe("HashSlot", func() { - - It("should calculate hash slots", func() { - tests := []struct { - key string - slot int - }{ - {"123456789", 12739}, - {"{}foo", 9500}, - {"foo{}", 5542}, - {"foo{}{bar}", 8363}, - {"", 10503}, - {"", 5176}, - {string([]byte{83, 153, 134, 118, 229, 214, 244, 75, 140, 37, 215, 215}), 5463}, - } - // Empty keys receive random slot. - rand.Seed(100) - - for _, test := range tests { - Expect(hashtag.Slot(test.key)).To(Equal(test.slot), "for %s", test.key) - } - }) - - It("should extract keys from tags", func() { - tests := []struct { - one, two string - }{ - {"foo{bar}", "bar"}, - {"{foo}bar", "foo"}, - {"{user1000}.following", "{user1000}.followers"}, - {"foo{{bar}}zap", "{bar"}, - {"foo{bar}{zap}", "bar"}, - } - - for _, test := range tests { - Expect(hashtag.Slot(test.one)).To(Equal(hashtag.Slot(test.two)), "for %s <-> %s", test.one, test.two) - } - }) - - }) - - Describe("Commands", func() { +var _ = Describe("ClusterClient", func() { + var client *redis.ClusterClient + describeClusterClient := func() { It("should CLUSTER SLOTS", func() { - res, err := cluster.primary().ClusterSlots().Result() + res, err := client.ClusterSlots().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(3)) @@ -251,73 +199,48 @@ var _ = Describe("Cluster", func() { }) It("should CLUSTER NODES", func() { - res, err := cluster.primary().ClusterNodes().Result() + res, err := client.ClusterNodes().Result() Expect(err).NotTo(HaveOccurred()) Expect(len(res)).To(BeNumerically(">", 400)) }) It("should CLUSTER INFO", func() { - res, err := cluster.primary().ClusterInfo().Result() + res, err := client.ClusterInfo().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(ContainSubstring("cluster_known_nodes:6")) }) It("should CLUSTER KEYSLOT", func() { - hashSlot, err := cluster.primary().ClusterKeySlot("somekey").Result() + hashSlot, err := client.ClusterKeySlot("somekey").Result() Expect(err).NotTo(HaveOccurred()) Expect(hashSlot).To(Equal(int64(hashtag.Slot("somekey")))) }) It("should CLUSTER COUNT-FAILURE-REPORTS", func() { - n, err := cluster.primary().ClusterCountFailureReports(cluster.nodeIds[0]).Result() + n, err := client.ClusterCountFailureReports(cluster.nodeIds[0]).Result() Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(0))) }) It("should CLUSTER COUNTKEYSINSLOT", func() { - n, err := cluster.primary().ClusterCountKeysInSlot(10).Result() + n, err := client.ClusterCountKeysInSlot(10).Result() Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(0))) }) - It("should CLUSTER DELSLOTS", func() { - res, err := cluster.primary().ClusterDelSlotsRange(16000, 16384-1).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal("OK")) - cluster.primary().ClusterAddSlotsRange(16000, 16384-1) - }) - It("should CLUSTER SAVECONFIG", func() { - res, err := cluster.primary().ClusterSaveConfig().Result() + res, err := client.ClusterSaveConfig().Result() Expect(err).NotTo(HaveOccurred()) Expect(res).To(Equal("OK")) }) It("should CLUSTER SLAVES", func() { - nodesList, err := cluster.primary().ClusterSlaves(cluster.nodeIds[0]).Result() + nodesList, err := client.ClusterSlaves(cluster.nodeIds[0]).Result() Expect(err).NotTo(HaveOccurred()) Expect(nodesList).Should(ContainElement(ContainSubstring("slave"))) Expect(nodesList).Should(HaveLen(1)) }) - // It("should CLUSTER READONLY", func() { - // res, err := cluster.primary().ReadOnly().Result() - // Expect(err).NotTo(HaveOccurred()) - // Expect(res).To(Equal("OK")) - // }) - - // It("should CLUSTER READWRITE", func() { - // res, err := cluster.primary().ReadWrite().Result() - // Expect(err).NotTo(HaveOccurred()) - // Expect(res).To(Equal("OK")) - // }) - }) -}) - -var _ = Describe("ClusterClient", func() { - var client *redis.ClusterClient - - describeClusterClient := func() { It("should GET/SET/DEL", func() { val, err := client.Get("A").Result() Expect(err).To(Equal(redis.Nil)) @@ -340,6 +263,18 @@ var _ = Describe("ClusterClient", func() { Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) }) + It("removes idle connections", func() { + stats := client.PoolStats() + Expect(stats.TotalConns).NotTo(BeZero()) + Expect(stats.FreeConns).NotTo(BeZero()) + + time.Sleep(2 * time.Second) + + stats = client.PoolStats() + Expect(stats.TotalConns).To(BeZero()) + Expect(stats.FreeConns).To(BeZero()) + }) + It("follows redirects", func() { Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) @@ -352,9 +287,9 @@ var _ = Describe("ClusterClient", func() { }) It("returns an error when there are no attempts left", func() { - client := cluster.clusterClient(&redis.ClusterOptions{ - MaxRedirects: -1, - }) + opt := redisClusterOptions() + opt.MaxRedirects = -1 + client := cluster.clusterClient(opt) slot := hashtag.Slot("A") Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) @@ -483,7 +418,7 @@ var _ = Describe("ClusterClient", func() { Describe("default ClusterClient", func() { BeforeEach(func() { - client = cluster.clusterClient(nil) + client = cluster.clusterClient(redisClusterOptions()) _ = client.ForEachMaster(func(master *redis.Client) error { return master.FlushDb().Err() @@ -499,9 +434,9 @@ var _ = Describe("ClusterClient", func() { Describe("ClusterClient with RouteByLatency", func() { BeforeEach(func() { - client = cluster.clusterClient(&redis.ClusterOptions{ - RouteByLatency: true, - }) + opt := redisClusterOptions() + opt.RouteByLatency = true + client = cluster.clusterClient(opt) _ = client.ForEachMaster(func(master *redis.Client) error { return master.FlushDb().Err() @@ -543,11 +478,13 @@ func BenchmarkRedisClusterPing(b *testing.B) { processes: make(map[string]*redisProcess, 6), clients: make(map[string]*redis.Client, 6), } + if err := startCluster(cluster); err != nil { b.Fatal(err) } defer stopCluster(cluster) - client := cluster.clusterClient(nil) + + client := cluster.clusterClient(redisClusterOptions()) defer client.Close() b.ResetTimer() diff --git a/commands_test.go b/commands_test.go index ea33e1b8f7..c89c333103 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1297,7 +1297,7 @@ var _ = Describe("Commands", func() { stats := client.PoolStats() Expect(stats.Requests).To(Equal(uint32(3))) - Expect(stats.Hits).To(Equal(uint32(2))) + Expect(stats.Hits).To(Equal(uint32(1))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) diff --git a/internal/hashtag/hashtag_test.go b/internal/hashtag/hashtag_test.go index 8132878d79..7f0fedf311 100644 --- a/internal/hashtag/hashtag_test.go +++ b/internal/hashtag/hashtag_test.go @@ -1,10 +1,18 @@ package hashtag import ( + "math/rand" + "testing" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) +func TestGinkgoSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "hashtag") +} + var _ = Describe("CRC16", func() { // http://redis.io/topics/cluster-spec#keys-distribution-model @@ -23,3 +31,44 @@ var _ = Describe("CRC16", func() { }) }) + +var _ = Describe("HashSlot", func() { + + It("should calculate hash slots", func() { + tests := []struct { + key string + slot int + }{ + {"123456789", 12739}, + {"{}foo", 9500}, + {"foo{}", 5542}, + {"foo{}{bar}", 8363}, + {"", 10503}, + {"", 5176}, + {string([]byte{83, 153, 134, 118, 229, 214, 244, 75, 140, 37, 215, 215}), 5463}, + } + // Empty keys receive random slot. + rand.Seed(100) + + for _, test := range tests { + Expect(Slot(test.key)).To(Equal(test.slot), "for %s", test.key) + } + }) + + It("should extract keys from tags", func() { + tests := []struct { + one, two string + }{ + {"foo{bar}", "bar"}, + {"{foo}bar", "foo"}, + {"{user1000}.following", "{user1000}.followers"}, + {"foo{{bar}}zap", "{bar"}, + {"foo{bar}{zap}", "bar"}, + } + + for _, test := range tests { + Expect(Slot(test.one)).To(Equal(Slot(test.two)), "for %s <-> %s", test.one, test.two) + } + }) + +}) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 55c1f9fda9..bd1c4130b8 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -25,8 +25,8 @@ var timers = sync.Pool{ }, } -// PoolStats contains pool state information and accumulated stats. -type PoolStats struct { +// Stats contains pool state information and accumulated stats. +type Stats struct { Requests uint32 // number of times a connection was requested by the pool Hits uint32 // number of times free connection was found in the pool Timeouts uint32 // number of times a wait timeout occurred @@ -41,7 +41,7 @@ type Pooler interface { Remove(*Conn, error) error Len() int FreeLen() int - Stats() *PoolStats + Stats() *Stats Close() error Closed() bool } @@ -64,7 +64,7 @@ type ConnPool struct { freeConnsMu sync.Mutex freeConns []*Conn - stats PoolStats + stats Stats _closed int32 // atomic lastErr atomic.Value @@ -173,16 +173,22 @@ func (p *ConnPool) Get() (*Conn, bool, error) { return nil, false, ErrPoolTimeout } - p.freeConnsMu.Lock() - cn := p.popFree() - p.freeConnsMu.Unlock() + for { + p.freeConnsMu.Lock() + cn := p.popFree() + p.freeConnsMu.Unlock() - if cn != nil { - atomic.AddUint32(&p.stats.Hits, 1) - if !cn.IsStale(p.idleTimeout) { - return cn, false, nil + if cn == nil { + break + } + + if cn.IsStale(p.idleTimeout) { + p.remove(cn, errConnStale) + continue } - _ = p.closeConn(cn, errConnStale) + + atomic.AddUint32(&p.stats.Hits, 1) + return cn, false, nil } newcn, err := p.NewConn() @@ -192,9 +198,6 @@ func (p *ConnPool) Get() (*Conn, bool, error) { } p.connsMu.Lock() - if cn != nil { - p.removeConn(cn) - } p.conns = append(p.conns, newcn) p.connsMu.Unlock() @@ -224,17 +227,13 @@ func (p *ConnPool) remove(cn *Conn, reason error) { _ = p.closeConn(cn, reason) p.connsMu.Lock() - p.removeConn(cn) - p.connsMu.Unlock() -} - -func (p *ConnPool) removeConn(cn *Conn) { for i, c := range p.conns { if c == cn { p.conns = append(p.conns[:i], p.conns[i+1:]...) break } } + p.connsMu.Unlock() } // Len returns total number of connections. @@ -253,14 +252,14 @@ func (p *ConnPool) FreeLen() int { return l } -func (p *ConnPool) Stats() *PoolStats { - stats := PoolStats{} - stats.Requests = atomic.LoadUint32(&p.stats.Requests) - stats.Hits = atomic.LoadUint32(&p.stats.Hits) - stats.Timeouts = atomic.LoadUint32(&p.stats.Timeouts) - stats.TotalConns = uint32(p.Len()) - stats.FreeConns = uint32(p.FreeLen()) - return &stats +func (p *ConnPool) Stats() *Stats { + return &Stats{ + Requests: atomic.LoadUint32(&p.stats.Requests), + Hits: atomic.LoadUint32(&p.stats.Hits), + Timeouts: atomic.LoadUint32(&p.stats.Timeouts), + TotalConns: uint32(p.Len()), + FreeConns: uint32(p.FreeLen()), + } } func (p *ConnPool) Closed() bool { diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index 0cf6c7c8a9..18ca616c3d 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -42,7 +42,7 @@ func (p *SingleConnPool) FreeLen() int { return 0 } -func (p *SingleConnPool) Stats() *PoolStats { +func (p *SingleConnPool) Stats() *Stats { return nil } diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index a2649e5d2e..ce45f4b44a 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -106,7 +106,9 @@ func (p *StickyConnPool) FreeLen() int { return 0 } -func (p *StickyConnPool) Stats() *PoolStats { return nil } +func (p *StickyConnPool) Stats() *Stats { + return nil +} func (p *StickyConnPool) Close() error { defer p.mx.Unlock() diff --git a/main_test.go b/main_test.go index 5208903fb6..a9afa8227e 100644 --- a/main_test.go +++ b/main_test.go @@ -108,8 +108,36 @@ func redisOptions() *redis.Options { WriteTimeout: 30 * time.Second, PoolSize: 10, PoolTimeout: 30 * time.Second, - IdleTimeout: time.Second, - IdleCheckFrequency: time.Second, + IdleTimeout: 500 * time.Millisecond, + IdleCheckFrequency: 500 * time.Millisecond, + } +} + +func redisClusterOptions() *redis.ClusterOptions { + return &redis.ClusterOptions{ + DialTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + PoolSize: 10, + PoolTimeout: 30 * time.Second, + IdleTimeout: 500 * time.Millisecond, + IdleCheckFrequency: 500 * time.Millisecond, + } +} + +func redisRingOptions() *redis.RingOptions { + return &redis.RingOptions{ + Addrs: map[string]string{ + "ringShardOne": ":" + ringShard1Port, + "ringShardTwo": ":" + ringShard2Port, + }, + DialTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + PoolSize: 10, + PoolTimeout: 30 * time.Second, + IdleTimeout: 500 * time.Millisecond, + IdleCheckFrequency: 500 * time.Millisecond, } } diff --git a/pool_test.go b/pool_test.go index 13edd700bf..2b0d2ad4c5 100644 --- a/pool_test.go +++ b/pool_test.go @@ -1,6 +1,8 @@ package redis_test import ( + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -20,7 +22,7 @@ var _ = Describe("pool", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("should respect max size", func() { + It("respects max size", func() { perform(1000, func(id int) { val, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) @@ -33,7 +35,7 @@ var _ = Describe("pool", func() { Expect(pool.Len()).To(Equal(pool.FreeLen())) }) - It("should respect max on multi", func() { + It("srespect max size on multi", func() { perform(1000, func(id int) { var ping *redis.StatusCmd @@ -58,7 +60,7 @@ var _ = Describe("pool", func() { Expect(pool.Len()).To(Equal(pool.FreeLen())) }) - It("should respect max on pipelines", func() { + It("respects max size on pipelines", func() { perform(1000, func(id int) { pipe := client.Pipeline() ping := pipe.Ping() @@ -76,7 +78,7 @@ var _ = Describe("pool", func() { Expect(pool.Len()).To(Equal(pool.FreeLen())) }) - It("should respect max on pubsub", func() { + It("respects max size on pubsub", func() { connPool := client.Pool() connPool.(*pool.ConnPool).DialLimiter = nil @@ -90,7 +92,7 @@ var _ = Describe("pool", func() { Expect(connPool.Len()).To(BeNumerically("<=", 10)) }) - It("should remove broken connections", func() { + It("removes broken connections", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.NetConn = &badConn{} @@ -113,7 +115,7 @@ var _ = Describe("pool", func() { Expect(stats.Timeouts).To(Equal(uint32(0))) }) - It("should reuse connections", func() { + It("reuses connections", func() { for i := 0; i < 100; i++ { val, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) @@ -129,4 +131,26 @@ var _ = Describe("pool", func() { Expect(stats.Hits).To(Equal(uint32(100))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) + + It("removes idle connections", func() { + stats := client.PoolStats() + Expect(stats).To(Equal(&redis.PoolStats{ + Requests: 1, + Hits: 0, + Timeouts: 0, + TotalConns: 1, + FreeConns: 1, + })) + + time.Sleep(2 * time.Second) + + stats = client.PoolStats() + Expect(stats).To(Equal(&redis.PoolStats{ + Requests: 1, + Hits: 0, + Timeouts: 0, + TotalConns: 0, + FreeConns: 0, + })) + }) }) diff --git a/ring.go b/ring.go index 570c53573f..b0bba58595 100644 --- a/ring.go +++ b/ring.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" "gopkg.in/redis.v4/internal" @@ -65,7 +66,7 @@ func (opt *RingOptions) clientOptions() *Options { type ringShard struct { Client *Client - down int + down int32 } func (shard *ringShard) String() string { @@ -80,7 +81,7 @@ func (shard *ringShard) String() string { func (shard *ringShard) IsDown() bool { const threshold = 3 - return shard.down >= threshold + return atomic.LoadInt32(&shard.down) >= threshold } func (shard *ringShard) IsUp() bool { @@ -91,7 +92,7 @@ func (shard *ringShard) IsUp() bool { func (shard *ringShard) Vote(up bool) bool { if up { changed := shard.IsDown() - shard.down = 0 + atomic.StoreInt32(&shard.down, 0) return changed } @@ -99,7 +100,7 @@ func (shard *ringShard) Vote(up bool) bool { return false } - shard.down++ + atomic.AddInt32(&shard.down, 1) return shard.IsDown() } @@ -157,6 +158,52 @@ func NewRing(opt *RingOptions) *Ring { return ring } +// PoolStats returns accumulated connection pool stats. +func (c *Ring) PoolStats() *PoolStats { + var acc PoolStats + for _, shard := range c.shards { + s := shard.Client.connPool.Stats() + acc.Requests += s.Requests + acc.Hits += s.Hits + acc.Timeouts += s.Timeouts + acc.TotalConns += s.TotalConns + acc.FreeConns += s.FreeConns + } + return &acc +} + +// ForEachShard concurrently calls the fn on each live shard in the ring. +// It returns the first error if any. +func (c *Ring) ForEachShard(fn func(client *Client) error) error { + var wg sync.WaitGroup + errCh := make(chan error, 1) + for _, shard := range c.shards { + if shard.IsDown() { + continue + } + + wg.Add(1) + go func(shard *ringShard) { + defer wg.Done() + err := fn(shard.Client) + if err != nil { + select { + case errCh <- err: + default: + } + } + }(shard) + } + wg.Wait() + + select { + case err := <-errCh: + return err + default: + return nil + } +} + func (c *Ring) cmdInfo(name string) *CommandInfo { c.cmdsInfoOnce.Do(func() { for _, shard := range c.shards { diff --git a/ring_test.go b/ring_test.go index a63f5c8e27..3c06686fe7 100644 --- a/ring_test.go +++ b/ring_test.go @@ -11,8 +11,9 @@ import ( "gopkg.in/redis.v4" ) -var _ = Describe("Redis ring", func() { +var _ = Describe("Redis Ring", func() { const heartbeat = 100 * time.Millisecond + var ring *redis.Ring setRingKeys := func() { @@ -23,20 +24,14 @@ var _ = Describe("Redis ring", func() { } BeforeEach(func() { - ring = redis.NewRing(&redis.RingOptions{ - Addrs: map[string]string{ - "ringShardOne": ":" + ringShard1Port, - "ringShardTwo": ":" + ringShard2Port, - }, - HeartbeatFrequency: heartbeat, - }) - - // Shards should not have any keys. - Expect(ringShard1.FlushDb().Err()).NotTo(HaveOccurred()) - Expect(ringShard1.Info().Val()).NotTo(ContainSubstring("keys=")) + opt := redisRingOptions() + opt.HeartbeatFrequency = heartbeat + ring = redis.NewRing(opt) - Expect(ringShard2.FlushDb().Err()).NotTo(HaveOccurred()) - Expect(ringShard2.Info().Val()).NotTo(ContainSubstring("keys=")) + err := ring.ForEachShard(func(cl *redis.Client) error { + return cl.FlushDb().Err() + }) + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { From 03da66c18a66a00ff0f796370438a58f6d2b3b13 Mon Sep 17 00:00:00 2001 From: Artem Chernyshev Date: Mon, 3 Oct 2016 12:38:13 +0300 Subject: [PATCH 0217/1746] Add new parameter to the RingOptions to override CommandInfo first key There is problem with `eval` and `evalsha` commands. `COMMAND INFO eval` returns first key position equals `0`. After that, redis ring chooses `eval` as a value for sharding. They, if you try to delete created value, ring may choose another shard and delete won't work. Eval command should be parsed, to be sharded properly, according to redis specs: http://redis.io/commands/command . I've introduced a new flag in the `RingOptions`, which will enable new behavior: `EnableKeyLocationParsing`. If it is enabled, `cmdFirstKey` will try to get key position using `cmd.getFirstKeyPos()`. This function is defined for `eval` and `evalsha` commands. If it has parameters, it will return `3`, otherwise it will return `0`. --- main_test.go | 1 + ring.go | 21 +++++++++++++++++++++ ring_test.go | 17 +++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/main_test.go b/main_test.go index a9afa8227e..d79a14395e 100644 --- a/main_test.go +++ b/main_test.go @@ -138,6 +138,7 @@ func redisRingOptions() *redis.RingOptions { PoolTimeout: 30 * time.Second, IdleTimeout: 500 * time.Millisecond, IdleCheckFrequency: 500 * time.Millisecond, + RouteByEvalKeys: true, } } diff --git a/ring.go b/ring.go index b0bba58595..08d52ef512 100644 --- a/ring.go +++ b/ring.go @@ -40,6 +40,9 @@ type RingOptions struct { PoolTimeout time.Duration IdleTimeout time.Duration IdleCheckFrequency time.Duration + + // RouteByEvalKeys flag to enable eval and evalsha key position parsing for sharding + RouteByEvalKeys bool } func (opt *RingOptions) init() { @@ -132,6 +135,8 @@ type Ring struct { cmdsInfoOnce *sync.Once closed bool + + routeByEvalKeys bool } var _ Cmdable = (*Ring)(nil) @@ -154,6 +159,7 @@ func NewRing(opt *RingOptions) *Ring { clopt.Addr = addr ring.addClient(name, NewClient(clopt)) } + ring.routeByEvalKeys = opt.RouteByEvalKeys go ring.heartbeat() return ring } @@ -221,7 +227,22 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { return c.cmdsInfo[name] } +func (c *Ring) getEvalFirstKey(cmd Cmder) string { + if c.routeByEvalKeys && cmd.arg(2) != "0" { + return cmd.arg(3) + } else { + return cmd.arg(0) + } +} + func (c *Ring) cmdFirstKey(cmd Cmder) string { + switch cmd.arg(0) { + case "eval": + return c.getEvalFirstKey(cmd) + case "evalsha": + return c.getEvalFirstKey(cmd) + } + cmdInfo := c.cmdInfo(cmd.arg(0)) if cmdInfo == nil { internal.Logf("info for cmd=%s not found", cmd.arg(0)) diff --git a/ring_test.go b/ring_test.go index 3c06686fe7..0ab258d858 100644 --- a/ring_test.go +++ b/ring_test.go @@ -89,6 +89,23 @@ var _ = Describe("Redis Ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=100")) }) + It("supports eval key search", func() { + script := redis.NewScript(` + local r = redis.call('SET', KEYS[1], ARGV[1]) + return r + `) + + var key string + for i := 0; i < 100; i++ { + key = fmt.Sprintf("key{%d}", i) + err := script.Run(ring, []string{key}, "value").Err() + Expect(err).NotTo(HaveOccurred()) + } + + Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=52")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=48")) + }) + Describe("pipelining", func() { It("returns an error when all shards are down", func() { ring := redis.NewRing(&redis.RingOptions{}) From 84ae9866599493e939021f55d40ded0e8f855cc0 Mon Sep 17 00:00:00 2001 From: yyoshiki41 Date: Thu, 6 Oct 2016 05:20:05 +0900 Subject: [PATCH 0218/1746] Remove Addr field from clusterNode --- cluster.go | 8 +++----- cluster_client_test.go | 34 +++++++++++++++++----------------- redis.go | 6 +++++- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/cluster.go b/cluster.go index ad56af70ed..9baad73aaa 100644 --- a/cluster.go +++ b/cluster.go @@ -13,7 +13,6 @@ import ( ) type clusterNode struct { - Addr string Client *Client Latency time.Duration } @@ -152,7 +151,7 @@ func (c *ClusterClient) nodeByAddr(addr string) (*clusterNode, error) { if !ok { node = c.newNode(addr) c.nodes[addr] = node - c.addrs = append(c.addrs, node.Addr) + c.addrs = append(c.addrs, addr) } return node, nil @@ -162,7 +161,6 @@ func (c *ClusterClient) newNode(addr string) *clusterNode { opt := c.opt.clientOptions() opt.Addr = addr return &clusterNode{ - Addr: addr, Client: NewClient(opt), } } @@ -313,7 +311,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { moved, ask, addr = errors.IsMoved(err) if moved || ask { master, _ := c.slotMasterNode(slot) - if moved && (master == nil || master.Addr != addr) { + if moved && (master == nil || master.Client.getAddr() != addr) { c.lazyReloadSlots() } @@ -425,7 +423,7 @@ func (c *ClusterClient) reloadSlots() { slots, err := node.Client.ClusterSlots().Result() if err != nil { - internal.Logf("ClusterSlots on addr=%q failed: %s", node.Addr, err) + internal.Logf("ClusterSlots on addr=%q failed: %s", node.Client.getAddr(), err) return } diff --git a/cluster_client_test.go b/cluster_client_test.go index 8277abdfb6..0a30d5dad4 100644 --- a/cluster_client_test.go +++ b/cluster_client_test.go @@ -8,7 +8,7 @@ import ( func (c *ClusterClient) SlotAddrs(slot int) []string { var addrs []string for _, n := range c.slotNodes(slot) { - addrs = append(addrs, n.Addr) + addrs = append(addrs, n.Client.getAddr()) } return addrs } @@ -51,22 +51,22 @@ var _ = Describe("ClusterClient", func() { It("should update slots cache", func() { populate() - Expect(subject.slots[0][0].Addr).To(Equal("127.0.0.1:7000")) - Expect(subject.slots[0][1].Addr).To(Equal("127.0.0.1:7004")) - Expect(subject.slots[4095][0].Addr).To(Equal("127.0.0.1:7000")) - Expect(subject.slots[4095][1].Addr).To(Equal("127.0.0.1:7004")) - Expect(subject.slots[4096][0].Addr).To(Equal("127.0.0.1:7001")) - Expect(subject.slots[4096][1].Addr).To(Equal("127.0.0.1:7005")) - Expect(subject.slots[8191][0].Addr).To(Equal("127.0.0.1:7001")) - Expect(subject.slots[8191][1].Addr).To(Equal("127.0.0.1:7005")) - Expect(subject.slots[8192][0].Addr).To(Equal("127.0.0.1:7002")) - Expect(subject.slots[8192][1].Addr).To(Equal("127.0.0.1:7006")) - Expect(subject.slots[12287][0].Addr).To(Equal("127.0.0.1:7002")) - Expect(subject.slots[12287][1].Addr).To(Equal("127.0.0.1:7006")) - Expect(subject.slots[12288][0].Addr).To(Equal("127.0.0.1:7003")) - Expect(subject.slots[12288][1].Addr).To(Equal("127.0.0.1:7007")) - Expect(subject.slots[16383][0].Addr).To(Equal("127.0.0.1:7003")) - Expect(subject.slots[16383][1].Addr).To(Equal("127.0.0.1:7007")) + Expect(subject.slots[0][0].Client.getAddr()).To(Equal("127.0.0.1:7000")) + Expect(subject.slots[0][1].Client.getAddr()).To(Equal("127.0.0.1:7004")) + Expect(subject.slots[4095][0].Client.getAddr()).To(Equal("127.0.0.1:7000")) + Expect(subject.slots[4095][1].Client.getAddr()).To(Equal("127.0.0.1:7004")) + Expect(subject.slots[4096][0].Client.getAddr()).To(Equal("127.0.0.1:7001")) + Expect(subject.slots[4096][1].Client.getAddr()).To(Equal("127.0.0.1:7005")) + Expect(subject.slots[8191][0].Client.getAddr()).To(Equal("127.0.0.1:7001")) + Expect(subject.slots[8191][1].Client.getAddr()).To(Equal("127.0.0.1:7005")) + Expect(subject.slots[8192][0].Client.getAddr()).To(Equal("127.0.0.1:7002")) + Expect(subject.slots[8192][1].Client.getAddr()).To(Equal("127.0.0.1:7006")) + Expect(subject.slots[12287][0].Client.getAddr()).To(Equal("127.0.0.1:7002")) + Expect(subject.slots[12287][1].Client.getAddr()).To(Equal("127.0.0.1:7006")) + Expect(subject.slots[12288][0].Client.getAddr()).To(Equal("127.0.0.1:7003")) + Expect(subject.slots[12288][1].Client.getAddr()).To(Equal("127.0.0.1:7007")) + Expect(subject.slots[16383][0].Client.getAddr()).To(Equal("127.0.0.1:7003")) + Expect(subject.slots[16383][1].Client.getAddr()).To(Equal("127.0.0.1:7007")) Expect(subject.addrs).To(Equal([]string{ "127.0.0.1:6379", "127.0.0.1:7003", diff --git a/redis.go b/redis.go index 4b535ff47d..60df328e61 100644 --- a/redis.go +++ b/redis.go @@ -24,7 +24,7 @@ type baseClient struct { } func (c *baseClient) String() string { - return fmt.Sprintf("Redis<%s db:%d>", c.opt.Addr, c.opt.DB) + return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB) } func (c *baseClient) conn() (*pool.Conn, bool, error) { @@ -140,6 +140,10 @@ func (c *baseClient) Close() error { return retErr } +func (c *baseClient) getAddr() string { + return c.opt.Addr +} + //------------------------------------------------------------------------------ // Client is a Redis client representing a pool of zero or more From ce3447602ff2e31bbdd843873950c9b5f1a10af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Ugov=C5=A1ek?= Date: Fri, 7 Oct 2016 15:39:45 +0200 Subject: [PATCH 0219/1746] Add missing argument to a `fmt.Errorf` call --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 46220d4051..a4ad4e2aa9 100644 --- a/parser.go +++ b/parser.go @@ -316,7 +316,7 @@ func commandInfoParser(rd *proto.Reader, n int64) (interface{}, error) { var err error if n != 6 { - return nil, fmt.Errorf("redis: got %d elements in COMMAND reply, wanted 6") + return nil, fmt.Errorf("redis: got %d elements in COMMAND reply, wanted 6", n) } cmd.Name, err = rd.ReadStringReply() From 3cc9b99fb8a9fe88d4d150d021554b5f41fb41d8 Mon Sep 17 00:00:00 2001 From: lijunfei Date: Sun, 9 Oct 2016 11:30:01 +0800 Subject: [PATCH 0220/1746] if readonly, read from master when slave is loading --- cluster.go | 11 +++++++++++ internal/errors/errors.go | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/cluster.go b/cluster.go index 9baad73aaa..e501154421 100644 --- a/cluster.go +++ b/cluster.go @@ -300,6 +300,17 @@ func (c *ClusterClient) Process(cmd Cmder) error { return nil } + // If slave is loading, read from master + if errors.IsLoading(cmd.Err()) && c.opt.ReadOnly { + trynode, err := c.slotMasterNode(slot) + if err == nil && trynode != node { + node = trynode + continue + } else { + break + } + } + // On network errors try random node. if errors.IsRetryable(err) { node, err = c.randomNode() diff --git a/internal/errors/errors.go b/internal/errors/errors.go index a17a3d37b6..a7e8604f2c 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -10,6 +10,8 @@ const Nil = RedisError("redis: nil") type RedisError string +var LoadingError RedisError = "LOADING Redis is loading the dataset in memory" + func (e RedisError) Error() string { return string(e) } func IsRetryable(err error) bool { @@ -65,3 +67,7 @@ func IsMoved(err error) (moved bool, ask bool, addr string) { addr = s[ind+1:] return } + +func IsLoading(err error) bool { + return err.Error() == string(LastIndexoadingError) +} From 1b06f9351ac74f0494ba0078db43f67ff91dd095 Mon Sep 17 00:00:00 2001 From: lijunfei Date: Sun, 9 Oct 2016 11:44:58 +0800 Subject: [PATCH 0221/1746] define loading error --- internal/errors/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/errors/errors.go b/internal/errors/errors.go index a7e8604f2c..49a80eaeaa 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -69,5 +69,5 @@ func IsMoved(err error) (moved bool, ask bool, addr string) { } func IsLoading(err error) bool { - return err.Error() == string(LastIndexoadingError) + return err.Error() == string(LoadingError) } From fcf53a2a787f0e1398a961fdccf877b81e97535b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Oct 2016 08:18:57 +0000 Subject: [PATCH 0222/1746] Better cluster loading handling. --- cluster.go | 41 ++++++++++++++++++++++++--------------- internal/errors/errors.go | 4 +--- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/cluster.go b/cluster.go index e501154421..7689a74e66 100644 --- a/cluster.go +++ b/cluster.go @@ -15,6 +15,11 @@ import ( type clusterNode struct { Client *Client Latency time.Duration + loading time.Time +} + +func (n *clusterNode) Loading() bool { + return !n.loading.IsZero() && time.Since(n.loading) < time.Minute } // ClusterClient is a Redis Cluster client representing a pool of zero @@ -218,10 +223,20 @@ func (c *ClusterClient) slotSlaveNode(slot int) (*clusterNode, error) { case 1: return nodes[0], nil case 2: - return nodes[1], nil + if slave := nodes[1]; !slave.Loading() { + return slave, nil + } + return nodes[0], nil default: - n := rand.Intn(len(nodes)-1) + 1 - return nodes[n], nil + var slave *clusterNode + for i := 0; i < 10; i++ { + n := rand.Intn(len(nodes)-1) + 1 + slave = nodes[n] + if !slave.Loading() { + break + } + } + return slave, nil } } @@ -287,28 +302,22 @@ func (c *ClusterClient) Process(cmd Cmder) error { pipe := node.Client.Pipeline() pipe.Process(NewCmd("ASKING")) pipe.Process(cmd) - _, _ = pipe.Exec() + _, err = pipe.Exec() pipe.Close() ask = false } else { - node.Client.Process(cmd) + err = node.Client.Process(cmd) } - // If there is no (real) error, we are done! - err := cmd.Err() + // If there is no (real) error - we are done. if err == nil { return nil } - // If slave is loading, read from master - if errors.IsLoading(cmd.Err()) && c.opt.ReadOnly { - trynode, err := c.slotMasterNode(slot) - if err == nil && trynode != node { - node = trynode - continue - } else { - break - } + // If slave is loading - read from master. + if c.opt.ReadOnly && errors.IsLoading(err) { + node.loading = time.Now() + continue } // On network errors try random node. diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 49a80eaeaa..c5abc0be58 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -10,8 +10,6 @@ const Nil = RedisError("redis: nil") type RedisError string -var LoadingError RedisError = "LOADING Redis is loading the dataset in memory" - func (e RedisError) Error() string { return string(e) } func IsRetryable(err error) bool { @@ -69,5 +67,5 @@ func IsMoved(err error) (moved bool, ask bool, addr string) { } func IsLoading(err error) bool { - return err.Error() == string(LoadingError) + return strings.HasPrefix(err.Error(), "LOADING") } From eeba1d7db110fcb3db5c469813abc4c62638c0ed Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Oct 2016 10:30:45 +0000 Subject: [PATCH 0223/1746] Select random node when there are no keys. --- cluster.go | 162 +++++++++++++++++++++++++-------------------------- command.go | 27 +++++++-- main_test.go | 1 - ring.go | 101 +++++++++++++++++--------------- 4 files changed, 155 insertions(+), 136 deletions(-) diff --git a/cluster.go b/cluster.go index 7689a74e66..6aea41b6fd 100644 --- a/cluster.go +++ b/cluster.go @@ -12,6 +12,72 @@ import ( "gopkg.in/redis.v4/internal/pool" ) +// ClusterOptions are used to configure a cluster client and should be +// passed to NewClusterClient. +type ClusterOptions struct { + // A seed list of host:port addresses of cluster nodes. + Addrs []string + + // The maximum number of retries before giving up. Command is retried + // on network errors and MOVED/ASK redirects. + // Default is 16. + MaxRedirects int + + // Enables read queries for a connection to a Redis Cluster slave node. + ReadOnly bool + + // Enables routing read-only queries to the closest master or slave node. + RouteByLatency bool + + // Following options are copied from Options struct. + + Password string + + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + + // PoolSize applies per cluster node and not for the whole cluster. + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration + IdleCheckFrequency time.Duration +} + +func (opt *ClusterOptions) init() { + if opt.MaxRedirects == -1 { + opt.MaxRedirects = 0 + } else if opt.MaxRedirects == 0 { + opt.MaxRedirects = 16 + } + + if opt.RouteByLatency { + opt.ReadOnly = true + } +} + +func (opt *ClusterOptions) clientOptions() *Options { + const disableIdleCheck = -1 + + return &Options{ + Password: opt.Password, + ReadOnly: opt.ReadOnly, + + DialTimeout: opt.DialTimeout, + ReadTimeout: opt.ReadTimeout, + WriteTimeout: opt.WriteTimeout, + + PoolSize: opt.PoolSize, + PoolTimeout: opt.PoolTimeout, + IdleTimeout: opt.IdleTimeout, + + // IdleCheckFrequency is not copied to disable reaper + IdleCheckFrequency: disableIdleCheck, + } +} + +//------------------------------------------------------------------------------ + type clusterNode struct { Client *Client Latency time.Duration @@ -36,8 +102,8 @@ type ClusterClient struct { slots [][]*clusterNode closed bool - cmdsInfo map[string]*CommandInfo cmdsInfoOnce *sync.Once + cmdsInfo map[string]*CommandInfo // Reports where slots reloading is in progress. reloading uint32 @@ -81,19 +147,22 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo { } c.cmdsInfoOnce = &sync.Once{} }) + if c.cmdsInfo == nil { + return nil + } return c.cmdsInfo[name] } func (c *ClusterClient) getNodes() map[string]*clusterNode { var nodes map[string]*clusterNode - c.mu.RLock() if !c.closed { nodes = make(map[string]*clusterNode, len(c.nodes)) + c.mu.RLock() for addr, node := range c.nodes { nodes[addr] = node } + c.mu.RUnlock() } - c.mu.RUnlock() return nodes } @@ -257,18 +326,11 @@ func (c *ClusterClient) slotClosestNode(slot int) (*clusterNode, error) { func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmdInfo(cmd.arg(0)) - if cmdInfo == nil { - internal.Logf("info for cmd=%s not found", cmd.arg(0)) + firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + if firstKey == "" { node, err := c.randomNode() - return 0, node, err + return -1, node, err } - - if cmdInfo.FirstKeyPos == -1 { - node, err := c.randomNode() - return 0, node, err - } - - firstKey := cmd.arg(int(cmdInfo.FirstKeyPos)) slot := hashtag.Slot(firstKey) if cmdInfo.ReadOnly && c.opt.ReadOnly { @@ -330,9 +392,11 @@ func (c *ClusterClient) Process(cmd Cmder) error { var addr string moved, ask, addr = errors.IsMoved(err) if moved || ask { - master, _ := c.slotMasterNode(slot) - if moved && (master == nil || master.Client.getAddr() != addr) { - c.lazyReloadSlots() + if slot >= 0 { + master, _ := c.slotMasterNode(slot) + if moved && (master == nil || master.Client.getAddr() != addr) { + c.lazyReloadSlots() + } } node, err = c.nodeByAddr(addr) @@ -609,69 +673,3 @@ func (c *ClusterClient) execClusterCmds( return failedCmds, retErr } - -//------------------------------------------------------------------------------ - -// ClusterOptions are used to configure a cluster client and should be -// passed to NewClusterClient. -type ClusterOptions struct { - // A seed list of host:port addresses of cluster nodes. - Addrs []string - - // The maximum number of retries before giving up. Command is retried - // on network errors and MOVED/ASK redirects. - // Default is 16. - MaxRedirects int - - // Enables read queries for a connection to a Redis Cluster slave node. - ReadOnly bool - - // Enables routing read-only queries to the closest master or slave node. - RouteByLatency bool - - // Following options are copied from Options struct. - - Password string - - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - - // PoolSize applies per cluster node and not for the whole cluster. - PoolSize int - PoolTimeout time.Duration - IdleTimeout time.Duration - IdleCheckFrequency time.Duration -} - -func (opt *ClusterOptions) init() { - if opt.MaxRedirects == -1 { - opt.MaxRedirects = 0 - } else if opt.MaxRedirects == 0 { - opt.MaxRedirects = 16 - } - - if opt.RouteByLatency { - opt.ReadOnly = true - } -} - -func (opt *ClusterOptions) clientOptions() *Options { - const disableIdleCheck = -1 - - return &Options{ - Password: opt.Password, - ReadOnly: opt.ReadOnly, - - DialTimeout: opt.DialTimeout, - ReadTimeout: opt.ReadTimeout, - WriteTimeout: opt.WriteTimeout, - - PoolSize: opt.PoolSize, - PoolTimeout: opt.PoolTimeout, - IdleTimeout: opt.IdleTimeout, - - // IdleCheckFrequency is not copied to disable reaper - IdleCheckFrequency: disableIdleCheck, - } -} diff --git a/command.go b/command.go index 8ac048acfe..7613203823 100644 --- a/command.go +++ b/command.go @@ -7,6 +7,8 @@ import ( "strings" "time" + "github.com/go-redis/redis/internal" + "gopkg.in/redis.v4/internal/pool" "gopkg.in/redis.v4/internal/proto" ) @@ -88,6 +90,22 @@ func cmdString(cmd Cmder, val interface{}) string { } +func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { + switch cmd.arg(0) { + case "eval", "evalsha": + if cmd.arg(2) != "0" { + return 3 + } else { + return 0 + } + } + if info == nil { + internal.Logf("info for cmd=%s not found", cmd.arg(0)) + return -1 + } + return int(info.FirstKeyPos) +} + //------------------------------------------------------------------------------ type baseCmd struct { @@ -109,12 +127,11 @@ func (cmd *baseCmd) args() []interface{} { } func (cmd *baseCmd) arg(pos int) string { - if len(cmd._args) > pos { - if s, ok := cmd._args[pos].(string); ok { - return s - } + if pos < 0 || pos >= len(cmd._args) { + return "" } - return "" + s, _ := cmd._args[pos].(string) + return s } func (cmd *baseCmd) readTimeout() *time.Duration { diff --git a/main_test.go b/main_test.go index d79a14395e..a9afa8227e 100644 --- a/main_test.go +++ b/main_test.go @@ -138,7 +138,6 @@ func redisRingOptions() *redis.RingOptions { PoolTimeout: 30 * time.Second, IdleTimeout: 500 * time.Millisecond, IdleCheckFrequency: 500 * time.Millisecond, - RouteByEvalKeys: true, } } diff --git a/ring.go b/ring.go index 08d52ef512..52badb3582 100644 --- a/ring.go +++ b/ring.go @@ -3,6 +3,8 @@ package redis import ( "errors" "fmt" + "math/rand" + "strconv" "sync" "sync/atomic" "time" @@ -40,9 +42,6 @@ type RingOptions struct { PoolTimeout time.Duration IdleTimeout time.Duration IdleCheckFrequency time.Duration - - // RouteByEvalKeys flag to enable eval and evalsha key position parsing for sharding - RouteByEvalKeys bool } func (opt *RingOptions) init() { @@ -131,12 +130,10 @@ type Ring struct { hash *consistenthash.Map shards map[string]*ringShard - cmdsInfo map[string]*CommandInfo cmdsInfoOnce *sync.Once + cmdsInfo map[string]*CommandInfo closed bool - - routeByEvalKeys bool } var _ Cmdable = (*Ring)(nil) @@ -159,7 +156,6 @@ func NewRing(opt *RingOptions) *Ring { clopt.Addr = addr ring.addClient(name, NewClient(clopt)) } - ring.routeByEvalKeys = opt.RouteByEvalKeys go ring.heartbeat() return ring } @@ -227,30 +223,6 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { return c.cmdsInfo[name] } -func (c *Ring) getEvalFirstKey(cmd Cmder) string { - if c.routeByEvalKeys && cmd.arg(2) != "0" { - return cmd.arg(3) - } else { - return cmd.arg(0) - } -} - -func (c *Ring) cmdFirstKey(cmd Cmder) string { - switch cmd.arg(0) { - case "eval": - return c.getEvalFirstKey(cmd) - case "evalsha": - return c.getEvalFirstKey(cmd) - } - - cmdInfo := c.cmdInfo(cmd.arg(0)) - if cmdInfo == nil { - internal.Logf("info for cmd=%s not found", cmd.arg(0)) - return "" - } - return cmd.arg(int(cmdInfo.FirstKeyPos)) -} - func (c *Ring) addClient(name string, cl *Client) { c.mu.Lock() c.hash.Add(name) @@ -258,14 +230,17 @@ func (c *Ring) addClient(name string, cl *Client) { c.mu.Unlock() } -func (c *Ring) getClient(key string) (*Client, error) { +func (c *Ring) shardByKey(key string) (*Client, error) { + key = hashtag.Key(key) + c.mu.RLock() if c.closed { + c.mu.RUnlock() return nil, pool.ErrClosed } - name := c.hash.Get(hashtag.Key(key)) + name := c.hash.Get(key) if name == "" { c.mu.RUnlock() return nil, errRingShardsDown @@ -276,8 +251,32 @@ func (c *Ring) getClient(key string) (*Client, error) { return cl, nil } +func (c *Ring) randomShard() (*Client, error) { + return c.shardByKey(strconv.Itoa(rand.Int())) +} + +func (c *Ring) shardByName(name string) (*Client, error) { + if name == "" { + return c.randomShard() + } + + c.mu.RLock() + cl := c.shards[name].Client + c.mu.RUnlock() + return cl, nil +} + +func (c *Ring) cmdShard(cmd Cmder) (*Client, error) { + cmdInfo := c.cmdInfo(cmd.arg(0)) + firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + if firstKey == "" { + return c.randomShard() + } + return c.shardByKey(firstKey) +} + func (c *Ring) Process(cmd Cmder) error { - cl, err := c.getClient(c.cmdFirstKey(cmd)) + cl, err := c.cmdShard(cmd) if err != nil { cmd.setErr(err) return err @@ -285,17 +284,18 @@ func (c *Ring) Process(cmd Cmder) error { return cl.baseClient.Process(cmd) } -// rebalance removes dead shards from the c. +// rebalance removes dead shards from the Ring. func (c *Ring) rebalance() { - defer c.mu.Unlock() - c.mu.Lock() - - c.hash = consistenthash.New(c.nreplicas, nil) + hash := consistenthash.New(c.nreplicas, nil) for name, shard := range c.shards { if shard.IsUp() { - c.hash.Add(name) + hash.Add(name) } } + + c.mu.Lock() + c.hash = hash + c.mu.Unlock() } // heartbeat monitors state of each shard in the ring. @@ -370,13 +370,10 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - name := c.hash.Get(hashtag.Key(c.cmdFirstKey(cmd))) - if name == "" { - cmd.setErr(errRingShardsDown) - if retErr == nil { - retErr = errRingShardsDown - } - continue + cmdInfo := c.cmdInfo(cmd.arg(0)) + name := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + if name != "" { + name = c.hash.Get(hashtag.Key(name)) } cmdsMap[name] = append(cmdsMap[name], cmd) } @@ -385,7 +382,15 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { failedCmdsMap := make(map[string][]Cmder) for name, cmds := range cmdsMap { - client := c.shards[name].Client + client, err := c.shardByName(name) + if err != nil { + setCmdsErr(cmds, err) + if retErr == nil { + retErr = err + } + continue + } + cn, _, err := client.conn() if err != nil { setCmdsErr(cmds, err) From f5245efa73e729fd7677b7623ad6a0191d4c1092 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Oct 2016 10:49:28 +0000 Subject: [PATCH 0224/1746] Prepare v5 release. --- .travis.yml | 4 +- CHANGELOG.md | 3 + bench_test.go | 95 ++++++------------------------ cluster.go | 8 +-- cluster_test.go | 4 +- command.go | 7 +-- command_test.go | 2 +- commands.go | 4 +- commands_test.go | 2 +- example_test.go | 2 +- export_test.go | 2 +- internal/pool/bench_test.go | 2 +- internal/pool/conn.go | 2 +- internal/pool/pool.go | 2 +- internal/pool/pool_test.go | 2 +- internal/proto/proto.go | 2 +- internal/proto/reader.go | 2 +- internal/proto/reader_test.go | 3 +- internal/proto/writebuffer_test.go | 3 +- iterator_test.go | 2 +- main_test.go | 4 +- options.go | 2 +- parser.go | 2 +- pipeline.go | 4 +- pipeline_test.go | 2 +- pool_test.go | 4 +- pubsub.go | 6 +- pubsub_test.go | 2 +- race_test.go | 4 +- redis.go | 8 +-- redis_test.go | 2 +- ring.go | 8 +-- ring_test.go | 2 +- sentinel.go | 4 +- sentinel_test.go | 2 +- tx.go | 8 +-- tx_test.go | 2 +- 37 files changed, 81 insertions(+), 138 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.travis.yml b/.travis.yml index 1d042e4c35..2e8f81337c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,5 @@ install: - go get gopkg.in/bsm/ratelimit.v1 - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega - - go get github.com/garyburd/redigo/redis - mkdir -p $HOME/gopath/src/gopkg.in - - mv $HOME/gopath/src/github.com/go-redis/redis $HOME/gopath/src/gopkg.in/redis.v4 - - cd $HOME/gopath/src/gopkg.in/redis.v4 + - mv `pwd` $HOME/gopath/src/gopkg.in/redis.v5 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..d3f8544978 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# v5 + + - *Important*. ClusterClient and Ring now chose random node/shard when command does not have any keys or command info is not fully available. Also clients use EVAL and EVALSHA keys to pick the right node. diff --git a/bench_test.go b/bench_test.go index 0c40cf3a45..0b576997ed 100644 --- a/bench_test.go +++ b/bench_test.go @@ -5,9 +5,7 @@ import ( "testing" "time" - redigo "github.com/garyburd/redigo/redis" - - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) func benchmarkRedisClient(poolSize int) *redis.Client { @@ -71,7 +69,7 @@ func BenchmarkRedisGetNil(b *testing.B) { }) } -func benchmarkSetGoRedis(b *testing.B, poolSize, payloadSize int) { +func benchmarkSetRedis(b *testing.B, poolSize, payloadSize int) { client := benchmarkRedisClient(poolSize) defer client.Close() @@ -88,93 +86,36 @@ func benchmarkSetGoRedis(b *testing.B, poolSize, payloadSize int) { }) } -func BenchmarkSetGoRedis10Conns64Bytes(b *testing.B) { - benchmarkSetGoRedis(b, 10, 64) -} - -func BenchmarkSetGoRedis100Conns64Bytes(b *testing.B) { - benchmarkSetGoRedis(b, 100, 64) -} - -func BenchmarkSetGoRedis10Conns1KB(b *testing.B) { - benchmarkSetGoRedis(b, 10, 1024) -} - -func BenchmarkSetGoRedis100Conns1KB(b *testing.B) { - benchmarkSetGoRedis(b, 100, 1024) -} - -func BenchmarkSetGoRedis10Conns10KB(b *testing.B) { - benchmarkSetGoRedis(b, 10, 10*1024) -} - -func BenchmarkSetGoRedis100Conns10KB(b *testing.B) { - benchmarkSetGoRedis(b, 100, 10*1024) -} - -func BenchmarkSetGoRedis10Conns1MB(b *testing.B) { - benchmarkSetGoRedis(b, 10, 1024*1024) -} - -func BenchmarkSetGoRedis100Conns1MB(b *testing.B) { - benchmarkSetGoRedis(b, 100, 1024*1024) -} - -func benchmarkSetRedigo(b *testing.B, poolSize, payloadSize int) { - pool := &redigo.Pool{ - Dial: func() (redigo.Conn, error) { - return redigo.DialTimeout("tcp", ":6379", time.Second, time.Second, time.Second) - }, - MaxActive: poolSize, - MaxIdle: poolSize, - } - defer pool.Close() - - value := string(bytes.Repeat([]byte{'1'}, payloadSize)) - - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - conn := pool.Get() - if _, err := conn.Do("SET", "key", value); err != nil { - b.Fatal(err) - } - conn.Close() - } - }) -} - -func BenchmarkSetRedigo10Conns64Bytes(b *testing.B) { - benchmarkSetRedigo(b, 10, 64) +func BenchmarkSetRedis10Conns64Bytes(b *testing.B) { + benchmarkSetRedis(b, 10, 64) } -func BenchmarkSetRedigo100Conns64Bytes(b *testing.B) { - benchmarkSetRedigo(b, 100, 64) +func BenchmarkSetRedis100Conns64Bytes(b *testing.B) { + benchmarkSetRedis(b, 100, 64) } -func BenchmarkSetRedigo10Conns1KB(b *testing.B) { - benchmarkSetRedigo(b, 10, 1024) +func BenchmarkSetRedis10Conns1KB(b *testing.B) { + benchmarkSetRedis(b, 10, 1024) } -func BenchmarkSetRedigo100Conns1KB(b *testing.B) { - benchmarkSetRedigo(b, 100, 1024) +func BenchmarkSetRedis100Conns1KB(b *testing.B) { + benchmarkSetRedis(b, 100, 1024) } -func BenchmarkSetRedigo10Conns10KB(b *testing.B) { - benchmarkSetRedigo(b, 10, 10*1024) +func BenchmarkSetRedis10Conns10KB(b *testing.B) { + benchmarkSetRedis(b, 10, 10*1024) } -func BenchmarkSetRedigo100Conns10KB(b *testing.B) { - benchmarkSetRedigo(b, 100, 10*1024) +func BenchmarkSetRedis100Conns10KB(b *testing.B) { + benchmarkSetRedis(b, 100, 10*1024) } -func BenchmarkSetRedigo10Conns1MB(b *testing.B) { - benchmarkSetRedigo(b, 10, 1024*1024) +func BenchmarkSetRedis10Conns1MB(b *testing.B) { + benchmarkSetRedis(b, 10, 1024*1024) } -func BenchmarkSetRedigo100Conns1MB(b *testing.B) { - benchmarkSetRedigo(b, 100, 1024*1024) +func BenchmarkSetRedis100Conns1MB(b *testing.B) { + benchmarkSetRedis(b, 100, 1024*1024) } func BenchmarkRedisSetGetBytes(b *testing.B) { diff --git a/cluster.go b/cluster.go index 6aea41b6fd..cc8b3007cc 100644 --- a/cluster.go +++ b/cluster.go @@ -6,10 +6,10 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v4/internal" - "gopkg.in/redis.v4/internal/errors" - "gopkg.in/redis.v4/internal/hashtag" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal/hashtag" + "gopkg.in/redis.v5/internal/pool" ) // ClusterOptions are used to configure a cluster client and should be diff --git a/cluster_test.go b/cluster_test.go index 74b260f8a5..367a6e34d1 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -13,8 +13,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" - "gopkg.in/redis.v4/internal/hashtag" + "gopkg.in/redis.v5" + "gopkg.in/redis.v5/internal/hashtag" ) type clusterScenario struct { diff --git a/command.go b/command.go index 7613203823..41fde30471 100644 --- a/command.go +++ b/command.go @@ -7,10 +7,9 @@ import ( "strings" "time" - "github.com/go-redis/redis/internal" - - "gopkg.in/redis.v4/internal/pool" - "gopkg.in/redis.v4/internal/proto" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/pool" + "gopkg.in/redis.v5/internal/proto" ) var ( diff --git a/command_test.go b/command_test.go index ab31f3f186..216a7b0c71 100644 --- a/command_test.go +++ b/command_test.go @@ -4,7 +4,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Cmd", func() { diff --git a/commands.go b/commands.go index f1173296ca..078242f76b 100644 --- a/commands.go +++ b/commands.go @@ -5,8 +5,8 @@ import ( "strconv" "time" - "gopkg.in/redis.v4/internal" - "gopkg.in/redis.v4/internal/errors" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/errors" ) func readTimeout(timeout time.Duration) time.Duration { diff --git a/commands_test.go b/commands_test.go index c89c333103..e613d4abcc 100644 --- a/commands_test.go +++ b/commands_test.go @@ -9,7 +9,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Commands", func() { diff --git a/example_test.go b/example_test.go index 71d06c4cc1..1aef66fcaa 100644 --- a/example_test.go +++ b/example_test.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var client *redis.Client diff --git a/export_test.go b/export_test.go index ff37e81746..d36f867155 100644 --- a/export_test.go +++ b/export_test.go @@ -3,7 +3,7 @@ package redis import ( "time" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal/pool" ) func (c *baseClient) Pool() pool.Pooler { diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index 663abc0bd2..be2c3053ae 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal/pool" ) func benchmarkPoolGetPut(b *testing.B, poolSize int) { diff --git a/internal/pool/conn.go b/internal/pool/conn.go index bb0922f384..785fc21451 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -4,7 +4,7 @@ import ( "net" "time" - "gopkg.in/redis.v4/internal/proto" + "gopkg.in/redis.v5/internal/proto" ) const defaultBufSize = 4096 diff --git a/internal/pool/pool.go b/internal/pool/pool.go index bd1c4130b8..389a3d22d2 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -10,7 +10,7 @@ import ( "gopkg.in/bsm/ratelimit.v1" - "gopkg.in/redis.v4/internal" + "gopkg.in/redis.v5/internal" ) var ( diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 425ce9280b..7ba35fd416 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal/pool" ) var _ = Describe("ConnPool", func() { diff --git a/internal/proto/proto.go b/internal/proto/proto.go index c63caaac18..00d5f6bae3 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "gopkg.in/redis.v4/internal/errors" + "gopkg.in/redis.v5/internal/errors" ) const ( diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 9d2f51ae3d..412eec6d5c 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -7,7 +7,7 @@ import ( "io" "strconv" - ierrors "gopkg.in/redis.v4/internal/errors" + ierrors "gopkg.in/redis.v5/internal/errors" ) type MultiBulkParse func(*Reader, int64) (interface{}, error) diff --git a/internal/proto/reader_test.go b/internal/proto/reader_test.go index 5169262661..421344be2f 100644 --- a/internal/proto/reader_test.go +++ b/internal/proto/reader_test.go @@ -7,7 +7,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4/internal/proto" + + "gopkg.in/redis.v5/internal/proto" ) var _ = Describe("Reader", func() { diff --git a/internal/proto/writebuffer_test.go b/internal/proto/writebuffer_test.go index 6316ded78d..36593af519 100644 --- a/internal/proto/writebuffer_test.go +++ b/internal/proto/writebuffer_test.go @@ -6,7 +6,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4/internal/proto" + + "gopkg.in/redis.v5/internal/proto" ) var _ = Describe("WriteBuffer", func() { diff --git a/iterator_test.go b/iterator_test.go index 7b301768f1..7200c14baa 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -6,7 +6,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("ScanIterator", func() { diff --git a/main_test.go b/main_test.go index a9afa8227e..67886e4bba 100644 --- a/main_test.go +++ b/main_test.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) const ( @@ -94,7 +94,7 @@ var _ = AfterSuite(func() { func TestGinkgoSuite(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "gopkg.in/redis.v4") + RunSpecs(t, "gopkg.in/redis.v5") } //------------------------------------------------------------------------------ diff --git a/options.go b/options.go index bfe2e8a2bf..376fa6bf00 100644 --- a/options.go +++ b/options.go @@ -4,7 +4,7 @@ import ( "net" "time" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal/pool" ) type Options struct { diff --git a/parser.go b/parser.go index a4ad4e2aa9..50b2a9565f 100644 --- a/parser.go +++ b/parser.go @@ -5,7 +5,7 @@ import ( "net" "strconv" - "gopkg.in/redis.v4/internal/proto" + "gopkg.in/redis.v5/internal/proto" ) // Implements proto.MultiBulkParse diff --git a/pipeline.go b/pipeline.go index bf80d408a8..fe226ece6c 100644 --- a/pipeline.go +++ b/pipeline.go @@ -4,8 +4,8 @@ import ( "sync" "sync/atomic" - "gopkg.in/redis.v4/internal/errors" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal/pool" ) // Pipeline implements pipelining as described in diff --git a/pipeline_test.go b/pipeline_test.go index c3a37f6651..74cdd38519 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Pipelining", func() { diff --git a/pool_test.go b/pool_test.go index 2b0d2ad4c5..1adea5e89e 100644 --- a/pool_test.go +++ b/pool_test.go @@ -6,8 +6,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5" + "gopkg.in/redis.v5/internal/pool" ) var _ = Describe("pool", func() { diff --git a/pubsub.go b/pubsub.go index 796bd4a25d..78136f5fcd 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,9 +5,9 @@ import ( "net" "time" - "gopkg.in/redis.v4/internal" - "gopkg.in/redis.v4/internal/errors" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal/pool" ) // PubSub implements Pub/Sub commands as described in diff --git a/pubsub_test.go b/pubsub_test.go index 957d3031e9..ecbea7687d 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -9,7 +9,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("PubSub", func() { diff --git a/race_test.go b/race_test.go index ca4fb7f297..cf093f25fd 100644 --- a/race_test.go +++ b/race_test.go @@ -11,8 +11,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5" + "gopkg.in/redis.v5/internal/pool" ) var _ = Describe("races", func() { diff --git a/redis.go b/redis.go index 60df328e61..9c8b490d14 100644 --- a/redis.go +++ b/redis.go @@ -1,12 +1,12 @@ -package redis // import "gopkg.in/redis.v4" +package redis // import "gopkg.in/redis.v5" import ( "fmt" "log" - "gopkg.in/redis.v4/internal" - "gopkg.in/redis.v4/internal/errors" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal/pool" ) // Redis nil reply, .e.g. when key does not exist. diff --git a/redis_test.go b/redis_test.go index 0b59547145..8d5f651a89 100644 --- a/redis_test.go +++ b/redis_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Client", func() { diff --git a/ring.go b/ring.go index 52badb3582..77193f327e 100644 --- a/ring.go +++ b/ring.go @@ -9,10 +9,10 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v4/internal" - "gopkg.in/redis.v4/internal/consistenthash" - "gopkg.in/redis.v4/internal/hashtag" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/consistenthash" + "gopkg.in/redis.v5/internal/hashtag" + "gopkg.in/redis.v5/internal/pool" ) var errRingShardsDown = errors.New("redis: all ring shards are down") diff --git a/ring_test.go b/ring_test.go index 0ab258d858..5e9d16f767 100644 --- a/ring_test.go +++ b/ring_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Redis Ring", func() { diff --git a/sentinel.go b/sentinel.go index 66f0974ce6..0849dda386 100644 --- a/sentinel.go +++ b/sentinel.go @@ -8,8 +8,8 @@ import ( "sync" "time" - "gopkg.in/redis.v4/internal" - "gopkg.in/redis.v4/internal/pool" + "gopkg.in/redis.v5/internal" + "gopkg.in/redis.v5/internal/pool" ) //------------------------------------------------------------------------------ diff --git a/sentinel_test.go b/sentinel_test.go index 165cce66ea..d7038575fe 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -4,7 +4,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Sentinel", func() { diff --git a/tx.go b/tx.go index 62e770183a..ba2fd933d4 100644 --- a/tx.go +++ b/tx.go @@ -4,10 +4,10 @@ import ( "errors" "fmt" - "gopkg.in/redis.v4/internal" - ierrors "gopkg.in/redis.v4/internal/errors" - "gopkg.in/redis.v4/internal/pool" - "gopkg.in/redis.v4/internal/proto" + "gopkg.in/redis.v5/internal" + ierrors "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal/pool" + "gopkg.in/redis.v5/internal/proto" ) // Redis transaction failed. diff --git a/tx_test.go b/tx_test.go index ddb9ddfa80..03d220bba1 100644 --- a/tx_test.go +++ b/tx_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v4" + "gopkg.in/redis.v5" ) var _ = Describe("Tx", func() { From 639950777c89b65d88003c6571b4c0119a198cd2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Oct 2016 11:12:32 +0000 Subject: [PATCH 0225/1746] More cluster tests. --- cluster.go | 9 ++- cluster_test.go | 150 +++++++++++++++++++++++++++++++++--------------- ring_test.go | 74 ++++++++++++++---------- 3 files changed, 157 insertions(+), 76 deletions(-) diff --git a/cluster.go b/cluster.go index cc8b3007cc..90fd4ce1a3 100644 --- a/cluster.go +++ b/cluster.go @@ -12,6 +12,8 @@ import ( "gopkg.in/redis.v5/internal/pool" ) +var errClusterNoNodes = errors.RedisError("redis: cluster has no nodes") + // ClusterOptions are used to configure a cluster client and should be // passed to NewClusterClient. type ClusterOptions struct { @@ -155,14 +157,14 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo { func (c *ClusterClient) getNodes() map[string]*clusterNode { var nodes map[string]*clusterNode + c.mu.RLock() if !c.closed { nodes = make(map[string]*clusterNode, len(c.nodes)) - c.mu.RLock() for addr, node := range c.nodes { nodes[addr] = node } - c.mu.RUnlock() } + c.mu.RUnlock() return nodes } @@ -261,6 +263,9 @@ func (c *ClusterClient) randomNode() (*clusterNode, error) { return nil, pool.ErrClosed } + if len(addrs) == 0 { + return nil, errClusterNoNodes + } n := rand.Intn(len(addrs)) node, err := c.nodeByAddr(addrs[n]) diff --git a/cluster_test.go b/cluster_test.go index 367a6e34d1..9639cf8245 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -301,6 +301,37 @@ var _ = Describe("ClusterClient", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) + It("distributes keys", func() { + for i := 0; i < 100; i++ { + err := client.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + + wanted := []string{"keys=31", "keys=29", "keys=40"} + for i, master := range cluster.masters() { + Expect(master.Info().Val()).To(ContainSubstring(wanted[i])) + } + }) + + It("distributes keys when using EVAL", func() { + script := redis.NewScript(` + local r = redis.call('SET', KEYS[1], ARGV[1]) + return r + `) + + var key string + for i := 0; i < 100; i++ { + key = fmt.Sprintf("key%d", i) + err := script.Run(client, []string{key}, "value").Err() + Expect(err).NotTo(HaveOccurred()) + } + + wanted := []string{"keys=31", "keys=29", "keys=40"} + for i, master := range cluster.masters() { + Expect(master.Info().Val()).To(ContainSubstring(wanted[i])) + } + }) + It("supports Watch", func() { var incr func(string) error @@ -342,60 +373,62 @@ var _ = Describe("ClusterClient", func() { Expect(n).To(Equal(int64(100))) }) - It("supports pipeline", func() { - slot := hashtag.Slot("A") - Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + Describe("pipeline", func() { + It("follows redirects", func() { + slot := hashtag.Slot("A") + Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) - pipe := client.Pipeline() - defer pipe.Close() + pipe := client.Pipeline() + defer pipe.Close() - keys := []string{"A", "B", "C", "D", "E", "F", "G"} + keys := []string{"A", "B", "C", "D", "E", "F", "G"} - for i, key := range keys { - pipe.Set(key, key+"_value", 0) - pipe.Expire(key, time.Duration(i+1)*time.Hour) - } - cmds, err := pipe.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(14)) + for i, key := range keys { + pipe.Set(key, key+"_value", 0) + pipe.Expire(key, time.Duration(i+1)*time.Hour) + } + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(14)) - for _, key := range keys { - pipe.Get(key) - pipe.TTL(key) - } - cmds, err = pipe.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(14)) - Expect(cmds[0].(*redis.StringCmd).Val()).To(Equal("A_value")) - Expect(cmds[1].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) - Expect(cmds[6].(*redis.StringCmd).Val()).To(Equal("D_value")) - Expect(cmds[7].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) - Expect(cmds[12].(*redis.StringCmd).Val()).To(Equal("G_value")) - Expect(cmds[13].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) - }) + for _, key := range keys { + pipe.Get(key) + pipe.TTL(key) + } + cmds, err = pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(14)) + Expect(cmds[0].(*redis.StringCmd).Val()).To(Equal("A_value")) + Expect(cmds[1].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) + Expect(cmds[6].(*redis.StringCmd).Val()).To(Equal("D_value")) + Expect(cmds[7].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) + Expect(cmds[12].(*redis.StringCmd).Val()).To(Equal("G_value")) + Expect(cmds[13].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) + }) - It("supports pipeline with missing keys", func() { - Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) - Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) + It("works with missing keys", func() { + Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) + Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) - var a, b, c *redis.StringCmd - cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { - a = pipe.Get("A") - b = pipe.Get("B") - c = pipe.Get("C") - return nil - }) - Expect(err).To(Equal(redis.Nil)) - Expect(cmds).To(HaveLen(3)) + var a, b, c *redis.StringCmd + cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { + a = pipe.Get("A") + b = pipe.Get("B") + c = pipe.Get("C") + return nil + }) + Expect(err).To(Equal(redis.Nil)) + Expect(cmds).To(HaveLen(3)) - Expect(a.Err()).NotTo(HaveOccurred()) - Expect(a.Val()).To(Equal("A_value")) + Expect(a.Err()).NotTo(HaveOccurred()) + Expect(a.Val()).To(Equal("A_value")) - Expect(b.Err()).To(Equal(redis.Nil)) - Expect(b.Val()).To(Equal("")) + Expect(b.Err()).To(Equal(redis.Nil)) + Expect(b.Val()).To(Equal("")) - Expect(c.Err()).NotTo(HaveOccurred()) - Expect(c.Val()).To(Equal("C_value")) + Expect(c.Err()).NotTo(HaveOccurred()) + Expect(c.Val()).To(Equal("C_value")) + }) }) It("calls fn for every master node", func() { @@ -451,6 +484,25 @@ var _ = Describe("ClusterClient", func() { describeClusterClient() }) + Describe("ClusterClient without nodes", func() { + BeforeEach(func() { + client = redis.NewClusterClient(&redis.ClusterOptions{}) + }) + + It("returns an error", func() { + err := client.Ping().Err() + Expect(err).To(MatchError("redis: cluster has no nodes")) + }) + + It("pipeline returns an error", func() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + Expect(err).To(MatchError("redis: cluster has no nodes")) + }) + }) + Describe("ClusterClient without valid nodes", func() { BeforeEach(func() { client = redis.NewClusterClient(&redis.ClusterOptions{ @@ -462,6 +514,14 @@ var _ = Describe("ClusterClient", func() { err := client.Ping().Err() Expect(err).To(MatchError("ERR This instance has cluster support disabled")) }) + + It("pipeline returns an error", func() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + }) }) }) diff --git a/ring_test.go b/ring_test.go index 5e9d16f767..1c1ae691b9 100644 --- a/ring_test.go +++ b/ring_test.go @@ -38,7 +38,7 @@ var _ = Describe("Redis Ring", func() { Expect(ring.Close()).NotTo(HaveOccurred()) }) - It("uses both shards", func() { + It("distributes keys", func() { setRingKeys() // Both shards should have some keys now. @@ -46,6 +46,23 @@ var _ = Describe("Redis Ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) }) + It("distributes keys when using EVAL", func() { + script := redis.NewScript(` + local r = redis.call('SET', KEYS[1], ARGV[1]) + return r + `) + + var key string + for i := 0; i < 100; i++ { + key = fmt.Sprintf("key%d", i) + err := script.Run(ring, []string{key}, "value").Err() + Expect(err).NotTo(HaveOccurred()) + } + + Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) + Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + }) + It("uses single shard when one of the shards is down", func() { // Stop ringShard2. Expect(ringShard2.Close()).NotTo(HaveOccurred()) @@ -89,34 +106,8 @@ var _ = Describe("Redis Ring", func() { Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=100")) }) - It("supports eval key search", func() { - script := redis.NewScript(` - local r = redis.call('SET', KEYS[1], ARGV[1]) - return r - `) - - var key string - for i := 0; i < 100; i++ { - key = fmt.Sprintf("key{%d}", i) - err := script.Run(ring, []string{key}, "value").Err() - Expect(err).NotTo(HaveOccurred()) - } - - Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=52")) - Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=48")) - }) - - Describe("pipelining", func() { - It("returns an error when all shards are down", func() { - ring := redis.NewRing(&redis.RingOptions{}) - _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { - pipe.Ping() - return nil - }) - Expect(err).To(MatchError("redis: all ring shards are down")) - }) - - It("uses both shards", func() { + Describe("pipeline", func() { + It("distributes keys", func() { pipe := ring.Pipeline() for i := 0; i < 100; i++ { err := pipe.Set(fmt.Sprintf("key%d", i), "value", 0).Err() @@ -175,3 +166,28 @@ var _ = Describe("Redis Ring", func() { }) }) }) + +var _ = Describe("empty Redis Ring", func() { + var ring *redis.Ring + + BeforeEach(func() { + ring = redis.NewRing(&redis.RingOptions{}) + }) + + AfterEach(func() { + Expect(ring.Close()).NotTo(HaveOccurred()) + }) + + It("returns an error", func() { + err := ring.Ping().Err() + Expect(err).To(MatchError("redis: all ring shards are down")) + }) + + It("pipeline returns an error", func() { + _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + Expect(err).To(MatchError("redis: all ring shards are down")) + }) +}) From 2c5b239ecb60e68f3373a41a4edae499b751e39d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Oct 2016 11:38:31 +0000 Subject: [PATCH 0226/1746] Remove internal errors package that clashes with std lib. --- cluster.go | 13 ++++++------- commands.go | 3 +-- internal/{errors => }/errors.go | 18 +++++++++--------- internal/proto/proto.go | 4 ++-- internal/proto/reader.go | 15 +++++++-------- pipeline.go | 4 ++-- pubsub.go | 3 +-- redis.go | 9 ++++----- tx.go | 6 ++---- 9 files changed, 34 insertions(+), 41 deletions(-) rename internal/{errors => }/errors.go (71%) diff --git a/cluster.go b/cluster.go index 90fd4ce1a3..876b6d0fbc 100644 --- a/cluster.go +++ b/cluster.go @@ -7,12 +7,11 @@ import ( "time" "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/errors" "gopkg.in/redis.v5/internal/hashtag" "gopkg.in/redis.v5/internal/pool" ) -var errClusterNoNodes = errors.RedisError("redis: cluster has no nodes") +var errClusterNoNodes = internal.RedisError("redis: cluster has no nodes") // ClusterOptions are used to configure a cluster client and should be // passed to NewClusterClient. @@ -382,20 +381,20 @@ func (c *ClusterClient) Process(cmd Cmder) error { } // If slave is loading - read from master. - if c.opt.ReadOnly && errors.IsLoading(err) { + if c.opt.ReadOnly && internal.IsLoadingError(err) { node.loading = time.Now() continue } // On network errors try random node. - if errors.IsRetryable(err) { + if internal.IsRetryableError(err) { node, err = c.randomNode() continue } var moved bool var addr string - moved, ask, addr = errors.IsMoved(err) + moved, ask, addr = internal.IsMovedError(err) if moved || ask { if slot >= 0 { master, _ := c.slotMasterNode(slot) @@ -650,11 +649,11 @@ func (c *ClusterClient) execClusterCmds( if err == nil { continue } - if errors.IsNetwork(err) { + if internal.IsNetworkError(err) { cmd.reset() failedCmds[nil] = append(failedCmds[nil], cmds[i:]...) break - } else if moved, ask, addr := errors.IsMoved(err); moved { + } else if moved, ask, addr := internal.IsMovedError(err); moved { c.lazyReloadSlots() cmd.reset() node, err := c.nodeByAddr(addr) diff --git a/commands.go b/commands.go index 078242f76b..8a2b9c5d35 100644 --- a/commands.go +++ b/commands.go @@ -6,7 +6,6 @@ import ( "time" "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/errors" ) func readTimeout(timeout time.Duration) time.Duration { @@ -1710,7 +1709,7 @@ func (c *cmdable) shutdown(modifier string) *StatusCmd { } } else { // Server did not quit. String reply contains the reason. - cmd.err = errors.RedisError(cmd.val) + cmd.err = internal.RedisError(cmd.val) cmd.val = "" } return cmd diff --git a/internal/errors/errors.go b/internal/errors.go similarity index 71% rename from internal/errors/errors.go rename to internal/errors.go index c5abc0be58..e94e123a5a 100644 --- a/internal/errors/errors.go +++ b/internal/errors.go @@ -1,4 +1,4 @@ -package errors +package internal import ( "io" @@ -12,16 +12,16 @@ type RedisError string func (e RedisError) Error() string { return string(e) } -func IsRetryable(err error) bool { - return IsNetwork(err) +func IsRetryableError(err error) bool { + return IsNetworkError(err) } -func IsInternal(err error) bool { +func IsInternalError(err error) bool { _, ok := err.(RedisError) return ok } -func IsNetwork(err error) bool { +func IsNetworkError(err error) bool { if err == io.EOF { return true } @@ -33,7 +33,7 @@ func IsBadConn(err error, allowTimeout bool) bool { if err == nil { return false } - if IsInternal(err) { + if IsInternalError(err) { return false } if allowTimeout { @@ -44,8 +44,8 @@ func IsBadConn(err error, allowTimeout bool) bool { return true } -func IsMoved(err error) (moved bool, ask bool, addr string) { - if !IsInternal(err) { +func IsMovedError(err error) (moved bool, ask bool, addr string) { + if !IsInternalError(err) { return } @@ -66,6 +66,6 @@ func IsMoved(err error) (moved bool, ask bool, addr string) { return } -func IsLoading(err error) bool { +func IsLoadingError(err error) bool { return strings.HasPrefix(err.Error(), "LOADING") } diff --git a/internal/proto/proto.go b/internal/proto/proto.go index 00d5f6bae3..1d975202e8 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -5,7 +5,7 @@ import ( "fmt" "strconv" - "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal" ) const ( @@ -18,7 +18,7 @@ const ( const defaultBufSize = 4096 -var errScanNil = errors.RedisError("redis: Scan(nil)") +const errScanNil = internal.RedisError("redis: Scan(nil)") func Scan(b []byte, val interface{}) error { switch v := val.(type) { diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 412eec6d5c..a98ddb603c 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -2,17 +2,16 @@ package proto import ( "bufio" - "errors" "fmt" "io" "strconv" - ierrors "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal" ) -type MultiBulkParse func(*Reader, int64) (interface{}, error) +const errEmptyReply = internal.RedisError("redis: reply is empty") -var errEmptyReply = errors.New("redis: reply is empty") +type MultiBulkParse func(*Reader, int64) (interface{}, error) type Reader struct { src *bufio.Reader @@ -59,7 +58,7 @@ func (p *Reader) ReadLine() ([]byte, error) { return nil, errEmptyReply } if isNilReply(line) { - return nil, ierrors.Nil + return nil, internal.Nil } return line, nil } @@ -209,7 +208,7 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) { func (p *Reader) parseBytesValue(line []byte) ([]byte, error) { if isNilReply(line) { - return nil, ierrors.Nil + return nil, internal.Nil } replyLen, err := strconv.Atoi(string(line[1:])) @@ -245,7 +244,7 @@ func isNilReply(b []byte) bool { } func parseErrorValue(line []byte) error { - return ierrors.RedisError(string(line[1:])) + return internal.RedisError(string(line[1:])) } func parseStatusValue(line []byte) ([]byte, error) { @@ -258,7 +257,7 @@ func parseIntValue(line []byte) (int64, error) { func parseArrayLen(line []byte) (int64, error) { if isNilReply(line) { - return 0, ierrors.Nil + return 0, internal.Nil } return parseIntValue(line) } diff --git a/pipeline.go b/pipeline.go index fe226ece6c..1cbc00f389 100644 --- a/pipeline.go +++ b/pipeline.go @@ -4,7 +4,7 @@ import ( "sync" "sync/atomic" - "gopkg.in/redis.v5/internal/errors" + "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/pool" ) @@ -100,7 +100,7 @@ func execCmds(cn *pool.Conn, cmds []Cmder) ([]Cmder, error) { if firstCmdErr == nil { firstCmdErr = err } - if errors.IsRetryable(err) { + if internal.IsRetryableError(err) { failedCmds = append(failedCmds, cmd) } } diff --git a/pubsub.go b/pubsub.go index 78136f5fcd..223518a029 100644 --- a/pubsub.go +++ b/pubsub.go @@ -6,7 +6,6 @@ import ( "time" "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/errors" "gopkg.in/redis.v5/internal/pool" ) @@ -212,7 +211,7 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { for { msgi, err := c.ReceiveTimeout(timeout) if err != nil { - if !errors.IsNetwork(err) { + if !internal.IsNetworkError(err) { return nil, err } diff --git a/redis.go b/redis.go index 9c8b490d14..113a6f3fdb 100644 --- a/redis.go +++ b/redis.go @@ -5,12 +5,11 @@ import ( "log" "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/errors" "gopkg.in/redis.v5/internal/pool" ) // Redis nil reply, .e.g. when key does not exist. -const Nil = errors.Nil +const Nil = internal.Nil func SetLogger(logger *log.Logger) { internal.Logger = logger @@ -42,7 +41,7 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { } func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { - if errors.IsBadConn(err, allowTimeout) { + if internal.IsBadConn(err, allowTimeout) { _ = c.connPool.Remove(cn, err) return false } @@ -101,7 +100,7 @@ func (c *baseClient) Process(cmd Cmder) error { if err := writeCmd(cn, cmd); err != nil { c.putConn(cn, err, false) cmd.setErr(err) - if err != nil && errors.IsRetryable(err) { + if err != nil && internal.IsRetryableError(err) { continue } return err @@ -109,7 +108,7 @@ func (c *baseClient) Process(cmd Cmder) error { err = cmd.readReply(cn) c.putConn(cn, err, readTimeout != nil) - if err != nil && errors.IsRetryable(err) { + if err != nil && internal.IsRetryableError(err) { continue } diff --git a/tx.go b/tx.go index ba2fd933d4..950c374fc7 100644 --- a/tx.go +++ b/tx.go @@ -1,19 +1,17 @@ package redis import ( - "errors" "fmt" "gopkg.in/redis.v5/internal" - ierrors "gopkg.in/redis.v5/internal/errors" "gopkg.in/redis.v5/internal/pool" "gopkg.in/redis.v5/internal/proto" ) // Redis transaction failed. -const TxFailedErr = ierrors.RedisError("redis: transaction failed") +const TxFailedErr = internal.RedisError("redis: transaction failed") -var errDiscard = errors.New("redis: Discard can be used only inside Exec") +var errDiscard = internal.RedisError("redis: Discard can be used only inside Exec") // Tx implements Redis transactions as described in // http://redis.io/topics/transactions. It's NOT safe for concurrent use From 0c5e0858956cc35566150366043a5f628ec66d02 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Oct 2016 11:51:21 +0000 Subject: [PATCH 0227/1746] Update readme. --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0b85a11b73..69f0b7a2f1 100644 --- a/README.md +++ b/README.md @@ -3,26 +3,26 @@ Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- [Pub/Sub](http://godoc.org/gopkg.in/redis.v4#PubSub). -- [Transactions](http://godoc.org/gopkg.in/redis.v4#Multi). -- [Pipelining](http://godoc.org/gopkg.in/redis.v4#Client.Pipeline). -- [Scripting](http://godoc.org/gopkg.in/redis.v4#Script). -- [Timeouts](http://godoc.org/gopkg.in/redis.v4#Options). -- [Redis Sentinel](http://godoc.org/gopkg.in/redis.v4#NewFailoverClient). -- [Redis Cluster](http://godoc.org/gopkg.in/redis.v4#NewClusterClient). -- [Ring](http://godoc.org/gopkg.in/redis.v4#NewRing). +- [Pub/Sub](http://godoc.org/gopkg.in/redis.v5#PubSub). +- [Transactions](http://godoc.org/gopkg.in/redis.v5#Multi). +- [Pipelining](http://godoc.org/gopkg.in/redis.v5#Client.Pipeline). +- [Scripting](http://godoc.org/gopkg.in/redis.v5#Script). +- [Timeouts](http://godoc.org/gopkg.in/redis.v5#Options). +- [Redis Sentinel](http://godoc.org/gopkg.in/redis.v5#NewFailoverClient). +- [Redis Cluster](http://godoc.org/gopkg.in/redis.v5#NewClusterClient). +- [Ring](http://godoc.org/gopkg.in/redis.v5#NewRing). - [Cache friendly](https://github.com/go-redis/cache). - [Rate limiting](https://github.com/go-redis/rate). - [Distributed Locks](https://github.com/bsm/redis-lock). -API docs: http://godoc.org/gopkg.in/redis.v4. -Examples: http://godoc.org/gopkg.in/redis.v4#pkg-examples. +API docs: http://godoc.org/gopkg.in/redis.v5. +Examples: http://godoc.org/gopkg.in/redis.v5#pkg-examples. ## Installation Install: - go get gopkg.in/redis.v4 + go get gopkg.in/redis.v5 ## Quickstart @@ -66,7 +66,7 @@ func ExampleClient() { ## Howto -Please go through [examples](http://godoc.org/gopkg.in/redis.v4#pkg-examples) to get an idea how to use this package. +Please go through [examples](http://godoc.org/gopkg.in/redis.v5#pkg-examples) to get an idea how to use this package. ## Look and feel From e6d4f5bd380bd84d65d4a156f309fcd970775f8d Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 12 Oct 2016 13:11:09 +0100 Subject: [PATCH 0228/1746] Remove static import --- README.md | 26 +++++++++++++++++++++++++- redis.go | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 69f0b7a2f1..db120c9897 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,31 @@ Examples: http://godoc.org/gopkg.in/redis.v5#pkg-examples. Install: - go get gopkg.in/redis.v5 +```shell +go get gopkg.in/redis.v5 +``` + +Import: + +```go +import "gopkg.in/redis.v5" +``` + +## Vendoring + +If you are using a vendoring tool with support for semantic versioning +e.g. [glide](https://github.com/Masterminds/glide), you can import this +package via its GitHub URL: + +```yaml +- package: github.com/go-redis/redis + version: ^5.0.0 +``` + +WARNING: please note that by importing `github.com/go-redis/redis` +directly (without semantic versioning constrol) you are in danger of +running in the breaking API changes. Use carefully and at your own +risk! ## Quickstart diff --git a/redis.go b/redis.go index 113a6f3fdb..e866a5ca5b 100644 --- a/redis.go +++ b/redis.go @@ -1,4 +1,4 @@ -package redis // import "gopkg.in/redis.v5" +package redis import ( "fmt" From 236c021d4ce13118d2c336058c11768b3bf7cf19 Mon Sep 17 00:00:00 2001 From: yyoshiki41 Date: Thu, 13 Oct 2016 17:27:43 +0900 Subject: [PATCH 0229/1746] simplify if condition --- cluster.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 876b6d0fbc..ff935fc4f0 100644 --- a/cluster.go +++ b/cluster.go @@ -653,7 +653,9 @@ func (c *ClusterClient) execClusterCmds( cmd.reset() failedCmds[nil] = append(failedCmds[nil], cmds[i:]...) break - } else if moved, ask, addr := internal.IsMovedError(err); moved { + } + moved, ask, addr := internal.IsMovedError(err) + if moved { c.lazyReloadSlots() cmd.reset() node, err := c.nodeByAddr(addr) From 8d2fb6e09bdaad89ed347f94f08242192edf1c6b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 13 Oct 2016 12:11:58 +0300 Subject: [PATCH 0230/1746] Simplify sync in pipeline. --- pipeline.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pipeline.go b/pipeline.go index 1cbc00f389..f3be4d5d8d 100644 --- a/pipeline.go +++ b/pipeline.go @@ -2,7 +2,6 @@ package redis import ( "sync" - "sync/atomic" "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/pool" @@ -17,10 +16,9 @@ type Pipeline struct { exec func([]Cmder) error - mu sync.Mutex // protects cmds - cmds []Cmder - - closed int32 + mu sync.Mutex + cmds []Cmder + closed bool } func (c *Pipeline) Process(cmd Cmder) error { @@ -32,20 +30,23 @@ func (c *Pipeline) Process(cmd Cmder) error { // Close closes the pipeline, releasing any open resources. func (c *Pipeline) Close() error { - atomic.StoreInt32(&c.closed, 1) - c.Discard() + c.mu.Lock() + c.discard() + c.closed = true + c.mu.Unlock() return nil } -func (c *Pipeline) isClosed() bool { - return atomic.LoadInt32(&c.closed) == 1 -} - // Discard resets the pipeline and discards queued commands. func (c *Pipeline) Discard() error { - defer c.mu.Unlock() c.mu.Lock() - if c.isClosed() { + err := c.discard() + c.mu.Unlock() + return err +} + +func (c *Pipeline) discard() error { + if c.closed { return pool.ErrClosed } c.cmds = c.cmds[:0] @@ -58,13 +59,13 @@ func (c *Pipeline) Discard() error { // Exec always returns list of commands and error of the first failed // command if any. func (c *Pipeline) Exec() ([]Cmder, error) { - if c.isClosed() { - return nil, pool.ErrClosed - } - defer c.mu.Unlock() c.mu.Lock() + if c.closed { + return nil, pool.ErrClosed + } + if len(c.cmds) == 0 { return c.cmds, nil } From 8558a92fa4100da0a8e6454de7cae6a49e6bcbbb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 13 Oct 2016 13:56:24 +0300 Subject: [PATCH 0231/1746] Retry multiple commands more conservatively. --- cluster.go | 13 +++++++++---- pipeline.go | 19 ++++++++----------- redis.go | 25 ++++++++++++++----------- ring.go | 39 ++++++++++++++++++++++----------------- 4 files changed, 53 insertions(+), 43 deletions(-) diff --git a/cluster.go b/cluster.go index ff935fc4f0..9d07d04f3e 100644 --- a/cluster.go +++ b/cluster.go @@ -649,28 +649,33 @@ func (c *ClusterClient) execClusterCmds( if err == nil { continue } - if internal.IsNetworkError(err) { + + if i == 0 && internal.IsNetworkError(err) { cmd.reset() - failedCmds[nil] = append(failedCmds[nil], cmds[i:]...) + failedCmds[nil] = append(failedCmds[nil], cmds...) break } + moved, ask, addr := internal.IsMovedError(err) if moved { c.lazyReloadSlots() - cmd.reset() + node, err := c.nodeByAddr(addr) if err != nil { setRetErr(err) continue } + + cmd.reset() failedCmds[node] = append(failedCmds[node], cmd) } else if ask { - cmd.reset() node, err := c.nodeByAddr(addr) if err != nil { setRetErr(err) continue } + + cmd.reset() failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) } else { setRetErr(err) diff --git a/pipeline.go b/pipeline.go index 1cbc00f389..ac8bcc2fc5 100644 --- a/pipeline.go +++ b/pipeline.go @@ -84,26 +84,23 @@ func (c *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { return cmds, err } -func execCmds(cn *pool.Conn, cmds []Cmder) ([]Cmder, error) { +func execCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) - return cmds, err + return true, err } - var firstCmdErr error - var failedCmds []Cmder - for _, cmd := range cmds { + for i, cmd := range cmds { err := cmd.readReply(cn) if err == nil { continue } - if firstCmdErr == nil { - firstCmdErr = err + if i == 0 && internal.IsNetworkError(err) { + return true, err } - if internal.IsRetryableError(err) { - failedCmds = append(failedCmds, cmd) + if firstErr == nil { + firstErr = err } } - - return failedCmds, firstCmdErr + return false, firstErr } diff --git a/redis.go b/redis.go index e866a5ca5b..aee3893375 100644 --- a/redis.go +++ b/redis.go @@ -197,28 +197,31 @@ func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { } func (c *Client) pipelineExec(cmds []Cmder) error { - var retErr error - failedCmds := cmds + var firstErr error for i := 0; i <= c.opt.MaxRetries; i++ { + if i > 0 { + resetCmds(cmds) + } + cn, _, err := c.conn() if err != nil { - setCmdsErr(failedCmds, err) + setCmdsErr(cmds, err) return err } - if i > 0 { - resetCmds(failedCmds) - } - failedCmds, err = execCmds(cn, failedCmds) + retry, err := execCmds(cn, cmds) c.putConn(cn, err, false) - if err != nil && retErr == nil { - retErr = err + if err == nil { + return nil } - if len(failedCmds) == 0 { + if firstErr == nil { + firstErr = err + } + if !retry { break } } - return retErr + return firstErr } func (c *Client) pubSub() *PubSub { diff --git a/ring.go b/ring.go index 77193f327e..9608534640 100644 --- a/ring.go +++ b/ring.go @@ -365,9 +365,7 @@ func (c *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } -func (c *Ring) pipelineExec(cmds []Cmder) error { - var retErr error - +func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { cmdInfo := c.cmdInfo(cmd.arg(0)) @@ -379,14 +377,18 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { } for i := 0; i <= c.opt.MaxRetries; i++ { - failedCmdsMap := make(map[string][]Cmder) + var failedCmdsMap map[string][]Cmder for name, cmds := range cmdsMap { + if i > 0 { + resetCmds(cmds) + } + client, err := c.shardByName(name) if err != nil { setCmdsErr(cmds, err) - if retErr == nil { - retErr = err + if firstErr == nil { + firstErr = err } continue } @@ -394,22 +396,25 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { cn, _, err := client.conn() if err != nil { setCmdsErr(cmds, err) - if retErr == nil { - retErr = err + if firstErr == nil { + firstErr = err } continue } - if i > 0 { - resetCmds(cmds) - } - failedCmds, err := execCmds(cn, cmds) + retry, err := execCmds(cn, cmds) client.putConn(cn, err, false) - if err != nil && retErr == nil { - retErr = err + if err == nil { + continue } - if len(failedCmds) > 0 { - failedCmdsMap[name] = failedCmds + if firstErr == nil { + firstErr = err + } + if retry { + if failedCmdsMap == nil { + failedCmdsMap = make(map[string][]Cmder) + } + failedCmdsMap[name] = cmds } } @@ -419,5 +424,5 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { cmdsMap = failedCmdsMap } - return retErr + return firstErr } From 20bc3ec5a69c72e9f223c6f1b2c1bde113938fab Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 13 Oct 2016 14:36:15 +0300 Subject: [PATCH 0232/1746] Refactor Tx using Pipeline to implement Cmdable interface. --- cluster_test.go | 4 +-- example_test.go | 4 +-- pool_test.go | 6 ++-- race_test.go | 4 +-- redis_test.go | 4 +-- tx.go | 75 +++++++++++++++++++------------------------------ tx_test.go | 24 ++++++++-------- 7 files changed, 52 insertions(+), 69 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 9639cf8245..4a3dd7fa4b 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -343,8 +343,8 @@ var _ = Describe("ClusterClient", func() { return err } - _, err = tx.MultiExec(func() error { - tx.Set(key, strconv.FormatInt(n+1, 10), 0) + _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) return err diff --git a/example_test.go b/example_test.go index 1aef66fcaa..0769aa05a8 100644 --- a/example_test.go +++ b/example_test.go @@ -190,8 +190,8 @@ func ExampleClient_Watch() { return err } - _, err = tx.MultiExec(func() error { - tx.Set(key, strconv.FormatInt(n+1, 10), 0) + _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) return err diff --git a/pool_test.go b/pool_test.go index 1adea5e89e..607fa463ec 100644 --- a/pool_test.go +++ b/pool_test.go @@ -35,13 +35,13 @@ var _ = Describe("pool", func() { Expect(pool.Len()).To(Equal(pool.FreeLen())) }) - It("srespect max size on multi", func() { + It("respects max size on multi", func() { perform(1000, func(id int) { var ping *redis.StatusCmd err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.MultiExec(func() error { - ping = tx.Ping() + cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + ping = pipe.Ping() return nil }) Expect(err).NotTo(HaveOccurred()) diff --git a/race_test.go b/race_test.go index cf093f25fd..d6503eb20a 100644 --- a/race_test.go +++ b/race_test.go @@ -222,8 +222,8 @@ var _ = Describe("races", func() { num, err := strconv.ParseInt(val, 10, 64) Expect(err).NotTo(HaveOccurred()) - cmds, err := tx.MultiExec(func() error { - tx.Set("key", strconv.FormatInt(num+1, 10), 0) + cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Set("key", strconv.FormatInt(num+1, 10), 0) return nil }) Expect(cmds).To(HaveLen(1)) diff --git a/redis_test.go b/redis_test.go index 8d5f651a89..8389e00830 100644 --- a/redis_test.go +++ b/redis_test.go @@ -67,8 +67,8 @@ var _ = Describe("Client", func() { It("should close Tx without closing the client", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.MultiExec(func() error { - tx.Ping() + _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() return nil }) return err diff --git a/tx.go b/tx.go index 950c374fc7..2a45990302 100644 --- a/tx.go +++ b/tx.go @@ -11,8 +11,6 @@ import ( // Redis transaction failed. const TxFailedErr = internal.RedisError("redis: transaction failed") -var errDiscard = internal.RedisError("redis: Discard can be used only inside Exec") - // Tx implements Redis transactions as described in // http://redis.io/topics/transactions. It's NOT safe for concurrent use // by multiple goroutines, because Exec resets list of watched keys. @@ -22,10 +20,11 @@ type Tx struct { statefulCmdable baseClient - cmds []Cmder closed bool } +var _ Cmdable = (*Tx)(nil) + func (c *Client) newTx() *Tx { tx := Tx{ baseClient: baseClient{ @@ -53,14 +52,6 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return retErr } -func (c *Tx) Process(cmd Cmder) error { - if c.cmds == nil { - return c.baseClient.Process(cmd) - } - c.cmds = append(c.cmds, cmd) - return nil -} - // close closes the transaction, releasing any open resources. func (c *Tx) close() error { if c.closed { @@ -98,16 +89,16 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd { return cmd } -// Discard discards queued commands. -func (c *Tx) Discard() error { - if c.cmds == nil { - return errDiscard +func (c *Tx) Pipeline() *Pipeline { + pipe := Pipeline{ + exec: c.exec, } - c.cmds = c.cmds[:1] - return nil + pipe.cmdable.process = pipe.Process + pipe.statefulCmdable.process = pipe.Process + return &pipe } -// MultiExec executes all previously queued commands in a transaction +// Pipelined executes commands queued in the fn in a transaction // and restores the connection state to normal. // // When using WATCH, EXEC will execute commands only if the watched keys @@ -116,36 +107,29 @@ func (c *Tx) Discard() error { // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. -func (c *Tx) MultiExec(fn func() error) ([]Cmder, error) { - if c.closed { - return nil, pool.ErrClosed - } - - c.cmds = []Cmder{NewStatusCmd("MULTI")} - if err := fn(); err != nil { - return nil, err - } - c.cmds = append(c.cmds, NewSliceCmd("EXEC")) - - cmds := c.cmds - c.cmds = nil +func (c *Tx) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) +} - if len(cmds) == 2 { - return []Cmder{}, nil +func (c *Tx) exec(cmds []Cmder) error { + if c.closed { + return pool.ErrClosed } - // Strip MULTI and EXEC commands. - retCmds := cmds[1 : len(cmds)-1] - cn, _, err := c.conn() if err != nil { - setCmdsErr(retCmds, err) - return retCmds, err + setCmdsErr(cmds, err) + return err } - err = c.execCmds(cn, cmds) + multiExec := make([]Cmder, 0, len(cmds)+2) + multiExec = append(multiExec, NewStatusCmd("MULTI")) + multiExec = append(multiExec, cmds...) + multiExec = append(multiExec, NewSliceCmd("EXEC")) + + err = c.execCmds(cn, multiExec) c.putConn(cn, err, false) - return retCmds, err + return err } func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { @@ -155,12 +139,11 @@ func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { return err } - statusCmd := NewStatusCmd() - // Omit last command (EXEC). cmdsLen := len(cmds) - 1 // Parse queued replies. + statusCmd := cmds[0] for i := 0; i < cmdsLen; i++ { if err := statusCmd.readReply(cn); err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) @@ -183,18 +166,18 @@ func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { return err } - var firstCmdErr error + var firstErr error // Parse replies. // Loop starts from 1 to omit MULTI cmd. for i := 1; i < cmdsLen; i++ { cmd := cmds[i] if err := cmd.readReply(cn); err != nil { - if firstCmdErr == nil { - firstCmdErr = err + if firstErr == nil { + firstErr = err } } } - return firstCmdErr + return firstErr } diff --git a/tx_test.go b/tx_test.go index 03d220bba1..9631e4cb57 100644 --- a/tx_test.go +++ b/tx_test.go @@ -33,8 +33,8 @@ var _ = Describe("Tx", func() { return err } - _, err = tx.MultiExec(func() error { - tx.Set(key, strconv.FormatInt(n+1, 10), 0) + _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) return err @@ -65,10 +65,10 @@ var _ = Describe("Tx", func() { It("should discard", func() { err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.MultiExec(func() error { - tx.Set("key1", "hello1", 0) - tx.Discard() - tx.Set("key2", "hello2", 0) + cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Set("key1", "hello1", 0) + pipe.Discard() + pipe.Set("key2", "hello2", 0) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -88,7 +88,7 @@ var _ = Describe("Tx", func() { It("should exec empty", func() { err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.MultiExec(func() error { return nil }) + cmds, err := tx.Pipelined(func(*redis.Pipeline) error { return nil }) Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(0)) return err @@ -104,9 +104,9 @@ var _ = Describe("Tx", func() { const N = 20000 err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.MultiExec(func() error { + cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { for i := 0; i < N; i++ { - tx.Incr("key") + pipe.Incr("key") } return nil }) @@ -135,8 +135,8 @@ var _ = Describe("Tx", func() { do := func() error { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.MultiExec(func() error { - tx.Ping() + _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() return nil }) return err @@ -162,7 +162,7 @@ var _ = Describe("Tx", func() { do := func() error { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.MultiExec(func() error { + _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { return nil }) return err From a65b760eec168123edcff019beefe3945e2c20ec Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 14 Oct 2016 10:37:30 +0300 Subject: [PATCH 0233/1746] Lowercase command name. --- command.go | 4 ++++ internal/util.go | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 internal/util.go diff --git a/command.go b/command.go index 41fde30471..992197ac20 100644 --- a/command.go +++ b/command.go @@ -146,6 +146,10 @@ func (cmd *baseCmd) setErr(e error) { } func newBaseCmd(args []interface{}) baseCmd { + if len(args) > 0 { + // Cmd name is expected to be in lower case. + args[0] = internal.ToLower(args[0].(string)) + } return baseCmd{_args: args} } diff --git a/internal/util.go b/internal/util.go new file mode 100644 index 0000000000..5986cc5a7d --- /dev/null +++ b/internal/util.go @@ -0,0 +1,27 @@ +package internal + +func ToLower(s string) string { + if isLower(s) { + return s + } + + b := make([]byte, len(s)) + for i := range b { + c := s[i] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + b[i] = c + } + return string(b) +} + +func isLower(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' { + return false + } + } + return true +} From 6f8957c5b7dece15d1ea5f592ec88f46ee2deffa Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 14 Oct 2016 14:21:25 +0300 Subject: [PATCH 0234/1746] Update changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f8544978..b722c9b528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ # v5 - *Important*. ClusterClient and Ring now chose random node/shard when command does not have any keys or command info is not fully available. Also clients use EVAL and EVALSHA keys to pick the right node. + - Tx is refactored using Pipeline API so it implements Cmdable interface. From dcdf129dd538a84efd0de3786b55370b30be1a79 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 14 Oct 2016 14:39:02 +0300 Subject: [PATCH 0235/1746] Add TimeCmd. --- command.go | 42 ++++++++++++++++++++++++++++++++++++++++++ commands.go | 7 +++---- commands_test.go | 6 +++--- parser.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/command.go b/command.go index 992197ac20..15a04c0e7f 100644 --- a/command.go +++ b/command.go @@ -355,6 +355,48 @@ func (cmd *DurationCmd) readReply(cn *pool.Conn) error { //------------------------------------------------------------------------------ +type TimeCmd struct { + baseCmd + + val time.Time +} + +func NewTimeCmd(args ...interface{}) *TimeCmd { + cmd := newBaseCmd(args) + return &TimeCmd{ + baseCmd: cmd, + } +} + +func (cmd *TimeCmd) reset() { + cmd.val = time.Time{} + cmd.err = nil +} + +func (cmd *TimeCmd) Val() time.Time { + return cmd.val +} + +func (cmd *TimeCmd) Result() (time.Time, error) { + return cmd.val, cmd.err +} + +func (cmd *TimeCmd) String() string { + return cmdString(cmd, cmd.val) +} + +func (cmd *TimeCmd) readReply(cn *pool.Conn) error { + v, err := cn.Rd.ReadArrayReply(timeParser) + if err != nil { + cmd.err = err + return err + } + cmd.val = v.(time.Time) + return nil +} + +//------------------------------------------------------------------------------ + type BoolCmd struct { baseCmd diff --git a/commands.go b/commands.go index 8a2b9c5d35..5ac7de7299 100644 --- a/commands.go +++ b/commands.go @@ -199,7 +199,7 @@ type Cmdable interface { ShutdownSave() *StatusCmd ShutdownNoSave() *StatusCmd SlaveOf(host, port string) *StatusCmd - Time() *StringSliceCmd + Time() *TimeCmd Eval(script string, keys []string, args ...interface{}) *Cmd EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd ScriptExists(scripts ...string) *BoolSliceCmd @@ -1741,9 +1741,8 @@ func (c *cmdable) Sync() { panic("not implemented") } -// TODO: add TimeCmd and use it here -func (c *cmdable) Time() *StringSliceCmd { - cmd := NewStringSliceCmd("time") +func (c *cmdable) Time() *TimeCmd { + cmd := NewTimeCmd("time") c.process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index e613d4abcc..2db90e2ac0 100644 --- a/commands_test.go +++ b/commands_test.go @@ -159,9 +159,9 @@ var _ = Describe("Commands", func() { }) It("should Time", func() { - time := client.Time() - Expect(time.Err()).NotTo(HaveOccurred()) - Expect(time.Val()).To(HaveLen(2)) + tm, err := client.Time().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(tm).To(BeTemporally("~", time.Now(), 3*time.Second)) }) }) diff --git a/parser.go b/parser.go index 50b2a9565f..578dbc9381 100644 --- a/parser.go +++ b/parser.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "strconv" + "time" "gopkg.in/redis.v5/internal/proto" ) @@ -364,6 +365,7 @@ func commandInfoParser(rd *proto.Reader, n int64) (interface{}, error) { return &cmd, nil } +// Implements proto.MultiBulkParse func commandInfoSliceParser(rd *proto.Reader, n int64) (interface{}, error) { m := make(map[string]*CommandInfo, n) for i := int64(0); i < n; i++ { @@ -377,3 +379,32 @@ func commandInfoSliceParser(rd *proto.Reader, n int64) (interface{}, error) { } return m, nil } + +// Implements proto.MultiBulkParse +func timeParser(rd *proto.Reader, n int64) (interface{}, error) { + if n != 2 { + fmt.Errorf("got %d elements, expected 2", n) + } + + secStr, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + microsecStr, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + sec, err := strconv.ParseInt(secStr, 10, 64) + if err != nil { + return nil, err + } + + microsec, err := strconv.ParseInt(microsecStr, 10, 64) + if err != nil { + return nil, err + } + + return time.Unix(sec, microsec*100), nil +} From 3996289fe27044d4666e1cda60ee16257fc6557c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 14 Oct 2016 14:42:32 +0300 Subject: [PATCH 0236/1746] internal/proto: use read prefix to indicate that method reads from the stream. --- internal/proto/reader.go | 6 +++--- parser.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/proto/reader.go b/internal/proto/reader.go index a98ddb603c..84f14880ba 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -77,7 +77,7 @@ func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { case IntReply: return parseIntValue(line) case StringReply: - return p.parseBytesValue(line) + return p.readBytesValue(line) case ArrayReply: n, err := parseArrayLen(line) if err != nil { @@ -112,7 +112,7 @@ func (p *Reader) ReadBytesReply() ([]byte, error) { case ErrorReply: return nil, parseErrorValue(line) case StringReply: - return p.parseBytesValue(line) + return p.readBytesValue(line) case StatusReply: return parseStatusValue(line) default: @@ -206,7 +206,7 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) { return keys, cursor, err } -func (p *Reader) parseBytesValue(line []byte) ([]byte, error) { +func (p *Reader) readBytesValue(line []byte) ([]byte, error) { if isNilReply(line) { return nil, internal.Nil } diff --git a/parser.go b/parser.go index 578dbc9381..10580abc9c 100644 --- a/parser.go +++ b/parser.go @@ -406,5 +406,5 @@ func timeParser(rd *proto.Reader, n int64) (interface{}, error) { return nil, err } - return time.Unix(sec, microsec*100), nil + return time.Unix(sec, microsec*1000), nil } From 50f1aff778822ec94fe7e0521c6f80a8886a003f Mon Sep 17 00:00:00 2001 From: Borys Piddubnyi Date: Fri, 21 Oct 2016 15:40:53 +0300 Subject: [PATCH 0237/1746] Fix "invalid expire time in set" for SetXX with expiration = 0 --- commands.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 8a2b9c5d35..b1510b064b 100644 --- a/commands.go +++ b/commands.go @@ -788,10 +788,14 @@ func (c *cmdable) SetNX(key string, value interface{}, expiration time.Duration) // Zero expiration means the key has no expiration time. func (c *cmdable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { var cmd *BoolCmd - if usePrecise(expiration) { - cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "xx") + if expiration == 0 { + cmd = NewBoolCmd("set", key, value, "xx") } else { - cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "xx") + if usePrecise(expiration) { + cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "xx") + } else { + cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "xx") + } } c.process(cmd) return cmd From cb63f1fd691953c8d4aeb4abfbba98cea7da1e9c Mon Sep 17 00:00:00 2001 From: Borys Piddubnyi Date: Fri, 21 Oct 2016 17:14:51 +0300 Subject: [PATCH 0238/1746] Add test for SetXX with expiration = 0 --- commands_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/commands_test.go b/commands_test.go index e613d4abcc..3f4f2c8610 100644 --- a/commands_test.go +++ b/commands_test.go @@ -996,6 +996,23 @@ var _ = Describe("Commands", func() { }) It("should SetXX", func() { + isSet, err := client.SetXX("key", "hello2", 0).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(isSet).To(Equal(false)) + + err = client.Set("key", "hello", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + isSet, err = client.SetXX("key", "hello2", 0).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(isSet).To(Equal(true)) + + val, err := client.Get("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("hello2")) + }) + + It("should SetXX with expiration", func() { isSet, err := client.SetXX("key", "hello2", time.Second).Result() Expect(err).NotTo(HaveOccurred()) Expect(isSet).To(Equal(false)) From ae523dd55201741e9a23324fa7657099b53b11c5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 6 Nov 2016 13:29:32 +0200 Subject: [PATCH 0239/1746] travis: run go vet --- .travis.yml | 1 + Makefile | 1 + iterator.go | 1 - parser.go | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e8f81337c..4c70460bfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ go: matrix: allow_failures: + - go: 1.4 - go: tip install: diff --git a/Makefile b/Makefile index 9ee35b2c69..4562692677 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ all: testdeps go test ./... go test ./... -short -race + go vet testdeps: testdata/redis/src/redis-server diff --git a/iterator.go b/iterator.go index d48ad7b895..e885985b95 100644 --- a/iterator.go +++ b/iterator.go @@ -71,7 +71,6 @@ func (it *ScanIterator) Next() bool { return true } } - return false } // Val returns the key/field at the current cursor position. diff --git a/parser.go b/parser.go index 10580abc9c..09714125ea 100644 --- a/parser.go +++ b/parser.go @@ -383,7 +383,7 @@ func commandInfoSliceParser(rd *proto.Reader, n int64) (interface{}, error) { // Implements proto.MultiBulkParse func timeParser(rd *proto.Reader, n int64) (interface{}, error) { if n != 2 { - fmt.Errorf("got %d elements, expected 2", n) + return nil, fmt.Errorf("got %d elements, expected 2", n) } secStr, err := rd.ReadStringReply() From 83208a1d9b93713c6bb1fb1cbc83ca4dea78b4de Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 8 Nov 2016 11:46:44 +0200 Subject: [PATCH 0240/1746] Use consistent cluster state when executing pipeline. --- cluster.go | 460 ++++++++++++++++++++++------------------- cluster_client_test.go | 91 -------- export_test.go | 15 ++ 3 files changed, 258 insertions(+), 308 deletions(-) delete mode 100644 cluster_client_test.go diff --git a/cluster.go b/cluster.go index 9d07d04f3e..e294a1a076 100644 --- a/cluster.go +++ b/cluster.go @@ -85,189 +85,132 @@ type clusterNode struct { loading time.Time } +func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { + opt := clOpt.clientOptions() + opt.Addr = addr + node := clusterNode{ + Client: NewClient(opt), + } + + if clOpt.RouteByLatency { + const probes = 10 + for i := 0; i < probes; i++ { + t1 := time.Now() + node.Client.Ping() + node.Latency += time.Since(t1) + } + node.Latency = node.Latency / probes + } + + return &node +} + func (n *clusterNode) Loading() bool { return !n.loading.IsZero() && time.Since(n.loading) < time.Minute } -// ClusterClient is a Redis Cluster client representing a pool of zero -// or more underlying connections. It's safe for concurrent use by -// multiple goroutines. -type ClusterClient struct { - cmdable +//------------------------------------------------------------------------------ +type clusterNodes struct { opt *ClusterOptions mu sync.RWMutex addrs []string nodes map[string]*clusterNode - slots [][]*clusterNode closed bool - - cmdsInfoOnce *sync.Once - cmdsInfo map[string]*CommandInfo - - // Reports where slots reloading is in progress. - reloading uint32 } -var _ Cmdable = (*ClusterClient)(nil) - -// NewClusterClient returns a Redis Cluster client as described in -// http://redis.io/topics/cluster-spec. -func NewClusterClient(opt *ClusterOptions) *ClusterClient { - opt.init() - - c := &ClusterClient{ +func newClusterNodes(opt *ClusterOptions) *clusterNodes { + return &clusterNodes{ opt: opt, nodes: make(map[string]*clusterNode), - - cmdsInfoOnce: new(sync.Once), } - c.cmdable.process = c.Process +} - for _, addr := range opt.Addrs { - _, _ = c.nodeByAddr(addr) - } - c.reloadSlots() +func (c *clusterNodes) Close() error { + c.mu.Lock() + defer c.mu.Unlock() - if opt.IdleCheckFrequency > 0 { - go c.reaper(opt.IdleCheckFrequency) + if c.closed { + return nil } + c.closed = true - return c -} - -func (c *ClusterClient) cmdInfo(name string) *CommandInfo { - c.cmdsInfoOnce.Do(func() { - for _, node := range c.nodes { - cmdsInfo, err := node.Client.Command().Result() - if err == nil { - c.cmdsInfo = cmdsInfo - return - } + var firstErr error + for _, node := range c.nodes { + if err := node.Client.Close(); err != nil && firstErr == nil { + firstErr = err } - c.cmdsInfoOnce = &sync.Once{} - }) - if c.cmdsInfo == nil { - return nil } - return c.cmdsInfo[name] + c.addrs = nil + c.nodes = nil + + return firstErr } -func (c *ClusterClient) getNodes() map[string]*clusterNode { - var nodes map[string]*clusterNode +func (c *clusterNodes) All() ([]*clusterNode, error) { c.mu.RLock() - if !c.closed { - nodes = make(map[string]*clusterNode, len(c.nodes)) - for addr, node := range c.nodes { - nodes[addr] = node - } - } - c.mu.RUnlock() - return nodes -} + defer c.mu.RUnlock() -func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { - node, err := c.slotMasterNode(hashtag.Slot(keys[0])) - if err != nil { - return err + if c.closed { + return nil, pool.ErrClosed } - return node.Client.Watch(fn, keys...) -} -// PoolStats returns accumulated connection pool stats. -func (c *ClusterClient) PoolStats() *PoolStats { - var acc PoolStats - for _, node := range c.getNodes() { - s := node.Client.connPool.Stats() - acc.Requests += s.Requests - acc.Hits += s.Hits - acc.Timeouts += s.Timeouts - acc.TotalConns += s.TotalConns - acc.FreeConns += s.FreeConns + var nodes []*clusterNode + for _, node := range c.nodes { + nodes = append(nodes, node) } - return &acc + return nodes, nil } -// Close closes the cluster client, releasing any open resources. -// -// It is rare to Close a ClusterClient, as the ClusterClient is meant -// to be long-lived and shared between many goroutines. -func (c *ClusterClient) Close() error { - c.mu.Lock() - if !c.closed { - c.closeClients() - c.addrs = nil - c.nodes = nil - c.slots = nil - c.cmdsInfo = nil - } - c.closed = true - c.mu.Unlock() - return nil -} +func (c *clusterNodes) Get(addr string) (*clusterNode, error) { + var node *clusterNode + var ok bool -func (c *ClusterClient) nodeByAddr(addr string) (*clusterNode, error) { c.mu.RLock() - node, ok := c.nodes[addr] + if !c.closed { + node, ok = c.nodes[addr] + } c.mu.RUnlock() if ok { return node, nil } - defer c.mu.Unlock() c.mu.Lock() + defer c.mu.Unlock() if c.closed { return nil, pool.ErrClosed } node, ok = c.nodes[addr] - if !ok { - node = c.newNode(addr) - c.nodes[addr] = node - c.addrs = append(c.addrs, addr) + if ok { + return node, nil } + c.addrs = append(c.addrs, addr) + node = newClusterNode(c.opt, addr) + c.nodes[addr] = node return node, nil } -func (c *ClusterClient) newNode(addr string) *clusterNode { - opt := c.opt.clientOptions() - opt.Addr = addr - return &clusterNode{ - Client: NewClient(opt), - } -} - -func (c *ClusterClient) slotNodes(slot int) (nodes []*clusterNode) { +func (c *clusterNodes) Random() (*clusterNode, error) { c.mu.RLock() - if slot < len(c.slots) { - nodes = c.slots[slot] - } + closed := c.closed + addrs := c.addrs c.mu.RUnlock() - return nodes -} -// randomNode returns random live node. -func (c *ClusterClient) randomNode() (*clusterNode, error) { + if closed { + return nil, pool.ErrClosed + } + if len(addrs) == 0 { + return nil, errClusterNoNodes + } + var nodeErr error for i := 0; i < 10; i++ { - c.mu.RLock() - closed := c.closed - addrs := c.addrs - c.mu.RUnlock() - - if closed { - return nil, pool.ErrClosed - } - - if len(addrs) == 0 { - return nil, errClusterNoNodes - } n := rand.Intn(len(addrs)) - - node, err := c.nodeByAddr(addrs[n]) + node, err := c.Get(addrs[n]) if err != nil { return nil, err } @@ -280,19 +223,50 @@ func (c *ClusterClient) randomNode() (*clusterNode, error) { return nil, nodeErr } -func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { +//------------------------------------------------------------------------------ + +type clusterState struct { + nodes *clusterNodes + slots [][]*clusterNode +} + +func newClusterState(nodes *clusterNodes, slots []ClusterSlot) (*clusterState, error) { + c := clusterState{ + nodes: nodes, + slots: make([][]*clusterNode, hashtag.SlotNumber), + } + + for _, slot := range slots { + var nodes []*clusterNode + for _, slotNode := range slot.Nodes { + node, err := c.nodes.Get(slotNode.Addr) + if err != nil { + return nil, err + } + nodes = append(nodes, node) + } + + for i := slot.Start; i <= slot.End; i++ { + c.slots[i] = nodes + } + } + + return &c, nil +} + +func (c *clusterState) slotMasterNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) if len(nodes) == 0 { - return c.randomNode() + return c.nodes.Random() } return nodes[0], nil } -func (c *ClusterClient) slotSlaveNode(slot int) (*clusterNode, error) { +func (c *clusterState) slotSlaveNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) switch len(nodes) { case 0: - return c.randomNode() + return c.nodes.Random() case 1: return nodes[0], nil case 2: @@ -313,10 +287,10 @@ func (c *ClusterClient) slotSlaveNode(slot int) (*clusterNode, error) { } } -func (c *ClusterClient) slotClosestNode(slot int) (*clusterNode, error) { +func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) if len(nodes) == 0 { - return c.randomNode() + return c.nodes.Random() } var node *clusterNode @@ -328,31 +302,108 @@ func (c *ClusterClient) slotClosestNode(slot int) (*clusterNode, error) { return node, nil } -func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { - cmdInfo := c.cmdInfo(cmd.arg(0)) +func (c *clusterState) slotNodes(slot int) []*clusterNode { + if slot < len(c.slots) { + return c.slots[slot] + } + return nil +} + +//------------------------------------------------------------------------------ + +// ClusterClient is a Redis Cluster client representing a pool of zero +// or more underlying connections. It's safe for concurrent use by +// multiple goroutines. +type ClusterClient struct { + cmdable + + opt *ClusterOptions + cmds map[string]*CommandInfo + nodes *clusterNodes + _state atomic.Value + + // Reports where slots reloading is in progress. + reloading uint32 + + closed bool +} + +var _ Cmdable = (*ClusterClient)(nil) + +// NewClusterClient returns a Redis Cluster client as described in +// http://redis.io/topics/cluster-spec. +func NewClusterClient(opt *ClusterOptions) *ClusterClient { + opt.init() + + c := &ClusterClient{ + opt: opt, + nodes: newClusterNodes(opt), + } + c.cmdable.process = c.Process + + // Add initial nodes. + for _, addr := range opt.Addrs { + _, _ = c.nodes.Get(addr) + } + + c.reloadSlots() + + if opt.IdleCheckFrequency > 0 { + go c.reaper(opt.IdleCheckFrequency) + } + + return c +} + +func (c *ClusterClient) state() *clusterState { + v := c._state.Load() + if v == nil { + return nil + } + return v.(*clusterState) +} + +func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { + cmdInfo := c.cmds[cmd.arg(0)] firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) if firstKey == "" { - node, err := c.randomNode() + node, err := c.nodes.Random() return -1, node, err } slot := hashtag.Slot(firstKey) if cmdInfo.ReadOnly && c.opt.ReadOnly { if c.opt.RouteByLatency { - node, err := c.slotClosestNode(slot) + node, err := state.slotClosestNode(slot) return slot, node, err } - node, err := c.slotSlaveNode(slot) + node, err := state.slotSlaveNode(slot) return slot, node, err } - node, err := c.slotMasterNode(slot) + node, err := state.slotMasterNode(slot) return slot, node, err } +func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { + node, err := c.state().slotMasterNode(hashtag.Slot(keys[0])) + if err != nil { + return err + } + return node.Client.Watch(fn, keys...) +} + +// Close closes the cluster client, releasing any open resources. +// +// It is rare to Close a ClusterClient, as the ClusterClient is meant +// to be long-lived and shared between many goroutines. +func (c *ClusterClient) Close() error { + return c.nodes.Close() +} + func (c *ClusterClient) Process(cmd Cmder) error { - slot, node, err := c.cmdSlotAndNode(cmd) + slot, node, err := c.cmdSlotAndNode(c.state(), cmd) if err != nil { cmd.setErr(err) return err @@ -388,7 +439,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { // On network errors try random node. if internal.IsRetryableError(err) { - node, err = c.randomNode() + node, err = c.nodes.Random() continue } @@ -397,17 +448,18 @@ func (c *ClusterClient) Process(cmd Cmder) error { moved, ask, addr = internal.IsMovedError(err) if moved || ask { if slot >= 0 { - master, _ := c.slotMasterNode(slot) + master, _ := c.state().slotMasterNode(slot) if moved && (master == nil || master.Client.getAddr() != addr) { c.lazyReloadSlots() } } - node, err = c.nodeByAddr(addr) + node, err = c.nodes.Get(addr) if err != nil { cmd.setErr(err) return err } + continue } @@ -420,14 +472,15 @@ func (c *ClusterClient) Process(cmd Cmder) error { // ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { - c.mu.RLock() - slots := c.slots - c.mu.RUnlock() + state := c.state() + if state == nil { + return nil + } var wg sync.WaitGroup visited := make(map[*clusterNode]struct{}) errCh := make(chan error, 1) - for _, nodes := range slots { + for _, nodes := range state.slots { if len(nodes) == 0 { continue } @@ -460,77 +513,59 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { } } -// closeClients closes all clients and returns the first error if there are any. -func (c *ClusterClient) closeClients() error { - var retErr error - for _, node := range c.nodes { - if err := node.Client.Close(); err != nil && retErr == nil { - retErr = err - } - } - return retErr -} - -func (c *ClusterClient) setSlots(cs []ClusterSlot) { - slots := make([][]*clusterNode, hashtag.SlotNumber) - for _, s := range cs { - var nodes []*clusterNode - for _, n := range s.Nodes { - node, err := c.nodeByAddr(n.Addr) - if err == nil { - nodes = append(nodes, node) - } - } - - for i := s.Start; i <= s.End; i++ { - slots[i] = nodes - } +// PoolStats returns accumulated connection pool stats. +func (c *ClusterClient) PoolStats() *PoolStats { + nodes, err := c.nodes.All() + if err != nil { + return nil } - c.mu.Lock() - if !c.closed { - c.slots = slots + var acc PoolStats + for _, node := range nodes { + s := node.Client.connPool.Stats() + acc.Requests += s.Requests + acc.Hits += s.Hits + acc.Timeouts += s.Timeouts + acc.TotalConns += s.TotalConns + acc.FreeConns += s.FreeConns } - c.mu.Unlock() + return &acc } func (c *ClusterClient) lazyReloadSlots() { if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { return } - go c.reloadSlots() + go func() { + c.reloadSlots() + atomic.StoreUint32(&c.reloading, 0) + }() } func (c *ClusterClient) reloadSlots() { - defer atomic.StoreUint32(&c.reloading, 0) - - node, err := c.randomNode() - if err != nil { - return - } + for i := 0; i < 10; i++ { + node, err := c.nodes.Random() + if err != nil { + return + } - slots, err := node.Client.ClusterSlots().Result() - if err != nil { - internal.Logf("ClusterSlots on addr=%q failed: %s", node.Client.getAddr(), err) - return - } + if c.cmds == nil { + cmds, err := node.Client.Command().Result() + if err == nil { + c.cmds = cmds + } + } - c.setSlots(slots) - if c.opt.RouteByLatency { - c.setNodesLatency() - } -} + slots, err := node.Client.ClusterSlots().Result() + if err != nil { + continue + } -func (c *ClusterClient) setNodesLatency() { - const n = 10 - for _, node := range c.getNodes() { - var latency time.Duration - for i := 0; i < n; i++ { - t1 := time.Now() - node.Client.Ping() - latency += time.Since(t1) + state, err := newClusterState(c.nodes, slots) + if err != nil { + return } - node.Latency = latency / n + c._state.Store(state) } } @@ -540,8 +575,8 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { defer ticker.Stop() for _ = range ticker.C { - nodes := c.getNodes() - if nodes == nil { + nodes, err := c.nodes.All() + if err != nil { break } @@ -584,9 +619,10 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { } } + state := c.state() cmdsMap := make(map[*clusterNode][]Cmder) for _, cmd := range cmds { - _, node, err := c.cmdSlotAndNode(cmd) + _, node, err := c.cmdSlotAndNode(state, cmd) if err != nil { cmd.setErr(err) setRetErr(err) @@ -599,16 +635,6 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { - if node == nil { - var err error - node, err = c.randomNode() - if err != nil { - setCmdsErr(cmds, err) - setRetErr(err) - continue - } - } - cn, _, err := node.Client.conn() if err != nil { setCmdsErr(cmds, err) @@ -660,7 +686,7 @@ func (c *ClusterClient) execClusterCmds( if moved { c.lazyReloadSlots() - node, err := c.nodeByAddr(addr) + node, err := c.nodes.Get(addr) if err != nil { setRetErr(err) continue @@ -669,7 +695,7 @@ func (c *ClusterClient) execClusterCmds( cmd.reset() failedCmds[node] = append(failedCmds[node], cmd) } else if ask { - node, err := c.nodeByAddr(addr) + node, err := c.nodes.Get(addr) if err != nil { setRetErr(err) continue diff --git a/cluster_client_test.go b/cluster_client_test.go deleted file mode 100644 index 0a30d5dad4..0000000000 --- a/cluster_client_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package redis - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func (c *ClusterClient) SlotAddrs(slot int) []string { - var addrs []string - for _, n := range c.slotNodes(slot) { - addrs = append(addrs, n.Client.getAddr()) - } - return addrs -} - -// SwapSlot swaps a slot's master/slave address -// for testing MOVED redirects -func (c *ClusterClient) SwapSlotNodes(slot int) []string { - c.mu.Lock() - nodes := c.slots[slot] - nodes[0], nodes[1] = nodes[1], nodes[0] - c.mu.Unlock() - return c.SlotAddrs(slot) -} - -var _ = Describe("ClusterClient", func() { - var subject *ClusterClient - - var populate = func() { - subject.setSlots([]ClusterSlot{ - {0, 4095, []ClusterNode{{"", "127.0.0.1:7000"}, {"", "127.0.0.1:7004"}}}, - {12288, 16383, []ClusterNode{{"", "127.0.0.1:7003"}, {"", "127.0.0.1:7007"}}}, - {4096, 8191, []ClusterNode{{"", "127.0.0.1:7001"}, {"", "127.0.0.1:7005"}}}, - {8192, 12287, []ClusterNode{{"", "127.0.0.1:7002"}, {"", "127.0.0.1:7006"}}}, - }) - } - - BeforeEach(func() { - subject = NewClusterClient(&ClusterOptions{ - Addrs: []string{"127.0.0.1:6379", "127.0.0.1:7003", "127.0.0.1:7006"}, - }) - }) - - AfterEach(func() { - _ = subject.Close() - }) - - It("should initialize", func() { - Expect(subject.addrs).To(HaveLen(3)) - }) - - It("should update slots cache", func() { - populate() - Expect(subject.slots[0][0].Client.getAddr()).To(Equal("127.0.0.1:7000")) - Expect(subject.slots[0][1].Client.getAddr()).To(Equal("127.0.0.1:7004")) - Expect(subject.slots[4095][0].Client.getAddr()).To(Equal("127.0.0.1:7000")) - Expect(subject.slots[4095][1].Client.getAddr()).To(Equal("127.0.0.1:7004")) - Expect(subject.slots[4096][0].Client.getAddr()).To(Equal("127.0.0.1:7001")) - Expect(subject.slots[4096][1].Client.getAddr()).To(Equal("127.0.0.1:7005")) - Expect(subject.slots[8191][0].Client.getAddr()).To(Equal("127.0.0.1:7001")) - Expect(subject.slots[8191][1].Client.getAddr()).To(Equal("127.0.0.1:7005")) - Expect(subject.slots[8192][0].Client.getAddr()).To(Equal("127.0.0.1:7002")) - Expect(subject.slots[8192][1].Client.getAddr()).To(Equal("127.0.0.1:7006")) - Expect(subject.slots[12287][0].Client.getAddr()).To(Equal("127.0.0.1:7002")) - Expect(subject.slots[12287][1].Client.getAddr()).To(Equal("127.0.0.1:7006")) - Expect(subject.slots[12288][0].Client.getAddr()).To(Equal("127.0.0.1:7003")) - Expect(subject.slots[12288][1].Client.getAddr()).To(Equal("127.0.0.1:7007")) - Expect(subject.slots[16383][0].Client.getAddr()).To(Equal("127.0.0.1:7003")) - Expect(subject.slots[16383][1].Client.getAddr()).To(Equal("127.0.0.1:7007")) - Expect(subject.addrs).To(Equal([]string{ - "127.0.0.1:6379", - "127.0.0.1:7003", - "127.0.0.1:7006", - "127.0.0.1:7000", - "127.0.0.1:7004", - "127.0.0.1:7007", - "127.0.0.1:7001", - "127.0.0.1:7005", - "127.0.0.1:7002", - })) - }) - - It("should close", func() { - populate() - Expect(subject.Close()).NotTo(HaveOccurred()) - Expect(subject.addrs).To(BeEmpty()) - Expect(subject.nodes).To(BeEmpty()) - Expect(subject.slots).To(BeEmpty()) - Expect(subject.Ping().Err().Error()).To(Equal("redis: client is closed")) - }) -}) diff --git a/export_test.go b/export_test.go index d36f867155..a366dceeea 100644 --- a/export_test.go +++ b/export_test.go @@ -17,3 +17,18 @@ func (c *PubSub) Pool() pool.Pooler { func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) { return c.receiveMessage(timeout) } + +func (c *ClusterClient) SlotAddrs(slot int) []string { + var addrs []string + for _, n := range c.state().slotNodes(slot) { + addrs = append(addrs, n.Client.getAddr()) + } + return addrs +} + +// SwapSlot swaps a slot's master/slave address for testing MOVED redirects. +func (c *ClusterClient) SwapSlotNodes(slot int) []string { + nodes := c.state().slots[slot] + nodes[0], nodes[1] = nodes[1], nodes[0] + return c.SlotAddrs(slot) +} From 62cd3b38ef5cbd3ba4af7aa8d463fe5fdcc88f8e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 9 Nov 2016 10:04:37 +0200 Subject: [PATCH 0241/1746] Limit allocation. --- internal/proto/reader.go | 43 ++++++++++++++++++++++++++++++---------- race_test.go | 9 ++++++--- redis_test.go | 5 ++--- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 84f14880ba..bd76e3a8c8 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -9,6 +9,8 @@ import ( "gopkg.in/redis.v5/internal" ) +const bytesAllocLimit = 1024 * 1024 // 1mb + const errEmptyReply = internal.RedisError("redis: reply is empty") type MultiBulkParse func(*Reader, int64) (interface{}, error) @@ -34,16 +36,7 @@ func (p *Reader) PeekBuffered() []byte { } func (p *Reader) ReadN(n int) ([]byte, error) { - // grow internal buffer, if necessary - if d := n - cap(p.buf); d > 0 { - p.buf = p.buf[:cap(p.buf)] - p.buf = append(p.buf, make([]byte, d)...) - } else { - p.buf = p.buf[:n] - } - - _, err := io.ReadFull(p.src, p.buf) - return p.buf, err + return readN(p.src, p.buf, n) } func (p *Reader) ReadLine() ([]byte, error) { @@ -225,6 +218,36 @@ func (p *Reader) readBytesValue(line []byte) ([]byte, error) { // -------------------------------------------------------------------- +func readN(r io.Reader, b []byte, n int) ([]byte, error) { + if n == 0 && b == nil { + return make([]byte, 0), nil + } + + if cap(b) >= n { + b = b[:n] + _, err := io.ReadFull(r, b) + return b, err + } + b = b[:cap(b)] + + pos := 0 + for pos < n { + diff := n - len(b) + if diff > bytesAllocLimit { + diff = bytesAllocLimit + } + b = append(b, make([]byte, diff)...) + + nn, err := io.ReadFull(r, b[pos:]) + if err != nil { + return nil, err + } + pos += nn + } + + return b, nil +} + func formatInt(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/race_test.go b/race_test.go index d6503eb20a..1e44bf03ef 100644 --- a/race_test.go +++ b/race_test.go @@ -104,7 +104,9 @@ var _ = Describe("races", func() { }) It("should handle big vals in Get", func() { - bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb + C, N = 4, 100 + + bigVal := bytes.Repeat([]byte{'*'}, 1<<17) // 128kb err := client.Set("key", bigVal, 0).Err() Expect(err).NotTo(HaveOccurred()) @@ -115,7 +117,7 @@ var _ = Describe("races", func() { perform(C, func(id int) { for i := 0; i < N; i++ { - got, err := client.Get("key").Result() + got, err := client.Get("key").Bytes() Expect(err).NotTo(HaveOccurred()) Expect(got).To(Equal(bigVal)) } @@ -124,7 +126,8 @@ var _ = Describe("races", func() { It("should handle big vals in Set", func() { C, N = 4, 100 - bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb + + bigVal := bytes.Repeat([]byte{'*'}, 1<<17) // 128kb perform(C, func(id int) { for i := 0; i < N; i++ { diff --git a/redis_test.go b/redis_test.go index 8389e00830..9bb4b6886f 100644 --- a/redis_test.go +++ b/redis_test.go @@ -185,7 +185,7 @@ var _ = Describe("Client", func() { }) It("should handle big vals", func() { - bigVal := string(bytes.Repeat([]byte{'*'}, 1<<17)) // 128kb + bigVal := bytes.Repeat([]byte{'*'}, 2e6) err := client.Set("key", bigVal, 0).Err() Expect(err).NotTo(HaveOccurred()) @@ -194,9 +194,8 @@ var _ = Describe("Client", func() { Expect(client.Close()).To(BeNil()) client = redis.NewClient(redisOptions()) - got, err := client.Get("key").Result() + got, err := client.Get("key").Bytes() Expect(err).NotTo(HaveOccurred()) - Expect(len(got)).To(Equal(len(bigVal))) Expect(got).To(Equal(bigVal)) }) From 019ff6eb38ef232dbf116aeb08780588be207417 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Fri, 11 Nov 2016 15:47:49 -0800 Subject: [PATCH 0242/1746] Add support for parsing redis:// and rediss:// URLs This includes setting up a default dialer that handles the ssl handshake. --- options.go | 72 ++++++++++++++++++++++++++++++++++++++ options_test.go | 92 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 options_test.go diff --git a/options.go b/options.go index 376fa6bf00..904f2f8d72 100644 --- a/options.go +++ b/options.go @@ -1,7 +1,12 @@ package redis import ( + "crypto/tls" + "errors" "net" + "net/url" + "strconv" + "strings" "time" "gopkg.in/redis.v5/internal/pool" @@ -58,6 +63,9 @@ type Options struct { // Enables read only queries on slave nodes. ReadOnly bool + + // Config to use when connecting via TLS + TLSConfig *tls.Config } func (opt *Options) init() { @@ -92,6 +100,70 @@ func (opt *Options) init() { } } +// ParseURL parses a redis URL into options that can be used to connect to redis +func ParseURL(redisURL string) (*Options, error) { + o := &Options{Network: "tcp"} + u, err := url.Parse(redisURL) + if err != nil { + return nil, err + } + + if u.Scheme != "redis" && u.Scheme != "rediss" { + return nil, errors.New("invalid redis URL scheme: " + u.Scheme) + } + + if u.User != nil { + if p, ok := u.User.Password(); ok { + o.Password = p + } + } + + if len(u.Query()) > 0 { + return nil, errors.New("no options supported") + } + + h, p, err := net.SplitHostPort(u.Host) + if err != nil { + h = u.Host + } + if h == "" { + h = "localhost" + } + if p == "" { + p = "6379" + } + o.Addr = net.JoinHostPort(h, p) + + f := strings.FieldsFunc(u.Path, func(r rune) bool { + return r == '/' + }) + switch len(f) { + case 0: + o.DB = 0 + case 1: + if o.DB, err = strconv.Atoi(f[0]); err != nil { + return nil, errors.New("Invalid redis database number: " + err.Error()) + } + default: + return nil, errors.New("invalid redis URL path: " + u.Path) + } + + if u.Scheme == "rediss" { + o.Dialer = func() (net.Conn, error) { + conn, err := net.DialTimeout(o.Network, o.Addr, o.DialTimeout) + if err != nil { + return nil, err + } + if o.TLSConfig == nil { + o.TLSConfig = &tls.Config{InsecureSkipVerify: true} + } + t := tls.Client(conn, o.TLSConfig) + return t, t.Handshake() + } + } + return o, nil +} + func newConnPool(opt *Options) *pool.ConnPool { return pool.NewConnPool( opt.Dialer, diff --git a/options_test.go b/options_test.go new file mode 100644 index 0000000000..c5518117da --- /dev/null +++ b/options_test.go @@ -0,0 +1,92 @@ +package redis + +import ( + "errors" + "testing" +) + +func TestParseURL(t *testing.T) { + cases := []struct { + u string + addr string + db int + dialer bool + err error + }{ + { + "redis://localhost:123/1", + "localhost:123", + 1, false, nil, + }, + { + "redis://localhost:123", + "localhost:123", + 0, false, nil, + }, + { + "redis://localhost/1", + "localhost:6379", + 1, false, nil, + }, + { + "redis://12345", + "12345:6379", + 0, false, nil, + }, + { + "rediss://localhost:123", + "localhost:123", + 0, true, nil, + }, + { + "redis://localhost/?abc=123", + "", + 0, false, errors.New("no options supported"), + }, + { + "http://google.com", + "", + 0, false, errors.New("invalid redis URL scheme: http"), + }, + { + "redis://localhost/1/2/3/4", + "", + 0, false, errors.New("invalid redis URL path: /1/2/3/4"), + }, + { + "12345", + "", + 0, false, errors.New("invalid redis URL scheme: "), + }, + { + "redis://localhost/iamadatabase", + "", + 0, false, errors.New("Invalid redis database number: strconv.ParseInt: parsing \"iamadatabase\": invalid syntax"), + }, + } + + for _, c := range cases { + t.Run(c.u, func(t *testing.T) { + o, err := ParseURL(c.u) + if c.err == nil && err != nil { + t.Fatalf("Expected err to be nil, but got: '%q'", err) + return + } + if c.err != nil && err != nil { + if c.err.Error() != err.Error() { + t.Fatalf("Expected err to be '%q', but got '%q'", c.err, err) + } + return + } + if o.Addr != c.addr { + t.Errorf("Expected Addr to be '%s', but got '%s'", c.addr, o.Addr) + } + if o.DB != c.db { + t.Errorf("Expecdted DB to be '%d', but got '%d'", c.db, o.DB) + } + if c.dialer && o.Dialer == nil { + t.Errorf("Expected a Dialer to be set, but isn't") + } + }) + } +} From 70eddf606d56a43f710b4e8caf26b5805d4739b7 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Mon, 14 Nov 2016 09:40:12 -0800 Subject: [PATCH 0243/1746] Place these tests behind a build tag Sub test support doesn't work for go < 1.7 --- options_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/options_test.go b/options_test.go index c5518117da..465bf0059b 100644 --- a/options_test.go +++ b/options_test.go @@ -1,3 +1,5 @@ +// +build go1.7 + package redis import ( From 4aa583b6f8bab0387497bddced7702e6cdb8abc6 Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Tue, 15 Nov 2016 10:27:20 -0800 Subject: [PATCH 0244/1746] Updates based on PR feedback --- options.go | 23 +++++++++-------------- options_test.go | 24 ++++++++++++------------ 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/options.go b/options.go index 904f2f8d72..9d8e917f7d 100644 --- a/options.go +++ b/options.go @@ -64,7 +64,7 @@ type Options struct { // Enables read only queries on slave nodes. ReadOnly bool - // Config to use when connecting via TLS + // TLS Config to use. When set TLS will be negotiated. TLSConfig *tls.Config } @@ -74,7 +74,12 @@ func (opt *Options) init() { } if opt.Dialer == nil { opt.Dialer = func() (net.Conn, error) { - return net.DialTimeout(opt.Network, opt.Addr, opt.DialTimeout) + conn, err := net.DialTimeout(opt.Network, opt.Addr, opt.DialTimeout) + if opt.TLSConfig == nil || err != nil { + return conn, err + } + t := tls.Client(conn, opt.TLSConfig) + return t, t.Handshake() } } if opt.PoolSize == 0 { @@ -142,24 +147,14 @@ func ParseURL(redisURL string) (*Options, error) { o.DB = 0 case 1: if o.DB, err = strconv.Atoi(f[0]); err != nil { - return nil, errors.New("Invalid redis database number: " + err.Error()) + return nil, errors.New("invalid redis database number: " + err.Error()) } default: return nil, errors.New("invalid redis URL path: " + u.Path) } if u.Scheme == "rediss" { - o.Dialer = func() (net.Conn, error) { - conn, err := net.DialTimeout(o.Network, o.Addr, o.DialTimeout) - if err != nil { - return nil, err - } - if o.TLSConfig == nil { - o.TLSConfig = &tls.Config{InsecureSkipVerify: true} - } - t := tls.Client(conn, o.TLSConfig) - return t, t.Handshake() - } + o.TLSConfig = &tls.Config{InsecureSkipVerify: true} } return o, nil } diff --git a/options_test.go b/options_test.go index 465bf0059b..effebd5a0f 100644 --- a/options_test.go +++ b/options_test.go @@ -9,11 +9,11 @@ import ( func TestParseURL(t *testing.T) { cases := []struct { - u string - addr string - db int - dialer bool - err error + u string + addr string + db int + tls bool + err error }{ { "redis://localhost:123/1", @@ -63,7 +63,7 @@ func TestParseURL(t *testing.T) { { "redis://localhost/iamadatabase", "", - 0, false, errors.New("Invalid redis database number: strconv.ParseInt: parsing \"iamadatabase\": invalid syntax"), + 0, false, errors.New("invalid redis database number: strconv.ParseInt: parsing \"iamadatabase\": invalid syntax"), }, } @@ -71,23 +71,23 @@ func TestParseURL(t *testing.T) { t.Run(c.u, func(t *testing.T) { o, err := ParseURL(c.u) if c.err == nil && err != nil { - t.Fatalf("Expected err to be nil, but got: '%q'", err) + t.Fatalf("unexpected error: '%q'", err) return } if c.err != nil && err != nil { if c.err.Error() != err.Error() { - t.Fatalf("Expected err to be '%q', but got '%q'", c.err, err) + t.Fatalf("got %q, expected %q", err, c.err) } return } if o.Addr != c.addr { - t.Errorf("Expected Addr to be '%s', but got '%s'", c.addr, o.Addr) + t.Errorf("got %q, want %q", o.Addr, c.addr) } if o.DB != c.db { - t.Errorf("Expecdted DB to be '%d', but got '%d'", c.db, o.DB) + t.Errorf("got %q, expected %q", o.DB, c.db) } - if c.dialer && o.Dialer == nil { - t.Errorf("Expected a Dialer to be set, but isn't") + if c.tls && o.TLSConfig == nil { + t.Errorf("got nil TLSConfig, expected a TLSConfig") } }) } From df2009821f8066d50a977f0a5df3a155aadcff5b Mon Sep 17 00:00:00 2001 From: Edward Muller Date: Wed, 16 Nov 2016 10:35:35 -0800 Subject: [PATCH 0245/1746] Default to secure I suspect that many people will need InsecureSkipVerify:true though, but that should be an explicit decision to expose security issues instead of papering over them. --- options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.go b/options.go index 9d8e917f7d..77ffbce3e7 100644 --- a/options.go +++ b/options.go @@ -154,7 +154,7 @@ func ParseURL(redisURL string) (*Options, error) { } if u.Scheme == "rediss" { - o.TLSConfig = &tls.Config{InsecureSkipVerify: true} + o.TLSConfig = &tls.Config{ServerName: h} } return o, nil } From a20665f042f007727466fe2643ac0870b03ae3ae Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 20 Nov 2016 09:50:49 +0200 Subject: [PATCH 0246/1746] Use string val in StringCmd. --- command.go | 14 ++++------ internal/proto/proto.go | 48 +++++++++++++---------------------- internal/proto/reader.go | 12 ++++++--- internal/proto/writebuffer.go | 4 ++- result.go | 2 +- 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/command.go b/command.go index 15a04c0e7f..0108f8ba23 100644 --- a/command.go +++ b/command.go @@ -457,7 +457,7 @@ func (cmd *BoolCmd) readReply(cn *pool.Conn) error { type StringCmd struct { baseCmd - val []byte + val string } func NewStringCmd(args ...interface{}) *StringCmd { @@ -466,12 +466,12 @@ func NewStringCmd(args ...interface{}) *StringCmd { } func (cmd *StringCmd) reset() { - cmd.val = nil + cmd.val = "" cmd.err = nil } func (cmd *StringCmd) Val() string { - return string(cmd.val) + return cmd.val } func (cmd *StringCmd) Result() (string, error) { @@ -479,7 +479,7 @@ func (cmd *StringCmd) Result() (string, error) { } func (cmd *StringCmd) Bytes() ([]byte, error) { - return cmd.val, cmd.err + return []byte(cmd.val), cmd.err } func (cmd *StringCmd) Int64() (int64, error) { @@ -520,11 +520,7 @@ func (cmd *StringCmd) readReply(cn *pool.Conn) error { cmd.err = err return err } - - new := make([]byte, len(b)) - copy(new, b) - cmd.val = new - + cmd.val = string(b) return nil } diff --git a/internal/proto/proto.go b/internal/proto/proto.go index 1d975202e8..e1c1ed4ae9 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -8,97 +8,85 @@ import ( "gopkg.in/redis.v5/internal" ) -const ( - ErrorReply = '-' - StatusReply = '+' - IntReply = ':' - StringReply = '$' - ArrayReply = '*' -) - -const defaultBufSize = 4096 - -const errScanNil = internal.RedisError("redis: Scan(nil)") - -func Scan(b []byte, val interface{}) error { +func Scan(s string, val interface{}) error { switch v := val.(type) { case nil: - return errScanNil + return internal.RedisError("redis: Scan(nil)") case *string: - *v = string(b) + *v = s return nil case *[]byte: - *v = b + *v = []byte(s) return nil case *int: var err error - *v, err = strconv.Atoi(string(b)) + *v, err = strconv.Atoi(s) return err case *int8: - n, err := strconv.ParseInt(string(b), 10, 8) + n, err := strconv.ParseInt(s, 10, 8) if err != nil { return err } *v = int8(n) return nil case *int16: - n, err := strconv.ParseInt(string(b), 10, 16) + n, err := strconv.ParseInt(s, 10, 16) if err != nil { return err } *v = int16(n) return nil case *int32: - n, err := strconv.ParseInt(string(b), 10, 32) + n, err := strconv.ParseInt(s, 10, 32) if err != nil { return err } *v = int32(n) return nil case *int64: - n, err := strconv.ParseInt(string(b), 10, 64) + n, err := strconv.ParseInt(s, 10, 64) if err != nil { return err } *v = n return nil case *uint: - n, err := strconv.ParseUint(string(b), 10, 64) + n, err := strconv.ParseUint(s, 10, 64) if err != nil { return err } *v = uint(n) return nil case *uint8: - n, err := strconv.ParseUint(string(b), 10, 8) + n, err := strconv.ParseUint(s, 10, 8) if err != nil { return err } *v = uint8(n) return nil case *uint16: - n, err := strconv.ParseUint(string(b), 10, 16) + n, err := strconv.ParseUint(s, 10, 16) if err != nil { return err } *v = uint16(n) return nil case *uint32: - n, err := strconv.ParseUint(string(b), 10, 32) + n, err := strconv.ParseUint(s, 10, 32) if err != nil { return err } *v = uint32(n) return nil case *uint64: - n, err := strconv.ParseUint(string(b), 10, 64) + n, err := strconv.ParseUint(s, 10, 64) if err != nil { return err } *v = n return nil case *float32: - n, err := strconv.ParseFloat(string(b), 32) + n, err := strconv.ParseFloat(s, 32) if err != nil { return err } @@ -106,14 +94,14 @@ func Scan(b []byte, val interface{}) error { return err case *float64: var err error - *v, err = strconv.ParseFloat(string(b), 64) + *v, err = strconv.ParseFloat(s, 64) return err case *bool: - *v = len(b) == 1 && b[0] == '1' + *v = len(s) == 1 && s[0] == '1' return nil default: if bu, ok := val.(encoding.BinaryUnmarshaler); ok { - return bu.UnmarshalBinary(b) + return bu.UnmarshalBinary([]byte(s)) } err := fmt.Errorf( "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", val) diff --git a/internal/proto/reader.go b/internal/proto/reader.go index bd76e3a8c8..7e72846394 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -11,7 +11,13 @@ import ( const bytesAllocLimit = 1024 * 1024 // 1mb -const errEmptyReply = internal.RedisError("redis: reply is empty") +const ( + ErrorReply = '-' + StatusReply = '+' + IntReply = ':' + StringReply = '$' + ArrayReply = '*' +) type MultiBulkParse func(*Reader, int64) (interface{}, error) @@ -23,7 +29,7 @@ type Reader struct { func NewReader(rd io.Reader) *Reader { return &Reader{ src: bufio.NewReader(rd), - buf: make([]byte, 0, defaultBufSize), + buf: make([]byte, 0, bufferSize), } } @@ -48,7 +54,7 @@ func (p *Reader) ReadLine() ([]byte, error) { return nil, bufio.ErrBufferFull } if len(line) == 0 { - return nil, errEmptyReply + return nil, internal.RedisError("redis: reply is empty") } if isNilReply(line) { return nil, internal.Nil diff --git a/internal/proto/writebuffer.go b/internal/proto/writebuffer.go index 8164f7f20e..1e0f8e6c1e 100644 --- a/internal/proto/writebuffer.go +++ b/internal/proto/writebuffer.go @@ -6,11 +6,13 @@ import ( "strconv" ) +const bufferSize = 4096 + type WriteBuffer struct{ b []byte } func NewWriteBuffer() *WriteBuffer { return &WriteBuffer{ - b: make([]byte, 0, defaultBufSize), + b: make([]byte, 0, bufferSize), } } diff --git a/result.go b/result.go index 481673d234..f5c7084cdf 100644 --- a/result.go +++ b/result.go @@ -51,7 +51,7 @@ func NewBoolResult(val bool, err error) *BoolCmd { } // NewStringResult returns a StringCmd initalised with val and err for testing -func NewStringResult(val []byte, err error) *StringCmd { +func NewStringResult(val string, err error) *StringCmd { var cmd StringCmd cmd.val = val cmd.setErr(err) From d4a3b1f28237694452d57be1672662cc6e535d74 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 26 Nov 2016 10:33:06 +0200 Subject: [PATCH 0247/1746] Add instrumentation example. --- README.md | 16 ---------------- example_test.go | 18 ++++++++++++++++++ internal/proto/proto.go | 14 ++++++-------- redis.go | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index db120c9897..0a976b2f1c 100644 --- a/README.md +++ b/README.md @@ -32,22 +32,6 @@ Import: import "gopkg.in/redis.v5" ``` -## Vendoring - -If you are using a vendoring tool with support for semantic versioning -e.g. [glide](https://github.com/Masterminds/glide), you can import this -package via its GitHub URL: - -```yaml -- package: github.com/go-redis/redis - version: ^5.0.0 -``` - -WARNING: please note that by importing `github.com/go-redis/redis` -directly (without semantic versioning constrol) you are in danger of -running in the breaking API changes. Use carefully and at your own -risk! - ## Quickstart ```go diff --git a/example_test.go b/example_test.go index 0769aa05a8..8f35f2f51d 100644 --- a/example_test.go +++ b/example_test.go @@ -331,3 +331,21 @@ func ExampleScanCmd_Iterator() { panic(err) } } + +func ExampleClient_instrumentation() { + client := redis.NewClient(&redis.Options{ + Addr: ":6379", + }) + client.WrapProcess(func(oldProcess func(cmd redis.Cmder) error) func(cmd redis.Cmder) error { + return func(cmd redis.Cmder) error { + start := time.Now() + err := oldProcess(cmd) + if err != nil { + fmt.Printf("command %s failed: %s", cmd, err) + } else { + fmt.Printf("command %q took %s", cmd, time.Since(start)) + } + return err + } + }) +} diff --git a/internal/proto/proto.go b/internal/proto/proto.go index e1c1ed4ae9..b51e4e86cb 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -8,8 +8,8 @@ import ( "gopkg.in/redis.v5/internal" ) -func Scan(s string, val interface{}) error { - switch v := val.(type) { +func Scan(s string, v interface{}) error { + switch v := v.(type) { case nil: return internal.RedisError("redis: Scan(nil)") case *string: @@ -99,12 +99,10 @@ func Scan(s string, val interface{}) error { case *bool: *v = len(s) == 1 && s[0] == '1' return nil + case encoding.BinaryUnmarshaler: + return v.UnmarshalBinary([]byte(s)) default: - if bu, ok := val.(encoding.BinaryUnmarshaler); ok { - return bu.UnmarshalBinary([]byte(s)) - } - err := fmt.Errorf( - "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", val) - return err + return fmt.Errorf( + "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", v) } } diff --git a/redis.go b/redis.go index aee3893375..72b8ee6e70 100644 --- a/redis.go +++ b/redis.go @@ -1,4 +1,4 @@ -package redis +package redis // import "gopkg.in/redis.v5" import ( "fmt" From 2fb4a785f3b163c2ff637295f5684adb44db204d Mon Sep 17 00:00:00 2001 From: Nathan Jones Date: Sun, 27 Nov 2016 12:27:58 +0000 Subject: [PATCH 0248/1746] Fixed Readme typo ZRangeBy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a976b2f1c..79f137df64 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Some corner cases: vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2 - vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeByScore{ + vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "-inf", Max: "+inf", Offset: 0, From 82f21639bf7180b247cb83de0d1cde203958e03b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 30 Nov 2016 12:39:14 +0200 Subject: [PATCH 0249/1746] Fix WrapProcess for Ring and Cluster. Add better example. --- commands.go | 17 +++------- commands_test.go | 47 +++++++++++++++++--------- example_instrumentation_test.go | 59 +++++++++++++++++++++++++++++++++ example_test.go | 18 ---------- pipeline_test.go | 2 +- redis.go | 16 +++++++++ ring.go | 26 +++++++-------- 7 files changed, 124 insertions(+), 61 deletions(-) create mode 100644 example_instrumentation_test.go diff --git a/commands.go b/commands.go index 45112842c5..0f06652d21 100644 --- a/commands.go +++ b/commands.go @@ -185,7 +185,6 @@ type Cmdable interface { ClientKill(ipPort string) *StatusCmd ClientList() *StringCmd ClientPause(dur time.Duration) *BoolCmd - ClientSetName(name string) *BoolCmd ConfigGet(parameter string) *SliceCmd ConfigResetStat() *StatusCmd ConfigSet(parameter, value string) *StatusCmd @@ -241,14 +240,6 @@ type cmdable struct { process func(cmd Cmder) error } -// WrapProcess replaces the process func. It takes a function createWrapper -// which is supplied by the user. createWrapper takes the old process func as -// an input and returns the new wrapper process func. createWrapper should -// use call the old process func within the new process func. -func (c *cmdable) WrapProcess(createWrapper func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { - c.process = createWrapper(c.process) -} - type statefulCmdable struct { process func(cmd Cmder) error } @@ -1625,15 +1616,15 @@ func (c *cmdable) ClientPause(dur time.Duration) *BoolCmd { return cmd } -// ClientSetName assigns a name to the one of many connections in the pool. -func (c *cmdable) ClientSetName(name string) *BoolCmd { +// ClientSetName assigns a name to the connection. +func (c *statefulCmdable) ClientSetName(name string) *BoolCmd { cmd := NewBoolCmd("client", "setname", name) c.process(cmd) return cmd } -// ClientGetName returns the name of the one of many connections in the pool. -func (c *Client) ClientGetName() *StringCmd { +// ClientGetName returns the name of the connection. +func (c *statefulCmdable) ClientGetName() *StringCmd { cmd := NewStringCmd("client", "getname") c.process(cmd) return cmd diff --git a/commands_test.go b/commands_test.go index 7da5f2a88a..dba37aac78 100644 --- a/commands_test.go +++ b/commands_test.go @@ -26,14 +26,20 @@ var _ = Describe("Commands", func() { Describe("server", func() { - // It("should Auth", func() { - // auth := client.Auth("password") - // Expect(auth.Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) - // Expect(auth.Val()).To(Equal("")) - // }) + It("should Auth", func() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Auth("password") + return nil + }) + Expect(err).To(MatchError("ERR Client sent AUTH, but no password is set")) + }) It("should Echo", func() { - echo := client.Echo("hello") + pipe := client.Pipeline() + echo := pipe.Echo("hello") + _, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(echo.Err()).NotTo(HaveOccurred()) Expect(echo.Val()).To(Equal("hello")) }) @@ -44,11 +50,15 @@ var _ = Describe("Commands", func() { Expect(ping.Val()).To(Equal("PONG")) }) - // It("should Select", func() { - // sel := client.Select(1) - // Expect(sel.Err()).NotTo(HaveOccurred()) - // Expect(sel.Val()).To(Equal("OK")) - // }) + It("should Select", func() { + pipe := client.Pipeline() + sel := pipe.Select(1) + _, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + + Expect(sel.Err()).NotTo(HaveOccurred()) + Expect(sel.Val()).To(Equal("OK")) + }) It("should BgRewriteAOF", func() { Skip("flaky test") @@ -84,13 +94,18 @@ var _ = Describe("Commands", func() { }) It("should ClientSetName and ClientGetName", func() { - isSet, err := client.ClientSetName("theclientname").Result() + pipe := client.Pipeline() + set := pipe.ClientSetName("theclientname") + get := pipe.ClientGetName() + _, err := pipe.Exec() Expect(err).NotTo(HaveOccurred()) - Expect(isSet).To(BeTrue()) - val, err := client.ClientGetName().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal("theclientname")) + Expect(set.Err()).NotTo(HaveOccurred()) + Expect(set.Val()).To(BeTrue()) + + Expect(get.Err()).NotTo(HaveOccurred()) + Expect(get.Val()).To(Equal("theclientname")) + }) It("should ConfigGet", func() { diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go new file mode 100644 index 0000000000..009138ebdf --- /dev/null +++ b/example_instrumentation_test.go @@ -0,0 +1,59 @@ +package redis_test + +import ( + "fmt" + "sync/atomic" + "time" + + redis "gopkg.in/redis.v5" +) + +func Example_instrumentation() { + ring := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "shard1": ":6379", + }, + }) + ring.ForEachShard(func(client *redis.Client) error { + wrapRedisProcess(client) + return nil + }) + + for { + ring.Ping() + } +} + +func wrapRedisProcess(client *redis.Client) { + const precision = time.Microsecond + var count, avgDur uint32 + + go func() { + for _ = range time.Tick(3 * time.Second) { + n := atomic.LoadUint32(&count) + dur := time.Duration(atomic.LoadUint32(&avgDur)) * precision + fmt.Printf("%s: processed=%d avg_dur=%s\n", client, n, dur) + } + }() + + client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error { + return func(cmd redis.Cmder) error { + start := time.Now() + err := oldProcess(cmd) + dur := time.Since(start) + + const decay = float64(1) / 100 + ms := float64(dur / precision) + for { + avg := atomic.LoadUint32(&avgDur) + newAvg := uint32((1-decay)*float64(avg) + decay*ms) + if atomic.CompareAndSwapUint32(&avgDur, avg, newAvg) { + break + } + } + atomic.AddUint32(&count, 1) + + return err + } + }) +} diff --git a/example_test.go b/example_test.go index 8f35f2f51d..0769aa05a8 100644 --- a/example_test.go +++ b/example_test.go @@ -331,21 +331,3 @@ func ExampleScanCmd_Iterator() { panic(err) } } - -func ExampleClient_instrumentation() { - client := redis.NewClient(&redis.Options{ - Addr: ":6379", - }) - client.WrapProcess(func(oldProcess func(cmd redis.Cmder) error) func(cmd redis.Cmder) error { - return func(cmd redis.Cmder) error { - start := time.Now() - err := oldProcess(cmd) - if err != nil { - fmt.Printf("command %s failed: %s", cmd, err) - } else { - fmt.Printf("command %q took %s", cmd, time.Since(start)) - } - return err - } - }) -} diff --git a/pipeline_test.go b/pipeline_test.go index 74cdd38519..01fcc09109 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -152,7 +152,7 @@ var _ = Describe("Pipelining", func() { const N = 1000 pipeline := client.Pipeline() - wg := &sync.WaitGroup{} + var wg sync.WaitGroup wg.Add(N) for i := 0; i < N; i++ { go func() { diff --git a/redis.go b/redis.go index 72b8ee6e70..1c5fdd1b8b 100644 --- a/redis.go +++ b/redis.go @@ -19,6 +19,7 @@ type baseClient struct { connPool pool.Pooler opt *Options + process func(Cmder) error onClose func() error // hook called when client is closed } @@ -78,6 +79,21 @@ func (c *baseClient) initConn(cn *pool.Conn) error { } func (c *baseClient) Process(cmd Cmder) error { + if c.process != nil { + return c.process(cmd) + } + return c.defaultProcess(cmd) +} + +// WrapProcess replaces the process func. It takes a function createWrapper +// which is supplied by the user. createWrapper takes the old process func as +// an input and returns the new wrapper process func. createWrapper should +// use call the old process func within the new process func. +func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { + c.process = fn(c.defaultProcess) +} + +func (c *baseClient) defaultProcess(cmd Cmder) error { for i := 0; i <= c.opt.MaxRetries; i++ { if i > 0 { cmd.reset() diff --git a/ring.go b/ring.go index 9608534640..53c3f113dc 100644 --- a/ring.go +++ b/ring.go @@ -230,7 +230,7 @@ func (c *Ring) addClient(name string, cl *Client) { c.mu.Unlock() } -func (c *Ring) shardByKey(key string) (*Client, error) { +func (c *Ring) shardByKey(key string) (*ringShard, error) { key = hashtag.Key(key) c.mu.RLock() @@ -246,27 +246,27 @@ func (c *Ring) shardByKey(key string) (*Client, error) { return nil, errRingShardsDown } - cl := c.shards[name].Client + shard := c.shards[name] c.mu.RUnlock() - return cl, nil + return shard, nil } -func (c *Ring) randomShard() (*Client, error) { +func (c *Ring) randomShard() (*ringShard, error) { return c.shardByKey(strconv.Itoa(rand.Int())) } -func (c *Ring) shardByName(name string) (*Client, error) { +func (c *Ring) shardByName(name string) (*ringShard, error) { if name == "" { return c.randomShard() } c.mu.RLock() - cl := c.shards[name].Client + shard := c.shards[name] c.mu.RUnlock() - return cl, nil + return shard, nil } -func (c *Ring) cmdShard(cmd Cmder) (*Client, error) { +func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { cmdInfo := c.cmdInfo(cmd.arg(0)) firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) if firstKey == "" { @@ -276,12 +276,12 @@ func (c *Ring) cmdShard(cmd Cmder) (*Client, error) { } func (c *Ring) Process(cmd Cmder) error { - cl, err := c.cmdShard(cmd) + shard, err := c.cmdShard(cmd) if err != nil { cmd.setErr(err) return err } - return cl.baseClient.Process(cmd) + return shard.Client.Process(cmd) } // rebalance removes dead shards from the Ring. @@ -384,7 +384,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { resetCmds(cmds) } - client, err := c.shardByName(name) + shard, err := c.shardByName(name) if err != nil { setCmdsErr(cmds, err) if firstErr == nil { @@ -393,7 +393,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { continue } - cn, _, err := client.conn() + cn, _, err := shard.Client.conn() if err != nil { setCmdsErr(cmds, err) if firstErr == nil { @@ -403,7 +403,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { } retry, err := execCmds(cn, cmds) - client.putConn(cn, err, false) + shard.Client.putConn(cn, err, false) if err == nil { continue } From b4efc45f1cb2db62a2e6895188c0a70a64c7377b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 3 Dec 2016 17:30:13 +0200 Subject: [PATCH 0250/1746] Set read/write timeouts more consistently. --- cluster.go | 44 ++++++++----- cluster_test.go | 121 +++++++++++++++++++++++++++++------ command.go | 2 +- internal/pool/conn.go | 30 +++------ internal/pool/pool.go | 10 +-- internal/pool/pool_sticky.go | 12 ++-- main_test.go | 21 +++--- options.go | 4 ++ pipeline.go | 25 +------- pipeline_test.go | 23 +++---- pubsub.go | 37 ++++++----- pubsub_test.go | 2 +- redis.go | 60 ++++++++++++----- redis_test.go | 83 ++++++++++++++++++++++-- ring.go | 11 ++-- sentinel.go | 4 +- tx.go | 12 ++-- tx_test.go | 34 +--------- 18 files changed, 340 insertions(+), 195 deletions(-) diff --git a/cluster.go b/cluster.go index e294a1a076..571d5c3660 100644 --- a/cluster.go +++ b/cluster.go @@ -387,7 +387,13 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl } func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { - node, err := c.state().slotMasterNode(hashtag.Slot(keys[0])) + var node *clusterNode + var err error + if len(keys) > 0 { + node, err = c.state().slotMasterNode(hashtag.Slot(keys[0])) + } else { + node, err = c.nodes.Random() + } if err != nil { return err } @@ -612,10 +618,10 @@ func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { } func (c *ClusterClient) pipelineExec(cmds []Cmder) error { - var retErr error - setRetErr := func(err error) { - if retErr == nil { - retErr = err + var firstErr error + setFirstErr := func(err error) { + if firstErr == nil { + firstErr = err } } @@ -625,7 +631,7 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { _, node, err := c.cmdSlotAndNode(state, cmd) if err != nil { cmd.setErr(err) - setRetErr(err) + setFirstErr(err) continue } cmdsMap[node] = append(cmdsMap[node], cmd) @@ -638,13 +644,13 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { cn, _, err := node.Client.conn() if err != nil { setCmdsErr(cmds, err) - setRetErr(err) + setFirstErr(err) continue } failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) if err != nil { - setRetErr(err) + setFirstErr(err) } node.Client.putConn(cn, err, false) } @@ -652,24 +658,28 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { cmdsMap = failedCmds } - return retErr + return firstErr } func (c *ClusterClient) execClusterCmds( cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) (map[*clusterNode][]Cmder, error) { + cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) return failedCmds, err } - var retErr error - setRetErr := func(err error) { - if retErr == nil { - retErr = err + var firstErr error + setFirstErr := func(err error) { + if firstErr == nil { + firstErr = err } } + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) + for i, cmd := range cmds { err := cmd.readReply(cn) if err == nil { @@ -688,7 +698,7 @@ func (c *ClusterClient) execClusterCmds( node, err := c.nodes.Get(addr) if err != nil { - setRetErr(err) + setFirstErr(err) continue } @@ -697,16 +707,16 @@ func (c *ClusterClient) execClusterCmds( } else if ask { node, err := c.nodes.Get(addr) if err != nil { - setRetErr(err) + setFirstErr(err) continue } cmd.reset() failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) } else { - setRetErr(err) + setFirstErr(err) } } - return failedCmds, retErr + return failedCmds, firstErr } diff --git a/cluster_test.go b/cluster_test.go index 4a3dd7fa4b..2c49f99bf3 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -483,45 +483,124 @@ var _ = Describe("ClusterClient", func() { describeClusterClient() }) +}) - Describe("ClusterClient without nodes", func() { - BeforeEach(func() { - client = redis.NewClusterClient(&redis.ClusterOptions{}) +var _ = Describe("ClusterClient without nodes", func() { + var client *redis.ClusterClient + + BeforeEach(func() { + client = redis.NewClusterClient(&redis.ClusterOptions{}) + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + It("returns an error", func() { + err := client.Ping().Err() + Expect(err).To(MatchError("redis: cluster has no nodes")) + }) + + It("pipeline returns an error", func() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil }) + Expect(err).To(MatchError("redis: cluster has no nodes")) + }) +}) + +var _ = Describe("ClusterClient without valid nodes", func() { + var client *redis.ClusterClient + + BeforeEach(func() { + client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{redisAddr}, + }) + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + It("returns an error", func() { + err := client.Ping().Err() + Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + }) + + It("pipeline returns an error", func() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + }) +}) + +var _ = Describe("ClusterClient timeout", func() { + var client *redis.ClusterClient - It("returns an error", func() { + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + testTimeout := func() { + It("Ping timeouts", func() { err := client.Ping().Err() - Expect(err).To(MatchError("redis: cluster has no nodes")) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) }) - It("pipeline returns an error", func() { + It("Pipeline timeouts", func() { _, err := client.Pipelined(func(pipe *redis.Pipeline) error { pipe.Ping() return nil }) - Expect(err).To(MatchError("redis: cluster has no nodes")) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) }) - }) - Describe("ClusterClient without valid nodes", func() { - BeforeEach(func() { - client = redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{redisAddr}, + It("Tx timeouts", func() { + err := client.Watch(func(tx *redis.Tx) error { + return tx.Ping().Err() }) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) }) - It("returns an error", func() { - err := client.Ping().Err() - Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + It("Tx Pipeline timeouts", func() { + err := client.Watch(func(tx *redis.Tx) error { + _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + return err + }) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) }) + } - It("pipeline returns an error", func() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { - pipe.Ping() - return nil - }) - Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + Context("read timeout", func() { + BeforeEach(func() { + opt := redisClusterOptions() + opt.ReadTimeout = time.Nanosecond + opt.WriteTimeout = -1 + client = cluster.clusterClient(opt) + }) + + testTimeout() + }) + + Context("write timeout", func() { + BeforeEach(func() { + opt := redisClusterOptions() + opt.ReadTimeout = time.Nanosecond + opt.WriteTimeout = -1 + client = cluster.clusterClient(opt) }) + + testTimeout() }) }) diff --git a/command.go b/command.go index 0108f8ba23..2bdf816717 100644 --- a/command.go +++ b/command.go @@ -64,7 +64,7 @@ func writeCmd(cn *pool.Conn, cmds ...Cmder) error { } } - _, err := cn.Write(cn.Wb.Bytes()) + _, err := cn.NetConn.Write(cn.Wb.Bytes()) return err } diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 785fc21451..b716cc2dbd 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -18,9 +18,6 @@ type Conn struct { Inited bool UsedAt time.Time - - ReadTimeout time.Duration - WriteTimeout time.Duration } func NewConn(netConn net.Conn) *Conn { @@ -30,7 +27,7 @@ func NewConn(netConn net.Conn) *Conn { UsedAt: time.Now(), } - cn.Rd = proto.NewReader(cn) + cn.Rd = proto.NewReader(cn.NetConn) return cn } @@ -38,28 +35,21 @@ func (cn *Conn) IsStale(timeout time.Duration) bool { return timeout > 0 && time.Since(cn.UsedAt) > timeout } -func (cn *Conn) Read(b []byte) (int, error) { +func (cn *Conn) SetReadTimeout(timeout time.Duration) error { cn.UsedAt = time.Now() - if cn.ReadTimeout != 0 { - cn.NetConn.SetReadDeadline(cn.UsedAt.Add(cn.ReadTimeout)) - } else { - cn.NetConn.SetReadDeadline(noDeadline) + if timeout > 0 { + return cn.NetConn.SetReadDeadline(cn.UsedAt.Add(timeout)) } - return cn.NetConn.Read(b) + return cn.NetConn.SetReadDeadline(noDeadline) + } -func (cn *Conn) Write(b []byte) (int, error) { +func (cn *Conn) SetWriteTimeout(timeout time.Duration) error { cn.UsedAt = time.Now() - if cn.WriteTimeout != 0 { - cn.NetConn.SetWriteDeadline(cn.UsedAt.Add(cn.WriteTimeout)) - } else { - cn.NetConn.SetWriteDeadline(noDeadline) + if timeout > 0 { + return cn.NetConn.SetWriteDeadline(cn.UsedAt.Add(timeout)) } - return cn.NetConn.Write(b) -} - -func (cn *Conn) RemoteAddr() net.Addr { - return cn.NetConn.RemoteAddr() + return cn.NetConn.SetWriteDeadline(noDeadline) } func (cn *Conn) Close() error { diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 389a3d22d2..b7e8977f88 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -266,19 +266,19 @@ func (p *ConnPool) Closed() bool { return atomic.LoadInt32(&p._closed) == 1 } -func (p *ConnPool) Close() (retErr error) { +func (p *ConnPool) Close() error { if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { return ErrClosed } p.connsMu.Lock() - // Close all connections. + var firstErr error for _, cn := range p.conns { if cn == nil { continue } - if err := p.closeConn(cn, ErrClosed); err != nil && retErr == nil { - retErr = err + if err := p.closeConn(cn, ErrClosed); err != nil && firstErr == nil { + firstErr = err } } p.conns = nil @@ -288,7 +288,7 @@ func (p *ConnPool) Close() (retErr error) { p.freeConns = nil p.freeConnsMu.Unlock() - return retErr + return firstErr } func (p *ConnPool) closeConn(cn *Conn, reason error) error { diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index ce45f4b44a..d25bf99a61 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -49,7 +49,7 @@ func (p *StickyConnPool) Get() (*Conn, bool, error) { return cn, true, nil } -func (p *StickyConnPool) put() (err error) { +func (p *StickyConnPool) putUpstream() (err error) { err = p.pool.Put(p.cn) p.cn = nil return err @@ -67,7 +67,7 @@ func (p *StickyConnPool) Put(cn *Conn) error { return nil } -func (p *StickyConnPool) remove(reason error) error { +func (p *StickyConnPool) removeUpstream(reason error) error { err := p.pool.Remove(p.cn, reason) p.cn = nil return err @@ -85,7 +85,7 @@ func (p *StickyConnPool) Remove(cn *Conn, reason error) error { if cn != nil && p.cn != cn { panic("p.cn != cn") } - return p.remove(reason) + return p.removeUpstream(reason) } func (p *StickyConnPool) Len() int { @@ -120,10 +120,10 @@ func (p *StickyConnPool) Close() error { var err error if p.cn != nil { if p.reusable { - err = p.put() + err = p.putUpstream() } else { - reason := errors.New("redis: sticky not reusable connection") - err = p.remove(reason) + reason := errors.New("redis: unreusable sticky connection") + err = p.removeUpstream(reason) } } return err diff --git a/main_test.go b/main_test.go index 67886e4bba..e48dfc90f8 100644 --- a/main_test.go +++ b/main_test.go @@ -2,6 +2,7 @@ package redis_test import ( "errors" + "fmt" "net" "os" "os/exec" @@ -159,8 +160,7 @@ func perform(n int, cbs ...func(int)) { func eventually(fn func() error, timeout time.Duration) error { var exit int32 - var retErr error - var mu sync.Mutex + errCh := make(chan error) done := make(chan struct{}) go func() { @@ -172,9 +172,10 @@ func eventually(fn func() error, timeout time.Duration) error { close(done) return } - mu.Lock() - retErr = err - mu.Unlock() + select { + case errCh <- err: + default: + } time.Sleep(timeout / 100) } }() @@ -184,10 +185,12 @@ func eventually(fn func() error, timeout time.Duration) error { return nil case <-time.After(timeout): atomic.StoreInt32(&exit, 1) - mu.Lock() - err := retErr - mu.Unlock() - return err + select { + case err := <-errCh: + return err + default: + return fmt.Errorf("timeout after %s", timeout) + } } } diff --git a/options.go b/options.go index 77ffbce3e7..a7f7fd7653 100644 --- a/options.go +++ b/options.go @@ -90,9 +90,13 @@ func (opt *Options) init() { } if opt.ReadTimeout == 0 { opt.ReadTimeout = 3 * time.Second + } else if opt.ReadTimeout == -1 { + opt.ReadTimeout = 0 } if opt.WriteTimeout == 0 { opt.WriteTimeout = opt.ReadTimeout + } else if opt.WriteTimeout == -1 { + opt.WriteTimeout = 0 } if opt.PoolTimeout == 0 { opt.PoolTimeout = opt.ReadTimeout + time.Second diff --git a/pipeline.go b/pipeline.go index fcd1f9999d..ef5510b2d3 100644 --- a/pipeline.go +++ b/pipeline.go @@ -1,9 +1,9 @@ package redis import ( + "errors" "sync" - "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/pool" ) @@ -67,7 +67,7 @@ func (c *Pipeline) Exec() ([]Cmder, error) { } if len(c.cmds) == 0 { - return c.cmds, nil + return nil, errors.New("redis: pipeline is empty") } cmds := c.cmds @@ -84,24 +84,3 @@ func (c *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { _ = c.Close() return cmds, err } - -func execCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { - if err := writeCmd(cn, cmds...); err != nil { - setCmdsErr(cmds, err) - return true, err - } - - for i, cmd := range cmds { - err := cmd.readReply(cn) - if err == nil { - continue - } - if i == 0 && internal.IsNetworkError(err) { - return true, err - } - if firstErr == nil { - firstErr = err - } - } - return false, firstErr -} diff --git a/pipeline_test.go b/pipeline_test.go index 01fcc09109..6c6fb96254 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -4,13 +4,13 @@ import ( "strconv" "sync" + "gopkg.in/redis.v5" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) -var _ = Describe("Pipelining", func() { +var _ = Describe("Pipeline", func() { var client *redis.Client BeforeEach(func() { @@ -51,15 +51,12 @@ var _ = Describe("Pipelining", func() { Expect(getNil.Val()).To(Equal("")) }) - It("should discard", func() { + It("discards queued commands", func() { pipeline := client.Pipeline() - pipeline.Get("key") pipeline.Discard() - cmds, err := pipeline.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(0)) - Expect(pipeline.Close()).NotTo(HaveOccurred()) + _, err := pipeline.Exec() + Expect(err).To(MatchError("redis: pipeline is empty")) }) It("should support block style", func() { @@ -84,12 +81,10 @@ var _ = Describe("Pipelining", func() { Expect(pipeline.Close()).NotTo(HaveOccurred()) }) - It("should pipeline with empty queue", func() { + It("returns an error when there are no commands", func() { pipeline := client.Pipeline() - cmds, err := pipeline.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(0)) - Expect(pipeline.Close()).NotTo(HaveOccurred()) + _, err := pipeline.Exec() + Expect(err).To(MatchError("redis: pipeline is empty")) }) It("should increment correctly", func() { diff --git a/pubsub.go b/pubsub.go index 223518a029..c9205a0310 100644 --- a/pubsub.go +++ b/pubsub.go @@ -35,12 +35,6 @@ func (c *PubSub) putConn(cn *pool.Conn, err error) { } func (c *PubSub) subscribe(redisCmd string, channels ...string) error { - cn, _, err := c.conn() - if err != nil { - return err - } - c.putConn(cn, err) - args := make([]interface{}, 1+len(channels)) args[0] = redisCmd for i, channel := range channels { @@ -48,7 +42,15 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { } cmd := NewSliceCmd(args...) - return writeCmd(cn, cmd) + cn, _, err := c.conn() + if err != nil { + return err + } + + cn.SetWriteTimeout(c.base.opt.WriteTimeout) + err = writeCmd(cn, cmd) + c.putConn(cn, err) + return err } // Subscribes the client to the specified channels. @@ -94,17 +96,21 @@ func (c *PubSub) Close() error { } func (c *PubSub) Ping(payload string) error { - cn, _, err := c.conn() - if err != nil { - return err - } - args := []interface{}{"PING"} if payload != "" { args = append(args, payload) } cmd := NewCmd(args...) - return writeCmd(cn, cmd) + + cn, _, err := c.conn() + if err != nil { + return err + } + + cn.SetWriteTimeout(c.base.opt.WriteTimeout) + err = writeCmd(cn, cmd) + c.putConn(cn, err) + return err } // Message received after a successful subscription to channel. @@ -176,13 +182,14 @@ func (c *PubSub) newMessage(reply []interface{}) (interface{}, error) { // is not received in time. This is low-level API and most clients // should use ReceiveMessage. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { + cmd := NewSliceCmd() + cn, _, err := c.conn() if err != nil { return nil, err } - cn.ReadTimeout = timeout - cmd := NewSliceCmd() + cn.SetReadTimeout(timeout) err = cmd.readReply(cn) c.putConn(cn, err) if err != nil { diff --git a/pubsub_test.go b/pubsub_test.go index ecbea7687d..f36621317e 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -315,7 +315,7 @@ var _ = Describe("PubSub", func() { Eventually(done).Should(Receive()) stats := client.PoolStats() - Expect(stats.Requests).To(Equal(uint32(4))) + Expect(stats.Requests).To(Equal(uint32(3))) Expect(stats.Hits).To(Equal(uint32(1))) } diff --git a/redis.go b/redis.go index 1c5fdd1b8b..6fcd5c4a07 100644 --- a/redis.go +++ b/redis.go @@ -3,6 +3,7 @@ package redis // import "gopkg.in/redis.v5" import ( "fmt" "log" + "time" "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/pool" @@ -105,14 +106,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { return err } - readTimeout := cmd.readTimeout() - if readTimeout != nil { - cn.ReadTimeout = *readTimeout - } else { - cn.ReadTimeout = c.opt.ReadTimeout - } - cn.WriteTimeout = c.opt.WriteTimeout - + cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmd); err != nil { c.putConn(cn, err, false) cmd.setErr(err) @@ -122,8 +116,9 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { return err } + cn.SetReadTimeout(c.cmdTimeout(cmd)) err = cmd.readReply(cn) - c.putConn(cn, err, readTimeout != nil) + c.putConn(cn, err, false) if err != nil && internal.IsRetryableError(err) { continue } @@ -134,6 +129,14 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { return cmd.Err() } +func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration { + if timeout := cmd.readTimeout(); timeout != nil { + return *timeout + } else { + return c.opt.ReadTimeout + } +} + func (c *baseClient) closed() bool { return c.connPool.Closed() } @@ -143,16 +146,16 @@ func (c *baseClient) closed() bool { // It is rare to Close a Client, as the Client is meant to be // long-lived and shared between many goroutines. func (c *baseClient) Close() error { - var retErr error + var firstErr error if c.onClose != nil { - if err := c.onClose(); err != nil && retErr == nil { - retErr = err + if err := c.onClose(); err != nil && firstErr == nil { + firstErr = err } } - if err := c.connPool.Close(); err != nil && retErr == nil { - retErr = err + if err := c.connPool.Close(); err != nil && firstErr == nil { + firstErr = err } - return retErr + return firstErr } func (c *baseClient) getAddr() string { @@ -225,7 +228,7 @@ func (c *Client) pipelineExec(cmds []Cmder) error { return err } - retry, err := execCmds(cn, cmds) + retry, err := c.execCmds(cn, cmds) c.putConn(cn, err, false) if err == nil { return nil @@ -240,6 +243,31 @@ func (c *Client) pipelineExec(cmds []Cmder) error { return firstErr } +func (c *Client) execCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { + cn.SetWriteTimeout(c.opt.WriteTimeout) + if err := writeCmd(cn, cmds...); err != nil { + setCmdsErr(cmds, err) + return true, err + } + + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) + + for i, cmd := range cmds { + err := cmd.readReply(cn) + if err == nil { + continue + } + if i == 0 && internal.IsNetworkError(err) { + return true, err + } + if firstErr == nil { + firstErr = err + } + } + return false, firstErr +} + func (c *Client) pubSub() *PubSub { return &PubSub{ base: baseClient{ diff --git a/redis_test.go b/redis_test.go index 9bb4b6886f..e15c871917 100644 --- a/redis_test.go +++ b/redis_test.go @@ -3,11 +3,12 @@ package redis_test import ( "bytes" "net" + "time" + + "gopkg.in/redis.v5" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("Client", func() { @@ -15,7 +16,7 @@ var _ = Describe("Client", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).To(BeNil()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -174,7 +175,7 @@ var _ = Describe("Client", func() { Expect(cn.UsedAt.After(createdAt)).To(BeTrue()) }) - It("should escape special chars", func() { + It("should process command with special chars", func() { set := client.Set("key", "hello1\r\nhello2\r\n", 0) Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) @@ -191,12 +192,84 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) // Reconnect to get new connection. - Expect(client.Close()).To(BeNil()) + Expect(client.Close()).NotTo(HaveOccurred()) client = redis.NewClient(redisOptions()) got, err := client.Get("key").Bytes() Expect(err).NotTo(HaveOccurred()) Expect(got).To(Equal(bigVal)) }) +}) + +var _ = Describe("Client timeout", func() { + var client *redis.Client + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + testTimeout := func() { + It("Ping timeouts", func() { + err := client.Ping().Err() + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + }) + + It("Pipeline timeouts", func() { + _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + }) + + It("Subscribe timeouts", func() { + _, err := client.Subscribe("_") + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + }) + + It("Tx timeouts", func() { + err := client.Watch(func(tx *redis.Tx) error { + return tx.Ping().Err() + }) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + }) + + It("Tx Pipeline timeouts", func() { + err := client.Watch(func(tx *redis.Tx) error { + _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + pipe.Ping() + return nil + }) + return err + }) + Expect(err).To(HaveOccurred()) + Expect(err.(net.Error).Timeout()).To(BeTrue()) + }) + } + + Context("read timeout", func() { + BeforeEach(func() { + opt := redisOptions() + opt.ReadTimeout = time.Nanosecond + opt.WriteTimeout = -1 + client = redis.NewClient(opt) + }) + + testTimeout() + }) + + Context("write timeout", func() { + BeforeEach(func() { + opt := redisOptions() + opt.ReadTimeout = -1 + opt.WriteTimeout = time.Nanosecond + client = redis.NewClient(opt) + }) + + testTimeout() + }) }) diff --git a/ring.go b/ring.go index 53c3f113dc..11945b4a5d 100644 --- a/ring.go +++ b/ring.go @@ -332,7 +332,7 @@ func (c *Ring) heartbeat() { // // It is rare to Close a Ring, as the Ring is meant to be long-lived // and shared between many goroutines. -func (c *Ring) Close() (retErr error) { +func (c *Ring) Close() error { defer c.mu.Unlock() c.mu.Lock() @@ -341,15 +341,16 @@ func (c *Ring) Close() (retErr error) { } c.closed = true + var firstErr error for _, shard := range c.shards { - if err := shard.Client.Close(); err != nil { - retErr = err + if err := shard.Client.Close(); err != nil && firstErr == nil { + firstErr = err } } c.hash = nil c.shards = nil - return retErr + return firstErr } func (c *Ring) Pipeline() *Pipeline { @@ -402,7 +403,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { continue } - retry, err := execCmds(cn, cmds) + retry, err := shard.Client.execCmds(cn, cmds) shard.Client.putConn(cn, err, false) if err == nil { continue diff --git a/sentinel.go b/sentinel.go index 0849dda386..2a3264776f 100644 --- a/sentinel.go +++ b/sentinel.go @@ -267,10 +267,10 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { if cn == nil { break } - if cn.RemoteAddr().String() != newMaster { + if cn.NetConn.RemoteAddr().String() != newMaster { err := fmt.Errorf( "sentinel: closing connection to the old master %s", - cn.RemoteAddr(), + cn.NetConn.RemoteAddr(), ) internal.Logf(err.Error()) d.pool.Remove(cn, err) diff --git a/tx.go b/tx.go index 2a45990302..772b3c9110 100644 --- a/tx.go +++ b/tx.go @@ -45,11 +45,11 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return err } } - retErr := fn(tx) - if err := tx.close(); err != nil && retErr == nil { - retErr = err + firstErr := fn(tx) + if err := tx.close(); err != nil && firstErr == nil { + firstErr = err } - return retErr + return firstErr } // close closes the transaction, releasing any open resources. @@ -133,12 +133,16 @@ func (c *Tx) exec(cmds []Cmder) error { } func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { + cn.SetWriteTimeout(c.opt.WriteTimeout) err := writeCmd(cn, cmds...) if err != nil { setCmdsErr(cmds[1:len(cmds)-1], err) return err } + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) + // Omit last command (EXEC). cmdsLen := len(cmds) - 1 diff --git a/tx_test.go b/tx_test.go index 9631e4cb57..156a890921 100644 --- a/tx_test.go +++ b/tx_test.go @@ -86,14 +86,12 @@ var _ = Describe("Tx", func() { Expect(get.Val()).To(Equal("hello2")) }) - It("should exec empty", func() { + It("returns an error when there are no commands", func() { err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(*redis.Pipeline) error { return nil }) - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(0)) + _, err := tx.Pipelined(func(*redis.Pipeline) error { return nil }) return err }) - Expect(err).NotTo(HaveOccurred()) + Expect(err).To(MatchError("redis: pipeline is empty")) v, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) @@ -150,30 +148,4 @@ var _ = Describe("Tx", func() { err = do() Expect(err).NotTo(HaveOccurred()) }) - - It("should recover from bad connection when there are no commands", func() { - // Put bad connection in the pool. - cn, _, err := client.Pool().Get() - Expect(err).NotTo(HaveOccurred()) - - cn.NetConn = &badConn{} - err = client.Pool().Put(cn) - Expect(err).NotTo(HaveOccurred()) - - do := func() error { - err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { - return nil - }) - return err - }, "key") - return err - } - - err = do() - Expect(err).To(MatchError("bad connection")) - - err = do() - Expect(err).NotTo(HaveOccurred()) - }) }) From e916395f6c328c17abce63b0a69d06c862951f11 Mon Sep 17 00:00:00 2001 From: "zezhou.yu" Date: Thu, 8 Dec 2016 01:29:17 +0800 Subject: [PATCH 0251/1746] evalsha with 0 key should return -1 pos --- command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command.go b/command.go index 2bdf816717..11150414d3 100644 --- a/command.go +++ b/command.go @@ -95,7 +95,7 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { if cmd.arg(2) != "0" { return 3 } else { - return 0 + return -1 } } if info == nil { From fb584d25dbb571e83e215de4498145754de9c022 Mon Sep 17 00:00:00 2001 From: "zezhou.yu" Date: Thu, 8 Dec 2016 01:34:22 +0800 Subject: [PATCH 0252/1746] fix cmdInfo nil pointer panic --- cluster.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 571d5c3660..673f3ff344 100644 --- a/cluster.go +++ b/cluster.go @@ -1,6 +1,7 @@ package redis import ( + "fmt" "math/rand" "sync" "sync/atomic" @@ -371,7 +372,9 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl return -1, node, err } slot := hashtag.Slot(firstKey) - + if cmdInfo == nil { + return -1, nil, internal.RedisError(fmt.Sprintf("cmdInfo of %s is nil", cmd.arg(0))) + } if cmdInfo.ReadOnly && c.opt.ReadOnly { if c.opt.RouteByLatency { node, err := state.slotClosestNode(slot) From 420337dc4a01f38e000d9f1c3823efe26721b185 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 9 Dec 2016 15:52:36 +0200 Subject: [PATCH 0253/1746] Simplify cmdInfo check. --- cluster.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index 673f3ff344..ec5a51f6b9 100644 --- a/cluster.go +++ b/cluster.go @@ -1,7 +1,6 @@ package redis import ( - "fmt" "math/rand" "sync" "sync/atomic" @@ -367,14 +366,12 @@ func (c *ClusterClient) state() *clusterState { func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmds[cmd.arg(0)] firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) - if firstKey == "" { + if firstKey == "" || cmdInfo == nil { node, err := c.nodes.Random() return -1, node, err } slot := hashtag.Slot(firstKey) - if cmdInfo == nil { - return -1, nil, internal.RedisError(fmt.Sprintf("cmdInfo of %s is nil", cmd.arg(0))) - } + if cmdInfo.ReadOnly && c.opt.ReadOnly { if c.opt.RouteByLatency { node, err := state.slotClosestNode(slot) From c7dfbb54af9a7a478eca47e75d0a536f39f48742 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 12 Dec 2016 17:30:08 +0200 Subject: [PATCH 0254/1746] Fix nil ptr in case when all nodes are unavailable. --- cluster.go | 53 ++++++++++++++++++++++++++++++++++++++++++++----- cluster_test.go | 27 +++++++++++++++++++++++++ tx.go | 6 ++---- 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/cluster.go b/cluster.go index ec5a51f6b9..f9a4fefb7c 100644 --- a/cluster.go +++ b/cluster.go @@ -156,7 +156,7 @@ func (c *clusterNodes) All() ([]*clusterNode, error) { return nil, pool.ErrClosed } - var nodes []*clusterNode + nodes := make([]*clusterNode, 0, len(c.nodes)) for _, node := range c.nodes { nodes = append(nodes, node) } @@ -208,7 +208,7 @@ func (c *clusterNodes) Random() (*clusterNode, error) { } var nodeErr error - for i := 0; i < 10; i++ { + for i := 0; i <= c.opt.MaxRedirects; i++ { n := rand.Intn(len(addrs)) node, err := c.Get(addrs[n]) if err != nil { @@ -446,6 +446,10 @@ func (c *ClusterClient) Process(cmd Cmder) error { // On network errors try random node. if internal.IsRetryableError(err) { node, err = c.nodes.Random() + if err != nil { + cmd.setErr(err) + return err + } continue } @@ -475,6 +479,39 @@ func (c *ClusterClient) Process(cmd Cmder) error { return cmd.Err() } +// ForEachNode concurrently calls the fn on each ever known node in the cluster. +// It returns the first error if any. +func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { + nodes, err := c.nodes.All() + if err != nil { + return err + } + + var wg sync.WaitGroup + errCh := make(chan error, 1) + for _, node := range nodes { + wg.Add(1) + go func(node *clusterNode) { + defer wg.Done() + err := fn(node.Client) + if err != nil { + select { + case errCh <- err: + default: + } + } + }(node) + } + wg.Wait() + + select { + case err := <-errCh: + return err + default: + return nil + } +} + // ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { @@ -649,10 +686,10 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { } failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) + node.Client.putConn(cn, err, false) if err != nil { setFirstErr(err) } - node.Client.putConn(cn, err, false) } cmdsMap = failedCmds @@ -686,9 +723,15 @@ func (c *ClusterClient) execClusterCmds( continue } - if i == 0 && internal.IsNetworkError(err) { + if i == 0 && internal.IsRetryableError(err) { + node, err := c.nodes.Random() + if err != nil { + setFirstErr(err) + continue + } + cmd.reset() - failedCmds[nil] = append(failedCmds[nil], cmds...) + failedCmds[node] = append(failedCmds[node], cmds...) break } diff --git a/cluster_test.go b/cluster_test.go index 2c49f99bf3..3406768011 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -602,6 +602,33 @@ var _ = Describe("ClusterClient timeout", func() { testTimeout() }) + + Context("network timeout", func() { + const pause = time.Second + + BeforeEach(func() { + opt := redisClusterOptions() + opt.ReadTimeout = 100 * time.Millisecond + opt.WriteTimeout = 100 * time.Millisecond + opt.MaxRedirects = 1 + client = cluster.clusterClient(opt) + + err := client.ForEachNode(func(client *redis.Client) error { + return client.ClientPause(pause).Err() + }) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Eventually(func() error { + return client.ForEachNode(func(client *redis.Client) error { + return client.Ping().Err() + }) + }, pause).ShouldNot(HaveOccurred()) + }) + + testTimeout() + }) }) //------------------------------------------------------------------------------ diff --git a/tx.go b/tx.go index 772b3c9110..30beda940b 100644 --- a/tx.go +++ b/tx.go @@ -176,10 +176,8 @@ func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { // Loop starts from 1 to omit MULTI cmd. for i := 1; i < cmdsLen; i++ { cmd := cmds[i] - if err := cmd.readReply(cn); err != nil { - if firstErr == nil { - firstErr = err - } + if err := cmd.readReply(cn); err != nil && firstErr == nil { + firstErr = err } } From 34122bffa897df4d7bf01cbd67e2e9d3f120e20c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 14 Dec 2016 12:12:50 +0200 Subject: [PATCH 0255/1746] Check that clients implement scripter interface. --- cluster.go | 2 -- commands.go | 5 +++++ redis.go | 2 -- ring.go | 2 -- script.go | 4 ++++ tx.go | 2 -- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cluster.go b/cluster.go index f9a4fefb7c..0feebb4822 100644 --- a/cluster.go +++ b/cluster.go @@ -328,8 +328,6 @@ type ClusterClient struct { closed bool } -var _ Cmdable = (*ClusterClient)(nil) - // NewClusterClient returns a Redis Cluster client as described in // http://redis.io/topics/cluster-spec. func NewClusterClient(opt *ClusterOptions) *ClusterClient { diff --git a/commands.go b/commands.go index 0f06652d21..5132a59797 100644 --- a/commands.go +++ b/commands.go @@ -236,6 +236,11 @@ type Cmdable interface { Command() *CommandsInfoCmd } +var _ Cmdable = (*Client)(nil) +var _ Cmdable = (*Tx)(nil) +var _ Cmdable = (*Ring)(nil) +var _ Cmdable = (*ClusterClient)(nil) + type cmdable struct { process func(cmd Cmder) error } diff --git a/redis.go b/redis.go index 6fcd5c4a07..bf465172ea 100644 --- a/redis.go +++ b/redis.go @@ -172,8 +172,6 @@ type Client struct { cmdable } -var _ Cmdable = (*Client)(nil) - func newClient(opt *Options, pool pool.Pooler) *Client { base := baseClient{opt: opt, connPool: pool} client := &Client{ diff --git a/ring.go b/ring.go index 11945b4a5d..96f7294cc4 100644 --- a/ring.go +++ b/ring.go @@ -136,8 +136,6 @@ type Ring struct { closed bool } -var _ Cmdable = (*Ring)(nil) - func NewRing(opt *RingOptions) *Ring { const nreplicas = 100 opt.init() diff --git a/script.go b/script.go index c5fc1d2bdc..e569af7b8c 100644 --- a/script.go +++ b/script.go @@ -14,6 +14,10 @@ type scripter interface { ScriptLoad(script string) *StringCmd } +var _ scripter = (*Client)(nil) +var _ scripter = (*Ring)(nil) +var _ scripter = (*ClusterClient)(nil) + type Script struct { src, hash string } diff --git a/tx.go b/tx.go index 30beda940b..c35032557f 100644 --- a/tx.go +++ b/tx.go @@ -23,8 +23,6 @@ type Tx struct { closed bool } -var _ Cmdable = (*Tx)(nil) - func (c *Client) newTx() *Tx { tx := Tx{ baseClient: baseClient{ From 865d501d07fc7461aa769741f4574351d9ecd993 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 13 Dec 2016 17:28:39 +0200 Subject: [PATCH 0256/1746] Add TxPipeline. --- cluster.go | 267 ++++++++++++++++++++------- cluster_test.go | 117 +++++++----- command.go | 251 +++++++------------------ internal/errors.go | 4 + internal/proto/reader.go | 12 +- internal/proto/{proto.go => scan.go} | 0 iterator.go | 1 - pipeline.go | 4 +- pipeline_test.go | 162 ++++------------ race_test.go | 31 ++++ redis.go | 194 +++++++++++++------ ring.go | 8 +- tx.go | 94 +--------- 13 files changed, 566 insertions(+), 579 deletions(-) rename internal/proto/{proto.go => scan.go} (100%) diff --git a/cluster.go b/cluster.go index f9a4fefb7c..e81a7eb839 100644 --- a/cluster.go +++ b/cluster.go @@ -1,6 +1,7 @@ package redis import ( + "fmt" "math/rand" "sync" "sync/atomic" @@ -9,6 +10,7 @@ import ( "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/hashtag" "gopkg.in/redis.v5/internal/pool" + "gopkg.in/redis.v5/internal/proto" ) var errClusterNoNodes = internal.RedisError("redis: cluster has no nodes") @@ -417,10 +419,6 @@ func (c *ClusterClient) Process(cmd Cmder) error { var ask bool for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { - if attempt > 0 { - cmd.reset() - } - if ask { pipe := node.Client.Pipeline() pipe.Process(NewCmd("ASKING")) @@ -655,111 +653,252 @@ func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { } func (c *ClusterClient) pipelineExec(cmds []Cmder) error { - var firstErr error - setFirstErr := func(err error) { - if firstErr == nil { - firstErr = err - } - } - - state := c.state() - cmdsMap := make(map[*clusterNode][]Cmder) - for _, cmd := range cmds { - _, node, err := c.cmdSlotAndNode(state, cmd) - if err != nil { - cmd.setErr(err) - setFirstErr(err) - continue - } - cmdsMap[node] = append(cmdsMap[node], cmd) + cmdsMap, err := c.mapCmdsByNode(cmds) + if err != nil { + return err } - for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { + for i := 0; i <= c.opt.MaxRedirects; i++ { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { cn, _, err := node.Client.conn() if err != nil { setCmdsErr(cmds, err) - setFirstErr(err) continue } - failedCmds, err = c.execClusterCmds(cn, cmds, failedCmds) + err = c.pipelineProcessCmds(cn, cmds, failedCmds) node.Client.putConn(cn, err, false) - if err != nil { - setFirstErr(err) - } } + if len(failedCmds) == 0 { + break + } cmdsMap = failedCmds } + var firstErr error + for _, cmd := range cmds { + if err := cmd.Err(); err != nil { + firstErr = err + break + } + } return firstErr } -func (c *ClusterClient) execClusterCmds( +func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, error) { + state := c.state() + cmdsMap := make(map[*clusterNode][]Cmder) + for _, cmd := range cmds { + _, node, err := c.cmdSlotAndNode(state, cmd) + if err != nil { + return nil, err + } + cmdsMap[node] = append(cmdsMap[node], cmd) + } + return cmdsMap, nil +} + +func (c *ClusterClient) pipelineProcessCmds( cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, -) (map[*clusterNode][]Cmder, error) { +) error { cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) - return failedCmds, err + return err } + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) + + return c.pipelineReadCmds(cn, cmds, failedCmds) +} + +func (c *ClusterClient) pipelineReadCmds( + cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, +) error { var firstErr error - setFirstErr := func(err error) { + for _, cmd := range cmds { + err := cmd.readReply(cn) + if err == nil { + continue + } + if firstErr == nil { firstErr = err } + + err = c.checkMovedErr(cmd, failedCmds) + if err != nil && firstErr == nil { + firstErr = err + } } + return firstErr +} - // Set read timeout for all commands. - cn.SetReadTimeout(c.opt.ReadTimeout) +func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]Cmder) error { + moved, ask, addr := internal.IsMovedError(cmd.Err()) + if moved { + c.lazyReloadSlots() - for i, cmd := range cmds { - err := cmd.readReply(cn) - if err == nil { + node, err := c.nodes.Get(addr) + if err != nil { + return err + } + + failedCmds[node] = append(failedCmds[node], cmd) + } + if ask { + node, err := c.nodes.Get(addr) + if err != nil { + return err + } + + failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) + } + return nil +} + +func (c *ClusterClient) TxPipeline() *Pipeline { + pipe := Pipeline{ + exec: c.txPipelineExec, + } + pipe.cmdable.process = pipe.Process + pipe.statefulCmdable.process = pipe.Process + return &pipe +} + +func (c *ClusterClient) TxPipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) +} + +func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { + cmdsMap, err := c.mapCmdsBySlot(cmds) + if err != nil { + return err + } + + for slot, cmds := range cmdsMap { + node, err := c.state().slotMasterNode(slot) + if err != nil { + setCmdsErr(cmds, err) continue } - if i == 0 && internal.IsRetryableError(err) { - node, err := c.nodes.Random() - if err != nil { - setFirstErr(err) - continue + cmdsMap := map[*clusterNode][]Cmder{node: cmds} + for i := 0; i <= c.opt.MaxRedirects; i++ { + failedCmds := make(map[*clusterNode][]Cmder) + + for node, cmds := range cmdsMap { + cn, _, err := node.Client.conn() + if err != nil { + setCmdsErr(cmds, err) + continue + } + + err = c.txPipelineProcessCmds(node, cn, cmds, failedCmds) + node.Client.putConn(cn, err, false) } - cmd.reset() - failedCmds[node] = append(failedCmds[node], cmds...) + if len(failedCmds) == 0 { + break + } + cmdsMap = failedCmds + } + } + + var firstErr error + for _, cmd := range cmds { + if err := cmd.Err(); err != nil { + firstErr = err break } + } + return firstErr +} - moved, ask, addr := internal.IsMovedError(err) - if moved { - c.lazyReloadSlots() +func (c *ClusterClient) mapCmdsBySlot(cmds []Cmder) (map[int][]Cmder, error) { + state := c.state() + cmdsMap := make(map[int][]Cmder) + for _, cmd := range cmds { + slot, _, err := c.cmdSlotAndNode(state, cmd) + if err != nil { + return nil, err + } + cmdsMap[slot] = append(cmdsMap[slot], cmd) + } + return cmdsMap, nil +} - node, err := c.nodes.Get(addr) - if err != nil { - setFirstErr(err) - continue - } +func (c *ClusterClient) txPipelineProcessCmds( + node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, +) error { + cn.SetWriteTimeout(c.opt.WriteTimeout) + if err := txPipelineWriteMulti(cn, cmds); err != nil { + setCmdsErr(cmds, err) + failedCmds[node] = cmds + return err + } - cmd.reset() - failedCmds[node] = append(failedCmds[node], cmd) - } else if ask { - node, err := c.nodes.Get(addr) - if err != nil { - setFirstErr(err) - continue - } + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) - cmd.reset() - failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) - } else { - setFirstErr(err) + if err := c.txPipelineReadQueued(cn, cmds, failedCmds); err != nil { + return err + } + + _, err := pipelineReadCmds(cn, cmds) + return err +} + +func (c *ClusterClient) txPipelineReadQueued( + cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, +) error { + var firstErr error + + // Parse queued replies. + var statusCmd StatusCmd + if err := statusCmd.readReply(cn); err != nil && firstErr == nil { + firstErr = err + } + + for _, cmd := range cmds { + err := statusCmd.readReply(cn) + if err == nil { + continue + } + + cmd.setErr(err) + if firstErr == nil { + firstErr = err + } + + err = c.checkMovedErr(cmd, failedCmds) + if err != nil && firstErr == nil { + firstErr = err } } - return failedCmds, firstErr + // Parse number of replies. + line, err := cn.Rd.ReadLine() + if err != nil { + if err == Nil { + err = TxFailedErr + } + return err + } + + switch line[0] { + case proto.ErrorReply: + return proto.ParseErrorReply(line) + case proto.ArrayReply: + // ok + default: + err := fmt.Errorf("redis: expected '*', but got line %q", line) + return err + } + + return firstErr } diff --git a/cluster_test.go b/cluster_test.go index 3406768011..d6707ef0f7 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -373,61 +373,86 @@ var _ = Describe("ClusterClient", func() { Expect(n).To(Equal(int64(100))) }) - Describe("pipeline", func() { - It("follows redirects", func() { - slot := hashtag.Slot("A") - Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + Describe("pipelining", func() { + var pipe *redis.Pipeline - pipe := client.Pipeline() - defer pipe.Close() + assertPipeline := func() { + It("follows redirects", func() { + slot := hashtag.Slot("A") + Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) - keys := []string{"A", "B", "C", "D", "E", "F", "G"} + keys := []string{"A", "B", "C", "D", "E", "F", "G"} - for i, key := range keys { - pipe.Set(key, key+"_value", 0) - pipe.Expire(key, time.Duration(i+1)*time.Hour) - } - cmds, err := pipe.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(14)) + for i, key := range keys { + pipe.Set(key, key+"_value", 0) + pipe.Expire(key, time.Duration(i+1)*time.Hour) + } + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(14)) - for _, key := range keys { - pipe.Get(key) - pipe.TTL(key) - } - cmds, err = pipe.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(14)) - Expect(cmds[0].(*redis.StringCmd).Val()).To(Equal("A_value")) - Expect(cmds[1].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) - Expect(cmds[6].(*redis.StringCmd).Val()).To(Equal("D_value")) - Expect(cmds[7].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) - Expect(cmds[12].(*redis.StringCmd).Val()).To(Equal("G_value")) - Expect(cmds[13].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) - }) + for _, key := range keys { + pipe.Get(key) + pipe.TTL(key) + } + cmds, err = pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(14)) + Expect(cmds[0].(*redis.StringCmd).Val()).To(Equal("A_value")) + Expect(cmds[1].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) + Expect(cmds[6].(*redis.StringCmd).Val()).To(Equal("D_value")) + Expect(cmds[7].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) + Expect(cmds[12].(*redis.StringCmd).Val()).To(Equal("G_value")) + Expect(cmds[13].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) + }) - It("works with missing keys", func() { - Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) - Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) + It("works with missing keys", func() { + Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) + Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) - var a, b, c *redis.StringCmd - cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { - a = pipe.Get("A") - b = pipe.Get("B") - c = pipe.Get("C") - return nil + var a, b, c *redis.StringCmd + cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { + a = pipe.Get("A") + b = pipe.Get("B") + c = pipe.Get("C") + return nil + }) + Expect(err).To(Equal(redis.Nil)) + Expect(cmds).To(HaveLen(3)) + + Expect(a.Err()).NotTo(HaveOccurred()) + Expect(a.Val()).To(Equal("A_value")) + + Expect(b.Err()).To(Equal(redis.Nil)) + Expect(b.Val()).To(Equal("")) + + Expect(c.Err()).NotTo(HaveOccurred()) + Expect(c.Val()).To(Equal("C_value")) + }) + } + + Describe("Pipeline", func() { + BeforeEach(func() { + pipe = client.Pipeline() }) - Expect(err).To(Equal(redis.Nil)) - Expect(cmds).To(HaveLen(3)) - Expect(a.Err()).NotTo(HaveOccurred()) - Expect(a.Val()).To(Equal("A_value")) + AfterEach(func() { + Expect(pipe.Close()).NotTo(HaveOccurred()) + }) + + assertPipeline() + }) - Expect(b.Err()).To(Equal(redis.Nil)) - Expect(b.Val()).To(Equal("")) + Describe("TxPipeline", func() { + BeforeEach(func() { + pipe = client.TxPipeline() + }) + + AfterEach(func() { + Expect(pipe.Close()).NotTo(HaveOccurred()) + }) - Expect(c.Err()).NotTo(HaveOccurred()) - Expect(c.Val()).To(Equal("C_value")) + assertPipeline() }) }) @@ -624,7 +649,7 @@ var _ = Describe("ClusterClient timeout", func() { return client.ForEachNode(func(client *redis.Client) error { return client.Ping().Err() }) - }, pause).ShouldNot(HaveOccurred()) + }, 2*pause).ShouldNot(HaveOccurred()) }) testTimeout() diff --git a/command.go b/command.go index 11150414d3..9809813190 100644 --- a/command.go +++ b/command.go @@ -36,7 +36,6 @@ type Cmder interface { readReply(*pool.Conn) error setErr(error) - reset() readTimeout() *time.Duration @@ -50,12 +49,6 @@ func setCmdsErr(cmds []Cmder, e error) { } } -func resetCmds(cmds []Cmder) { - for _, cmd := range cmds { - cmd.reset() - } -} - func writeCmd(cn *pool.Conn, cmds ...Cmder) error { cn.Wb.Reset() for _, cmd := range cmds { @@ -167,11 +160,6 @@ func NewCmd(args ...interface{}) *Cmd { } } -func (cmd *Cmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *Cmd) Val() interface{} { return cmd.val } @@ -185,16 +173,13 @@ func (cmd *Cmd) String() string { } func (cmd *Cmd) readReply(cn *pool.Conn) error { - val, err := cn.Rd.ReadReply(sliceParser) - if err != nil { - cmd.err = err + cmd.val, cmd.err = cn.Rd.ReadReply(sliceParser) + if cmd.err != nil { return cmd.err } - if b, ok := val.([]byte); ok { + if b, ok := cmd.val.([]byte); ok { // Bytes must be copied, because underlying memory is reused. cmd.val = string(b) - } else { - cmd.val = val } return nil } @@ -212,11 +197,6 @@ func NewSliceCmd(args ...interface{}) *SliceCmd { return &SliceCmd{baseCmd: cmd} } -func (cmd *SliceCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *SliceCmd) Val() []interface{} { return cmd.val } @@ -230,10 +210,10 @@ func (cmd *SliceCmd) String() string { } func (cmd *SliceCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(sliceParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(sliceParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.([]interface{}) return nil @@ -252,11 +232,6 @@ func NewStatusCmd(args ...interface{}) *StatusCmd { return &StatusCmd{baseCmd: cmd} } -func (cmd *StatusCmd) reset() { - cmd.val = "" - cmd.err = nil -} - func (cmd *StatusCmd) Val() string { return cmd.val } @@ -287,11 +262,6 @@ func NewIntCmd(args ...interface{}) *IntCmd { return &IntCmd{baseCmd: cmd} } -func (cmd *IntCmd) reset() { - cmd.val = 0 - cmd.err = nil -} - func (cmd *IntCmd) Val() int64 { return cmd.val } @@ -326,11 +296,6 @@ func NewDurationCmd(precision time.Duration, args ...interface{}) *DurationCmd { } } -func (cmd *DurationCmd) reset() { - cmd.val = 0 - cmd.err = nil -} - func (cmd *DurationCmd) Val() time.Duration { return cmd.val } @@ -344,10 +309,10 @@ func (cmd *DurationCmd) String() string { } func (cmd *DurationCmd) readReply(cn *pool.Conn) error { - n, err := cn.Rd.ReadIntReply() - if err != nil { - cmd.err = err - return err + var n int64 + n, cmd.err = cn.Rd.ReadIntReply() + if cmd.err != nil { + return cmd.err } cmd.val = time.Duration(n) * cmd.precision return nil @@ -368,11 +333,6 @@ func NewTimeCmd(args ...interface{}) *TimeCmd { } } -func (cmd *TimeCmd) reset() { - cmd.val = time.Time{} - cmd.err = nil -} - func (cmd *TimeCmd) Val() time.Time { return cmd.val } @@ -386,10 +346,10 @@ func (cmd *TimeCmd) String() string { } func (cmd *TimeCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(timeParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(timeParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.(time.Time) return nil @@ -408,11 +368,6 @@ func NewBoolCmd(args ...interface{}) *BoolCmd { return &BoolCmd{baseCmd: cmd} } -func (cmd *BoolCmd) reset() { - cmd.val = false - cmd.err = nil -} - func (cmd *BoolCmd) Val() bool { return cmd.val } @@ -428,27 +383,29 @@ func (cmd *BoolCmd) String() string { var ok = []byte("OK") func (cmd *BoolCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadReply(nil) + var v interface{} + v, cmd.err = cn.Rd.ReadReply(nil) // `SET key value NX` returns nil when key already exists. But // `SETNX key value` returns bool (0/1). So convert nil to bool. // TODO: is this okay? - if err == Nil { + if cmd.err == Nil { cmd.val = false + cmd.err = nil return nil } - if err != nil { - cmd.err = err - return err + if cmd.err != nil { + return cmd.err } - switch vv := v.(type) { + switch v := v.(type) { case int64: - cmd.val = vv == 1 + cmd.val = v == 1 return nil case []byte: - cmd.val = bytes.Equal(vv, ok) + cmd.val = bytes.Equal(v, ok) return nil default: - return fmt.Errorf("got %T, wanted int64 or string", v) + cmd.err = fmt.Errorf("got %T, wanted int64 or string", v) + return cmd.err } } @@ -465,11 +422,6 @@ func NewStringCmd(args ...interface{}) *StringCmd { return &StringCmd{baseCmd: cmd} } -func (cmd *StringCmd) reset() { - cmd.val = "" - cmd.err = nil -} - func (cmd *StringCmd) Val() string { return cmd.val } @@ -515,13 +467,8 @@ func (cmd *StringCmd) String() string { } func (cmd *StringCmd) readReply(cn *pool.Conn) error { - b, err := cn.Rd.ReadBytesReply() - if err != nil { - cmd.err = err - return err - } - cmd.val = string(b) - return nil + cmd.val, cmd.err = cn.Rd.ReadStringReply() + return cmd.err } //------------------------------------------------------------------------------ @@ -537,11 +484,6 @@ func NewFloatCmd(args ...interface{}) *FloatCmd { return &FloatCmd{baseCmd: cmd} } -func (cmd *FloatCmd) reset() { - cmd.val = 0 - cmd.err = nil -} - func (cmd *FloatCmd) Val() float64 { return cmd.val } @@ -572,11 +514,6 @@ func NewStringSliceCmd(args ...interface{}) *StringSliceCmd { return &StringSliceCmd{baseCmd: cmd} } -func (cmd *StringSliceCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *StringSliceCmd) Val() []string { return cmd.val } @@ -590,10 +527,10 @@ func (cmd *StringSliceCmd) String() string { } func (cmd *StringSliceCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(stringSliceParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(stringSliceParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.([]string) return nil @@ -612,11 +549,6 @@ func NewBoolSliceCmd(args ...interface{}) *BoolSliceCmd { return &BoolSliceCmd{baseCmd: cmd} } -func (cmd *BoolSliceCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *BoolSliceCmd) Val() []bool { return cmd.val } @@ -630,10 +562,10 @@ func (cmd *BoolSliceCmd) String() string { } func (cmd *BoolSliceCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(boolSliceParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(boolSliceParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.([]bool) return nil @@ -652,11 +584,6 @@ func NewStringStringMapCmd(args ...interface{}) *StringStringMapCmd { return &StringStringMapCmd{baseCmd: cmd} } -func (cmd *StringStringMapCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *StringStringMapCmd) Val() map[string]string { return cmd.val } @@ -670,10 +597,10 @@ func (cmd *StringStringMapCmd) String() string { } func (cmd *StringStringMapCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(stringStringMapParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(stringStringMapParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.(map[string]string) return nil @@ -704,16 +631,11 @@ func (cmd *StringIntMapCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *StringIntMapCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *StringIntMapCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(stringIntMapParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(stringIntMapParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.(map[string]int64) return nil @@ -732,11 +654,6 @@ func NewZSliceCmd(args ...interface{}) *ZSliceCmd { return &ZSliceCmd{baseCmd: cmd} } -func (cmd *ZSliceCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *ZSliceCmd) Val() []Z { return cmd.val } @@ -750,10 +667,10 @@ func (cmd *ZSliceCmd) String() string { } func (cmd *ZSliceCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(zSliceParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(zSliceParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.([]Z) return nil @@ -775,12 +692,6 @@ func NewScanCmd(args ...interface{}) *ScanCmd { } } -func (cmd *ScanCmd) reset() { - cmd.cursor = 0 - cmd.page = nil - cmd.err = nil -} - func (cmd *ScanCmd) Val() (keys []string, cursor uint64) { return cmd.page, cmd.cursor } @@ -794,14 +705,8 @@ func (cmd *ScanCmd) String() string { } func (cmd *ScanCmd) readReply(cn *pool.Conn) error { - page, cursor, err := cn.Rd.ReadScanReply() - if err != nil { - cmd.err = err - return cmd.err - } - cmd.page = page - cmd.cursor = cursor - return nil + cmd.page, cmd.cursor, cmd.err = cn.Rd.ReadScanReply() + return cmd.err } //------------------------------------------------------------------------------ @@ -840,16 +745,11 @@ func (cmd *ClusterSlotsCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *ClusterSlotsCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *ClusterSlotsCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(clusterSlotsParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(clusterSlotsParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.([]ClusterSlot) return nil @@ -913,11 +813,6 @@ func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { } } -func (cmd *GeoLocationCmd) reset() { - cmd.locations = nil - cmd.err = nil -} - func (cmd *GeoLocationCmd) Val() []GeoLocation { return cmd.locations } @@ -931,12 +826,12 @@ func (cmd *GeoLocationCmd) String() string { } func (cmd *GeoLocationCmd) readReply(cn *pool.Conn) error { - reply, err := cn.Rd.ReadArrayReply(newGeoLocationSliceParser(cmd.q)) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(newGeoLocationSliceParser(cmd.q)) + if cmd.err != nil { + return cmd.err } - cmd.locations = reply.([]GeoLocation) + cmd.locations = v.([]GeoLocation) return nil } @@ -969,18 +864,13 @@ func (cmd *GeoPosCmd) String() string { return cmdString(cmd, cmd.positions) } -func (cmd *GeoPosCmd) reset() { - cmd.positions = nil - cmd.err = nil -} - func (cmd *GeoPosCmd) readReply(cn *pool.Conn) error { - reply, err := cn.Rd.ReadArrayReply(geoPosSliceParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(geoPosSliceParser) + if cmd.err != nil { + return cmd.err } - cmd.positions = reply.([]*GeoPos) + cmd.positions = v.([]*GeoPos) return nil } @@ -1019,16 +909,11 @@ func (cmd *CommandsInfoCmd) String() string { return cmdString(cmd, cmd.val) } -func (cmd *CommandsInfoCmd) reset() { - cmd.val = nil - cmd.err = nil -} - func (cmd *CommandsInfoCmd) readReply(cn *pool.Conn) error { - v, err := cn.Rd.ReadArrayReply(commandInfoSliceParser) - if err != nil { - cmd.err = err - return err + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(commandInfoSliceParser) + if cmd.err != nil { + return cmd.err } cmd.val = v.(map[string]*CommandInfo) return nil diff --git a/internal/errors.go b/internal/errors.go index e94e123a5a..67b29aec3e 100644 --- a/internal/errors.go +++ b/internal/errors.go @@ -69,3 +69,7 @@ func IsMovedError(err error) (moved bool, ask bool, addr string) { func IsLoadingError(err error) bool { return strings.HasPrefix(err.Error(), "LOADING") } + +func IsExecAbortError(err error) bool { + return strings.HasPrefix(err.Error(), "EXECABORT") +} diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 7e72846394..9aed3def17 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -70,7 +70,7 @@ func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { switch line[0] { case ErrorReply: - return nil, parseErrorValue(line) + return nil, ParseErrorReply(line) case StatusReply: return parseStatusValue(line) case IntReply: @@ -94,7 +94,7 @@ func (p *Reader) ReadIntReply() (int64, error) { } switch line[0] { case ErrorReply: - return 0, parseErrorValue(line) + return 0, ParseErrorReply(line) case IntReply: return parseIntValue(line) default: @@ -109,7 +109,7 @@ func (p *Reader) ReadBytesReply() ([]byte, error) { } switch line[0] { case ErrorReply: - return nil, parseErrorValue(line) + return nil, ParseErrorReply(line) case StringReply: return p.readBytesValue(line) case StatusReply: @@ -142,7 +142,7 @@ func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { } switch line[0] { case ErrorReply: - return nil, parseErrorValue(line) + return nil, ParseErrorReply(line) case ArrayReply: n, err := parseArrayLen(line) if err != nil { @@ -161,7 +161,7 @@ func (p *Reader) ReadArrayLen() (int64, error) { } switch line[0] { case ErrorReply: - return 0, parseErrorValue(line) + return 0, ParseErrorReply(line) case ArrayReply: return parseArrayLen(line) default: @@ -272,7 +272,7 @@ func isNilReply(b []byte) bool { b[1] == '-' && b[2] == '1' } -func parseErrorValue(line []byte) error { +func ParseErrorReply(line []byte) error { return internal.RedisError(string(line[1:])) } diff --git a/internal/proto/proto.go b/internal/proto/scan.go similarity index 100% rename from internal/proto/proto.go rename to internal/proto/scan.go diff --git a/iterator.go b/iterator.go index e885985b95..4f75360819 100644 --- a/iterator.go +++ b/iterator.go @@ -58,7 +58,6 @@ func (it *ScanIterator) Next() bool { } else { it.ScanCmd._args[2] = it.ScanCmd.cursor } - it.ScanCmd.reset() it.client.process(it.ScanCmd) if it.ScanCmd.Err() != nil { return false diff --git a/pipeline.go b/pipeline.go index ef5510b2d3..a0a00e209e 100644 --- a/pipeline.go +++ b/pipeline.go @@ -7,6 +7,8 @@ import ( "gopkg.in/redis.v5/internal/pool" ) +type pipelineExecer func([]Cmder) error + // Pipeline implements pipelining as described in // http://redis.io/topics/pipelining. It's safe for concurrent use // by multiple goroutines. @@ -14,7 +16,7 @@ type Pipeline struct { cmdable statefulCmdable - exec func([]Cmder) error + exec pipelineExecer mu sync.Mutex cmds []Cmder diff --git a/pipeline_test.go b/pipeline_test.go index 6c6fb96254..14ba784c68 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -1,17 +1,15 @@ package redis_test import ( - "strconv" - "sync" - "gopkg.in/redis.v5" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -var _ = Describe("Pipeline", func() { +var _ = Describe("pipelining", func() { var client *redis.Client + var pipe *redis.Pipeline BeforeEach(func() { client = redis.NewClient(redisOptions()) @@ -22,44 +20,7 @@ var _ = Describe("Pipeline", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("should pipeline", func() { - set := client.Set("key2", "hello2", 0) - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) - - pipeline := client.Pipeline() - set = pipeline.Set("key1", "hello1", 0) - get := pipeline.Get("key2") - incr := pipeline.Incr("key3") - getNil := pipeline.Get("key4") - - cmds, err := pipeline.Exec() - Expect(err).To(Equal(redis.Nil)) - Expect(cmds).To(HaveLen(4)) - Expect(pipeline.Close()).NotTo(HaveOccurred()) - - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) - - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal("hello2")) - - Expect(incr.Err()).NotTo(HaveOccurred()) - Expect(incr.Val()).To(Equal(int64(1))) - - Expect(getNil.Err()).To(Equal(redis.Nil)) - Expect(getNil.Val()).To(Equal("")) - }) - - It("discards queued commands", func() { - pipeline := client.Pipeline() - pipeline.Get("key") - pipeline.Discard() - _, err := pipeline.Exec() - Expect(err).To(MatchError("redis: pipeline is empty")) - }) - - It("should support block style", func() { + It("supports block style", func() { var get *redis.StringCmd cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { get = pipe.Get("foo") @@ -72,98 +33,47 @@ var _ = Describe("Pipeline", func() { Expect(get.Val()).To(Equal("")) }) - It("should handle vals/err", func() { - pipeline := client.Pipeline() - - get := pipeline.Get("key") - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal("")) - Expect(pipeline.Close()).NotTo(HaveOccurred()) - }) - - It("returns an error when there are no commands", func() { - pipeline := client.Pipeline() - _, err := pipeline.Exec() - Expect(err).To(MatchError("redis: pipeline is empty")) - }) - - It("should increment correctly", func() { - const N = 20000 - key := "TestPipelineIncr" - pipeline := client.Pipeline() - for i := 0; i < N; i++ { - pipeline.Incr(key) - } - - cmds, err := pipeline.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(pipeline.Close()).NotTo(HaveOccurred()) - - Expect(len(cmds)).To(Equal(20000)) - for _, cmd := range cmds { - Expect(cmd.Err()).NotTo(HaveOccurred()) - } - - get := client.Get(key) - Expect(get.Err()).NotTo(HaveOccurred()) - Expect(get.Val()).To(Equal(strconv.Itoa(N))) - }) - - It("should PipelineEcho", func() { - const N = 1000 - - wg := &sync.WaitGroup{} - wg.Add(N) - for i := 0; i < N; i++ { - go func(i int) { - defer GinkgoRecover() - defer wg.Done() - - pipeline := client.Pipeline() + assertPipeline := func() { + It("returns an error when there are no commands", func() { + _, err := pipe.Exec() + Expect(err).To(MatchError("redis: pipeline is empty")) + }) - msg1 := "echo" + strconv.Itoa(i) - msg2 := "echo" + strconv.Itoa(i+1) + It("discards queued commands", func() { + pipe.Get("key") + pipe.Discard() + _, err := pipe.Exec() + Expect(err).To(MatchError("redis: pipeline is empty")) + }) - echo1 := pipeline.Echo(msg1) - echo2 := pipeline.Echo(msg2) + It("handles val/err", func() { + err := client.Set("key", "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) - cmds, err := pipeline.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(2)) + get := pipe.Get("key") + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(1)) - Expect(echo1.Err()).NotTo(HaveOccurred()) - Expect(echo1.Val()).To(Equal(msg1)) + val, err := get.Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal("value")) + }) + } - Expect(echo2.Err()).NotTo(HaveOccurred()) - Expect(echo2.Val()).To(Equal(msg2)) + Describe("Pipeline", func() { + BeforeEach(func() { + pipe = client.Pipeline() + }) - Expect(pipeline.Close()).NotTo(HaveOccurred()) - }(i) - } - wg.Wait() + assertPipeline() }) - It("should be thread-safe", func() { - const N = 1000 - - pipeline := client.Pipeline() - var wg sync.WaitGroup - wg.Add(N) - for i := 0; i < N; i++ { - go func() { - defer GinkgoRecover() - - pipeline.Ping() - wg.Done() - }() - } - wg.Wait() - - cmds, err := pipeline.Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(cmds).To(HaveLen(N)) + Describe("TxPipeline", func() { + BeforeEach(func() { + pipe = client.TxPipeline() + }) - Expect(pipeline.Close()).NotTo(HaveOccurred()) + assertPipeline() }) - }) diff --git a/race_test.go b/race_test.go index 1e44bf03ef..2f5e7b31df 100644 --- a/race_test.go +++ b/race_test.go @@ -245,4 +245,35 @@ var _ = Describe("races", func() { Expect(val).To(Equal(int64(C * N))) }) + It("should Pipeline", func() { + perform(C, func(id int) { + pipe := client.Pipeline() + for i := 0; i < N; i++ { + pipe.Echo(fmt.Sprint(i)) + } + + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(N)) + + for i := 0; i < N; i++ { + Expect(cmds[i].(*redis.StringCmd).Val()).To(Equal(fmt.Sprint(i))) + } + }) + }) + + It("should Pipeline", func() { + pipe := client.Pipeline() + perform(N, func(id int) { + pipe.Incr("key") + }) + + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(N)) + + n, err := client.Get("key").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(N))) + }) }) diff --git a/redis.go b/redis.go index 6fcd5c4a07..7bfd0abf75 100644 --- a/redis.go +++ b/redis.go @@ -7,6 +7,7 @@ import ( "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/pool" + "gopkg.in/redis.v5/internal/proto" ) // Redis nil reply, .e.g. when key does not exist. @@ -96,10 +97,6 @@ func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func( func (c *baseClient) defaultProcess(cmd Cmder) error { for i := 0; i <= c.opt.MaxRetries; i++ { - if i > 0 { - cmd.reset() - } - cn, _, err := c.conn() if err != nil { cmd.setErr(err) @@ -162,6 +159,129 @@ func (c *baseClient) getAddr() string { return c.opt.Addr } +type pipelineProcessor func(*pool.Conn, []Cmder) (bool, error) + +func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { + return func(cmds []Cmder) error { + var firstErr error + for i := 0; i <= c.opt.MaxRetries; i++ { + cn, _, err := c.conn() + if err != nil { + setCmdsErr(cmds, err) + return err + } + + canRetry, err := p(cn, cmds) + c.putConn(cn, err, false) + if err == nil { + return nil + } + if firstErr == nil { + firstErr = err + } + if !canRetry || !internal.IsRetryableError(err) { + break + } + } + return firstErr + } +} + +func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { + cn.SetWriteTimeout(c.opt.WriteTimeout) + if err := writeCmd(cn, cmds...); err != nil { + setCmdsErr(cmds, err) + return true, err + } + + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) + return pipelineReadCmds(cn, cmds) +} + +func pipelineReadCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { + for i, cmd := range cmds { + err := cmd.readReply(cn) + if err == nil { + continue + } + if i == 0 { + retry = true + } + if firstErr == nil { + firstErr = err + } + } + return false, firstErr +} + +func (c *baseClient) txPipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) { + cn.SetWriteTimeout(c.opt.WriteTimeout) + if err := txPipelineWriteMulti(cn, cmds); err != nil { + setCmdsErr(cmds, err) + return true, err + } + + // Set read timeout for all commands. + cn.SetReadTimeout(c.opt.ReadTimeout) + + if err := c.txPipelineReadQueued(cn, cmds); err != nil { + return false, err + } + + _, err := pipelineReadCmds(cn, cmds) + return false, err +} + +func txPipelineWriteMulti(cn *pool.Conn, cmds []Cmder) error { + multiExec := make([]Cmder, 0, len(cmds)+2) + multiExec = append(multiExec, NewStatusCmd("MULTI")) + multiExec = append(multiExec, cmds...) + multiExec = append(multiExec, NewSliceCmd("EXEC")) + return writeCmd(cn, multiExec...) +} + +func (c *baseClient) txPipelineReadQueued(cn *pool.Conn, cmds []Cmder) error { + var firstErr error + + // Parse queued replies. + var statusCmd StatusCmd + if err := statusCmd.readReply(cn); err != nil && firstErr == nil { + firstErr = err + } + + for _, cmd := range cmds { + err := statusCmd.readReply(cn) + if err != nil { + cmd.setErr(err) + if firstErr == nil { + firstErr = err + } + } + } + + // Parse number of replies. + line, err := cn.Rd.ReadLine() + if err != nil { + if err == Nil { + err = TxFailedErr + } + return err + } + + switch line[0] { + case proto.ErrorReply: + return proto.ParseErrorReply(line) + case proto.ArrayReply: + // ok + default: + err := fmt.Errorf("redis: expected '*', but got line %q", line) + return err + } + + return nil +} + //------------------------------------------------------------------------------ // Client is a Redis client representing a pool of zero or more @@ -202,70 +322,30 @@ func (c *Client) PoolStats() *PoolStats { } } +func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) +} + func (c *Client) Pipeline() *Pipeline { pipe := Pipeline{ - exec: c.pipelineExec, + exec: c.pipelineExecer(c.pipelineProcessCmds), } pipe.cmdable.process = pipe.Process pipe.statefulCmdable.process = pipe.Process return &pipe } -func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) -} - -func (c *Client) pipelineExec(cmds []Cmder) error { - var firstErr error - for i := 0; i <= c.opt.MaxRetries; i++ { - if i > 0 { - resetCmds(cmds) - } - - cn, _, err := c.conn() - if err != nil { - setCmdsErr(cmds, err) - return err - } - - retry, err := c.execCmds(cn, cmds) - c.putConn(cn, err, false) - if err == nil { - return nil - } - if firstErr == nil { - firstErr = err - } - if !retry { - break - } - } - return firstErr +func (c *Client) TxPipelined(fn func(*Pipeline) error) ([]Cmder, error) { + return c.TxPipeline().pipelined(fn) } -func (c *Client) execCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { - cn.SetWriteTimeout(c.opt.WriteTimeout) - if err := writeCmd(cn, cmds...); err != nil { - setCmdsErr(cmds, err) - return true, err - } - - // Set read timeout for all commands. - cn.SetReadTimeout(c.opt.ReadTimeout) - - for i, cmd := range cmds { - err := cmd.readReply(cn) - if err == nil { - continue - } - if i == 0 && internal.IsNetworkError(err) { - return true, err - } - if firstErr == nil { - firstErr = err - } +func (c *Client) TxPipeline() *Pipeline { + pipe := Pipeline{ + exec: c.pipelineExecer(c.txPipelineProcessCmds), } - return false, firstErr + pipe.cmdable.process = pipe.Process + pipe.statefulCmdable.process = pipe.Process + return &pipe } func (c *Client) pubSub() *PubSub { diff --git a/ring.go b/ring.go index 11945b4a5d..116ca17107 100644 --- a/ring.go +++ b/ring.go @@ -381,10 +381,6 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { var failedCmdsMap map[string][]Cmder for name, cmds := range cmdsMap { - if i > 0 { - resetCmds(cmds) - } - shard, err := c.shardByName(name) if err != nil { setCmdsErr(cmds, err) @@ -403,7 +399,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { continue } - retry, err := shard.Client.execCmds(cn, cmds) + canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds) shard.Client.putConn(cn, err, false) if err == nil { continue @@ -411,7 +407,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { if firstErr == nil { firstErr = err } - if retry { + if canRetry && internal.IsRetryableError(err) { if failedCmdsMap == nil { failedCmdsMap = make(map[string][]Cmder) } diff --git a/tx.go b/tx.go index 30beda940b..af65fe9791 100644 --- a/tx.go +++ b/tx.go @@ -1,11 +1,8 @@ package redis import ( - "fmt" - "gopkg.in/redis.v5/internal" "gopkg.in/redis.v5/internal/pool" - "gopkg.in/redis.v5/internal/proto" ) // Redis transaction failed. @@ -19,8 +16,6 @@ type Tx struct { cmdable statefulCmdable baseClient - - closed bool } var _ Cmdable = (*Tx)(nil) @@ -41,26 +36,20 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { tx := c.newTx() if len(keys) > 0 { if err := tx.Watch(keys...).Err(); err != nil { - _ = tx.close() + _ = tx.Close() return err } } firstErr := fn(tx) - if err := tx.close(); err != nil && firstErr == nil { + if err := tx.Close(); err != nil && firstErr == nil { firstErr = err } return firstErr } // close closes the transaction, releasing any open resources. -func (c *Tx) close() error { - if c.closed { - return nil - } - c.closed = true - if err := c.Unwatch().Err(); err != nil { - internal.Logf("Unwatch failed: %s", err) - } +func (c *Tx) Close() error { + _ = c.Unwatch().Err() return c.baseClient.Close() } @@ -91,7 +80,7 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd { func (c *Tx) Pipeline() *Pipeline { pipe := Pipeline{ - exec: c.exec, + exec: c.pipelineExecer(c.txPipelineProcessCmds), } pipe.cmdable.process = pipe.Process pipe.statefulCmdable.process = pipe.Process @@ -110,76 +99,3 @@ func (c *Tx) Pipeline() *Pipeline { func (c *Tx) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } - -func (c *Tx) exec(cmds []Cmder) error { - if c.closed { - return pool.ErrClosed - } - - cn, _, err := c.conn() - if err != nil { - setCmdsErr(cmds, err) - return err - } - - multiExec := make([]Cmder, 0, len(cmds)+2) - multiExec = append(multiExec, NewStatusCmd("MULTI")) - multiExec = append(multiExec, cmds...) - multiExec = append(multiExec, NewSliceCmd("EXEC")) - - err = c.execCmds(cn, multiExec) - c.putConn(cn, err, false) - return err -} - -func (c *Tx) execCmds(cn *pool.Conn, cmds []Cmder) error { - cn.SetWriteTimeout(c.opt.WriteTimeout) - err := writeCmd(cn, cmds...) - if err != nil { - setCmdsErr(cmds[1:len(cmds)-1], err) - return err - } - - // Set read timeout for all commands. - cn.SetReadTimeout(c.opt.ReadTimeout) - - // Omit last command (EXEC). - cmdsLen := len(cmds) - 1 - - // Parse queued replies. - statusCmd := cmds[0] - for i := 0; i < cmdsLen; i++ { - if err := statusCmd.readReply(cn); err != nil { - setCmdsErr(cmds[1:len(cmds)-1], err) - return err - } - } - - // Parse number of replies. - line, err := cn.Rd.ReadLine() - if err != nil { - if err == Nil { - err = TxFailedErr - } - setCmdsErr(cmds[1:len(cmds)-1], err) - return err - } - if line[0] != proto.ArrayReply { - err := fmt.Errorf("redis: expected '*', but got line %q", line) - setCmdsErr(cmds[1:len(cmds)-1], err) - return err - } - - var firstErr error - - // Parse replies. - // Loop starts from 1 to omit MULTI cmd. - for i := 1; i < cmdsLen; i++ { - cmd := cmds[i] - if err := cmd.readReply(cn); err != nil && firstErr == nil { - firstErr = err - } - } - - return firstErr -} From 4ba635e15f06c5b2e02114cb7ed0e9783cfbd753 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 16 Dec 2016 14:19:53 +0200 Subject: [PATCH 0257/1746] Add func doc. --- cluster.go | 1 + redis.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cluster.go b/cluster.go index 5a8d25c147..23d091b84b 100644 --- a/cluster.go +++ b/cluster.go @@ -759,6 +759,7 @@ func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]C return nil } +// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. func (c *ClusterClient) TxPipeline() *Pipeline { pipe := Pipeline{ exec: c.txPipelineExec, diff --git a/redis.go b/redis.go index 4dfdfbd031..6a3b04c6d2 100644 --- a/redis.go +++ b/redis.go @@ -337,6 +337,7 @@ func (c *Client) TxPipelined(fn func(*Pipeline) error) ([]Cmder, error) { return c.TxPipeline().pipelined(fn) } +// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. func (c *Client) TxPipeline() *Pipeline { pipe := Pipeline{ exec: c.pipelineExecer(c.txPipelineProcessCmds), From 6cd7a09b225902ecb92c6851e0aad63d8331ac3e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 16 Dec 2016 14:05:21 +0200 Subject: [PATCH 0258/1746] Use first slot/shard when key is not defined. --- cluster.go | 17 +++++++++-------- ring.go | 3 --- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cluster.go b/cluster.go index 5a8d25c147..b898c0d3a3 100644 --- a/cluster.go +++ b/cluster.go @@ -258,10 +258,10 @@ func newClusterState(nodes *clusterNodes, slots []ClusterSlot) (*clusterState, e func (c *clusterState) slotMasterNode(slot int) (*clusterNode, error) { nodes := c.slotNodes(slot) - if len(nodes) == 0 { - return c.nodes.Random() + if len(nodes) > 0 { + return nodes[0], nil } - return nodes[0], nil + return c.nodes.Random() } func (c *clusterState) slotSlaveNode(slot int) (*clusterNode, error) { @@ -364,15 +364,16 @@ func (c *ClusterClient) state() *clusterState { } func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { - cmdInfo := c.cmds[cmd.arg(0)] - firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) - if firstKey == "" || cmdInfo == nil { + if state == nil { node, err := c.nodes.Random() - return -1, node, err + return 0, node, err } + + cmdInfo := c.cmds[cmd.arg(0)] + firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) slot := hashtag.Slot(firstKey) - if cmdInfo.ReadOnly && c.opt.ReadOnly { + if cmdInfo != nil && cmdInfo.ReadOnly && c.opt.ReadOnly { if c.opt.RouteByLatency { node, err := state.slotClosestNode(slot) return slot, node, err diff --git a/ring.go b/ring.go index 356684f31a..aff019daec 100644 --- a/ring.go +++ b/ring.go @@ -267,9 +267,6 @@ func (c *Ring) shardByName(name string) (*ringShard, error) { func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { cmdInfo := c.cmdInfo(cmd.arg(0)) firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) - if firstKey == "" { - return c.randomShard() - } return c.shardByKey(firstKey) } From c17f58f7a0ba7eeae87a153a47a2a51aeea8043b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 16 Dec 2016 15:43:37 +0200 Subject: [PATCH 0259/1746] Remove dial limiter. --- .travis.yml | 1 - internal/pool/bench_test.go | 2 -- internal/pool/pool.go | 39 +++---------------------------------- internal/pool/pool_test.go | 17 ---------------- pool_test.go | 2 -- race_test.go | 2 -- 6 files changed, 3 insertions(+), 60 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c70460bfd..41e06f8365 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ matrix: - go: tip install: - - go get gopkg.in/bsm/ratelimit.v1 - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega - mkdir -p $HOME/gopath/src/gopkg.in diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index be2c3053ae..9184fe7421 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -10,7 +10,6 @@ import ( func benchmarkPoolGetPut(b *testing.B, poolSize int) { connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) - connPool.DialLimiter = nil b.ResetTimer() @@ -41,7 +40,6 @@ func BenchmarkPoolGetPut1000Conns(b *testing.B) { func benchmarkPoolGetRemove(b *testing.B, poolSize int) { connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) - connPool.DialLimiter = nil removeReason := errors.New("benchmark") b.ResetTimer() diff --git a/internal/pool/pool.go b/internal/pool/pool.go index b7e8977f88..6a0e057737 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -8,8 +8,6 @@ import ( "sync/atomic" "time" - "gopkg.in/bsm/ratelimit.v1" - "gopkg.in/redis.v5/internal" ) @@ -49,9 +47,8 @@ type Pooler interface { type dialer func() (net.Conn, error) type ConnPool struct { - _dial dialer - DialLimiter *ratelimit.RateLimiter - OnClose func(*Conn) error + dial dialer + OnClose func(*Conn) error poolTimeout time.Duration idleTimeout time.Duration @@ -74,8 +71,7 @@ var _ Pooler = (*ConnPool)(nil) func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout, idleCheckFrequency time.Duration) *ConnPool { p := &ConnPool{ - _dial: dial, - DialLimiter: ratelimit.New(3*poolSize, time.Second), + dial: dial, poolTimeout: poolTimeout, idleTimeout: idleTimeout, @@ -90,23 +86,6 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout, idleCheckF return p } -func (p *ConnPool) dial() (net.Conn, error) { - if p.DialLimiter != nil && p.DialLimiter.Limit() { - err := fmt.Errorf( - "redis: you open connections too fast (last_error=%q)", - p.loadLastErr(), - ) - return nil, err - } - - cn, err := p._dial() - if err != nil { - p.storeLastErr(err.Error()) - return nil, err - } - return cn, nil -} - func (p *ConnPool) NewConn() (*Conn, error) { netConn, err := p.dial() if err != nil { @@ -292,7 +271,6 @@ func (p *ConnPool) Close() error { } func (p *ConnPool) closeConn(cn *Conn, reason error) error { - p.storeLastErr(reason.Error()) if p.OnClose != nil { _ = p.OnClose(cn) } @@ -356,17 +334,6 @@ func (p *ConnPool) reaper(frequency time.Duration) { } } -func (p *ConnPool) storeLastErr(err string) { - p.lastErr.Store(err) -} - -func (p *ConnPool) loadLastErr() string { - if v := p.lastErr.Load(); v != nil { - return v.(string) - } - return "" -} - //------------------------------------------------------------------------------ var idleCheckFrequency atomic.Value diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 7ba35fd416..ece9b8f544 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -23,21 +23,6 @@ var _ = Describe("ConnPool", func() { connPool.Close() }) - It("rate limits dial", func() { - var rateErr error - for i := 0; i < 1000; i++ { - cn, _, err := connPool.Get() - if err != nil { - rateErr = err - break - } - - _ = connPool.Remove(cn, errors.New("test")) - } - - Expect(rateErr).To(MatchError(`redis: you open connections too fast (last_error="test")`)) - }) - It("should unblock client when conn is removed", func() { // Reserve one connection. cn, _, err := connPool.Get() @@ -220,7 +205,6 @@ var _ = Describe("race", func() { It("does not happen on Get, Put, and Remove", func() { connPool = pool.NewConnPool( dummyDialer, 10, time.Minute, time.Millisecond, time.Millisecond) - connPool.DialLimiter = nil perform(C, func(id int) { for i := 0; i < N; i++ { @@ -244,7 +228,6 @@ var _ = Describe("race", func() { It("does not happen on Get and PopFree", func() { connPool = pool.NewConnPool( dummyDialer, 10, time.Minute, time.Second, time.Millisecond) - connPool.DialLimiter = nil perform(C, func(id int) { for i := 0; i < N; i++ { diff --git a/pool_test.go b/pool_test.go index 607fa463ec..08760296da 100644 --- a/pool_test.go +++ b/pool_test.go @@ -7,7 +7,6 @@ import ( . "github.com/onsi/gomega" "gopkg.in/redis.v5" - "gopkg.in/redis.v5/internal/pool" ) var _ = Describe("pool", func() { @@ -80,7 +79,6 @@ var _ = Describe("pool", func() { It("respects max size on pubsub", func() { connPool := client.Pool() - connPool.(*pool.ConnPool).DialLimiter = nil perform(1000, func(id int) { pubsub, err := client.Subscribe() diff --git a/race_test.go b/race_test.go index 2f5e7b31df..34973fe124 100644 --- a/race_test.go +++ b/race_test.go @@ -12,7 +12,6 @@ import ( . "github.com/onsi/gomega" "gopkg.in/redis.v5" - "gopkg.in/redis.v5/internal/pool" ) var _ = Describe("races", func() { @@ -139,7 +138,6 @@ var _ = Describe("races", func() { It("should PubSub", func() { connPool := client.Pool() - connPool.(*pool.ConnPool).DialLimiter = nil perform(C, func(id int) { for i := 0; i < N; i++ { From cd7431c40a33673573f9fd2fd3067b8ee7aa2462 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 16 Dec 2016 16:26:48 +0200 Subject: [PATCH 0260/1746] Fix cluster pipeline tests. --- cluster.go | 25 +++++++++++----- cluster_test.go | 78 ++++++++++++++++++++++++++++--------------------- 2 files changed, 62 insertions(+), 41 deletions(-) diff --git a/cluster.go b/cluster.go index 6e3c302308..76899eac2f 100644 --- a/cluster.go +++ b/cluster.go @@ -95,18 +95,22 @@ func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { } if clOpt.RouteByLatency { - const probes = 10 - for i := 0; i < probes; i++ { - t1 := time.Now() - node.Client.Ping() - node.Latency += time.Since(t1) - } - node.Latency = node.Latency / probes + node.updateLatency() } return &node } +func (n *clusterNode) updateLatency() { + const probes = 10 + for i := 0; i < probes; i++ { + start := time.Now() + n.Client.Ping() + n.Latency += time.Since(start) + } + n.Latency = n.Latency / probes +} + func (n *clusterNode) Loading() bool { return !n.loading.IsZero() && time.Since(n.loading) < time.Minute } @@ -290,6 +294,8 @@ func (c *clusterState) slotSlaveNode(slot int) (*clusterNode, error) { } func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { + const threshold = time.Millisecond + nodes := c.slotNodes(slot) if len(nodes) == 0 { return c.nodes.Random() @@ -297,7 +303,10 @@ func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { var node *clusterNode for _, n := range nodes { - if node == nil || n.Latency < node.Latency { + if n.Loading() { + continue + } + if node == nil || node.Latency-n.Latency > threshold { node = n } } diff --git a/cluster_test.go b/cluster_test.go index d6707ef0f7..589ef98049 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -6,15 +6,14 @@ import ( "strconv" "strings" "sync" - "testing" "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "gopkg.in/redis.v5" "gopkg.in/redis.v5/internal/hashtag" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) type clusterScenario struct { @@ -24,10 +23,6 @@ type clusterScenario struct { clients map[string]*redis.Client } -func (s *clusterScenario) primary() *redis.Client { - return s.clients[s.ports[0]] -} - func (s *clusterScenario) masters() []*redis.Client { result := make([]*redis.Client, 3) for pos, port := range s.ports[:3] { @@ -157,6 +152,9 @@ func slotEqual(s1, s2 redis.ClusterSlot) bool { if s1.End != s2.End { return false } + if len(s1.Nodes) != len(s2.Nodes) { + return false + } for i, n1 := range s1.Nodes { if n1.Addr != s2.Nodes[i].Addr { return false @@ -182,9 +180,10 @@ func stopCluster(scenario *clusterScenario) error { //------------------------------------------------------------------------------ var _ = Describe("ClusterClient", func() { + var opt *redis.ClusterOptions var client *redis.ClusterClient - describeClusterClient := func() { + assertClusterClient := func() { It("should CLUSTER SLOTS", func() { res, err := client.ClusterSlots().Result() Expect(err).NotTo(HaveOccurred()) @@ -377,11 +376,13 @@ var _ = Describe("ClusterClient", func() { var pipe *redis.Pipeline assertPipeline := func() { - It("follows redirects", func() { - slot := hashtag.Slot("A") - Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + keys := []string{"A", "B", "C", "D", "E", "F", "G"} - keys := []string{"A", "B", "C", "D", "E", "F", "G"} + It("follows redirects", func() { + for _, key := range keys { + slot := hashtag.Slot(key) + client.SwapSlotNodes(slot) + } for i, key := range keys { pipe.Set(key, key+"_value", 0) @@ -391,6 +392,15 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(14)) + if opt.RouteByLatency { + return + } + + for _, key := range keys { + slot := hashtag.Slot(key) + client.SwapSlotNodes(slot) + } + for _, key := range keys { pipe.Get(key) pipe.TTL(key) @@ -398,25 +408,26 @@ var _ = Describe("ClusterClient", func() { cmds, err = pipe.Exec() Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(14)) - Expect(cmds[0].(*redis.StringCmd).Val()).To(Equal("A_value")) - Expect(cmds[1].(*redis.DurationCmd).Val()).To(BeNumerically("~", 1*time.Hour, time.Second)) - Expect(cmds[6].(*redis.StringCmd).Val()).To(Equal("D_value")) - Expect(cmds[7].(*redis.DurationCmd).Val()).To(BeNumerically("~", 4*time.Hour, time.Second)) - Expect(cmds[12].(*redis.StringCmd).Val()).To(Equal("G_value")) - Expect(cmds[13].(*redis.DurationCmd).Val()).To(BeNumerically("~", 7*time.Hour, time.Second)) + + for i, key := range keys { + get := cmds[i*2].(*redis.StringCmd) + Expect(get.Val()).To(Equal(key + "_value")) + + ttl := cmds[(i*2)+1].(*redis.DurationCmd) + Expect(ttl.Val()).To(BeNumerically("~", time.Duration(i+1)*time.Hour, time.Second)) + } }) It("works with missing keys", func() { - Expect(client.Set("A", "A_value", 0).Err()).NotTo(HaveOccurred()) - Expect(client.Set("C", "C_value", 0).Err()).NotTo(HaveOccurred()) - - var a, b, c *redis.StringCmd - cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { - a = pipe.Get("A") - b = pipe.Get("B") - c = pipe.Get("C") - return nil - }) + pipe.Set("A", "A_value", 0) + pipe.Set("C", "C_value", 0) + _, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + + a := pipe.Get("A") + b := pipe.Get("B") + c := pipe.Get("C") + cmds, err := pipe.Exec() Expect(err).To(Equal(redis.Nil)) Expect(cmds).To(HaveLen(3)) @@ -476,7 +487,8 @@ var _ = Describe("ClusterClient", func() { Describe("default ClusterClient", func() { BeforeEach(func() { - client = cluster.clusterClient(redisClusterOptions()) + opt = redisClusterOptions() + client = cluster.clusterClient(opt) _ = client.ForEachMaster(func(master *redis.Client) error { return master.FlushDb().Err() @@ -487,12 +499,12 @@ var _ = Describe("ClusterClient", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - describeClusterClient() + assertClusterClient() }) Describe("ClusterClient with RouteByLatency", func() { BeforeEach(func() { - opt := redisClusterOptions() + opt = redisClusterOptions() opt.RouteByLatency = true client = cluster.clusterClient(opt) @@ -506,7 +518,7 @@ var _ = Describe("ClusterClient", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - describeClusterClient() + assertClusterClient() }) }) From ce1ddaa30c8e603f1a396c0373d23be08a14c9cc Mon Sep 17 00:00:00 2001 From: Back Yu Date: Wed, 21 Dec 2016 01:03:12 +0800 Subject: [PATCH 0261/1746] Update commands.go Let HSet and HSetNX can use value as interface{} . --- commands.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/commands.go b/commands.go index 5132a59797..89a70b606f 100644 --- a/commands.go +++ b/commands.go @@ -109,8 +109,8 @@ type Cmdable interface { HLen(key string) *IntCmd HMGet(key string, fields ...string) *SliceCmd HMSet(key string, fields map[string]string) *StatusCmd - HSet(key, field, value string) *BoolCmd - HSetNX(key, field, value string) *BoolCmd + HSet(key, field string, value interface{}) *BoolCmd + HSetNX(key, field string, value interface{}) *BoolCmd HVals(key string) *StringSliceCmd BLPop(timeout time.Duration, keys ...string) *StringSliceCmd BRPop(timeout time.Duration, keys ...string) *StringSliceCmd @@ -892,13 +892,13 @@ func (c *cmdable) HMSet(key string, fields map[string]string) *StatusCmd { return cmd } -func (c *cmdable) HSet(key, field, value string) *BoolCmd { +func (c *cmdable) HSet(key, field string, value interface{}) *BoolCmd { cmd := NewBoolCmd("hset", key, field, value) c.process(cmd) return cmd } -func (c *cmdable) HSetNX(key, field, value string) *BoolCmd { +func (c *cmdable) HSetNX(key, field string, value interface{}) *BoolCmd { cmd := NewBoolCmd("hsetnx", key, field, value) c.process(cmd) return cmd From c9f896d6a3ecec360b8a05367a87648ffb368b8b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 21 Dec 2016 12:55:45 +0200 Subject: [PATCH 0262/1746] Remove changelog. --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index b722c9b528..0000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,4 +0,0 @@ -# v5 - - - *Important*. ClusterClient and Ring now chose random node/shard when command does not have any keys or command info is not fully available. Also clients use EVAL and EVALSHA keys to pick the right node. - - Tx is refactored using Pipeline API so it implements Cmdable interface. From aa895c2c94662537dc982815a391b4d9dd6db641 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 22 Dec 2016 13:14:34 +0200 Subject: [PATCH 0263/1746] Add TxPipeline examples. --- example_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/example_test.go b/example_test.go index 0769aa05a8..45be04395c 100644 --- a/example_test.go +++ b/example_test.go @@ -160,20 +160,56 @@ func ExampleClient_Scan() { func ExampleClient_Pipelined() { var incr *redis.IntCmd _, err := client.Pipelined(func(pipe *redis.Pipeline) error { - incr = pipe.Incr("counter1") - pipe.Expire("counter1", time.Hour) + incr = pipe.Incr("pipelined_counter") + pipe.Expire("pipelined_counter", time.Hour) return nil }) fmt.Println(incr.Val(), err) // Output: 1 } -func ExamplePipeline() { +func ExampleClient_Pipeline() { pipe := client.Pipeline() - defer pipe.Close() - incr := pipe.Incr("counter2") - pipe.Expire("counter2", time.Hour) + incr := pipe.Incr("pipeline_counter") + pipe.Expire("pipeline_counter", time.Hour) + + // Execute + // + // INCR pipeline_counter + // EXPIRE pipeline_counts 3600 + // + // using one client-server roundtrip. + _, err := pipe.Exec() + fmt.Println(incr.Val(), err) + // Output: 1 +} + +func ExampleClient_TxPipelined() { + var incr *redis.IntCmd + _, err := client.TxPipelined(func(pipe *redis.Pipeline) error { + incr = pipe.Incr("tx_pipelined_counter") + pipe.Expire("tx_pipelined_counter", time.Hour) + return nil + }) + fmt.Println(incr.Val(), err) + // Output: 1 +} + +func ExampleClient_TxPipeline() { + pipe := client.TxPipeline() + + incr := pipe.Incr("tx_pipeline_counter") + pipe.Expire("tx_pipeline_counter", time.Hour) + + // Execute + // + // MULTI + // INCR pipeline_counter + // EXPIRE pipeline_counts 3600 + // EXEC + // + // using one client-server roundtrip. _, err := pipe.Exec() fmt.Println(incr.Val(), err) // Output: 1 From c939d2283e13a7edd37c6ea560120a901e533fcc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 22 Dec 2016 13:26:00 +0200 Subject: [PATCH 0264/1746] Allow creating PubSub without channels. --- redis.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/redis.go b/redis.go index 6a3b04c6d2..3e9ee07e43 100644 --- a/redis.go +++ b/redis.go @@ -359,11 +359,23 @@ func (c *Client) pubSub() *PubSub { // Subscribe subscribes the client to the specified channels. func (c *Client) Subscribe(channels ...string) (*PubSub, error) { pubsub := c.pubSub() - return pubsub, pubsub.Subscribe(channels...) + if len(channels) > 0 { + if err := pubsub.Subscribe(channels...); err != nil { + pubsub.Close() + return nil, err + } + } + return pubsub, nil } // PSubscribe subscribes the client to the given patterns. func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { pubsub := c.pubSub() - return pubsub, pubsub.PSubscribe(channels...) + if len(channels) > 0 { + if err := pubsub.PSubscribe(channels...); err != nil { + pubsub.Close() + return nil, err + } + } + return pubsub, nil } From 0f05d8df9d44296646139f186f9253a5bcebd983 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 22 Dec 2016 13:42:05 +0200 Subject: [PATCH 0265/1746] Add unlink command. --- commands.go | 12 ++++++++++++ commands_test.go | 27 ++++++++++++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/commands.go b/commands.go index 89a70b606f..5bbcc1044c 100644 --- a/commands.go +++ b/commands.go @@ -47,6 +47,7 @@ type Cmdable interface { Ping() *StatusCmd Quit() *StatusCmd Del(keys ...string) *IntCmd + Unlink(keys ...string) *IntCmd Dump(key string) *StringCmd Exists(key string) *BoolCmd Expire(key string, expiration time.Duration) *BoolCmd @@ -292,6 +293,17 @@ func (c *cmdable) Del(keys ...string) *IntCmd { return cmd } +func (c *cmdable) Unlink(keys ...string) *IntCmd { + args := make([]interface{}, 1+len(keys)) + args[0] = "unlink" + for i, key := range keys { + args[1+i] = key + } + cmd := NewIntCmd(args...) + c.process(cmd) + return cmd +} + func (c *cmdable) Dump(key string) *StringCmd { cmd := NewStringCmd("dump", key) c.process(cmd) diff --git a/commands_test.go b/commands_test.go index dba37aac78..226300e8ae 100644 --- a/commands_test.go +++ b/commands_test.go @@ -199,16 +199,25 @@ var _ = Describe("Commands", func() { Describe("keys", func() { It("should Del", func() { - set := client.Set("key1", "Hello", 0) - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) - set = client.Set("key2", "World", 0) - Expect(set.Err()).NotTo(HaveOccurred()) - Expect(set.Val()).To(Equal("OK")) + err := client.Set("key1", "Hello", 0).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.Set("key2", "World", 0).Err() + Expect(err).NotTo(HaveOccurred()) - del := client.Del("key1", "key2", "key3") - Expect(del.Err()).NotTo(HaveOccurred()) - Expect(del.Val()).To(Equal(int64(2))) + n, err := client.Del("key1", "key2", "key3").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(2))) + }) + + It("should Unlink", func() { + err := client.Set("key1", "Hello", 0).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.Set("key2", "World", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + n, err := client.Unlink("key1", "key2", "key3").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(2))) }) It("should Dump", func() { From 40594f7b0234b31955bd5b0036576a440ddc429b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 23 Dec 2016 13:21:14 +0200 Subject: [PATCH 0266/1746] Update readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 79f137df64..cae990a4e4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. - [Pub/Sub](http://godoc.org/gopkg.in/redis.v5#PubSub). - [Transactions](http://godoc.org/gopkg.in/redis.v5#Multi). -- [Pipelining](http://godoc.org/gopkg.in/redis.v5#Client.Pipeline). +- [Pipeline](https://godoc.org/gopkg.in/redis.v5#example-Client-Pipeline) and [TxPipeline](https://godoc.org/gopkg.in/redis.v5#example-Client-TxPipeline). - [Scripting](http://godoc.org/gopkg.in/redis.v5#Script). - [Timeouts](http://godoc.org/gopkg.in/redis.v5#Options). - [Redis Sentinel](http://godoc.org/gopkg.in/redis.v5#NewFailoverClient). From 362efb0c4949154c2c2a381660498be85a0da339 Mon Sep 17 00:00:00 2001 From: Sukharev Maxim Date: Fri, 30 Dec 2016 11:47:43 +0700 Subject: [PATCH 0267/1746] Test on WrapProcess --- example_instrumentation_test.go | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go index 009138ebdf..a627586bae 100644 --- a/example_instrumentation_test.go +++ b/example_instrumentation_test.go @@ -5,6 +5,9 @@ import ( "sync/atomic" "time" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + redis "gopkg.in/redis.v5" ) @@ -57,3 +60,35 @@ func wrapRedisProcess(client *redis.Client) { } }) } + +var _ = Describe("Instrumentation", func() { + var client *redis.Client + + BeforeEach(func() { + client = redis.NewClient(redisOptions()) + Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + Describe("WrapProcess", func() { + + It("should call for client", func() { + wrapperFnCalled := false + + client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error { + return func(cmd redis.Cmder) error { + wrapperFnCalled = true + return oldProcess(cmd) + } + }) + + client.Ping() + + Expect(wrapperFnCalled).To(Equal(true)) + }) + + }) +}) From 9556378547ca70fd22bb7236b05483d368c47c89 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 30 Dec 2016 12:58:04 +0200 Subject: [PATCH 0268/1746] Fix Client process instrumentation. --- example_instrumentation_test.go | 35 --------------------------------- redis.go | 12 ++++++----- redis_test.go | 15 ++++++++++++++ 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go index a627586bae..009138ebdf 100644 --- a/example_instrumentation_test.go +++ b/example_instrumentation_test.go @@ -5,9 +5,6 @@ import ( "sync/atomic" "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - redis "gopkg.in/redis.v5" ) @@ -60,35 +57,3 @@ func wrapRedisProcess(client *redis.Client) { } }) } - -var _ = Describe("Instrumentation", func() { - var client *redis.Client - - BeforeEach(func() { - client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - Expect(client.Close()).NotTo(HaveOccurred()) - }) - - Describe("WrapProcess", func() { - - It("should call for client", func() { - wrapperFnCalled := false - - client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error { - return func(cmd redis.Cmder) error { - wrapperFnCalled = true - return oldProcess(cmd) - } - }) - - client.Ping() - - Expect(wrapperFnCalled).To(Equal(true)) - }) - - }) -}) diff --git a/redis.go b/redis.go index 3e9ee07e43..894294d802 100644 --- a/redis.go +++ b/redis.go @@ -293,12 +293,14 @@ type Client struct { } func newClient(opt *Options, pool pool.Pooler) *Client { - base := baseClient{opt: opt, connPool: pool} - client := &Client{ - baseClient: base, - cmdable: cmdable{base.Process}, + client := Client{ + baseClient: baseClient{ + opt: opt, + connPool: pool, + }, } - return client + client.cmdable.process = client.Process + return &client } // NewClient returns a client to the Redis Server specified by Options. diff --git a/redis_test.go b/redis_test.go index e15c871917..69c68df74e 100644 --- a/redis_test.go +++ b/redis_test.go @@ -199,6 +199,21 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(got).To(Equal(bigVal)) }) + + It("should call WrapProcess", func() { + var wrapperFnCalled bool + + client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error { + return func(cmd redis.Cmder) error { + wrapperFnCalled = true + return oldProcess(cmd) + } + }) + + Expect(client.Ping().Err()).NotTo(HaveOccurred()) + + Expect(wrapperFnCalled).To(BeTrue()) + }) }) var _ = Describe("Client timeout", func() { From 3eb7c870405ecd0ddbb4ba50ede6492c25a2b7ca Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 30 Dec 2016 13:01:03 +0200 Subject: [PATCH 0269/1746] Update readme. --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cae990a4e4..1127944b89 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,23 @@ -# Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) +# Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=v5)](https://travis-ci.org/go-redis/redis) Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- [Pub/Sub](http://godoc.org/gopkg.in/redis.v5#PubSub). -- [Transactions](http://godoc.org/gopkg.in/redis.v5#Multi). +- [Pub/Sub](https://godoc.org/gopkg.in/redis.v5#PubSub). +- [Transactions](https://godoc.org/gopkg.in/redis.v5#Multi). - [Pipeline](https://godoc.org/gopkg.in/redis.v5#example-Client-Pipeline) and [TxPipeline](https://godoc.org/gopkg.in/redis.v5#example-Client-TxPipeline). -- [Scripting](http://godoc.org/gopkg.in/redis.v5#Script). -- [Timeouts](http://godoc.org/gopkg.in/redis.v5#Options). -- [Redis Sentinel](http://godoc.org/gopkg.in/redis.v5#NewFailoverClient). -- [Redis Cluster](http://godoc.org/gopkg.in/redis.v5#NewClusterClient). -- [Ring](http://godoc.org/gopkg.in/redis.v5#NewRing). +- [Scripting](https://godoc.org/gopkg.in/redis.v5#Script). +- [Timeouts](https://godoc.org/gopkg.in/redis.v5#Options). +- [Redis Sentinel](https://godoc.org/gopkg.in/redis.v5#NewFailoverClient). +- [Redis Cluster](https://godoc.org/gopkg.in/redis.v5#NewClusterClient). +- [Ring](https://godoc.org/gopkg.in/redis.v5#NewRing). +- [Instrumentation](https://godoc.org/gopkg.in/redis.v5#ex-package--Instrumentation). - [Cache friendly](https://github.com/go-redis/cache). - [Rate limiting](https://github.com/go-redis/rate). - [Distributed Locks](https://github.com/bsm/redis-lock). -API docs: http://godoc.org/gopkg.in/redis.v5. -Examples: http://godoc.org/gopkg.in/redis.v5#pkg-examples. +API docs: https://godoc.org/gopkg.in/redis.v5. +Examples: https://godoc.org/gopkg.in/redis.v5#pkg-examples. ## Installation @@ -74,7 +75,7 @@ func ExampleClient() { ## Howto -Please go through [examples](http://godoc.org/gopkg.in/redis.v5#pkg-examples) to get an idea how to use this package. +Please go through [examples](https://godoc.org/gopkg.in/redis.v5#pkg-examples) to get an idea how to use this package. ## Look and feel From 7eeb6810769099c2d54bc6add982357a9206b57c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 3 Jan 2017 12:21:22 +0200 Subject: [PATCH 0270/1746] Add Eval test. --- commands_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/commands_test.go b/commands_test.go index 226300e8ae..caa2e35e8c 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2852,6 +2852,20 @@ var _ = Describe("Commands", func() { }) + Describe("Eval", func() { + + It("returns keys and values", func() { + vals, err := client.Eval( + "return {KEYS[1],ARGV[1]}", + []string{"key"}, + "hello", + ).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]interface{}{"key", "hello"})) + }) + + }) + }) type numberStruct struct { From 10c56cede39fec28a4daab451ee35a982edc12d5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 3 Jan 2017 12:44:06 +0200 Subject: [PATCH 0271/1746] Remove Scanner in favor of ScanCmd. --- command.go | 15 ++++++++++++--- commands.go | 44 ++++++++++++++++---------------------------- iterator.go | 43 ++++++++++++++++--------------------------- 3 files changed, 44 insertions(+), 58 deletions(-) diff --git a/command.go b/command.go index 9809813190..f8417079ac 100644 --- a/command.go +++ b/command.go @@ -683,12 +683,14 @@ type ScanCmd struct { page []string cursor uint64 + + process func(cmd Cmder) error } -func NewScanCmd(args ...interface{}) *ScanCmd { - cmd := newBaseCmd(args) +func NewScanCmd(process func(cmd Cmder) error, args ...interface{}) *ScanCmd { return &ScanCmd{ - baseCmd: cmd, + baseCmd: newBaseCmd(args), + process: process, } } @@ -709,6 +711,13 @@ func (cmd *ScanCmd) readReply(cn *pool.Conn) error { return cmd.err } +// Iterator creates a new ScanIterator. +func (cmd *ScanCmd) Iterator() *ScanIterator { + return &ScanIterator{ + cmd: cmd, + } +} + //------------------------------------------------------------------------------ type ClusterNode struct { diff --git a/commands.go b/commands.go index 5bbcc1044c..2e50cda1d0 100644 --- a/commands.go +++ b/commands.go @@ -71,10 +71,10 @@ type Cmdable interface { SortInterfaces(key string, sort Sort) *SliceCmd TTL(key string) *DurationCmd Type(key string) *StatusCmd - Scan(cursor uint64, match string, count int64) Scanner - SScan(key string, cursor uint64, match string, count int64) Scanner - HScan(key string, cursor uint64, match string, count int64) Scanner - ZScan(key string, cursor uint64, match string, count int64) Scanner + Scan(cursor uint64, match string, count int64) *ScanCmd + SScan(key string, cursor uint64, match string, count int64) *ScanCmd + HScan(key string, cursor uint64, match string, count int64) *ScanCmd + ZScan(key string, cursor uint64, match string, count int64) *ScanCmd Append(key, value string) *IntCmd BitCount(key string, bitCount *BitCount) *IntCmd BitOpAnd(destKey string, keys ...string) *IntCmd @@ -515,7 +515,7 @@ func (c *cmdable) Type(key string) *StatusCmd { return cmd } -func (c *cmdable) Scan(cursor uint64, match string, count int64) Scanner { +func (c *cmdable) Scan(cursor uint64, match string, count int64) *ScanCmd { args := []interface{}{"scan", cursor} if match != "" { args = append(args, "match", match) @@ -523,15 +523,12 @@ func (c *cmdable) Scan(cursor uint64, match string, count int64) Scanner { if count > 0 { args = append(args, "count", count) } - cmd := NewScanCmd(args...) + cmd := NewScanCmd(c.process, args...) c.process(cmd) - return Scanner{ - client: c, - ScanCmd: cmd, - } + return cmd } -func (c *cmdable) SScan(key string, cursor uint64, match string, count int64) Scanner { +func (c *cmdable) SScan(key string, cursor uint64, match string, count int64) *ScanCmd { args := []interface{}{"sscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -539,15 +536,12 @@ func (c *cmdable) SScan(key string, cursor uint64, match string, count int64) Sc if count > 0 { args = append(args, "count", count) } - cmd := NewScanCmd(args...) + cmd := NewScanCmd(c.process, args...) c.process(cmd) - return Scanner{ - client: c, - ScanCmd: cmd, - } + return cmd } -func (c *cmdable) HScan(key string, cursor uint64, match string, count int64) Scanner { +func (c *cmdable) HScan(key string, cursor uint64, match string, count int64) *ScanCmd { args := []interface{}{"hscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -555,15 +549,12 @@ func (c *cmdable) HScan(key string, cursor uint64, match string, count int64) Sc if count > 0 { args = append(args, "count", count) } - cmd := NewScanCmd(args...) + cmd := NewScanCmd(c.process, args...) c.process(cmd) - return Scanner{ - client: c, - ScanCmd: cmd, - } + return cmd } -func (c *cmdable) ZScan(key string, cursor uint64, match string, count int64) Scanner { +func (c *cmdable) ZScan(key string, cursor uint64, match string, count int64) *ScanCmd { args := []interface{}{"zscan", key, cursor} if match != "" { args = append(args, "match", match) @@ -571,12 +562,9 @@ func (c *cmdable) ZScan(key string, cursor uint64, match string, count int64) Sc if count > 0 { args = append(args, "count", count) } - cmd := NewScanCmd(args...) + cmd := NewScanCmd(c.process, args...) c.process(cmd) - return Scanner{ - client: c, - ScanCmd: cmd, - } + return cmd } //------------------------------------------------------------------------------ diff --git a/iterator.go b/iterator.go index 4f75360819..5d4bedfe5d 100644 --- a/iterator.go +++ b/iterator.go @@ -2,30 +2,18 @@ package redis import "sync" -type Scanner struct { - client *cmdable - *ScanCmd -} - -// Iterator creates a new ScanIterator. -func (s Scanner) Iterator() *ScanIterator { - return &ScanIterator{ - Scanner: s, - } -} - // ScanIterator is used to incrementally iterate over a collection of elements. // It's safe for concurrent use by multiple goroutines. type ScanIterator struct { - mu sync.Mutex // protects Scanner and pos - Scanner + mu sync.Mutex // protects Scanner and pos + cmd *ScanCmd pos int } // Err returns the last iterator error, if any. func (it *ScanIterator) Err() error { it.mu.Lock() - err := it.ScanCmd.Err() + err := it.cmd.Err() it.mu.Unlock() return err } @@ -36,37 +24,38 @@ func (it *ScanIterator) Next() bool { defer it.mu.Unlock() // Instantly return on errors. - if it.ScanCmd.Err() != nil { + if it.cmd.Err() != nil { return false } // Advance cursor, check if we are still within range. - if it.pos < len(it.ScanCmd.page) { + if it.pos < len(it.cmd.page) { it.pos++ return true } for { // Return if there is no more data to fetch. - if it.ScanCmd.cursor == 0 { + if it.cmd.cursor == 0 { return false } // Fetch next page. - if it.ScanCmd._args[0] == "scan" { - it.ScanCmd._args[1] = it.ScanCmd.cursor + if it.cmd._args[0] == "scan" { + it.cmd._args[1] = it.cmd.cursor } else { - it.ScanCmd._args[2] = it.ScanCmd.cursor + it.cmd._args[2] = it.cmd.cursor } - it.client.process(it.ScanCmd) - if it.ScanCmd.Err() != nil { + + err := it.cmd.process(it.cmd) + if err != nil { return false } it.pos = 1 - // Redis can occasionally return empty page - if len(it.ScanCmd.page) > 0 { + // Redis can occasionally return empty page. + if len(it.cmd.page) > 0 { return true } } @@ -76,8 +65,8 @@ func (it *ScanIterator) Next() bool { func (it *ScanIterator) Val() string { var v string it.mu.Lock() - if it.ScanCmd.Err() == nil && it.pos > 0 && it.pos <= len(it.ScanCmd.page) { - v = it.ScanCmd.page[it.pos-1] + if it.cmd.Err() == nil && it.pos > 0 && it.pos <= len(it.cmd.page) { + v = it.cmd.page[it.pos-1] } it.mu.Unlock() return v From 2f247ebe83446fe246462259a33dfda68d06c8bf Mon Sep 17 00:00:00 2001 From: Sukharev Maxim Date: Wed, 11 Jan 2017 10:32:10 +0700 Subject: [PATCH 0272/1746] Clone and WithContext #471 --- redis.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/redis.go b/redis.go index 894294d802..dc779f47d7 100644 --- a/redis.go +++ b/redis.go @@ -1,6 +1,7 @@ package redis // import "gopkg.in/redis.v5" import ( + "context" "fmt" "log" "time" @@ -23,6 +24,8 @@ type baseClient struct { process func(Cmder) error onClose func() error // hook called when client is closed + + ctx context.Context } func (c *baseClient) String() string { @@ -309,6 +312,29 @@ func NewClient(opt *Options) *Client { return newClient(opt, newConnPool(opt)) } +func (c *Client) Clone() *Client { + c2 := new(Client) + *c2 = *c + c2.cmdable.process = c2.Process + return c2 +} + +func (c *Client) Context() context.Context { + if c.ctx != nil { + return c.ctx + } + return context.Background() +} + +func (c *Client) WithContext(ctx context.Context) *Client { + if ctx == nil { + panic("nil context") + } + c2 := c.Clone() + c2.ctx = ctx + return c2 +} + // PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { s := c.connPool.Stats() From 69554c0ec551c811c5ac7751b75013276be0bdfb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 13 Jan 2017 13:39:59 +0200 Subject: [PATCH 0273/1746] Reduce number of allocations. --- bench_test.go | 2 +- cluster.go | 2 +- command.go | 101 ++++++++++++++++++++++++--------------- internal/proto/reader.go | 67 +++++++++++++++++++------- internal/proto/scan.go | 35 +++++++------- internal/safe.go | 7 +++ internal/unsafe.go | 14 ++++++ internal/util.go | 2 +- parser.go | 14 +----- result.go | 2 +- ring.go | 4 +- 11 files changed, 158 insertions(+), 92 deletions(-) create mode 100644 internal/safe.go create mode 100644 internal/unsafe.go diff --git a/bench_test.go b/bench_test.go index 0b576997ed..2b4d45c584 100644 --- a/bench_test.go +++ b/bench_test.go @@ -37,7 +37,7 @@ func BenchmarkRedisPing(b *testing.B) { }) } -func BenchmarkRedisSet(b *testing.B) { +func BenchmarkRedisSetString(b *testing.B) { client := benchmarkRedisClient(10) defer client.Close() diff --git a/cluster.go b/cluster.go index 76899eac2f..f2832a4b0b 100644 --- a/cluster.go +++ b/cluster.go @@ -378,7 +378,7 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl return 0, node, err } - cmdInfo := c.cmds[cmd.arg(0)] + cmdInfo := c.cmds[cmd.name()] firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) slot := hashtag.Slot(firstKey) diff --git a/command.go b/command.go index f8417079ac..2adb629949 100644 --- a/command.go +++ b/command.go @@ -33,6 +33,7 @@ var ( type Cmder interface { args() []interface{} arg(int) string + name() string readReply(*pool.Conn) error setErr(error) @@ -83,7 +84,7 @@ func cmdString(cmd Cmder, val interface{}) string { } func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { - switch cmd.arg(0) { + switch cmd.name() { case "eval", "evalsha": if cmd.arg(2) != "0" { return 3 @@ -92,7 +93,7 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { } } if info == nil { - internal.Logf("info for cmd=%s not found", cmd.arg(0)) + internal.Logf("info for cmd=%s not found", cmd.name()) return -1 } return int(info.FirstKeyPos) @@ -126,6 +127,16 @@ func (cmd *baseCmd) arg(pos int) string { return s } +func (cmd *baseCmd) name() string { + if len(cmd._args) > 0 { + // Cmd name must be lower cased. + s := internal.ToLower(cmd.arg(0)) + cmd._args[0] = s + return s + } + return "" +} + func (cmd *baseCmd) readTimeout() *time.Duration { return cmd._readTimeout } @@ -156,7 +167,7 @@ type Cmd struct { func NewCmd(args ...interface{}) *Cmd { return &Cmd{ - baseCmd: newBaseCmd(args), + baseCmd: baseCmd{_args: args}, } } @@ -193,8 +204,9 @@ type SliceCmd struct { } func NewSliceCmd(args ...interface{}) *SliceCmd { - cmd := newBaseCmd(args) - return &SliceCmd{baseCmd: cmd} + return &SliceCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *SliceCmd) Val() []interface{} { @@ -228,8 +240,9 @@ type StatusCmd struct { } func NewStatusCmd(args ...interface{}) *StatusCmd { - cmd := newBaseCmd(args) - return &StatusCmd{baseCmd: cmd} + return &StatusCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *StatusCmd) Val() string { @@ -258,8 +271,9 @@ type IntCmd struct { } func NewIntCmd(args ...interface{}) *IntCmd { - cmd := newBaseCmd(args) - return &IntCmd{baseCmd: cmd} + return &IntCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *IntCmd) Val() int64 { @@ -289,10 +303,9 @@ type DurationCmd struct { } func NewDurationCmd(precision time.Duration, args ...interface{}) *DurationCmd { - cmd := newBaseCmd(args) return &DurationCmd{ + baseCmd: baseCmd{_args: args}, precision: precision, - baseCmd: cmd, } } @@ -327,9 +340,8 @@ type TimeCmd struct { } func NewTimeCmd(args ...interface{}) *TimeCmd { - cmd := newBaseCmd(args) return &TimeCmd{ - baseCmd: cmd, + baseCmd: baseCmd{_args: args}, } } @@ -364,8 +376,9 @@ type BoolCmd struct { } func NewBoolCmd(args ...interface{}) *BoolCmd { - cmd := newBaseCmd(args) - return &BoolCmd{baseCmd: cmd} + return &BoolCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *BoolCmd) Val() bool { @@ -414,16 +427,17 @@ func (cmd *BoolCmd) readReply(cn *pool.Conn) error { type StringCmd struct { baseCmd - val string + val []byte } func NewStringCmd(args ...interface{}) *StringCmd { - cmd := newBaseCmd(args) - return &StringCmd{baseCmd: cmd} + return &StringCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *StringCmd) Val() string { - return cmd.val + return internal.BytesToString(cmd.val) } func (cmd *StringCmd) Result() (string, error) { @@ -467,7 +481,7 @@ func (cmd *StringCmd) String() string { } func (cmd *StringCmd) readReply(cn *pool.Conn) error { - cmd.val, cmd.err = cn.Rd.ReadStringReply() + cmd.val, cmd.err = cn.Rd.ReadBytesReply() return cmd.err } @@ -480,8 +494,9 @@ type FloatCmd struct { } func NewFloatCmd(args ...interface{}) *FloatCmd { - cmd := newBaseCmd(args) - return &FloatCmd{baseCmd: cmd} + return &FloatCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *FloatCmd) Val() float64 { @@ -510,8 +525,9 @@ type StringSliceCmd struct { } func NewStringSliceCmd(args ...interface{}) *StringSliceCmd { - cmd := newBaseCmd(args) - return &StringSliceCmd{baseCmd: cmd} + return &StringSliceCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *StringSliceCmd) Val() []string { @@ -545,8 +561,9 @@ type BoolSliceCmd struct { } func NewBoolSliceCmd(args ...interface{}) *BoolSliceCmd { - cmd := newBaseCmd(args) - return &BoolSliceCmd{baseCmd: cmd} + return &BoolSliceCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *BoolSliceCmd) Val() []bool { @@ -580,8 +597,9 @@ type StringStringMapCmd struct { } func NewStringStringMapCmd(args ...interface{}) *StringStringMapCmd { - cmd := newBaseCmd(args) - return &StringStringMapCmd{baseCmd: cmd} + return &StringStringMapCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *StringStringMapCmd) Val() map[string]string { @@ -615,8 +633,9 @@ type StringIntMapCmd struct { } func NewStringIntMapCmd(args ...interface{}) *StringIntMapCmd { - cmd := newBaseCmd(args) - return &StringIntMapCmd{baseCmd: cmd} + return &StringIntMapCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *StringIntMapCmd) Val() map[string]int64 { @@ -650,8 +669,9 @@ type ZSliceCmd struct { } func NewZSliceCmd(args ...interface{}) *ZSliceCmd { - cmd := newBaseCmd(args) - return &ZSliceCmd{baseCmd: cmd} + return &ZSliceCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *ZSliceCmd) Val() []Z { @@ -689,7 +709,7 @@ type ScanCmd struct { func NewScanCmd(process func(cmd Cmder) error, args ...interface{}) *ScanCmd { return &ScanCmd{ - baseCmd: newBaseCmd(args), + baseCmd: baseCmd{_args: args}, process: process, } } @@ -738,8 +758,9 @@ type ClusterSlotsCmd struct { } func NewClusterSlotsCmd(args ...interface{}) *ClusterSlotsCmd { - cmd := newBaseCmd(args) - return &ClusterSlotsCmd{baseCmd: cmd} + return &ClusterSlotsCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *ClusterSlotsCmd) Val() []ClusterSlot { @@ -857,8 +878,9 @@ type GeoPosCmd struct { } func NewGeoPosCmd(args ...interface{}) *GeoPosCmd { - cmd := newBaseCmd(args) - return &GeoPosCmd{baseCmd: cmd} + return &GeoPosCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *GeoPosCmd) Val() []*GeoPos { @@ -902,8 +924,9 @@ type CommandsInfoCmd struct { } func NewCommandsInfoCmd(args ...interface{}) *CommandsInfoCmd { - cmd := newBaseCmd(args) - return &CommandsInfoCmd{baseCmd: cmd} + return &CommandsInfoCmd{ + baseCmd: baseCmd{_args: args}, + } } func (cmd *CommandsInfoCmd) Val() map[string]*CommandInfo { diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 9aed3def17..ee811c856f 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -74,7 +74,7 @@ func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { case StatusReply: return parseStatusValue(line) case IntReply: - return parseIntValue(line) + return parseInt(line[1:], 10, 64) case StringReply: return p.readBytesValue(line) case ArrayReply: @@ -96,13 +96,13 @@ func (p *Reader) ReadIntReply() (int64, error) { case ErrorReply: return 0, ParseErrorReply(line) case IntReply: - return parseIntValue(line) + return parseInt(line[1:], 10, 64) default: return 0, fmt.Errorf("redis: can't parse int reply: %.100q", line) } } -func (p *Reader) ReadBytesReply() ([]byte, error) { +func (p *Reader) ReadTmpBytesReply() ([]byte, error) { line, err := p.ReadLine() if err != nil { return nil, err @@ -119,8 +119,18 @@ func (p *Reader) ReadBytesReply() ([]byte, error) { } } +func (r *Reader) ReadBytesReply() ([]byte, error) { + b, err := r.ReadTmpBytesReply() + if err != nil { + return nil, err + } + cp := make([]byte, len(b)) + copy(cp, b) + return cp, nil +} + func (p *Reader) ReadStringReply() (string, error) { - b, err := p.ReadBytesReply() + b, err := p.ReadTmpBytesReply() if err != nil { return "", err } @@ -128,11 +138,11 @@ func (p *Reader) ReadStringReply() (string, error) { } func (p *Reader) ReadFloatReply() (float64, error) { - s, err := p.ReadStringReply() + b, err := p.ReadTmpBytesReply() if err != nil { return 0, err } - return strconv.ParseFloat(s, 64) + return parseFloat(b, 64) } func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { @@ -178,12 +188,7 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) { return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n) } - s, err := p.ReadStringReply() - if err != nil { - return nil, 0, err - } - - cursor, err := strconv.ParseUint(s, 10, 64) + cursor, err := p.ReadUint() if err != nil { return nil, 0, err } @@ -222,6 +227,22 @@ func (p *Reader) readBytesValue(line []byte) ([]byte, error) { return b[:replyLen], nil } +func (r *Reader) ReadInt() (int64, error) { + b, err := r.ReadTmpBytesReply() + if err != nil { + return 0, err + } + return parseInt(b, 10, 64) +} + +func (r *Reader) ReadUint() (uint64, error) { + b, err := r.ReadTmpBytesReply() + if err != nil { + return 0, err + } + return parseUint(b, 10, 64) +} + // -------------------------------------------------------------------- func readN(r io.Reader, b []byte, n int) ([]byte, error) { @@ -280,13 +301,25 @@ func parseStatusValue(line []byte) ([]byte, error) { return line[1:], nil } -func parseIntValue(line []byte) (int64, error) { - return strconv.ParseInt(string(line[1:]), 10, 64) -} - func parseArrayLen(line []byte) (int64, error) { if isNilReply(line) { return 0, internal.Nil } - return parseIntValue(line) + return parseInt(line[1:], 10, 64) +} + +func atoi(b []byte) (int, error) { + return strconv.Atoi(internal.BytesToString(b)) +} + +func parseInt(b []byte, base int, bitSize int) (int64, error) { + return strconv.ParseInt(internal.BytesToString(b), base, bitSize) +} + +func parseUint(b []byte, base int, bitSize int) (uint64, error) { + return strconv.ParseUint(internal.BytesToString(b), base, bitSize) +} + +func parseFloat(b []byte, bitSize int) (float64, error) { + return strconv.ParseFloat(internal.BytesToString(b), bitSize) } diff --git a/internal/proto/scan.go b/internal/proto/scan.go index b51e4e86cb..67ea521cd3 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -3,90 +3,89 @@ package proto import ( "encoding" "fmt" - "strconv" "gopkg.in/redis.v5/internal" ) -func Scan(s string, v interface{}) error { +func Scan(b []byte, v interface{}) error { switch v := v.(type) { case nil: return internal.RedisError("redis: Scan(nil)") case *string: - *v = s + *v = internal.BytesToString(b) return nil case *[]byte: - *v = []byte(s) + *v = b return nil case *int: var err error - *v, err = strconv.Atoi(s) + *v, err = atoi(b) return err case *int8: - n, err := strconv.ParseInt(s, 10, 8) + n, err := parseInt(b, 10, 8) if err != nil { return err } *v = int8(n) return nil case *int16: - n, err := strconv.ParseInt(s, 10, 16) + n, err := parseInt(b, 10, 16) if err != nil { return err } *v = int16(n) return nil case *int32: - n, err := strconv.ParseInt(s, 10, 32) + n, err := parseInt(b, 10, 32) if err != nil { return err } *v = int32(n) return nil case *int64: - n, err := strconv.ParseInt(s, 10, 64) + n, err := parseInt(b, 10, 64) if err != nil { return err } *v = n return nil case *uint: - n, err := strconv.ParseUint(s, 10, 64) + n, err := parseUint(b, 10, 64) if err != nil { return err } *v = uint(n) return nil case *uint8: - n, err := strconv.ParseUint(s, 10, 8) + n, err := parseUint(b, 10, 8) if err != nil { return err } *v = uint8(n) return nil case *uint16: - n, err := strconv.ParseUint(s, 10, 16) + n, err := parseUint(b, 10, 16) if err != nil { return err } *v = uint16(n) return nil case *uint32: - n, err := strconv.ParseUint(s, 10, 32) + n, err := parseUint(b, 10, 32) if err != nil { return err } *v = uint32(n) return nil case *uint64: - n, err := strconv.ParseUint(s, 10, 64) + n, err := parseUint(b, 10, 64) if err != nil { return err } *v = n return nil case *float32: - n, err := strconv.ParseFloat(s, 32) + n, err := parseFloat(b, 32) if err != nil { return err } @@ -94,13 +93,13 @@ func Scan(s string, v interface{}) error { return err case *float64: var err error - *v, err = strconv.ParseFloat(s, 64) + *v, err = parseFloat(b, 64) return err case *bool: - *v = len(s) == 1 && s[0] == '1' + *v = len(b) == 1 && b[0] == '1' return nil case encoding.BinaryUnmarshaler: - return v.UnmarshalBinary([]byte(s)) + return v.UnmarshalBinary(b) default: return fmt.Errorf( "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", v) diff --git a/internal/safe.go b/internal/safe.go new file mode 100644 index 0000000000..dc5f4cc8a4 --- /dev/null +++ b/internal/safe.go @@ -0,0 +1,7 @@ +// +build appengine + +package internal + +func BytesToString(b []byte) string { + return string(b) +} diff --git a/internal/unsafe.go b/internal/unsafe.go new file mode 100644 index 0000000000..94e4a9d35d --- /dev/null +++ b/internal/unsafe.go @@ -0,0 +1,14 @@ +// +build !appengine + +package internal + +import ( + "reflect" + "unsafe" +) + +func BytesToString(b []byte) string { + bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + strHeader := reflect.StringHeader{bytesHeader.Data, bytesHeader.Len} + return *(*string)(unsafe.Pointer(&strHeader)) +} diff --git a/internal/util.go b/internal/util.go index 5986cc5a7d..06623383a0 100644 --- a/internal/util.go +++ b/internal/util.go @@ -13,7 +13,7 @@ func ToLower(s string) string { } b[i] = c } - return string(b) + return BytesToString(b) } func isLower(s string) bool { diff --git a/parser.go b/parser.go index 09714125ea..bba6096afd 100644 --- a/parser.go +++ b/parser.go @@ -386,22 +386,12 @@ func timeParser(rd *proto.Reader, n int64) (interface{}, error) { return nil, fmt.Errorf("got %d elements, expected 2", n) } - secStr, err := rd.ReadStringReply() + sec, err := rd.ReadInt() if err != nil { return nil, err } - microsecStr, err := rd.ReadStringReply() - if err != nil { - return nil, err - } - - sec, err := strconv.ParseInt(secStr, 10, 64) - if err != nil { - return nil, err - } - - microsec, err := strconv.ParseInt(microsecStr, 10, 64) + microsec, err := rd.ReadInt() if err != nil { return nil, err } diff --git a/result.go b/result.go index f5c7084cdf..28cea5ca83 100644 --- a/result.go +++ b/result.go @@ -53,7 +53,7 @@ func NewBoolResult(val bool, err error) *BoolCmd { // NewStringResult returns a StringCmd initalised with val and err for testing func NewStringResult(val string, err error) *StringCmd { var cmd StringCmd - cmd.val = val + cmd.val = []byte(val) cmd.setErr(err) return &cmd } diff --git a/ring.go b/ring.go index aff019daec..4eb57c132a 100644 --- a/ring.go +++ b/ring.go @@ -265,7 +265,7 @@ func (c *Ring) shardByName(name string) (*ringShard, error) { } func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { - cmdInfo := c.cmdInfo(cmd.arg(0)) + cmdInfo := c.cmdInfo(cmd.name()) firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) return c.shardByKey(firstKey) } @@ -364,7 +364,7 @@ func (c *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - cmdInfo := c.cmdInfo(cmd.arg(0)) + cmdInfo := c.cmdInfo(cmd.name()) name := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) if name != "" { name = c.hash.Get(hashtag.Key(name)) From b9ab636be44bb604f10ba411fea670e043dc7f43 Mon Sep 17 00:00:00 2001 From: Sukharev Maxim Date: Fri, 13 Jan 2017 11:11:07 +0700 Subject: [PATCH 0274/1746] Context methods only for go1.7+ --- redis.go | 29 +---------------------------- redis_context.go | 35 +++++++++++++++++++++++++++++++++++ redis_no_context.go | 15 +++++++++++++++ 3 files changed, 51 insertions(+), 28 deletions(-) create mode 100644 redis_context.go create mode 100644 redis_no_context.go diff --git a/redis.go b/redis.go index dc779f47d7..32b83d5aeb 100644 --- a/redis.go +++ b/redis.go @@ -1,7 +1,6 @@ package redis // import "gopkg.in/redis.v5" import ( - "context" "fmt" "log" "time" @@ -18,16 +17,6 @@ func SetLogger(logger *log.Logger) { internal.Logger = logger } -type baseClient struct { - connPool pool.Pooler - opt *Options - - process func(Cmder) error - onClose func() error // hook called when client is closed - - ctx context.Context -} - func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB) } @@ -312,29 +301,13 @@ func NewClient(opt *Options) *Client { return newClient(opt, newConnPool(opt)) } -func (c *Client) Clone() *Client { +func (c *Client) copy() *Client { c2 := new(Client) *c2 = *c c2.cmdable.process = c2.Process return c2 } -func (c *Client) Context() context.Context { - if c.ctx != nil { - return c.ctx - } - return context.Background() -} - -func (c *Client) WithContext(ctx context.Context) *Client { - if ctx == nil { - panic("nil context") - } - c2 := c.Clone() - c2.ctx = ctx - return c2 -} - // PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { s := c.connPool.Stats() diff --git a/redis_context.go b/redis_context.go new file mode 100644 index 0000000000..bfa62bc5a2 --- /dev/null +++ b/redis_context.go @@ -0,0 +1,35 @@ +// +build go1.7 + +package redis + +import ( + "context" + + "gopkg.in/redis.v5/internal/pool" +) + +type baseClient struct { + connPool pool.Pooler + opt *Options + + process func(Cmder) error + onClose func() error // hook called when client is closed + + ctx context.Context +} + +func (c *Client) Context() context.Context { + if c.ctx != nil { + return c.ctx + } + return context.Background() +} + +func (c *Client) WithContext(ctx context.Context) *Client { + if ctx == nil { + panic("nil context") + } + c2 := c.copy() + c2.ctx = ctx + return c2 +} diff --git a/redis_no_context.go b/redis_no_context.go new file mode 100644 index 0000000000..78a5a637c5 --- /dev/null +++ b/redis_no_context.go @@ -0,0 +1,15 @@ +// +build !go1.7 + +package redis + +import ( + "gopkg.in/redis.v5/internal/pool" +) + +type baseClient struct { + connPool pool.Pooler + opt *Options + + process func(Cmder) error + onClose func() error // hook called when client is closed +} From 63bac70a1913f5b2b0f9d811fdaa928707d60889 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 26 Jan 2017 15:51:34 +0200 Subject: [PATCH 0275/1746] Add ZRemRangeByLex. --- commands.go | 7 +++++++ commands_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/commands.go b/commands.go index 2e50cda1d0..676fad3477 100644 --- a/commands.go +++ b/commands.go @@ -170,6 +170,7 @@ type Cmdable interface { ZRem(key string, members ...interface{}) *IntCmd ZRemRangeByRank(key string, start, stop int64) *IntCmd ZRemRangeByScore(key, min, max string) *IntCmd + ZRemRangeByLex(key, min, max string) *IntCmd ZRevRange(key string, start, stop int64) *StringSliceCmd ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd @@ -1468,6 +1469,12 @@ func (c *cmdable) ZRemRangeByScore(key, min, max string) *IntCmd { return cmd } +func (c *cmdable) ZRemRangeByLex(key, min, max string) *IntCmd { + cmd := NewIntCmd("zremrangebylex", key, min, max) + c.process(cmd) + return cmd +} + func (c *cmdable) ZRevRange(key string, start, stop int64) *StringSliceCmd { cmd := NewStringSliceCmd("zrevrange", key, start, stop) c.process(cmd) diff --git a/commands_test.go b/commands_test.go index caa2e35e8c..269288588a 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2429,6 +2429,33 @@ var _ = Describe("Commands", func() { Expect(val).To(Equal([]redis.Z{{2, "two"}, {3, "three"}})) }) + It("should ZRemRangeByLex", func() { + zz := []redis.Z{ + {0, "aaaa"}, + {0, "b"}, + {0, "c"}, + {0, "d"}, + {0, "e"}, + {0, "foo"}, + {0, "zap"}, + {0, "zip"}, + {0, "ALPHA"}, + {0, "alpha"}, + } + for _, z := range zz { + err := client.ZAdd("zset", z).Err() + Expect(err).NotTo(HaveOccurred()) + } + + n, err := client.ZRemRangeByLex("zset", "[alpha", "[omega").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(6))) + + vals, err := client.ZRange("zset", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]string{"ALPHA", "aaaa", "zap", "zip"})) + }) + It("should ZRevRange", func() { zAdd := client.ZAdd("zset", redis.Z{1, "one"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) From 3fa2fb8dc00b63abee8f1ba89e7f3119f8df5ffc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 26 Jan 2017 15:59:44 +0200 Subject: [PATCH 0276/1746] ObjectIdleTime accepts one key. --- commands.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/commands.go b/commands.go index 2e50cda1d0..ce5f92dd7f 100644 --- a/commands.go +++ b/commands.go @@ -57,7 +57,7 @@ type Cmdable interface { Move(key string, db int64) *BoolCmd ObjectRefCount(keys ...string) *IntCmd ObjectEncoding(keys ...string) *StringCmd - ObjectIdleTime(keys ...string) *DurationCmd + ObjectIdleTime(key string) *DurationCmd Persist(key string) *BoolCmd PExpire(key string, expiration time.Duration) *BoolCmd PExpireAt(key string, tm time.Time) *BoolCmd @@ -378,14 +378,8 @@ func (c *cmdable) ObjectEncoding(keys ...string) *StringCmd { return cmd } -func (c *cmdable) ObjectIdleTime(keys ...string) *DurationCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "object" - args[1] = "idletime" - for i, key := range keys { - args[2+i] = key - } - cmd := NewDurationCmd(time.Second, args...) +func (c *cmdable) ObjectIdleTime(key string) *DurationCmd { + cmd := NewDurationCmd(time.Second, "object", "idletime", key) c.process(cmd) return cmd } From 8ddd2beaee34736502ae7cc96860fb6ec3f52352 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 26 Jan 2017 16:34:09 +0200 Subject: [PATCH 0277/1746] Fix error message for Go 1.8. --- options.go | 3 ++- options_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index a7f7fd7653..bab7d4d594 100644 --- a/options.go +++ b/options.go @@ -3,6 +3,7 @@ package redis import ( "crypto/tls" "errors" + "fmt" "net" "net/url" "strconv" @@ -151,7 +152,7 @@ func ParseURL(redisURL string) (*Options, error) { o.DB = 0 case 1: if o.DB, err = strconv.Atoi(f[0]); err != nil { - return nil, errors.New("invalid redis database number: " + err.Error()) + return nil, fmt.Errorf("invalid redis database number: %q", f[0]) } default: return nil, errors.New("invalid redis URL path: " + u.Path) diff --git a/options_test.go b/options_test.go index effebd5a0f..6a4af7169a 100644 --- a/options_test.go +++ b/options_test.go @@ -63,7 +63,7 @@ func TestParseURL(t *testing.T) { { "redis://localhost/iamadatabase", "", - 0, false, errors.New("invalid redis database number: strconv.ParseInt: parsing \"iamadatabase\": invalid syntax"), + 0, false, errors.New(`invalid redis database number: "iamadatabase"`), }, } From 308ebee45771b60fdefbbc8e7f0d69d0e233734a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 28 Jan 2017 10:53:10 +0200 Subject: [PATCH 0278/1746] Fix defer order. --- internal/pool/pool_sticky.go | 39 ++++++++++++++++++++---------------- pipeline.go | 2 +- ring.go | 2 +- sentinel.go | 2 +- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index d25bf99a61..9fb9971c78 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -11,7 +11,7 @@ type StickyConnPool struct { cn *Conn closed bool - mx sync.Mutex + mu sync.Mutex } var _ Pooler = (*StickyConnPool)(nil) @@ -24,15 +24,15 @@ func NewStickyConnPool(pool *ConnPool, reusable bool) *StickyConnPool { } func (p *StickyConnPool) First() *Conn { - p.mx.Lock() + p.mu.Lock() cn := p.cn - p.mx.Unlock() + p.mu.Unlock() return cn } func (p *StickyConnPool) Get() (*Conn, bool, error) { - defer p.mx.Unlock() - p.mx.Lock() + p.mu.Lock() + defer p.mu.Unlock() if p.closed { return nil, false, ErrClosed @@ -56,8 +56,9 @@ func (p *StickyConnPool) putUpstream() (err error) { } func (p *StickyConnPool) Put(cn *Conn) error { - defer p.mx.Unlock() - p.mx.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { return ErrClosed } @@ -74,8 +75,9 @@ func (p *StickyConnPool) removeUpstream(reason error) error { } func (p *StickyConnPool) Remove(cn *Conn, reason error) error { - defer p.mx.Unlock() - p.mx.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { return nil } @@ -89,8 +91,9 @@ func (p *StickyConnPool) Remove(cn *Conn, reason error) error { } func (p *StickyConnPool) Len() int { - defer p.mx.Unlock() - p.mx.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.cn == nil { return 0 } @@ -98,8 +101,9 @@ func (p *StickyConnPool) Len() int { } func (p *StickyConnPool) FreeLen() int { - defer p.mx.Unlock() - p.mx.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.cn == nil { return 1 } @@ -111,8 +115,9 @@ func (p *StickyConnPool) Stats() *Stats { } func (p *StickyConnPool) Close() error { - defer p.mx.Unlock() - p.mx.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { return ErrClosed } @@ -130,8 +135,8 @@ func (p *StickyConnPool) Close() error { } func (p *StickyConnPool) Closed() bool { - p.mx.Lock() + p.mu.Lock() closed := p.closed - p.mx.Unlock() + p.mu.Unlock() return closed } diff --git a/pipeline.go b/pipeline.go index a0a00e209e..13b29e1a6d 100644 --- a/pipeline.go +++ b/pipeline.go @@ -61,8 +61,8 @@ func (c *Pipeline) discard() error { // Exec always returns list of commands and error of the first failed // command if any. func (c *Pipeline) Exec() ([]Cmder, error) { - defer c.mu.Unlock() c.mu.Lock() + defer c.mu.Unlock() if c.closed { return nil, pool.ErrClosed diff --git a/ring.go b/ring.go index 4eb57c132a..6e5f62aef2 100644 --- a/ring.go +++ b/ring.go @@ -328,8 +328,8 @@ func (c *Ring) heartbeat() { // It is rare to Close a Ring, as the Ring is meant to be long-lived // and shared between many goroutines. func (c *Ring) Close() error { - defer c.mu.Unlock() c.mu.Lock() + defer c.mu.Unlock() if c.closed { return nil diff --git a/sentinel.go b/sentinel.go index 2a3264776f..77b892d5f1 100644 --- a/sentinel.go +++ b/sentinel.go @@ -162,8 +162,8 @@ func (d *sentinelFailover) Pool() *pool.ConnPool { } func (d *sentinelFailover) MasterAddr() (string, error) { - defer d.mu.Unlock() d.mu.Lock() + defer d.mu.Unlock() // Try last working sentinel. if d.sentinel != nil { From ce4fd8b6774e97e61a1f432af3ed49b70437ebe4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 8 Feb 2017 11:24:09 +0200 Subject: [PATCH 0279/1746] Fix ReceiveMessage to work without any subscriptions. --- Makefile | 1 + command.go | 2 +- internal/pool/conn.go | 62 +++++++++++----- internal/pool/pool.go | 7 +- internal/pool/pool_single.go | 8 -- internal/pool/pool_sticky.go | 23 ------ internal/pool/pool_test.go | 2 +- internal/proto/reader.go | 29 +++++--- internal/proto/reader_test.go | 14 ++-- .../proto/{writebuffer.go => write_buffer.go} | 8 +- ...itebuffer_test.go => write_buffer_test.go} | 8 +- pool_test.go | 2 +- pubsub.go | 73 +++++++++++-------- pubsub_test.go | 32 +++++++- redis.go | 4 - redis_test.go | 8 +- sentinel.go | 8 +- tx_test.go | 2 +- 18 files changed, 164 insertions(+), 129 deletions(-) rename internal/proto/{writebuffer.go => write_buffer.go} (95%) rename internal/proto/{writebuffer_test.go => write_buffer_test.go} (94%) diff --git a/Makefile b/Makefile index 4562692677..50fdc55a1a 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,5 @@ testdata/redis: wget -qO- https://github.com/antirez/redis/archive/unstable.tar.gz | tar xvz --strip-components=1 -C $@ testdata/redis/src/redis-server: testdata/redis + sed -i 's/libjemalloc.a/libjemalloc.a -lrt/g' $ 0 && time.Since(cn.UsedAt) > timeout + return timeout > 0 && time.Since(cn.UsedAt()) > timeout } func (cn *Conn) SetReadTimeout(timeout time.Duration) error { - cn.UsedAt = time.Now() + now := time.Now() + cn.SetUsedAt(now) if timeout > 0 { - return cn.NetConn.SetReadDeadline(cn.UsedAt.Add(timeout)) + return cn.netConn.SetReadDeadline(now.Add(timeout)) } - return cn.NetConn.SetReadDeadline(noDeadline) - + return cn.netConn.SetReadDeadline(noDeadline) } func (cn *Conn) SetWriteTimeout(timeout time.Duration) error { - cn.UsedAt = time.Now() + now := time.Now() + cn.SetUsedAt(now) if timeout > 0 { - return cn.NetConn.SetWriteDeadline(cn.UsedAt.Add(timeout)) + return cn.netConn.SetWriteDeadline(now.Add(timeout)) } - return cn.NetConn.SetWriteDeadline(noDeadline) + return cn.netConn.SetWriteDeadline(noDeadline) +} + +func (cn *Conn) Write(b []byte) (int, error) { + return cn.netConn.Write(b) +} + +func (cn *Conn) RemoteAddr() net.Addr { + return cn.netConn.RemoteAddr() } func (cn *Conn) Close() error { - return cn.NetConn.Close() + return cn.netConn.Close() } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 6a0e057737..4033e58dee 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -41,7 +41,6 @@ type Pooler interface { FreeLen() int Stats() *Stats Close() error - Closed() bool } type dialer func() (net.Conn, error) @@ -132,7 +131,7 @@ func (p *ConnPool) popFree() *Conn { // Get returns existed connection from the pool or creates a new one. func (p *ConnPool) Get() (*Conn, bool, error) { - if p.Closed() { + if p.closed() { return nil, false, ErrClosed } @@ -241,7 +240,7 @@ func (p *ConnPool) Stats() *Stats { } } -func (p *ConnPool) Closed() bool { +func (p *ConnPool) closed() bool { return atomic.LoadInt32(&p._closed) == 1 } @@ -318,7 +317,7 @@ func (p *ConnPool) reaper(frequency time.Duration) { defer ticker.Stop() for _ = range ticker.C { - if p.Closed() { + if p.closed() { break } n, err := p.ReapStaleConns() diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index 18ca616c3d..22eaba9d40 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -12,10 +12,6 @@ func NewSingleConnPool(cn *Conn) *SingleConnPool { } } -func (p *SingleConnPool) First() *Conn { - return p.cn -} - func (p *SingleConnPool) Get() (*Conn, bool, error) { return p.cn, false, nil } @@ -49,7 +45,3 @@ func (p *SingleConnPool) Stats() *Stats { func (p *SingleConnPool) Close() error { return nil } - -func (p *SingleConnPool) Closed() bool { - return false -} diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 9fb9971c78..7426cd260c 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -23,13 +23,6 @@ func NewStickyConnPool(pool *ConnPool, reusable bool) *StickyConnPool { } } -func (p *StickyConnPool) First() *Conn { - p.mu.Lock() - cn := p.cn - p.mu.Unlock() - return cn -} - func (p *StickyConnPool) Get() (*Conn, bool, error) { p.mu.Lock() defer p.mu.Unlock() @@ -62,9 +55,6 @@ func (p *StickyConnPool) Put(cn *Conn) error { if p.closed { return ErrClosed } - if p.cn != cn { - panic("p.cn != cn") - } return nil } @@ -81,12 +71,6 @@ func (p *StickyConnPool) Remove(cn *Conn, reason error) error { if p.closed { return nil } - if p.cn == nil { - panic("p.cn == nil") - } - if cn != nil && p.cn != cn { - panic("p.cn != cn") - } return p.removeUpstream(reason) } @@ -133,10 +117,3 @@ func (p *StickyConnPool) Close() error { } return err } - -func (p *StickyConnPool) Closed() bool { - p.mu.Lock() - closed := p.closed - p.mu.Unlock() - return closed -} diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index ece9b8f544..f24f855f74 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -100,7 +100,7 @@ var _ = Describe("conns reaper", func() { for i := 0; i < 3; i++ { cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) - cn.UsedAt = time.Now().Add(-2 * idleTimeout) + cn.SetUsedAt(time.Now().Add(-2 * idleTimeout)) conns = append(conns, cn) idleConns = append(idleConns, cn) } diff --git a/internal/proto/reader.go b/internal/proto/reader.go index ee811c856f..78f32317cf 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -26,13 +26,17 @@ type Reader struct { buf []byte } -func NewReader(rd io.Reader) *Reader { +func NewReader(rd io.Reader, buf []byte) *Reader { return &Reader{ src: bufio.NewReader(rd), - buf: make([]byte, 0, bufferSize), + buf: buf, } } +func (r *Reader) Reset(rd io.Reader) { + r.src.Reset(rd) +} + func (p *Reader) PeekBuffered() []byte { if n := p.src.Buffered(); n != 0 { b, _ := p.src.Peek(n) @@ -42,7 +46,12 @@ func (p *Reader) PeekBuffered() []byte { } func (p *Reader) ReadN(n int) ([]byte, error) { - return readN(p.src, p.buf, n) + b, err := readN(p.src, p.buf, n) + if err != nil { + return nil, err + } + p.buf = b + return b, nil } func (p *Reader) ReadLine() ([]byte, error) { @@ -72,11 +81,11 @@ func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { case ErrorReply: return nil, ParseErrorReply(line) case StatusReply: - return parseStatusValue(line) + return parseStatusValue(line), nil case IntReply: return parseInt(line[1:], 10, 64) case StringReply: - return p.readBytesValue(line) + return p.readTmpBytesValue(line) case ArrayReply: n, err := parseArrayLen(line) if err != nil { @@ -111,9 +120,9 @@ func (p *Reader) ReadTmpBytesReply() ([]byte, error) { case ErrorReply: return nil, ParseErrorReply(line) case StringReply: - return p.readBytesValue(line) + return p.readTmpBytesValue(line) case StatusReply: - return parseStatusValue(line) + return parseStatusValue(line), nil default: return nil, fmt.Errorf("redis: can't parse string reply: %.100q", line) } @@ -210,7 +219,7 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) { return keys, cursor, err } -func (p *Reader) readBytesValue(line []byte) ([]byte, error) { +func (p *Reader) readTmpBytesValue(line []byte) ([]byte, error) { if isNilReply(line) { return nil, internal.Nil } @@ -297,8 +306,8 @@ func ParseErrorReply(line []byte) error { return internal.RedisError(string(line[1:])) } -func parseStatusValue(line []byte) ([]byte, error) { - return line[1:], nil +func parseStatusValue(line []byte) []byte { + return line[1:] } func parseArrayLen(line []byte) (int64, error) { diff --git a/internal/proto/reader_test.go b/internal/proto/reader_test.go index 421344be2f..4835a625e0 100644 --- a/internal/proto/reader_test.go +++ b/internal/proto/reader_test.go @@ -5,27 +5,27 @@ import ( "strings" "testing" + "gopkg.in/redis.v5/internal/proto" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5/internal/proto" ) var _ = Describe("Reader", func() { It("should read n bytes", func() { - data, err := proto.NewReader(strings.NewReader("ABCDEFGHIJKLMNO")).ReadN(10) + data, err := proto.NewReader(strings.NewReader("ABCDEFGHIJKLMNO"), nil).ReadN(10) Expect(err).NotTo(HaveOccurred()) Expect(len(data)).To(Equal(10)) Expect(string(data)).To(Equal("ABCDEFGHIJ")) - data, err = proto.NewReader(strings.NewReader(strings.Repeat("x", 8192))).ReadN(6000) + data, err = proto.NewReader(strings.NewReader(strings.Repeat("x", 8192)), nil).ReadN(6000) Expect(err).NotTo(HaveOccurred()) Expect(len(data)).To(Equal(6000)) }) It("should read lines", func() { - p := proto.NewReader(strings.NewReader("$5\r\nhello\r\n")) + p := proto.NewReader(strings.NewReader("$5\r\nhello\r\n"), nil) data, err := p.ReadLine() Expect(err).NotTo(HaveOccurred()) @@ -59,11 +59,11 @@ func BenchmarkReader_ParseReply_Slice(b *testing.B) { } func benchmarkParseReply(b *testing.B, reply string, m proto.MultiBulkParse, wanterr bool) { - buf := &bytes.Buffer{} + buf := new(bytes.Buffer) for i := 0; i < b.N; i++ { buf.WriteString(reply) } - p := proto.NewReader(buf) + p := proto.NewReader(buf, nil) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/internal/proto/writebuffer.go b/internal/proto/write_buffer.go similarity index 95% rename from internal/proto/writebuffer.go rename to internal/proto/write_buffer.go index 1e0f8e6c1e..93fb367741 100644 --- a/internal/proto/writebuffer.go +++ b/internal/proto/write_buffer.go @@ -8,11 +8,13 @@ import ( const bufferSize = 4096 -type WriteBuffer struct{ b []byte } +type WriteBuffer struct { + b []byte +} -func NewWriteBuffer() *WriteBuffer { +func NewWriteBuffer(b []byte) *WriteBuffer { return &WriteBuffer{ - b: make([]byte, 0, bufferSize), + b: b, } } diff --git a/internal/proto/writebuffer_test.go b/internal/proto/write_buffer_test.go similarity index 94% rename from internal/proto/writebuffer_test.go rename to internal/proto/write_buffer_test.go index 36593af519..fd70cd7f98 100644 --- a/internal/proto/writebuffer_test.go +++ b/internal/proto/write_buffer_test.go @@ -4,17 +4,17 @@ import ( "testing" "time" + "gopkg.in/redis.v5/internal/proto" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5/internal/proto" ) var _ = Describe("WriteBuffer", func() { var buf *proto.WriteBuffer BeforeEach(func() { - buf = proto.NewWriteBuffer() + buf = proto.NewWriteBuffer(nil) }) It("should reset", func() { @@ -53,7 +53,7 @@ var _ = Describe("WriteBuffer", func() { }) func BenchmarkWriteBuffer_Append(b *testing.B) { - buf := proto.NewWriteBuffer() + buf := proto.NewWriteBuffer(nil) args := []interface{}{"hello", "world", "foo", "bar"} for i := 0; i < b.N; i++ { diff --git a/pool_test.go b/pool_test.go index 08760296da..683f7c81f6 100644 --- a/pool_test.go +++ b/pool_test.go @@ -93,7 +93,7 @@ var _ = Describe("pool", func() { It("removes broken connections", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.NetConn = &badConn{} + cn.SetNetConn(&badConn{}) Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) err = client.Ping().Err() diff --git a/pubsub.go b/pubsub.go index c9205a0310..f98566e89a 100644 --- a/pubsub.go +++ b/pubsub.go @@ -3,6 +3,7 @@ package redis import ( "fmt" "net" + "sync" "time" "gopkg.in/redis.v5/internal" @@ -14,7 +15,9 @@ import ( // multiple goroutines. type PubSub struct { base baseClient + cmd *Cmd + mu sync.Mutex channels []string patterns []string } @@ -150,31 +153,40 @@ func (p *Pong) String() string { return "Pong" } -func (c *PubSub) newMessage(reply []interface{}) (interface{}, error) { - switch kind := reply[0].(string); kind { - case "subscribe", "unsubscribe", "psubscribe", "punsubscribe": - return &Subscription{ - Kind: kind, - Channel: reply[1].(string), - Count: int(reply[2].(int64)), - }, nil - case "message": - return &Message{ - Channel: reply[1].(string), - Payload: reply[2].(string), - }, nil - case "pmessage": - return &Message{ - Pattern: reply[1].(string), - Channel: reply[2].(string), - Payload: reply[3].(string), - }, nil - case "pong": +func (c *PubSub) newMessage(reply interface{}) (interface{}, error) { + switch reply := reply.(type) { + case string: return &Pong{ - Payload: reply[1].(string), + Payload: reply, }, nil + case []interface{}: + switch kind := reply[0].(string); kind { + case "subscribe", "unsubscribe", "psubscribe", "punsubscribe": + return &Subscription{ + Kind: kind, + Channel: reply[1].(string), + Count: int(reply[2].(int64)), + }, nil + case "message": + return &Message{ + Channel: reply[1].(string), + Payload: reply[2].(string), + }, nil + case "pmessage": + return &Message{ + Pattern: reply[1].(string), + Channel: reply[2].(string), + Payload: reply[3].(string), + }, nil + case "pong": + return &Pong{ + Payload: reply[1].(string), + }, nil + default: + return nil, fmt.Errorf("redis: unsupported pubsub message: %q", kind) + } default: - return nil, fmt.Errorf("redis: unsupported pubsub notification: %q", kind) + return nil, fmt.Errorf("redis: unsupported pubsub message: %#v", reply) } } @@ -182,7 +194,9 @@ func (c *PubSub) newMessage(reply []interface{}) (interface{}, error) { // is not received in time. This is low-level API and most clients // should use ReceiveMessage. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { - cmd := NewSliceCmd() + if c.cmd == nil { + c.cmd = NewCmd() + } cn, _, err := c.conn() if err != nil { @@ -190,13 +204,13 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { } cn.SetReadTimeout(timeout) - err = cmd.readReply(cn) + err = c.cmd.readReply(cn) c.putConn(cn, err) if err != nil { return nil, err } - return c.newMessage(cmd.Val()) + return c.newMessage(c.cmd.Val()) } // Receive returns a message as a Subscription, Message, Pong or error. @@ -225,14 +239,14 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { errNum++ if errNum < 3 { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - err := c.Ping("") + err := c.Ping("hello") if err != nil { internal.Logf("PubSub.Ping failed: %s", err) } } } else { - // 3 consequent errors - connection is bad - // and/or Redis Server is down. + // 3 consequent errors - connection is broken or + // Redis Server is down. // Sleep to not exceed max number of open connections. time.Sleep(time.Second) } @@ -256,9 +270,6 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { } func (c *PubSub) resubscribe() { - if c.base.closed() { - return - } if len(c.channels) > 0 { if err := c.Subscribe(c.channels...); err != nil { internal.Logf("Subscribe failed: %s", err) diff --git a/pubsub_test.go b/pubsub_test.go index f36621317e..9490688325 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -288,12 +288,13 @@ var _ = Describe("PubSub", func() { }) expectReceiveMessageOnError := func(pubsub *redis.PubSub) { - cn1, _, err := pubsub.Pool().Get() + cn, _, err := pubsub.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn1.NetConn = &badConn{ + cn.SetNetConn(&badConn{ readErr: io.EOF, writeErr: io.EOF, - } + }) + pubsub.Pool().Put(cn) done := make(chan bool, 1) go func() { @@ -315,7 +316,7 @@ var _ = Describe("PubSub", func() { Eventually(done).Should(Receive()) stats := client.PoolStats() - Expect(stats.Requests).To(Equal(uint32(3))) + Expect(stats.Requests).To(Equal(uint32(4))) Expect(stats.Hits).To(Equal(uint32(1))) } @@ -362,4 +363,27 @@ var _ = Describe("PubSub", func() { wg.Wait() }) + It("should ReceiveMessage without a subscription", func() { + timeout := 100 * time.Millisecond + + pubsub, err := client.Subscribe() + Expect(err).NotTo(HaveOccurred()) + defer pubsub.Close() + + go func() { + defer GinkgoRecover() + + time.Sleep(2 * timeout) + err = pubsub.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + + err := client.Publish("mychannel", "hello").Err() + Expect(err).NotTo(HaveOccurred()) + }() + + msg, err := pubsub.ReceiveMessageTimeout(timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("hello")) + }) }) diff --git a/redis.go b/redis.go index 32b83d5aeb..1ddd754c5e 100644 --- a/redis.go +++ b/redis.go @@ -126,10 +126,6 @@ func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration { } } -func (c *baseClient) closed() bool { - return c.connPool.Closed() -} - // Close closes the client, releasing any open resources. // // It is rare to Close a Client, as the Client is meant to be diff --git a/redis_test.go b/redis_test.go index 69c68df74e..c7ee7be5e3 100644 --- a/redis_test.go +++ b/redis_test.go @@ -148,7 +148,7 @@ var _ = Describe("Client", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.NetConn = &badConn{} + cn.SetNetConn(&badConn{}) err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) @@ -160,11 +160,11 @@ var _ = Describe("Client", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn.UsedAt).NotTo(BeZero()) - createdAt := cn.UsedAt + createdAt := cn.UsedAt() err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) - Expect(cn.UsedAt.Equal(createdAt)).To(BeTrue()) + Expect(cn.UsedAt().Equal(createdAt)).To(BeTrue()) err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) @@ -172,7 +172,7 @@ var _ = Describe("Client", func() { cn, _, err = client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) - Expect(cn.UsedAt.After(createdAt)).To(BeTrue()) + Expect(cn.UsedAt().After(createdAt)).To(BeTrue()) }) It("should process command with special chars", func() { diff --git a/sentinel.go b/sentinel.go index 77b892d5f1..b39d365346 100644 --- a/sentinel.go +++ b/sentinel.go @@ -258,7 +258,7 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { // closeOldConns closes connections to the old master after failover switch. func (d *sentinelFailover) closeOldConns(newMaster string) { // Good connections that should be put back to the pool. They - // can't be put immediately, because pool.First will return them + // can't be put immediately, because pool.PopFree will return them // again on next iteration. cnsToPut := make([]*pool.Conn, 0) @@ -267,10 +267,10 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { if cn == nil { break } - if cn.NetConn.RemoteAddr().String() != newMaster { + if cn.RemoteAddr().String() != newMaster { err := fmt.Errorf( "sentinel: closing connection to the old master %s", - cn.NetConn.RemoteAddr(), + cn.RemoteAddr(), ) internal.Logf(err.Error()) d.pool.Remove(cn, err) @@ -289,8 +289,10 @@ func (d *sentinelFailover) listen(sentinel *sentinelClient) { for { if pubsub == nil { pubsub = sentinel.PubSub() + if err := pubsub.Subscribe("+switch-master"); err != nil { internal.Logf("sentinel: Subscribe failed: %s", err) + pubsub.Close() d.resetSentinel() return } diff --git a/tx_test.go b/tx_test.go index 156a890921..5ca22fdf1d 100644 --- a/tx_test.go +++ b/tx_test.go @@ -127,7 +127,7 @@ var _ = Describe("Tx", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) - cn.NetConn = &badConn{} + cn.SetNetConn(&badConn{}) err = client.Pool().Put(cn) Expect(err).NotTo(HaveOccurred()) From 4cbe497190412f1b3e95ecc14b8652486a9b92dd Mon Sep 17 00:00:00 2001 From: Eyal Post Date: Fri, 10 Feb 2017 00:32:52 +0200 Subject: [PATCH 0280/1746] ObjectRefCount and ObjectEncoding accept one key --- commands.go | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/commands.go b/commands.go index f346c174ad..77384cb0e6 100644 --- a/commands.go +++ b/commands.go @@ -55,8 +55,8 @@ type Cmdable interface { Keys(pattern string) *StringSliceCmd Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd Move(key string, db int64) *BoolCmd - ObjectRefCount(keys ...string) *IntCmd - ObjectEncoding(keys ...string) *StringCmd + ObjectRefCount(key string) *IntCmd + ObjectEncoding(key string) *StringCmd ObjectIdleTime(key string) *DurationCmd Persist(key string) *BoolCmd PExpire(key string, expiration time.Duration) *BoolCmd @@ -355,26 +355,14 @@ func (c *cmdable) Move(key string, db int64) *BoolCmd { return cmd } -func (c *cmdable) ObjectRefCount(keys ...string) *IntCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "object" - args[1] = "refcount" - for i, key := range keys { - args[2+i] = key - } - cmd := NewIntCmd(args...) +func (c *cmdable) ObjectRefCount(key string) *IntCmd { + cmd := NewIntCmd("object", "refcount", key) c.process(cmd) return cmd } -func (c *cmdable) ObjectEncoding(keys ...string) *StringCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "object" - args[1] = "encoding" - for i, key := range keys { - args[2+i] = key - } - cmd := NewStringCmd(args...) +func (c *cmdable) ObjectEncoding(key string) *StringCmd { + cmd := NewStringCmd("object", "encoding", key) c.process(cmd) return cmd } From 7c0cf90fb85ba92c18d2cc70b37f5484193296c6 Mon Sep 17 00:00:00 2001 From: Eyal Post Date: Fri, 10 Feb 2017 12:15:25 +0200 Subject: [PATCH 0281/1746] Support for multi keys in Exists --- commands.go | 13 +++++++++++++ commands_test.go | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/commands.go b/commands.go index 77384cb0e6..8ea5def1f0 100644 --- a/commands.go +++ b/commands.go @@ -50,6 +50,8 @@ type Cmdable interface { Unlink(keys ...string) *IntCmd Dump(key string) *StringCmd Exists(key string) *BoolCmd + // TODO: merge with Exists in v6 + ExistsMulti(keys ...string) *IntCmd Expire(key string, expiration time.Duration) *BoolCmd ExpireAt(key string, tm time.Time) *BoolCmd Keys(pattern string) *StringSliceCmd @@ -317,6 +319,17 @@ func (c *cmdable) Exists(key string) *BoolCmd { return cmd } +func (c *cmdable) ExistsMulti(keys ...string) *IntCmd { + args := make([]interface{}, 1+len(keys)) + args[0] = "exists" + for i, key := range keys { + args[1+i] = key + } + cmd := NewIntCmd(args...) + c.process(cmd) + return cmd +} + func (c *cmdable) Expire(key string, expiration time.Duration) *BoolCmd { cmd := NewBoolCmd("expire", key, formatSec(expiration)) c.process(cmd) diff --git a/commands_test.go b/commands_test.go index 269288588a..ecfc012cb8 100644 --- a/commands_test.go +++ b/commands_test.go @@ -242,6 +242,14 @@ var _ = Describe("Commands", func() { exists = client.Exists("key2") Expect(exists.Err()).NotTo(HaveOccurred()) Expect(exists.Val()).To(Equal(false)) + + existsMul := client.ExistsMulti("key1", "key2") + Expect(existsMul.Err()).NotTo(HaveOccurred()) + Expect(existsMul.Val()).To(Equal(int64(1))) + + existsMul = client.ExistsMulti("key1", "key1") + Expect(existsMul.Err()).NotTo(HaveOccurred()) + Expect(existsMul.Val()).To(Equal(int64(2))) }) It("should Expire", func() { From 5de14cbf582662a9819e15bcae7c39800dc1a720 Mon Sep 17 00:00:00 2001 From: eyalpost Date: Wed, 15 Feb 2017 17:15:24 +0200 Subject: [PATCH 0282/1746] ensure wait.done to avoid deadlock in test --- pubsub_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pubsub_test.go b/pubsub_test.go index 9490688325..ba26be7b76 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -347,11 +347,11 @@ var _ = Describe("PubSub", func() { defer GinkgoRecover() wg.Done() - + defer wg.Done() + _, err := pubsub.ReceiveMessage() Expect(err).To(MatchError("redis: client is closed")) - wg.Done() }() wg.Wait() From 6b8c6b3fe9eb1cb80e9ef6d414e6fe5d44314857 Mon Sep 17 00:00:00 2001 From: Nate Bosscher Date: Thu, 16 Feb 2017 11:15:34 -0500 Subject: [PATCH 0283/1746] Added implementation for WAIT command Reference: https://redis.io/commands/wait --- commands.go | 7 +++++++ commands_test.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/commands.go b/commands.go index 8ea5def1f0..bc93df9f13 100644 --- a/commands.go +++ b/commands.go @@ -273,6 +273,13 @@ func (c *cmdable) Ping() *StatusCmd { return cmd } +func (c *cmdable) Wait(numSlaves int, timeout time.Duration) *IntCmd { + + cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Second)) + c.process(cmd) + return cmd +} + func (c *cmdable) Quit() *StatusCmd { panic("not implemented") } diff --git a/commands_test.go b/commands_test.go index ecfc012cb8..66359d2ae1 100644 --- a/commands_test.go +++ b/commands_test.go @@ -50,6 +50,13 @@ var _ = Describe("Commands", func() { Expect(ping.Val()).To(Equal("PONG")) }) + It("should Wait", func() { + // assume testing on single redis instance + wait := client.Wait(0, time.Minute) + Expect(wait.Err()).NotTo(HaveOccurred()) + Expect(wait.Val()).To(Equal(int64(0))) + }) + It("should Select", func() { pipe := client.Pipeline() sel := pipe.Select(1) From 681a1fe646c8e1f2ce21a0ede91e1a7a89a36c5e Mon Sep 17 00:00:00 2001 From: Back Yu Date: Wed, 1 Feb 2017 16:36:33 +0800 Subject: [PATCH 0284/1746] Add ScanSlice. --- command.go | 4 +++ internal/proto/scan.go | 33 +++++++++++++++++ internal/proto/scan_test.go | 70 +++++++++++++++++++++++++++++++++++++ internal/util.go | 20 +++++++++++ 4 files changed, 127 insertions(+) create mode 100644 internal/proto/scan_test.go diff --git a/command.go b/command.go index dc13e82d89..e28eb7581c 100644 --- a/command.go +++ b/command.go @@ -542,6 +542,10 @@ func (cmd *StringSliceCmd) String() string { return cmdString(cmd, cmd.val) } +func (cmd *StringSliceCmd) ScanSlice(container interface{}) error { + return proto.ScanSlice(cmd.Val(), container) +} + func (cmd *StringSliceCmd) readReply(cn *pool.Conn) error { var v interface{} v, cmd.err = cn.Rd.ReadArrayReply(stringSliceParser) diff --git a/internal/proto/scan.go b/internal/proto/scan.go index 67ea521cd3..9745e95183 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -3,6 +3,7 @@ package proto import ( "encoding" "fmt" + "reflect" "gopkg.in/redis.v5/internal" ) @@ -105,3 +106,35 @@ func Scan(b []byte, v interface{}) error { "redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", v) } } + +// Scan a string slice into a custom container +// Example: +// var container []YourStruct; ScanSlice([]string{""},&container) +// var container []*YourStruct; ScanSlice([]string{""},&container) +func ScanSlice(sSlice []string, container interface{}) error { + val := reflect.ValueOf(container) + + if !val.IsValid() { + return fmt.Errorf("redis: ScanSlice(nil)") + } + + // Check the if the container is pointer + if val.Kind() != reflect.Ptr { + return fmt.Errorf("redis: ScanSlice(non-pointer %T)", container) + } + + // slice of the Ptr + val = val.Elem() + // if the container is slice + if val.Kind() != reflect.Slice { + return fmt.Errorf("redis: Wrong object type `%T` for ScanSlice(), need *[]*Type or *[]Type", container) + } + + for index, s := range sSlice { + elem := internal.SliceNextElem(val) + if err := Scan([]byte(s), elem.Addr().Interface()); err != nil { + return fmt.Errorf("redis: ScanSlice failed at index of %d => %s, %s", index, s, err.Error()) + } + } + return nil +} diff --git a/internal/proto/scan_test.go b/internal/proto/scan_test.go new file mode 100644 index 0000000000..a393e39469 --- /dev/null +++ b/internal/proto/scan_test.go @@ -0,0 +1,70 @@ +package proto + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +type testScanSliceStruct struct { + ID int + Name string +} + +func (this *testScanSliceStruct) MarshalBinary() (data []byte, err error) { + return json.Marshal(data) +} + +func (this *testScanSliceStruct) UnmarshalBinary(data []byte) error { + return json.Unmarshal(data, this) +} + +var _ = Describe("ScanSlice", func() { + + // Base string array for test. + strAry := []string{`{"ID":-1,"Name":"Back Yu"}`, `{"ID":1,"Name":"szyhf"}`} + // Validate json bytes of container if ScanSlice success + equalJson := Equal([]byte(`[{"ID":-1,"Name":"Back Yu"},{"ID":1,"Name":"szyhf"}]`)) + + It("var testContainer []testScanSliceStruct", func() { + var testContainer []testScanSliceStruct + err := ScanSlice(strAry, &testContainer) + Expect(err).NotTo(HaveOccurred()) + + jsonBytes, err := json.Marshal(testContainer) + Expect(err).NotTo(HaveOccurred()) + Expect(jsonBytes).Should(equalJson) + }) + + It("testContainer := new([]testScanSliceStruct)", func() { + testContainer := new([]testScanSliceStruct) + err := ScanSlice(strAry, testContainer) + Expect(err).NotTo(HaveOccurred()) + + jsonBytes, err := json.Marshal(testContainer) + Expect(err).NotTo(HaveOccurred()) + Expect(jsonBytes).Should(equalJson) + }) + + It("var testContainer []*testScanSliceStruct", func() { + var testContainer []*testScanSliceStruct + err := ScanSlice(strAry, &testContainer) + Expect(err).NotTo(HaveOccurred()) + + jsonBytes, err := json.Marshal(testContainer) + Expect(err).NotTo(HaveOccurred()) + Expect(jsonBytes).Should(equalJson) + }) + + It("testContainer := new([]*testScanSliceStruct)", func() { + testContainer := new([]*testScanSliceStruct) + err := ScanSlice(strAry, testContainer) + Expect(err).NotTo(HaveOccurred()) + + jsonBytes, err := json.Marshal(testContainer) + Expect(err).NotTo(HaveOccurred()) + Expect(jsonBytes).Should(equalJson) + }) + +}) diff --git a/internal/util.go b/internal/util.go index 06623383a0..520596fd97 100644 --- a/internal/util.go +++ b/internal/util.go @@ -1,5 +1,7 @@ package internal +import "reflect" + func ToLower(s string) string { if isLower(s) { return s @@ -25,3 +27,21 @@ func isLower(s string) bool { } return true } + +func SliceNextElem(v reflect.Value) reflect.Value { + if v.Len() < v.Cap() { + v.Set(v.Slice(0, v.Len()+1)) + return v.Index(v.Len() - 1) + } + + elemType := v.Type().Elem() + + if elemType.Kind() == reflect.Ptr { + elem := reflect.New(elemType.Elem()) + v.Set(reflect.Append(v, elem)) + return elem.Elem() + } + + v.Set(reflect.Append(v, reflect.Zero(elemType))) + return v.Index(v.Len() - 1) +} From 335956cc9a44435ed8f3773932c76c5c9306cc33 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Feb 2017 12:10:47 +0200 Subject: [PATCH 0285/1746] Cleanup code a bit. --- commands_test.go | 23 ++++++++----- internal/proto/scan.go | 33 +++++++----------- internal/proto/scan_test.go | 68 +++++++++++++------------------------ 3 files changed, 49 insertions(+), 75 deletions(-) diff --git a/commands_test.go b/commands_test.go index 66359d2ae1..a524aafc31 100644 --- a/commands_test.go +++ b/commands_test.go @@ -250,11 +250,11 @@ var _ = Describe("Commands", func() { Expect(exists.Err()).NotTo(HaveOccurred()) Expect(exists.Val()).To(Equal(false)) - existsMul := client.ExistsMulti("key1", "key2") + existsMul := client.ExistsMulti("key1", "key2") Expect(existsMul.Err()).NotTo(HaveOccurred()) Expect(existsMul.Val()).To(Equal(int64(1))) - existsMul = client.ExistsMulti("key1", "key1") + existsMul = client.ExistsMulti("key1", "key1") Expect(existsMul.Err()).NotTo(HaveOccurred()) Expect(existsMul.Val()).To(Equal(int64(2))) }) @@ -1264,14 +1264,19 @@ var _ = Describe("Commands", func() { }) It("should HVals", func() { - hSet := client.HSet("hash", "key1", "hello1") - Expect(hSet.Err()).NotTo(HaveOccurred()) - hSet = client.HSet("hash", "key2", "hello2") - Expect(hSet.Err()).NotTo(HaveOccurred()) + err := client.HSet("hash", "key1", "hello1").Err() + Expect(err).NotTo(HaveOccurred()) + err = client.HSet("hash", "key2", "hello2").Err() + Expect(err).NotTo(HaveOccurred()) - hVals := client.HVals("hash") - Expect(hVals.Err()).NotTo(HaveOccurred()) - Expect(hVals.Val()).To(Equal([]string{"hello1", "hello2"})) + v, err := client.HVals("hash").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(v).To(Equal([]string{"hello1", "hello2"})) + + var slice []string + err = client.HVals("hash").ScanSlice(&slice) + Expect(err).NotTo(HaveOccurred()) + Expect(slice).To(Equal([]string{"hello1", "hello2"})) }) }) diff --git a/internal/proto/scan.go b/internal/proto/scan.go index 9745e95183..f3c75d9031 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -107,34 +107,25 @@ func Scan(b []byte, v interface{}) error { } } -// Scan a string slice into a custom container -// Example: -// var container []YourStruct; ScanSlice([]string{""},&container) -// var container []*YourStruct; ScanSlice([]string{""},&container) -func ScanSlice(sSlice []string, container interface{}) error { - val := reflect.ValueOf(container) - - if !val.IsValid() { +func ScanSlice(data []string, slice interface{}) error { + v := reflect.ValueOf(slice) + if !v.IsValid() { return fmt.Errorf("redis: ScanSlice(nil)") } - - // Check the if the container is pointer - if val.Kind() != reflect.Ptr { - return fmt.Errorf("redis: ScanSlice(non-pointer %T)", container) + if v.Kind() != reflect.Ptr { + return fmt.Errorf("redis: ScanSlice(non-pointer %T)", slice) } - - // slice of the Ptr - val = val.Elem() - // if the container is slice - if val.Kind() != reflect.Slice { - return fmt.Errorf("redis: Wrong object type `%T` for ScanSlice(), need *[]*Type or *[]Type", container) + v = v.Elem() + if v.Kind() != reflect.Slice { + return fmt.Errorf("redis: ScanSlice(non-slice %T)", slice) } - for index, s := range sSlice { - elem := internal.SliceNextElem(val) + for i, s := range data { + elem := internal.SliceNextElem(v) if err := Scan([]byte(s), elem.Addr().Interface()); err != nil { - return fmt.Errorf("redis: ScanSlice failed at index of %d => %s, %s", index, s, err.Error()) + return fmt.Errorf("redis: ScanSlice(index=%d value=%q) failed: %s", i, s, err) } } + return nil } diff --git a/internal/proto/scan_test.go b/internal/proto/scan_test.go index a393e39469..fadcd0561f 100644 --- a/internal/proto/scan_test.go +++ b/internal/proto/scan_test.go @@ -12,59 +12,37 @@ type testScanSliceStruct struct { Name string } -func (this *testScanSliceStruct) MarshalBinary() (data []byte, err error) { - return json.Marshal(data) +func (s *testScanSliceStruct) MarshalBinary() ([]byte, error) { + return json.Marshal(s) } -func (this *testScanSliceStruct) UnmarshalBinary(data []byte) error { - return json.Unmarshal(data, this) +func (s *testScanSliceStruct) UnmarshalBinary(b []byte) error { + return json.Unmarshal(b, s) } var _ = Describe("ScanSlice", func() { - - // Base string array for test. - strAry := []string{`{"ID":-1,"Name":"Back Yu"}`, `{"ID":1,"Name":"szyhf"}`} - // Validate json bytes of container if ScanSlice success - equalJson := Equal([]byte(`[{"ID":-1,"Name":"Back Yu"},{"ID":1,"Name":"szyhf"}]`)) - - It("var testContainer []testScanSliceStruct", func() { - var testContainer []testScanSliceStruct - err := ScanSlice(strAry, &testContainer) - Expect(err).NotTo(HaveOccurred()) - - jsonBytes, err := json.Marshal(testContainer) - Expect(err).NotTo(HaveOccurred()) - Expect(jsonBytes).Should(equalJson) - }) - - It("testContainer := new([]testScanSliceStruct)", func() { - testContainer := new([]testScanSliceStruct) - err := ScanSlice(strAry, testContainer) - Expect(err).NotTo(HaveOccurred()) - - jsonBytes, err := json.Marshal(testContainer) - Expect(err).NotTo(HaveOccurred()) - Expect(jsonBytes).Should(equalJson) + data := []string{ + `{"ID":-1,"Name":"Back Yu"}`, + `{"ID":1,"Name":"szyhf"}`, + } + + It("[]testScanSliceStruct", func() { + var slice []testScanSliceStruct + err := ScanSlice(data, &slice) + Expect(err).NotTo(HaveOccurred()) + Expect(slice).To(Equal([]testScanSliceStruct{ + {-1, "Back Yu"}, + {1, "szyhf"}, + })) }) It("var testContainer []*testScanSliceStruct", func() { - var testContainer []*testScanSliceStruct - err := ScanSlice(strAry, &testContainer) - Expect(err).NotTo(HaveOccurred()) - - jsonBytes, err := json.Marshal(testContainer) + var slice []*testScanSliceStruct + err := ScanSlice(data, &slice) Expect(err).NotTo(HaveOccurred()) - Expect(jsonBytes).Should(equalJson) + Expect(slice).To(Equal([]*testScanSliceStruct{ + {-1, "Back Yu"}, + {1, "szyhf"}, + })) }) - - It("testContainer := new([]*testScanSliceStruct)", func() { - testContainer := new([]*testScanSliceStruct) - err := ScanSlice(strAry, testContainer) - Expect(err).NotTo(HaveOccurred()) - - jsonBytes, err := json.Marshal(testContainer) - Expect(err).NotTo(HaveOccurred()) - Expect(jsonBytes).Should(equalJson) - }) - }) From c2156b59f35203765d67e97a03fb1e7703367aec Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Feb 2017 12:28:01 +0200 Subject: [PATCH 0286/1746] Fix Go 1.4 build. --- pubsub_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pubsub_test.go b/pubsub_test.go index ba26be7b76..22f3130732 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -6,10 +6,10 @@ import ( "sync" "time" + "gopkg.in/redis.v5" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("PubSub", func() { @@ -348,10 +348,13 @@ var _ = Describe("PubSub", func() { wg.Done() defer wg.Done() - - _, err := pubsub.ReceiveMessage() - Expect(err).To(MatchError("redis: client is closed")) + _, err := pubsub.ReceiveMessage() + Expect(err).To(HaveOccurred()) + Expect(err).To(SatisfyAny( + MatchError("redis: client is closed"), + MatchError("use of closed network connection"), // Go 1.4 + )) }() wg.Wait() From 6b6f5ca13362bb9b77d2cfd7bec085ce531e7c57 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 19 Feb 2017 09:42:45 +0200 Subject: [PATCH 0287/1746] Separate read and write buffers for PubSub. --- internal/pool/conn.go | 5 ++--- internal/proto/reader.go | 4 ++-- internal/proto/reader_test.go | 8 ++++---- internal/proto/write_buffer.go | 4 ++-- internal/proto/write_buffer_test.go | 4 ++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 8a58f9aac0..2a135ae979 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -21,12 +21,11 @@ type Conn struct { } func NewConn(netConn net.Conn) *Conn { - buf := make([]byte, 4096) cn := &Conn{ netConn: netConn, - Wb: proto.NewWriteBuffer(buf), + Wb: proto.NewWriteBuffer(), } - cn.Rd = proto.NewReader(cn.netConn, buf) + cn.Rd = proto.NewReader(cn.netConn) cn.SetUsedAt(time.Now()) return cn } diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 78f32317cf..e5dc95edb6 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -26,10 +26,10 @@ type Reader struct { buf []byte } -func NewReader(rd io.Reader, buf []byte) *Reader { +func NewReader(rd io.Reader) *Reader { return &Reader{ src: bufio.NewReader(rd), - buf: buf, + buf: make([]byte, 4096), } } diff --git a/internal/proto/reader_test.go b/internal/proto/reader_test.go index 4835a625e0..63046464ec 100644 --- a/internal/proto/reader_test.go +++ b/internal/proto/reader_test.go @@ -14,18 +14,18 @@ import ( var _ = Describe("Reader", func() { It("should read n bytes", func() { - data, err := proto.NewReader(strings.NewReader("ABCDEFGHIJKLMNO"), nil).ReadN(10) + data, err := proto.NewReader(strings.NewReader("ABCDEFGHIJKLMNO")).ReadN(10) Expect(err).NotTo(HaveOccurred()) Expect(len(data)).To(Equal(10)) Expect(string(data)).To(Equal("ABCDEFGHIJ")) - data, err = proto.NewReader(strings.NewReader(strings.Repeat("x", 8192)), nil).ReadN(6000) + data, err = proto.NewReader(strings.NewReader(strings.Repeat("x", 8192))).ReadN(6000) Expect(err).NotTo(HaveOccurred()) Expect(len(data)).To(Equal(6000)) }) It("should read lines", func() { - p := proto.NewReader(strings.NewReader("$5\r\nhello\r\n"), nil) + p := proto.NewReader(strings.NewReader("$5\r\nhello\r\n")) data, err := p.ReadLine() Expect(err).NotTo(HaveOccurred()) @@ -63,7 +63,7 @@ func benchmarkParseReply(b *testing.B, reply string, m proto.MultiBulkParse, wan for i := 0; i < b.N; i++ { buf.WriteString(reply) } - p := proto.NewReader(buf, nil) + p := proto.NewReader(buf) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/internal/proto/write_buffer.go b/internal/proto/write_buffer.go index 93fb367741..019e64c873 100644 --- a/internal/proto/write_buffer.go +++ b/internal/proto/write_buffer.go @@ -12,9 +12,9 @@ type WriteBuffer struct { b []byte } -func NewWriteBuffer(b []byte) *WriteBuffer { +func NewWriteBuffer() *WriteBuffer { return &WriteBuffer{ - b: b, + b: make([]byte, 0, 4096), } } diff --git a/internal/proto/write_buffer_test.go b/internal/proto/write_buffer_test.go index fd70cd7f98..c4438e4a92 100644 --- a/internal/proto/write_buffer_test.go +++ b/internal/proto/write_buffer_test.go @@ -14,7 +14,7 @@ var _ = Describe("WriteBuffer", func() { var buf *proto.WriteBuffer BeforeEach(func() { - buf = proto.NewWriteBuffer(nil) + buf = proto.NewWriteBuffer() }) It("should reset", func() { @@ -53,7 +53,7 @@ var _ = Describe("WriteBuffer", func() { }) func BenchmarkWriteBuffer_Append(b *testing.B) { - buf := proto.NewWriteBuffer(nil) + buf := proto.NewWriteBuffer() args := []interface{}{"hello", "world", "foo", "bar"} for i := 0; i < b.N; i++ { From 56ddaf11991c389c7c0119141926e900258760d0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Feb 2017 15:42:12 +0200 Subject: [PATCH 0288/1746] Fix HMSet to accept interface{} value. --- commands.go | 4 ++-- commands_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index bc93df9f13..21d5aa965a 100644 --- a/commands.go +++ b/commands.go @@ -111,7 +111,7 @@ type Cmdable interface { HKeys(key string) *StringSliceCmd HLen(key string) *IntCmd HMGet(key string, fields ...string) *SliceCmd - HMSet(key string, fields map[string]string) *StatusCmd + HMSet(key string, fields map[string]interface{}) *StatusCmd HSet(key, field string, value interface{}) *BoolCmd HSetNX(key, field string, value interface{}) *BoolCmd HVals(key string) *StringSliceCmd @@ -880,7 +880,7 @@ func (c *cmdable) HMGet(key string, fields ...string) *SliceCmd { return cmd } -func (c *cmdable) HMSet(key string, fields map[string]string) *StatusCmd { +func (c *cmdable) HMSet(key string, fields map[string]interface{}) *StatusCmd { args := make([]interface{}, 2+len(fields)*2) args[0] = "hmset" args[1] = key diff --git a/commands_test.go b/commands_test.go index a524aafc31..261918620d 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1223,7 +1223,7 @@ var _ = Describe("Commands", func() { }) It("should HMSet", func() { - ok, err := client.HMSet("hash", map[string]string{ + ok, err := client.HMSet("hash", map[string]interface{}{ "key1": "hello1", "key2": "hello2", }).Result() From d95ce53b0de76684a6d3fc0263e3c97a33087c23 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Feb 2017 16:32:39 +0200 Subject: [PATCH 0289/1746] Replace Exists with ExistsMulti. --- commands.go | 12 ++---------- commands_test.go | 42 +++++++++++++++++++++--------------------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/commands.go b/commands.go index 21d5aa965a..9fcfadcc14 100644 --- a/commands.go +++ b/commands.go @@ -49,9 +49,7 @@ type Cmdable interface { Del(keys ...string) *IntCmd Unlink(keys ...string) *IntCmd Dump(key string) *StringCmd - Exists(key string) *BoolCmd - // TODO: merge with Exists in v6 - ExistsMulti(keys ...string) *IntCmd + Exists(keys ...string) *IntCmd Expire(key string, expiration time.Duration) *BoolCmd ExpireAt(key string, tm time.Time) *BoolCmd Keys(pattern string) *StringSliceCmd @@ -320,13 +318,7 @@ func (c *cmdable) Dump(key string) *StringCmd { return cmd } -func (c *cmdable) Exists(key string) *BoolCmd { - cmd := NewBoolCmd("exists", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) ExistsMulti(keys ...string) *IntCmd { +func (c *cmdable) Exists(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) args[0] = "exists" for i, key := range keys { diff --git a/commands_test.go b/commands_test.go index 261918620d..29d6cb6c65 100644 --- a/commands_test.go +++ b/commands_test.go @@ -242,21 +242,21 @@ var _ = Describe("Commands", func() { Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) - exists := client.Exists("key1") - Expect(exists.Err()).NotTo(HaveOccurred()) - Expect(exists.Val()).To(Equal(true)) + n, err := client.Exists("key1").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) - exists = client.Exists("key2") - Expect(exists.Err()).NotTo(HaveOccurred()) - Expect(exists.Val()).To(Equal(false)) + n, err = client.Exists("key2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) - existsMul := client.ExistsMulti("key1", "key2") - Expect(existsMul.Err()).NotTo(HaveOccurred()) - Expect(existsMul.Val()).To(Equal(int64(1))) + n, err = client.Exists("key1", "key2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) - existsMul = client.ExistsMulti("key1", "key1") - Expect(existsMul.Err()).NotTo(HaveOccurred()) - Expect(existsMul.Val()).To(Equal(int64(2))) + n, err = client.Exists("key1", "key1").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(2))) }) It("should Expire", func() { @@ -286,17 +286,17 @@ var _ = Describe("Commands", func() { Expect(set.Err()).NotTo(HaveOccurred()) Expect(set.Val()).To(Equal("OK")) - exists := client.Exists("key") - Expect(exists.Err()).NotTo(HaveOccurred()) - Expect(exists.Val()).To(Equal(true)) + n, err := client.Exists("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) expireAt := client.ExpireAt("key", time.Now().Add(-time.Hour)) Expect(expireAt.Err()).NotTo(HaveOccurred()) Expect(expireAt.Val()).To(Equal(true)) - exists = client.Exists("key") - Expect(exists.Err()).NotTo(HaveOccurred()) - Expect(exists.Val()).To(Equal(false)) + n, err = client.Exists("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) }) It("should Keys", func() { @@ -675,9 +675,9 @@ var _ = Describe("Commands", func() { Describe("strings", func() { It("should Append", func() { - exists := client.Exists("key") - Expect(exists.Err()).NotTo(HaveOccurred()) - Expect(exists.Val()).To(Equal(false)) + n, err := client.Exists("key").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) append := client.Append("key", "Hello") Expect(append.Err()).NotTo(HaveOccurred()) From 8040d63c4f268fee385d48d6fdb8db0d32685265 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 18 Feb 2017 16:42:34 +0200 Subject: [PATCH 0290/1746] Remove gopkg.in --- .travis.yml | 4 +--- bench_test.go | 2 +- cluster.go | 8 ++++---- cluster_test.go | 4 ++-- command.go | 6 +++--- command_test.go | 4 ++-- commands.go | 2 +- commands_test.go | 2 +- example_instrumentation_test.go | 2 +- example_test.go | 2 +- export_test.go | 2 +- internal/pool/bench_test.go | 2 +- internal/pool/conn.go | 2 +- internal/pool/pool.go | 2 +- internal/pool/pool_test.go | 4 ++-- internal/proto/reader.go | 2 +- internal/proto/reader_test.go | 2 +- internal/proto/scan.go | 2 +- internal/proto/write_buffer_test.go | 2 +- iterator_test.go | 4 ++-- main_test.go | 6 +++--- options.go | 2 +- parser.go | 2 +- pipeline.go | 2 +- pipeline_test.go | 2 +- pool_test.go | 4 ++-- pubsub.go | 4 ++-- pubsub_test.go | 2 +- race_test.go | 4 ++-- redis.go | 8 ++++---- redis_context.go | 2 +- redis_no_context.go | 2 +- redis_test.go | 2 +- ring.go | 8 ++++---- ring_test.go | 4 ++-- sentinel.go | 4 ++-- sentinel_test.go | 4 ++-- tx.go | 4 ++-- tx_test.go | 4 ++-- 39 files changed, 64 insertions(+), 66 deletions(-) diff --git a/.travis.yml b/.travis.yml index 41e06f8365..f8e0d652ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ services: go: - 1.4 - - 1.6 - 1.7 + - 1.8 - tip matrix: @@ -18,5 +18,3 @@ matrix: install: - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega - - mkdir -p $HOME/gopath/src/gopkg.in - - mv `pwd` $HOME/gopath/src/gopkg.in/redis.v5 diff --git a/bench_test.go b/bench_test.go index 2b4d45c584..021dba96b4 100644 --- a/bench_test.go +++ b/bench_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "gopkg.in/redis.v5" + "github.com/go-redis/redis" ) func benchmarkRedisClient(poolSize int) *redis.Client { diff --git a/cluster.go b/cluster.go index f2832a4b0b..d6d4e8426b 100644 --- a/cluster.go +++ b/cluster.go @@ -7,10 +7,10 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/hashtag" - "gopkg.in/redis.v5/internal/pool" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/hashtag" + "github.com/go-redis/redis/internal/pool" + "github.com/go-redis/redis/internal/proto" ) var errClusterNoNodes = internal.RedisError("redis: cluster has no nodes") diff --git a/cluster_test.go b/cluster_test.go index 589ef98049..4a7c4c1e24 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "gopkg.in/redis.v5" - "gopkg.in/redis.v5/internal/hashtag" + "github.com/go-redis/redis" + "github.com/go-redis/redis/internal/hashtag" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/command.go b/command.go index e28eb7581c..6ebb45c9c4 100644 --- a/command.go +++ b/command.go @@ -7,9 +7,9 @@ import ( "strings" "time" - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/pool" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/pool" + "github.com/go-redis/redis/internal/proto" ) var ( diff --git a/command_test.go b/command_test.go index 216a7b0c71..920f14b7c4 100644 --- a/command_test.go +++ b/command_test.go @@ -1,10 +1,10 @@ package redis_test import ( + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("Cmd", func() { diff --git a/commands.go b/commands.go index 9fcfadcc14..d8e5132288 100644 --- a/commands.go +++ b/commands.go @@ -5,7 +5,7 @@ import ( "strconv" "time" - "gopkg.in/redis.v5/internal" + "github.com/go-redis/redis/internal" ) func readTimeout(timeout time.Duration) time.Duration { diff --git a/commands_test.go b/commands_test.go index 29d6cb6c65..ec97c929eb 100644 --- a/commands_test.go +++ b/commands_test.go @@ -9,7 +9,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v5" + "github.com/go-redis/redis" ) var _ = Describe("Commands", func() { diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go index 009138ebdf..08fba0bfd0 100644 --- a/example_instrumentation_test.go +++ b/example_instrumentation_test.go @@ -5,7 +5,7 @@ import ( "sync/atomic" "time" - redis "gopkg.in/redis.v5" + "github.com/go-redis/redis" ) func Example_instrumentation() { diff --git a/example_test.go b/example_test.go index 45be04395c..3ecd0ba485 100644 --- a/example_test.go +++ b/example_test.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "gopkg.in/redis.v5" + "github.com/go-redis/redis" ) var client *redis.Client diff --git a/export_test.go b/export_test.go index a366dceeea..d043b0df80 100644 --- a/export_test.go +++ b/export_test.go @@ -3,7 +3,7 @@ package redis import ( "time" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal/pool" ) func (c *baseClient) Pool() pool.Pooler { diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index 9184fe7421..610e12c7c7 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal/pool" ) func benchmarkPoolGetPut(b *testing.B, poolSize int) { diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 2a135ae979..8af51d9de6 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -5,7 +5,7 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal/proto" ) var noDeadline = time.Time{} diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 4033e58dee..f89f9b3c86 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v5/internal" + "github.com/go-redis/redis/internal" ) var ( diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index f24f855f74..c2983dd06c 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" + "github.com/go-redis/redis/internal/pool" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5/internal/pool" ) var _ = Describe("ConnPool", func() { diff --git a/internal/proto/reader.go b/internal/proto/reader.go index e5dc95edb6..2159cf639a 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -6,7 +6,7 @@ import ( "io" "strconv" - "gopkg.in/redis.v5/internal" + "github.com/go-redis/redis/internal" ) const bytesAllocLimit = 1024 * 1024 // 1mb diff --git a/internal/proto/reader_test.go b/internal/proto/reader_test.go index 63046464ec..8d2d71be95 100644 --- a/internal/proto/reader_test.go +++ b/internal/proto/reader_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal/proto" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/internal/proto/scan.go b/internal/proto/scan.go index f3c75d9031..a73a369d5f 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -5,7 +5,7 @@ import ( "fmt" "reflect" - "gopkg.in/redis.v5/internal" + "github.com/go-redis/redis/internal" ) func Scan(b []byte, v interface{}) error { diff --git a/internal/proto/write_buffer_test.go b/internal/proto/write_buffer_test.go index c4438e4a92..84799ff3b2 100644 --- a/internal/proto/write_buffer_test.go +++ b/internal/proto/write_buffer_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal/proto" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/iterator_test.go b/iterator_test.go index 7200c14baa..954165cc6f 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -3,10 +3,10 @@ package redis_test import ( "fmt" + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("ScanIterator", func() { diff --git a/main_test.go b/main_test.go index e48dfc90f8..7c5a6a969f 100644 --- a/main_test.go +++ b/main_test.go @@ -12,10 +12,10 @@ import ( "testing" "time" + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) const ( @@ -95,7 +95,7 @@ var _ = AfterSuite(func() { func TestGinkgoSuite(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "gopkg.in/redis.v5") + RunSpecs(t, "go-redis") } //------------------------------------------------------------------------------ diff --git a/options.go b/options.go index bab7d4d594..1edc963c98 100644 --- a/options.go +++ b/options.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal/pool" ) type Options struct { diff --git a/parser.go b/parser.go index bba6096afd..ea73fb7803 100644 --- a/parser.go +++ b/parser.go @@ -6,7 +6,7 @@ import ( "strconv" "time" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal/proto" ) // Implements proto.MultiBulkParse diff --git a/pipeline.go b/pipeline.go index 13b29e1a6d..9e3ba0e683 100644 --- a/pipeline.go +++ b/pipeline.go @@ -4,7 +4,7 @@ import ( "errors" "sync" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal/pool" ) type pipelineExecer func([]Cmder) error diff --git a/pipeline_test.go b/pipeline_test.go index 14ba784c68..563bb66bdd 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -1,7 +1,7 @@ package redis_test import ( - "gopkg.in/redis.v5" + "github.com/go-redis/redis" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pool_test.go b/pool_test.go index 683f7c81f6..3cd5585b29 100644 --- a/pool_test.go +++ b/pool_test.go @@ -3,10 +3,10 @@ package redis_test import ( "time" + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("pool", func() { diff --git a/pubsub.go b/pubsub.go index f98566e89a..29a6823955 100644 --- a/pubsub.go +++ b/pubsub.go @@ -6,8 +6,8 @@ import ( "sync" "time" - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/pool" ) // PubSub implements Pub/Sub commands as described in diff --git a/pubsub_test.go b/pubsub_test.go index 22f3130732..4c74845515 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "gopkg.in/redis.v5" + "github.com/go-redis/redis" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/race_test.go b/race_test.go index 34973fe124..4529cf4963 100644 --- a/race_test.go +++ b/race_test.go @@ -8,10 +8,10 @@ import ( "testing" "time" + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("races", func() { diff --git a/redis.go b/redis.go index 1ddd754c5e..04c9a7b84f 100644 --- a/redis.go +++ b/redis.go @@ -1,13 +1,13 @@ -package redis // import "gopkg.in/redis.v5" +package redis import ( "fmt" "log" "time" - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/pool" - "gopkg.in/redis.v5/internal/proto" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/pool" + "github.com/go-redis/redis/internal/proto" ) // Redis nil reply, .e.g. when key does not exist. diff --git a/redis_context.go b/redis_context.go index bfa62bc5a2..6ec811ca5c 100644 --- a/redis_context.go +++ b/redis_context.go @@ -5,7 +5,7 @@ package redis import ( "context" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal/pool" ) type baseClient struct { diff --git a/redis_no_context.go b/redis_no_context.go index 78a5a637c5..0752192f15 100644 --- a/redis_no_context.go +++ b/redis_no_context.go @@ -3,7 +3,7 @@ package redis import ( - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal/pool" ) type baseClient struct { diff --git a/redis_test.go b/redis_test.go index c7ee7be5e3..3f47e91e81 100644 --- a/redis_test.go +++ b/redis_test.go @@ -5,7 +5,7 @@ import ( "net" "time" - "gopkg.in/redis.v5" + "github.com/go-redis/redis" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/ring.go b/ring.go index 6e5f62aef2..961b778051 100644 --- a/ring.go +++ b/ring.go @@ -9,10 +9,10 @@ import ( "sync/atomic" "time" - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/consistenthash" - "gopkg.in/redis.v5/internal/hashtag" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/consistenthash" + "github.com/go-redis/redis/internal/hashtag" + "github.com/go-redis/redis/internal/pool" ) var errRingShardsDown = errors.New("redis: all ring shards are down") diff --git a/ring_test.go b/ring_test.go index 1c1ae691b9..21adab2baa 100644 --- a/ring_test.go +++ b/ring_test.go @@ -5,10 +5,10 @@ import ( "fmt" "time" + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("Redis Ring", func() { diff --git a/sentinel.go b/sentinel.go index b39d365346..8070b46424 100644 --- a/sentinel.go +++ b/sentinel.go @@ -8,8 +8,8 @@ import ( "sync" "time" - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/pool" ) //------------------------------------------------------------------------------ diff --git a/sentinel_test.go b/sentinel_test.go index d7038575fe..04bbabd1e8 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -1,10 +1,10 @@ package redis_test import ( + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("Sentinel", func() { diff --git a/tx.go b/tx.go index 60d57cd2f6..762bddc90d 100644 --- a/tx.go +++ b/tx.go @@ -1,8 +1,8 @@ package redis import ( - "gopkg.in/redis.v5/internal" - "gopkg.in/redis.v5/internal/pool" + "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/pool" ) // Redis transaction failed. diff --git a/tx_test.go b/tx_test.go index 5ca22fdf1d..583ea05752 100644 --- a/tx_test.go +++ b/tx_test.go @@ -4,10 +4,10 @@ import ( "strconv" "sync" + "github.com/go-redis/redis" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "gopkg.in/redis.v5" ) var _ = Describe("Tx", func() { From c7a6d68f33a5aba2e91ef04e8cfa43960d37912b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 19 Feb 2017 09:56:12 +0200 Subject: [PATCH 0291/1746] readme: remove gopkg.in --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1127944b89..beaffe523b 100644 --- a/README.md +++ b/README.md @@ -3,34 +3,34 @@ Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. -- [Pub/Sub](https://godoc.org/gopkg.in/redis.v5#PubSub). -- [Transactions](https://godoc.org/gopkg.in/redis.v5#Multi). -- [Pipeline](https://godoc.org/gopkg.in/redis.v5#example-Client-Pipeline) and [TxPipeline](https://godoc.org/gopkg.in/redis.v5#example-Client-TxPipeline). -- [Scripting](https://godoc.org/gopkg.in/redis.v5#Script). -- [Timeouts](https://godoc.org/gopkg.in/redis.v5#Options). -- [Redis Sentinel](https://godoc.org/gopkg.in/redis.v5#NewFailoverClient). -- [Redis Cluster](https://godoc.org/gopkg.in/redis.v5#NewClusterClient). -- [Ring](https://godoc.org/gopkg.in/redis.v5#NewRing). -- [Instrumentation](https://godoc.org/gopkg.in/redis.v5#ex-package--Instrumentation). +- [Pub/Sub](https://godoc.org/github.com/go-redis/redis#PubSub). +- [Transactions](https://godoc.org/github.com/go-redis/redis#Multi). +- [Pipeline](https://godoc.org/github.com/go-redis/redis#example-Client-Pipeline) and [TxPipeline](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline). +- [Scripting](https://godoc.org/github.com/go-redis/redis#Script). +- [Timeouts](https://godoc.org/github.com/go-redis/redis#Options). +- [Redis Sentinel](https://godoc.org/github.com/go-redis/redis#NewFailoverClient). +- [Redis Cluster](https://godoc.org/github.com/go-redis/redis#NewClusterClient). +- [Ring](https://godoc.org/github.com/go-redis/redis#NewRing). +- [Instrumentation](https://godoc.org/github.com/go-redis/redis#ex-package--Instrumentation). - [Cache friendly](https://github.com/go-redis/cache). - [Rate limiting](https://github.com/go-redis/rate). - [Distributed Locks](https://github.com/bsm/redis-lock). -API docs: https://godoc.org/gopkg.in/redis.v5. -Examples: https://godoc.org/gopkg.in/redis.v5#pkg-examples. +API docs: https://godoc.org/github.com/go-redis/redis. +Examples: https://godoc.org/github.com/go-redis/redis#pkg-examples. ## Installation Install: ```shell -go get gopkg.in/redis.v5 +go get -u github.com/go-redis/redis ``` Import: ```go -import "gopkg.in/redis.v5" +import "github.com/go-redis/redis" ``` ## Quickstart @@ -75,7 +75,7 @@ func ExampleClient() { ## Howto -Please go through [examples](https://godoc.org/gopkg.in/redis.v5#pkg-examples) to get an idea how to use this package. +Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-examples) to get an idea how to use this package. ## Look and feel From 30412d7652bcc0a9843b982d3036db123f124e4e Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Fri, 17 Feb 2017 10:12:06 +0000 Subject: [PATCH 0292/1746] Added support for universal client --- cluster.go | 8 +-- cluster_test.go | 8 ++- example_test.go | 28 ++++++++++ universal.go | 134 ++++++++++++++++++++++++++++++++++++++++++++++ universal_test.go | 41 ++++++++++++++ 5 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 universal.go create mode 100644 universal_test.go diff --git a/cluster.go b/cluster.go index d6d4e8426b..52f357a46a 100644 --- a/cluster.go +++ b/cluster.go @@ -34,7 +34,8 @@ type ClusterOptions struct { // Following options are copied from Options struct. - Password string + MaxRetries int + Password string DialTimeout time.Duration ReadTimeout time.Duration @@ -63,8 +64,9 @@ func (opt *ClusterOptions) clientOptions() *Options { const disableIdleCheck = -1 return &Options{ - Password: opt.Password, - ReadOnly: opt.ReadOnly, + MaxRetries: opt.MaxRetries, + Password: opt.Password, + ReadOnly: opt.ReadOnly, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, diff --git a/cluster_test.go b/cluster_test.go index 4a7c4c1e24..49cb13cac5 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -39,12 +39,16 @@ func (s *clusterScenario) slaves() []*redis.Client { return result } -func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.ClusterClient { +func (s *clusterScenario) addrs() []string { addrs := make([]string, len(s.ports)) for i, port := range s.ports { addrs[i] = net.JoinHostPort("127.0.0.1", port) } - opt.Addrs = addrs + return addrs +} + +func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.ClusterClient { + opt.Addrs = s.addrs() return redis.NewClusterClient(opt) } diff --git a/example_test.go b/example_test.go index 3ecd0ba485..7899ab01ee 100644 --- a/example_test.go +++ b/example_test.go @@ -367,3 +367,31 @@ func ExampleScanCmd_Iterator() { panic(err) } } + +func ExampleNewUniversalClient_simple() { + client := redis.NewUniversalClient(&redis.UniversalOptions{ + Addrs: []string{":6379"}, + }) + defer client.Close() + + client.Ping() +} + +func ExampleNewUniversalClient_failover() { + client := redis.NewUniversalClient(&redis.UniversalOptions{ + MasterName: "master", + Addrs: []string{":26379"}, + }) + defer client.Close() + + client.Ping() +} + +func ExampleNewUniversalClient_cluster() { + client := redis.NewUniversalClient(&redis.UniversalOptions{ + Addrs: []string{":7000", ":7001", ":7002", ":7003", ":7004", ":7005"}, + }) + defer client.Close() + + client.Ping() +} diff --git a/universal.go b/universal.go new file mode 100644 index 0000000000..3c13b19674 --- /dev/null +++ b/universal.go @@ -0,0 +1,134 @@ +package redis + +import "time" + +// UniversalOptions information is required by UniversalClient to establish +// connections. +type UniversalOptions struct { + // Either a single address or a seed list of host:port addresses + // of cluster/sentinel nodes. + Addrs []string + + // The sentinel master name. + // Only failover clients. + MasterName string + + // Database to be selected after connecting to the server. + // Only single-node and failover clients. + DB int + + // Enables read only queries on slave nodes. + // Only cluster and single-node clients. + ReadOnly bool + + // Only cluster clients. + + MaxRedirects int + RouteByLatency bool + + // Common options + + MaxRetries int + Password string + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration + IdleCheckFrequency time.Duration +} + +func (o *UniversalOptions) cluster() *ClusterOptions { + if len(o.Addrs) == 0 { + o.Addrs = []string{"127.0.0.1:6379"} + } + + return &ClusterOptions{ + Addrs: o.Addrs, + MaxRedirects: o.MaxRedirects, + RouteByLatency: o.RouteByLatency, + ReadOnly: o.ReadOnly, + + MaxRetries: o.MaxRetries, + Password: o.Password, + DialTimeout: o.DialTimeout, + ReadTimeout: o.ReadTimeout, + WriteTimeout: o.WriteTimeout, + PoolSize: o.PoolSize, + PoolTimeout: o.PoolTimeout, + IdleTimeout: o.IdleTimeout, + IdleCheckFrequency: o.IdleCheckFrequency, + } +} + +func (o *UniversalOptions) failover() *FailoverOptions { + if len(o.Addrs) == 0 { + o.Addrs = []string{"127.0.0.1:26379"} + } + + return &FailoverOptions{ + SentinelAddrs: o.Addrs, + MasterName: o.MasterName, + DB: o.DB, + + MaxRetries: o.MaxRetries, + Password: o.Password, + DialTimeout: o.DialTimeout, + ReadTimeout: o.ReadTimeout, + WriteTimeout: o.WriteTimeout, + PoolSize: o.PoolSize, + PoolTimeout: o.PoolTimeout, + IdleTimeout: o.IdleTimeout, + IdleCheckFrequency: o.IdleCheckFrequency, + } +} + +func (o *UniversalOptions) simple() *Options { + addr := "127.0.0.1:6379" + if len(o.Addrs) > 0 { + addr = o.Addrs[0] + } + + return &Options{ + Addr: addr, + DB: o.DB, + ReadOnly: o.ReadOnly, + + MaxRetries: o.MaxRetries, + Password: o.Password, + DialTimeout: o.DialTimeout, + ReadTimeout: o.ReadTimeout, + WriteTimeout: o.WriteTimeout, + PoolSize: o.PoolSize, + PoolTimeout: o.PoolTimeout, + IdleTimeout: o.IdleTimeout, + IdleCheckFrequency: o.IdleCheckFrequency, + } +} + +// -------------------------------------------------------------------- + +// UniversalClient is an abstract client which - based on the provided options - +// can connect to either clusters, or sentinel-backed failover instances or simple +// single-instance servers. This can be useful for testing cluster-specific +// applications locally. +type UniversalClient interface { + Cmdable + Close() error +} + +// NewUniversalClient returns a new multi client. The type of client returned depends +// on the following three conditions: +// +// 1. if a MasterName is passed a sentinel-backed FailoverClient will be returned +// 2. if the number of Addrs is two or more, a ClusterClient will be returned +// 3. otherwise, a single-node redis Client will be returned. +func NewUniversalClient(opts *UniversalOptions) UniversalClient { + if opts.MasterName != "" { + return NewFailoverClient(opts.failover()) + } else if len(opts.Addrs) > 1 { + return NewClusterClient(opts.cluster()) + } + return NewClient(opts.simple()) +} diff --git a/universal_test.go b/universal_test.go new file mode 100644 index 0000000000..b05736c44f --- /dev/null +++ b/universal_test.go @@ -0,0 +1,41 @@ +package redis_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "gopkg.in/redis.v5" +) + +var _ = Describe("UniversalClient", func() { + var client redis.UniversalClient + + AfterEach(func() { + if client != nil { + Expect(client.Close()).To(Succeed()) + } + }) + + It("should connect to failover servers", func() { + client = redis.NewUniversalClient(&redis.UniversalOptions{ + MasterName: sentinelName, + Addrs: []string{":" + sentinelPort}, + }) + Expect(client.Ping().Err()).NotTo(HaveOccurred()) + }) + + It("should connect to simple servers", func() { + client = redis.NewUniversalClient(&redis.UniversalOptions{ + Addrs: []string{redisAddr}, + }) + Expect(client.Ping().Err()).NotTo(HaveOccurred()) + }) + + It("should connect to clusters", func() { + client = redis.NewUniversalClient(&redis.UniversalOptions{ + Addrs: cluster.addrs(), + }) + Expect(client.Ping().Err()).NotTo(HaveOccurred()) + }) + +}) From 271ccda18781782da1ce4f8b8ec4866d7e1f4782 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Mon, 20 Feb 2017 14:13:02 +0000 Subject: [PATCH 0293/1746] Update import --- README.md | 2 +- universal_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index beaffe523b..abd430c502 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=v5)](https://travis-ci.org/go-redis/redis) +# Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) Supports: diff --git a/universal_test.go b/universal_test.go index b05736c44f..2a0850dea2 100644 --- a/universal_test.go +++ b/universal_test.go @@ -4,7 +4,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "gopkg.in/redis.v5" + "github.com/go-redis/redis" ) var _ = Describe("UniversalClient", func() { From f605e59ade6da5562434f765d8acb80fd83a1b9b Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Tue, 21 Feb 2017 12:13:25 +0000 Subject: [PATCH 0294/1746] Expose Process on UniversalClient --- universal.go | 1 + 1 file changed, 1 insertion(+) diff --git a/universal.go b/universal.go index 3c13b19674..02ed51abd9 100644 --- a/universal.go +++ b/universal.go @@ -115,6 +115,7 @@ func (o *UniversalOptions) simple() *Options { // applications locally. type UniversalClient interface { Cmdable + Process(cmd Cmder) error Close() error } From 50f52107804d10282d4ec7df76d71b69391ac791 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 23 Feb 2017 15:19:43 +0200 Subject: [PATCH 0295/1746] internal/pool: remove unused var --- internal/pool/pool.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index f89f9b3c86..34ec539a99 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -332,19 +332,3 @@ func (p *ConnPool) reaper(frequency time.Duration) { ) } } - -//------------------------------------------------------------------------------ - -var idleCheckFrequency atomic.Value - -func SetIdleCheckFrequency(d time.Duration) { - idleCheckFrequency.Store(d) -} - -func getIdleCheckFrequency() time.Duration { - v := idleCheckFrequency.Load() - if v == nil { - return time.Minute - } - return v.(time.Duration) -} From 58e9c55d8e82abbc9233ddb1e3c11007983c6d36 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 23 Feb 2017 15:29:38 +0200 Subject: [PATCH 0296/1746] Use simple PING for compatibility --- pubsub.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pubsub.go b/pubsub.go index 29a6823955..d183c70abd 100644 --- a/pubsub.go +++ b/pubsub.go @@ -98,10 +98,10 @@ func (c *PubSub) Close() error { return c.base.Close() } -func (c *PubSub) Ping(payload string) error { +func (c *PubSub) Ping(payload ...string) error { args := []interface{}{"PING"} - if payload != "" { - args = append(args, payload) + if len(payload) == 1 { + args = append(args, payload[0]) } cmd := NewCmd(args...) @@ -239,7 +239,7 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { errNum++ if errNum < 3 { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - err := c.Ping("hello") + err := c.Ping() if err != nil { internal.Logf("PubSub.Ping failed: %s", err) } From c86c141c389296f6b279ccebc75b7594a3f7f1fb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 23 Feb 2017 15:29:38 +0200 Subject: [PATCH 0297/1746] Use simple PING for compatibility --- pubsub.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pubsub.go b/pubsub.go index f98566e89a..6c18229614 100644 --- a/pubsub.go +++ b/pubsub.go @@ -98,10 +98,10 @@ func (c *PubSub) Close() error { return c.base.Close() } -func (c *PubSub) Ping(payload string) error { +func (c *PubSub) Ping(payload ...string) error { args := []interface{}{"PING"} - if payload != "" { - args = append(args, payload) + if len(payload) == 1 { + args = append(args, payload[0]) } cmd := NewCmd(args...) @@ -239,7 +239,7 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { errNum++ if errNum < 3 { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - err := c.Ping("hello") + err := c.Ping() if err != nil { internal.Logf("PubSub.Ping failed: %s", err) } From 892fb8d573f4e03aabc53100df0473a24ace8516 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Fri, 24 Feb 2017 18:03:21 +0900 Subject: [PATCH 0298/1746] the timeout of WAIT command is in milliseconds. --- commands.go | 2 +- commands_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index bc93df9f13..e2dc0caa14 100644 --- a/commands.go +++ b/commands.go @@ -275,7 +275,7 @@ func (c *cmdable) Ping() *StatusCmd { func (c *cmdable) Wait(numSlaves int, timeout time.Duration) *IntCmd { - cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Second)) + cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Millisecond)) c.process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index a524aafc31..7e4182e673 100644 --- a/commands_test.go +++ b/commands_test.go @@ -52,9 +52,11 @@ var _ = Describe("Commands", func() { It("should Wait", func() { // assume testing on single redis instance - wait := client.Wait(0, time.Minute) + start := time.Now() + wait := client.Wait(1, time.Second) Expect(wait.Err()).NotTo(HaveOccurred()) Expect(wait.Val()).To(Equal(int64(0))) + Expect(time.Now()).To(BeTemporally("~", start.Add(time.Second), 800*time.Millisecond)) }) It("should Select", func() { From b8fb4a11502ff6fa5810e59e771e4caf72cec5a6 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Fri, 24 Feb 2017 18:03:21 +0900 Subject: [PATCH 0299/1746] the timeout of WAIT command is in milliseconds. --- commands.go | 2 +- commands_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index d8e5132288..f941da8382 100644 --- a/commands.go +++ b/commands.go @@ -273,7 +273,7 @@ func (c *cmdable) Ping() *StatusCmd { func (c *cmdable) Wait(numSlaves int, timeout time.Duration) *IntCmd { - cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Second)) + cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Millisecond)) c.process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index ec97c929eb..5f060ecf93 100644 --- a/commands_test.go +++ b/commands_test.go @@ -52,9 +52,11 @@ var _ = Describe("Commands", func() { It("should Wait", func() { // assume testing on single redis instance - wait := client.Wait(0, time.Minute) + start := time.Now() + wait := client.Wait(1, time.Second) Expect(wait.Err()).NotTo(HaveOccurred()) Expect(wait.Val()).To(Equal(int64(0))) + Expect(time.Now()).To(BeTemporally("~", start.Add(time.Second), 800*time.Millisecond)) }) It("should Select", func() { From d9f1dc2386f0324418a3aa10af2bef4133b58a25 Mon Sep 17 00:00:00 2001 From: "Giovanni T. Parra" Date: Fri, 24 Feb 2017 22:01:11 -0300 Subject: [PATCH 0300/1746] Mention SETEX in the documentation. To help people looking for it specifically. --- commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.go b/commands.go index e2dc0caa14..b96da7b77c 100644 --- a/commands.go +++ b/commands.go @@ -734,6 +734,7 @@ func (c *cmdable) MSetNX(pairs ...interface{}) *BoolCmd { // Redis `SET key value [expiration]` command. // +// Use expiration for `SETEX`-like behavior. // Zero expiration means the key has no expiration time. func (c *cmdable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { args := make([]interface{}, 3, 4) From 518c1051693a15326f82e764722facf6d025b4c7 Mon Sep 17 00:00:00 2001 From: Yin Jifeng Date: Wed, 1 Mar 2017 22:41:18 +0800 Subject: [PATCH 0301/1746] StringCmd: remove an unnessary type casting this fixes up 69554c0e --- command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command.go b/command.go index e28eb7581c..8ad982b0c3 100644 --- a/command.go +++ b/command.go @@ -445,7 +445,7 @@ func (cmd *StringCmd) Result() (string, error) { } func (cmd *StringCmd) Bytes() ([]byte, error) { - return []byte(cmd.val), cmd.err + return cmd.val, cmd.err } func (cmd *StringCmd) Int64() (int64, error) { From 216ec11a0e8175294eb2f983ef73bbe94be6d7ec Mon Sep 17 00:00:00 2001 From: yuekui Date: Fri, 3 Mar 2017 15:45:40 -0800 Subject: [PATCH 0302/1746] Fix wrong usage of timer Reset(), which could cause service frozen during master switch. --- internal/pool/pool.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 4033e58dee..c97875f4e6 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -19,7 +19,9 @@ var ( var timers = sync.Pool{ New: func() interface{} { - return time.NewTimer(0) + t := time.NewTimer(time.Hour) + t.Stop() + return t }, } @@ -95,12 +97,13 @@ func (p *ConnPool) NewConn() (*Conn, error) { func (p *ConnPool) PopFree() *Conn { timer := timers.Get().(*time.Timer) - if !timer.Reset(p.poolTimeout) { - <-timer.C - } + timer.Reset(p.poolTimeout) select { case p.queue <- struct{}{}: + if !timer.Stop() { + <-timer.C + } timers.Put(timer) case <-timer.C: timers.Put(timer) @@ -138,12 +141,13 @@ func (p *ConnPool) Get() (*Conn, bool, error) { atomic.AddUint32(&p.stats.Requests, 1) timer := timers.Get().(*time.Timer) - if !timer.Reset(p.poolTimeout) { - <-timer.C - } + timer.Reset(p.poolTimeout) select { case p.queue <- struct{}{}: + if !timer.Stop() { + <-timer.C + } timers.Put(timer) case <-timer.C: timers.Put(timer) From 15998effbe309a5f2011aaecfe58c870c70bd5f5 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 4 Mar 2017 13:04:27 +0200 Subject: [PATCH 0303/1746] Don't panic if cluster state is nil. --- cluster.go | 93 +++++++++++++++++++++++++++++++------------------ cluster_test.go | 2 +- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/cluster.go b/cluster.go index f2832a4b0b..a5a16db706 100644 --- a/cluster.go +++ b/cluster.go @@ -14,6 +14,7 @@ import ( ) var errClusterNoNodes = internal.RedisError("redis: cluster has no nodes") +var errNilClusterState = internal.RedisError("redis: cannot load cluster slots") // ClusterOptions are used to configure a cluster client and should be // passed to NewClusterClient. @@ -355,7 +356,14 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { _, _ = c.nodes.Get(addr) } - c.reloadSlots() + // Preload cluster slots. + for i := 0; i < 10; i++ { + state, err := c.reloadSlots() + if err == nil { + c._state.Store(state) + break + } + } if opt.IdleCheckFrequency > 0 { go c.reaper(opt.IdleCheckFrequency) @@ -366,10 +374,11 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { func (c *ClusterClient) state() *clusterState { v := c._state.Load() - if v == nil { - return nil + if v != nil { + return v.(*clusterState) } - return v.(*clusterState) + c.lazyReloadSlots() + return nil } func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { @@ -397,10 +406,12 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl } func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { + state := c.state() + var node *clusterNode var err error - if len(keys) > 0 { - node, err = c.state().slotMasterNode(hashtag.Slot(keys[0])) + if state != nil && len(keys) > 0 { + node, err = state.slotMasterNode(hashtag.Slot(keys[0])) } else { node, err = c.nodes.Random() } @@ -463,8 +474,9 @@ func (c *ClusterClient) Process(cmd Cmder) error { var addr string moved, ask, addr = internal.IsMovedError(err) if moved || ask { - if slot >= 0 { - master, _ := c.state().slotMasterNode(slot) + state := c.state() + if state != nil && slot >= 0 { + master, _ := state.slotMasterNode(slot) if moved && (master == nil || master.Client.getAddr() != addr) { c.lazyReloadSlots() } @@ -523,7 +535,7 @@ func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { state := c.state() if state == nil { - return nil + return errNilClusterState } var wg sync.WaitGroup @@ -564,12 +576,13 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { // PoolStats returns accumulated connection pool stats. func (c *ClusterClient) PoolStats() *PoolStats { + var acc PoolStats + nodes, err := c.nodes.All() if err != nil { - return nil + return &acc } - var acc PoolStats for _, node := range nodes { s := node.Client.connPool.Stats() acc.Requests += s.Requests @@ -585,37 +598,46 @@ func (c *ClusterClient) lazyReloadSlots() { if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { return } + go func() { - c.reloadSlots() + for i := 0; i < 1000; i++ { + state, err := c.reloadSlots() + if err == pool.ErrClosed { + break + } + if err == nil { + c._state.Store(state) + break + } + time.Sleep(time.Millisecond) + } + + time.Sleep(3 * time.Second) atomic.StoreUint32(&c.reloading, 0) }() } -func (c *ClusterClient) reloadSlots() { - for i := 0; i < 10; i++ { - node, err := c.nodes.Random() - if err != nil { - return - } - - if c.cmds == nil { - cmds, err := node.Client.Command().Result() - if err == nil { - c.cmds = cmds - } - } +func (c *ClusterClient) reloadSlots() (*clusterState, error) { + node, err := c.nodes.Random() + if err != nil { + return nil, err + } - slots, err := node.Client.ClusterSlots().Result() + // TODO: fix race + if c.cmds == nil { + cmds, err := node.Client.Command().Result() if err != nil { - continue + return nil, err } + c.cmds = cmds + } - state, err := newClusterState(c.nodes, slots) - if err != nil { - return - } - c._state.Store(state) + slots, err := node.Client.ClusterSlots().Result() + if err != nil { + return nil, err } + + return newClusterState(c.nodes, slots) } // reaper closes idle connections to the cluster. @@ -789,8 +811,13 @@ func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { return err } + state := c.state() + if state == nil { + return errNilClusterState + } + for slot, cmds := range cmdsMap { - node, err := c.state().slotMasterNode(slot) + node, err := state.slotMasterNode(slot) if err != nil { setCmdsErr(cmds, err) continue diff --git a/cluster_test.go b/cluster_test.go index 589ef98049..28314bc2fd 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -578,7 +578,7 @@ var _ = Describe("ClusterClient timeout", func() { var client *redis.ClusterClient AfterEach(func() { - Expect(client.Close()).NotTo(HaveOccurred()) + _ = client.Close() }) testTimeout := func() { From ab1e8de9ea33fe63e2de8467a4ee5a98ff3524f2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 20 Mar 2017 12:15:21 +0200 Subject: [PATCH 0304/1746] Add ability to read client options --- cluster.go | 5 +++++ redis.go | 5 +++++ ring.go | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/cluster.go b/cluster.go index a5a16db706..7d7b02602e 100644 --- a/cluster.go +++ b/cluster.go @@ -372,6 +372,11 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } +// Options returns read-only Options that were used to create the client. +func (c *ClusterClient) Options() *ClusterOptions { + return c.opt +} + func (c *ClusterClient) state() *clusterState { v := c._state.Load() if v != nil { diff --git a/redis.go b/redis.go index 1ddd754c5e..74d677b0c5 100644 --- a/redis.go +++ b/redis.go @@ -21,6 +21,11 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB) } +// Options returns read-only Options that were used to create the client. +func (c *baseClient) Options() *Options { + return c.opt +} + func (c *baseClient) conn() (*pool.Conn, bool, error) { cn, isNew, err := c.connPool.Get() if err != nil { diff --git a/ring.go b/ring.go index 6e5f62aef2..9b03f97f03 100644 --- a/ring.go +++ b/ring.go @@ -158,6 +158,11 @@ func NewRing(opt *RingOptions) *Ring { return ring } +// Options returns read-only Options that were used to create the client. +func (c *Ring) Options() *RingOptions { + return c.opt +} + // PoolStats returns accumulated connection pool stats. func (c *Ring) PoolStats() *PoolStats { var acc PoolStats From 14f25982cbf6c4821fd7fa5a06224cedf1770850 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 24 Mar 2017 12:48:32 +0200 Subject: [PATCH 0305/1746] Remove manual strconv --- commands.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/commands.go b/commands.go index cac75fe9b2..fb0e9105fb 100644 --- a/commands.go +++ b/commands.go @@ -2,7 +2,6 @@ package redis import ( "io" - "strconv" "time" "github.com/go-redis/redis/internal" @@ -19,24 +18,24 @@ func usePrecise(dur time.Duration) bool { return dur < time.Second || dur%time.Second != 0 } -func formatMs(dur time.Duration) string { +func formatMs(dur time.Duration) int64 { if dur > 0 && dur < time.Millisecond { internal.Logf( "specified duration is %s, but minimal supported value is %s", dur, time.Millisecond, ) } - return strconv.FormatInt(int64(dur/time.Millisecond), 10) + return int64(dur / time.Millisecond) } -func formatSec(dur time.Duration) string { +func formatSec(dur time.Duration) int64 { if dur > 0 && dur < time.Second { internal.Logf( "specified duration is %s, but minimal supported value is %s", dur, time.Second, ) } - return strconv.FormatInt(int64(dur/time.Second), 10) + return int64(dur / time.Second) } type Cmdable interface { @@ -1340,7 +1339,7 @@ func (c *cmdable) ZInterStore(destination string, store ZStore, keys ...string) args := make([]interface{}, 3+len(keys)) args[0] = "zinterstore" args[1] = destination - args[2] = strconv.Itoa(len(keys)) + args[2] = len(keys) for i, key := range keys { args[3+i] = key } @@ -1536,7 +1535,7 @@ func (c *cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd args := make([]interface{}, 3+len(keys)) args[0] = "zunionstore" args[1] = dest - args[2] = strconv.Itoa(len(keys)) + args[2] = len(keys) for i, key := range keys { args[3+i] = key } @@ -1755,7 +1754,7 @@ func (c *cmdable) Eval(script string, keys []string, args ...interface{}) *Cmd { cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "eval" cmdArgs[1] = script - cmdArgs[2] = strconv.Itoa(len(keys)) + cmdArgs[2] = len(keys) for i, key := range keys { cmdArgs[3+i] = key } @@ -1772,7 +1771,7 @@ func (c *cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd cmdArgs := make([]interface{}, 3+len(keys)+len(args)) cmdArgs[0] = "evalsha" cmdArgs[1] = sha1 - cmdArgs[2] = strconv.Itoa(len(keys)) + cmdArgs[2] = len(keys) for i, key := range keys { cmdArgs[3+i] = key } @@ -1984,7 +1983,7 @@ func (c *cmdable) ClusterAddSlots(slots ...int) *StatusCmd { args[0] = "cluster" args[1] = "addslots" for i, num := range slots { - args[2+i] = strconv.Itoa(num) + args[2+i] = num } cmd := NewStatusCmd(args...) c.process(cmd) From 5a99b806bd516fdaff2395019c2590357c15e544 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 24 Mar 2017 12:48:43 +0200 Subject: [PATCH 0306/1746] Add cluster benchmark --- cluster_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/cluster_test.go b/cluster_test.go index 53a3363153..d9db8975e9 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -1,6 +1,7 @@ package redis_test import ( + "bytes" "fmt" "net" "strconv" @@ -704,3 +705,36 @@ func BenchmarkRedisClusterPing(b *testing.B) { } }) } + +func BenchmarkRedisClusterSetString(b *testing.B) { + if testing.Short() { + b.Skip("skipping in short mode") + } + + cluster := &clusterScenario{ + ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, + nodeIds: make([]string, 6), + processes: make(map[string]*redisProcess, 6), + clients: make(map[string]*redis.Client, 6), + } + + if err := startCluster(cluster); err != nil { + b.Fatal(err) + } + defer stopCluster(cluster) + + client := cluster.clusterClient(redisClusterOptions()) + defer client.Close() + + value := string(bytes.Repeat([]byte{'1'}, 10000)) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := client.Set("key", value, 0).Err(); err != nil { + b.Fatal(err) + } + } + }) +} From 18dcec21440a5c6e96da08fcdce732bb9436c190 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 24 Mar 2017 13:33:35 +0200 Subject: [PATCH 0307/1746] Cleanup code --- command.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/command.go b/command.go index 2cc3b64acf..0209ebde49 100644 --- a/command.go +++ b/command.go @@ -149,14 +149,6 @@ func (cmd *baseCmd) setErr(e error) { cmd.err = e } -func newBaseCmd(args []interface{}) baseCmd { - if len(args) > 0 { - // Cmd name is expected to be in lower case. - args[0] = internal.ToLower(args[0].(string)) - } - return baseCmd{_args: args} -} - //------------------------------------------------------------------------------ type Cmd struct { @@ -840,9 +832,8 @@ func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { if q.Sort != "" { args = append(args, q.Sort) } - cmd := newBaseCmd(args) return &GeoLocationCmd{ - baseCmd: cmd, + baseCmd: baseCmd{_args: args}, q: q, } } From ef95182d291700e010dbed10c0998dc58575ad11 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 31 Mar 2017 15:11:11 +0300 Subject: [PATCH 0308/1746] Code cleanup --- cluster.go | 2 -- commands_test.go | 1 + internal/pool/pool.go | 1 - internal/proto/write_buffer.go | 2 -- parser.go | 26 -------------------------- pubsub.go | 2 -- redis_test.go | 2 ++ 7 files changed, 3 insertions(+), 33 deletions(-) diff --git a/cluster.go b/cluster.go index 0e97da10e2..251214c1f4 100644 --- a/cluster.go +++ b/cluster.go @@ -338,8 +338,6 @@ type ClusterClient struct { // Reports where slots reloading is in progress. reloading uint32 - - closed bool } // NewClusterClient returns a Redis Cluster client as described in diff --git a/commands_test.go b/commands_test.go index 5f060ecf93..a123133413 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2316,6 +2316,7 @@ var _ = Describe("Commands", func() { zAdd = client.ZAdd("zset", redis.Z{0, "b"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) zAdd = client.ZAdd("zset", redis.Z{0, "c"}) + Expect(zAdd.Err()).NotTo(HaveOccurred()) zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "-", diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 47cedfa12e..1e5fac6fd5 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -65,7 +65,6 @@ type ConnPool struct { stats Stats _closed int32 // atomic - lastErr atomic.Value } var _ Pooler = (*ConnPool)(nil) diff --git a/internal/proto/write_buffer.go b/internal/proto/write_buffer.go index 019e64c873..096b6d76af 100644 --- a/internal/proto/write_buffer.go +++ b/internal/proto/write_buffer.go @@ -6,8 +6,6 @@ import ( "strconv" ) -const bufferSize = 4096 - type WriteBuffer struct { b []byte } diff --git a/parser.go b/parser.go index ea73fb7803..1d7ec630e4 100644 --- a/parser.go +++ b/parser.go @@ -30,19 +30,6 @@ func sliceParser(rd *proto.Reader, n int64) (interface{}, error) { return vals, nil } -// Implements proto.MultiBulkParse -func intSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - ints := make([]int64, 0, n) - for i := int64(0); i < n; i++ { - n, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - ints = append(ints, n) - } - return ints, nil -} - // Implements proto.MultiBulkParse func boolSliceParser(rd *proto.Reader, n int64) (interface{}, error) { bools := make([]bool, 0, n) @@ -72,19 +59,6 @@ func stringSliceParser(rd *proto.Reader, n int64) (interface{}, error) { return ss, nil } -// Implements proto.MultiBulkParse -func floatSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - nn := make([]float64, 0, n) - for i := int64(0); i < n; i++ { - n, err := rd.ReadFloatReply() - if err != nil { - return nil, err - } - nn = append(nn, n) - } - return nn, nil -} - // Implements proto.MultiBulkParse func stringStringMapParser(rd *proto.Reader, n int64) (interface{}, error) { m := make(map[string]string, n/2) diff --git a/pubsub.go b/pubsub.go index d183c70abd..b90d72712f 100644 --- a/pubsub.go +++ b/pubsub.go @@ -3,7 +3,6 @@ package redis import ( "fmt" "net" - "sync" "time" "github.com/go-redis/redis/internal" @@ -17,7 +16,6 @@ type PubSub struct { base baseClient cmd *Cmd - mu sync.Mutex channels []string patterns []string } diff --git a/redis_test.go b/redis_test.go index 3f47e91e81..64cb2a4fc3 100644 --- a/redis_test.go +++ b/redis_test.go @@ -59,6 +59,7 @@ var _ = Describe("Client", func() { It("should close pubsub without closing the client", func() { pubsub, err := client.Subscribe() + Expect(err).NotTo(HaveOccurred()) Expect(pubsub.Close()).NotTo(HaveOccurred()) _, err = pubsub.Receive() @@ -92,6 +93,7 @@ var _ = Describe("Client", func() { It("should close pubsub when client is closed", func() { pubsub, err := client.Subscribe() + Expect(err).NotTo(HaveOccurred()) Expect(client.Close()).NotTo(HaveOccurred()) _, err = pubsub.Receive() From 80673992e6ffabb0bbea74d44cf4bff373e4c2b2 Mon Sep 17 00:00:00 2001 From: Poloskin Valentin Georgievich Date: Sun, 2 Apr 2017 17:10:47 +0300 Subject: [PATCH 0309/1746] gofmt --- cluster.go | 2 +- example_instrumentation_test.go | 2 +- internal/pool/pool.go | 2 +- ring.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 251214c1f4..9bc650659c 100644 --- a/cluster.go +++ b/cluster.go @@ -650,7 +650,7 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { ticker := time.NewTicker(idleCheckFrequency) defer ticker.Stop() - for _ = range ticker.C { + for range ticker.C { nodes, err := c.nodes.All() if err != nil { break diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go index 08fba0bfd0..02051f9c9c 100644 --- a/example_instrumentation_test.go +++ b/example_instrumentation_test.go @@ -29,7 +29,7 @@ func wrapRedisProcess(client *redis.Client) { var count, avgDur uint32 go func() { - for _ = range time.Tick(3 * time.Second) { + for range time.Tick(3 * time.Second) { n := atomic.LoadUint32(&count) dur := time.Duration(atomic.LoadUint32(&avgDur)) * precision fmt.Printf("%s: processed=%d avg_dur=%s\n", client, n, dur) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 1e5fac6fd5..45743fed70 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -319,7 +319,7 @@ func (p *ConnPool) reaper(frequency time.Duration) { ticker := time.NewTicker(frequency) defer ticker.Stop() - for _ = range ticker.C { + for range ticker.C { if p.closed() { break } diff --git a/ring.go b/ring.go index 6b4d77b1e2..eb5add1064 100644 --- a/ring.go +++ b/ring.go @@ -302,7 +302,7 @@ func (c *Ring) rebalance() { func (c *Ring) heartbeat() { ticker := time.NewTicker(c.opt.HeartbeatFrequency) defer ticker.Stop() - for _ = range ticker.C { + for range ticker.C { var rebalance bool c.mu.RLock() From dd76391eb986c2ea0aba41e9edde61ff7e0d4b28 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 5 Apr 2017 15:32:37 +0300 Subject: [PATCH 0310/1746] Tweak readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index abd430c502..99ca9b6b81 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,8 @@ BenchmarkRedisPing-4 200000 6983 ns/op 116 B/op BenchmarkRedisClusterPing-4 100000 11535 ns/op 117 B/op 4 allocs/op ``` -## Shameless plug +## See also -Check my [PostgreSQL client for Go](https://github.com/go-pg/pg). +- [Golang PostgreSQL ORM](https://github.com/go-pg/pg) +- [Golang msgpack](https://github.com/vmihailenco/msgpack) +- [Golang message task queue](https://github.com/go-msgqueue/msgqueue) From 2e4944712d9d049bf534f94e6d91fe25c8e1b8df Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 8 Apr 2017 09:35:56 +0300 Subject: [PATCH 0311/1746] Document Script.Run. Fixes #536 --- script.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script.go b/script.go index e569af7b8c..133117b20d 100644 --- a/script.go +++ b/script.go @@ -47,6 +47,8 @@ func (s *Script) EvalSha(c scripter, keys []string, args ...interface{}) *Cmd { return c.EvalSha(s.hash, keys, args...) } +// Run optimistically uses EVALSHA to run the script. If script does not exist +// it is retried using EVAL. func (s *Script) Run(c scripter, keys []string, args ...interface{}) *Cmd { r := s.EvalSha(c, keys, args...) if err := r.Err(); err != nil && strings.HasPrefix(err.Error(), "NOSCRIPT ") { From 34ea5f98ebe4e3e4df85809ecc5818bb9f5e77c3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 11 Apr 2017 16:18:35 +0300 Subject: [PATCH 0312/1746] Add Channel helper --- pubsub.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pubsub.go b/pubsub.go index b90d72712f..e681949b2f 100644 --- a/pubsub.go +++ b/pubsub.go @@ -280,6 +280,26 @@ func (c *PubSub) resubscribe() { } } +// Channel returns a channel for concurrently receiving messages. +// The channel is closed with PubSub. +func (c *PubSub) Channel() <-chan *Message { + ch := make(chan *Message, 100) + go func() { + for { + msg, err := c.ReceiveMessage() + if err != nil { + if err == pool.ErrClosed { + break + } + continue + } + ch <- msg + } + close(ch) + }() + return ch +} + func remove(ss []string, es ...string) []string { if len(es) == 0 { return ss[:0] From 8d52a95269db9577017b2827d8ca238388ed202f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 11 Apr 2017 16:53:55 +0300 Subject: [PATCH 0313/1746] Simplify PubSub API --- example_test.go | 12 +++--------- pool_test.go | 3 +-- pubsub.go | 16 ++++----------- pubsub_test.go | 52 ++++++++++++++++++------------------------------- race_test.go | 5 ++--- redis.go | 18 ++++++----------- redis_test.go | 13 ++++++------- 7 files changed, 41 insertions(+), 78 deletions(-) diff --git a/example_test.go b/example_test.go index 7899ab01ee..98ee2dd098 100644 --- a/example_test.go +++ b/example_test.go @@ -258,13 +258,10 @@ func ExampleClient_Watch() { } func ExamplePubSub() { - pubsub, err := client.Subscribe("mychannel1") - if err != nil { - panic(err) - } + pubsub := client.Subscribe("mychannel1") defer pubsub.Close() - err = client.Publish("mychannel1", "hello").Err() + err := client.Publish("mychannel1", "hello").Err() if err != nil { panic(err) } @@ -279,10 +276,7 @@ func ExamplePubSub() { } func ExamplePubSub_Receive() { - pubsub, err := client.Subscribe("mychannel2") - if err != nil { - panic(err) - } + pubsub := client.Subscribe("mychannel2") defer pubsub.Close() n, err := client.Publish("mychannel2", "hello").Result() diff --git a/pool_test.go b/pool_test.go index 3cd5585b29..c6731e4866 100644 --- a/pool_test.go +++ b/pool_test.go @@ -81,8 +81,7 @@ var _ = Describe("pool", func() { connPool := client.Pool() perform(1000, func(id int) { - pubsub, err := client.Subscribe() - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("test") Expect(pubsub.Close()).NotTo(HaveOccurred()) }) diff --git a/pubsub.go b/pubsub.go index e681949b2f..8705a137c4 100644 --- a/pubsub.go +++ b/pubsub.go @@ -57,18 +57,14 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { // Subscribes the client to the specified channels. func (c *PubSub) Subscribe(channels ...string) error { err := c.subscribe("SUBSCRIBE", channels...) - if err == nil { - c.channels = appendIfNotExists(c.channels, channels...) - } + c.channels = appendIfNotExists(c.channels, channels...) return err } // Subscribes the client to the given patterns. func (c *PubSub) PSubscribe(patterns ...string) error { err := c.subscribe("PSUBSCRIBE", patterns...) - if err == nil { - c.patterns = appendIfNotExists(c.patterns, patterns...) - } + c.patterns = appendIfNotExists(c.patterns, patterns...) return err } @@ -76,9 +72,7 @@ func (c *PubSub) PSubscribe(patterns ...string) error { // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { err := c.subscribe("UNSUBSCRIBE", channels...) - if err == nil { - c.channels = remove(c.channels, channels...) - } + c.channels = remove(c.channels, channels...) return err } @@ -86,9 +80,7 @@ func (c *PubSub) Unsubscribe(channels ...string) error { // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { err := c.subscribe("PUNSUBSCRIBE", patterns...) - if err == nil { - c.patterns = remove(c.patterns, patterns...) - } + c.patterns = remove(c.patterns, patterns...) return err } diff --git a/pubsub_test.go b/pubsub_test.go index 4c74845515..0164805ab4 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -25,8 +25,7 @@ var _ = Describe("PubSub", func() { }) It("should support pattern matching", func() { - pubsub, err := client.PSubscribe("mychannel*") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.PSubscribe("mychannel*") defer pubsub.Close() { @@ -77,8 +76,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(channels).To(BeEmpty()) - pubsub, err := client.Subscribe("mychannel", "mychannel2") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel", "mychannel2") defer pubsub.Close() channels, err = client.PubSubChannels("mychannel*").Result() @@ -95,8 +93,7 @@ var _ = Describe("PubSub", func() { }) It("should return the numbers of subscribers", func() { - pubsub, err := client.Subscribe("mychannel", "mychannel2") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel", "mychannel2") defer pubsub.Close() channels, err := client.PubSubNumSub("mychannel", "mychannel2", "mychannel3").Result() @@ -113,8 +110,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(num).To(Equal(int64(0))) - pubsub, err := client.PSubscribe("*") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.PSubscribe("*") defer pubsub.Close() num, err = client.PubSubNumPat().Result() @@ -123,8 +119,7 @@ var _ = Describe("PubSub", func() { }) It("should pub/sub", func() { - pubsub, err := client.Subscribe("mychannel", "mychannel2") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel", "mychannel2") defer pubsub.Close() { @@ -200,11 +195,10 @@ var _ = Describe("PubSub", func() { }) It("should ping/pong", func() { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel") defer pubsub.Close() - _, err = pubsub.ReceiveTimeout(time.Second) + _, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) err = pubsub.Ping("") @@ -217,11 +211,10 @@ var _ = Describe("PubSub", func() { }) It("should ping/pong with payload", func() { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel") defer pubsub.Close() - _, err = pubsub.ReceiveTimeout(time.Second) + _, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) err = pubsub.Ping("hello") @@ -234,11 +227,10 @@ var _ = Describe("PubSub", func() { }) It("should multi-ReceiveMessage", func() { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel") defer pubsub.Close() - err = client.Publish("mychannel", "hello").Err() + err := client.Publish("mychannel", "hello").Err() Expect(err).NotTo(HaveOccurred()) err = client.Publish("mychannel", "world").Err() @@ -258,8 +250,7 @@ var _ = Describe("PubSub", func() { It("should ReceiveMessage after timeout", func() { timeout := 100 * time.Millisecond - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel") defer pubsub.Close() done := make(chan bool, 1) @@ -321,24 +312,21 @@ var _ = Describe("PubSub", func() { } It("Subscribe should reconnect on ReceiveMessage error", func() { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel") defer pubsub.Close() expectReceiveMessageOnError(pubsub) }) It("PSubscribe should reconnect on ReceiveMessage error", func() { - pubsub, err := client.PSubscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.PSubscribe("mychannel") defer pubsub.Close() expectReceiveMessageOnError(pubsub) }) It("should return on Close", func() { - pubsub, err := client.Subscribe("mychannel") - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe("mychannel") defer pubsub.Close() var wg sync.WaitGroup @@ -360,8 +348,7 @@ var _ = Describe("PubSub", func() { wg.Wait() wg.Add(1) - err = pubsub.Close() - Expect(err).NotTo(HaveOccurred()) + Expect(pubsub.Close()).NotTo(HaveOccurred()) wg.Wait() }) @@ -369,18 +356,17 @@ var _ = Describe("PubSub", func() { It("should ReceiveMessage without a subscription", func() { timeout := 100 * time.Millisecond - pubsub, err := client.Subscribe() - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe() defer pubsub.Close() go func() { defer GinkgoRecover() time.Sleep(2 * timeout) - err = pubsub.Subscribe("mychannel") + err := pubsub.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) - err := client.Publish("mychannel", "hello").Err() + err = client.Publish("mychannel", "hello").Err() Expect(err).NotTo(HaveOccurred()) }() diff --git a/race_test.go b/race_test.go index 4529cf4963..3af7226f03 100644 --- a/race_test.go +++ b/race_test.go @@ -141,8 +141,7 @@ var _ = Describe("races", func() { perform(C, func(id int) { for i := 0; i < N; i++ { - pubsub, err := client.Subscribe(fmt.Sprintf("mychannel%d", id)) - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe(fmt.Sprintf("mychannel%d", id)) go func() { defer GinkgoRecover() @@ -152,7 +151,7 @@ var _ = Describe("races", func() { Expect(err).NotTo(HaveOccurred()) }() - _, err = pubsub.ReceiveMessage() + _, err := pubsub.ReceiveMessage() Expect(err.Error()).To(ContainSubstring("closed")) val := "echo" + strconv.Itoa(i) diff --git a/redis.go b/redis.go index 262e676148..b20b34b962 100644 --- a/redis.go +++ b/redis.go @@ -359,25 +359,19 @@ func (c *Client) pubSub() *PubSub { } // Subscribe subscribes the client to the specified channels. -func (c *Client) Subscribe(channels ...string) (*PubSub, error) { +func (c *Client) Subscribe(channels ...string) *PubSub { pubsub := c.pubSub() if len(channels) > 0 { - if err := pubsub.Subscribe(channels...); err != nil { - pubsub.Close() - return nil, err - } + _ = pubsub.Subscribe(channels...) } - return pubsub, nil + return pubsub } // PSubscribe subscribes the client to the given patterns. -func (c *Client) PSubscribe(channels ...string) (*PubSub, error) { +func (c *Client) PSubscribe(channels ...string) *PubSub { pubsub := c.pubSub() if len(channels) > 0 { - if err := pubsub.PSubscribe(channels...); err != nil { - pubsub.Close() - return nil, err - } + _ = pubsub.PSubscribe(channels...) } - return pubsub, nil + return pubsub } diff --git a/redis_test.go b/redis_test.go index 64cb2a4fc3..f044722270 100644 --- a/redis_test.go +++ b/redis_test.go @@ -58,11 +58,10 @@ var _ = Describe("Client", func() { }) It("should close pubsub without closing the client", func() { - pubsub, err := client.Subscribe() - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe() Expect(pubsub.Close()).NotTo(HaveOccurred()) - _, err = pubsub.Receive() + _, err := pubsub.Receive() Expect(err).To(MatchError("redis: client is closed")) Expect(client.Ping().Err()).NotTo(HaveOccurred()) }) @@ -92,11 +91,10 @@ var _ = Describe("Client", func() { }) It("should close pubsub when client is closed", func() { - pubsub, err := client.Subscribe() - Expect(err).NotTo(HaveOccurred()) + pubsub := client.Subscribe() Expect(client.Close()).NotTo(HaveOccurred()) - _, err = pubsub.Receive() + _, err := pubsub.Receive() Expect(err).To(HaveOccurred()) Expect(pubsub.Close()).NotTo(HaveOccurred()) @@ -242,7 +240,8 @@ var _ = Describe("Client timeout", func() { }) It("Subscribe timeouts", func() { - _, err := client.Subscribe("_") + pubsub := client.Subscribe() + err := pubsub.Subscribe("_") Expect(err).To(HaveOccurred()) Expect(err.(net.Error).Timeout()).To(BeTrue()) }) From 346c00d485fc72b92d5bd935535ba322df2be5d3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 11 Apr 2017 17:29:31 +0300 Subject: [PATCH 0314/1746] Add PubSub support to Ring --- ring.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ring.go b/ring.go index eb5add1064..69715b6b78 100644 --- a/ring.go +++ b/ring.go @@ -177,6 +177,34 @@ func (c *Ring) PoolStats() *PoolStats { return &acc } +// Subscribe subscribes the client to the specified channels. +func (c *Ring) Subscribe(channels ...string) *PubSub { + if len(channels) == 0 { + panic("at least one channel is required") + } + + shard, err := c.shardByKey(channels[0]) + if err != nil { + // TODO: return PubSub with sticky error + panic(err) + } + return shard.Client.Subscribe(channels...) +} + +// PSubscribe subscribes the client to the given patterns. +func (c *Ring) PSubscribe(channels ...string) *PubSub { + if len(channels) == 0 { + panic("at least one channel is required") + } + + shard, err := c.shardByKey(channels[0]) + if err != nil { + // TODO: return PubSub with sticky error + panic(err) + } + return shard.Client.PSubscribe(channels...) +} + // ForEachShard concurrently calls the fn on each live shard in the ring. // It returns the first error if any. func (c *Ring) ForEachShard(fn func(client *Client) error) error { From e737f3e6dddb064b079f54c5e41835da7712f85b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 12 Apr 2017 13:00:20 +0300 Subject: [PATCH 0315/1746] Fix publish command info --- command.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/command.go b/command.go index 0209ebde49..543aeac18e 100644 --- a/command.go +++ b/command.go @@ -91,6 +91,8 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { } else { return -1 } + case "publish": + return 1 } if info == nil { internal.Logf("info for cmd=%s not found", cmd.name()) From 7475b2fb0a52f233f31acf660e56be4eb7932b2a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 12 Apr 2017 13:09:09 +0300 Subject: [PATCH 0316/1746] Increase default pool size --- options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.go b/options.go index 1edc963c98..e92e11c5d4 100644 --- a/options.go +++ b/options.go @@ -84,7 +84,7 @@ func (opt *Options) init() { } } if opt.PoolSize == 0 { - opt.PoolSize = 10 + opt.PoolSize = 100 } if opt.DialTimeout == 0 { opt.DialTimeout = 5 * time.Second From d910e514987bd67322cb2b95e2a0242eb933ded9 Mon Sep 17 00:00:00 2001 From: shaoguang Date: Wed, 12 Apr 2017 20:11:20 +0800 Subject: [PATCH 0317/1746] pipelineReadCmds always retry=true, maybe bug --- redis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.go b/redis.go index b20b34b962..7fbb7fd38d 100644 --- a/redis.go +++ b/redis.go @@ -205,7 +205,7 @@ func pipelineReadCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) firstErr = err } } - return false, firstErr + return } func (c *baseClient) txPipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) { From 7c3f4d045eb91b3da0b83c5ed07ee901b414dd79 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 15 Apr 2017 13:40:20 +0300 Subject: [PATCH 0318/1746] Lower case custom command --- example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_test.go b/example_test.go index 98ee2dd098..6c3d6bdb57 100644 --- a/example_test.go +++ b/example_test.go @@ -332,7 +332,7 @@ func ExampleScript() { func Example_customCommand() { Get := func(client *redis.Client, key string) *redis.StringCmd { - cmd := redis.NewStringCmd("GET", key) + cmd := redis.NewStringCmd("get", key) client.Process(cmd) return cmd } From d9dd5852abb4bb35aa86318813c9c284a4124124 Mon Sep 17 00:00:00 2001 From: Yin Jifeng Date: Mon, 17 Apr 2017 17:30:40 +0800 Subject: [PATCH 0319/1746] doc: mismatched IdleTimeout default value --- options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.go b/options.go index e92e11c5d4..29b6c5f02c 100644 --- a/options.go +++ b/options.go @@ -55,7 +55,7 @@ type Options struct { PoolTimeout time.Duration // Amount of time after which client closes idle connections. // Should be less than server's timeout. - // Default is to not close idle connections. + // Default is 5 minutes. IdleTimeout time.Duration // Frequency of idle checks. // Default is 1 minute. From 66e06285b2cb9316abc0e36da73dd39da6781a7d Mon Sep 17 00:00:00 2001 From: Yin Jifeng Date: Mon, 17 Apr 2017 19:29:41 +0800 Subject: [PATCH 0320/1746] doc: update more Options default value --- options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index 29b6c5f02c..03c64c784a 100644 --- a/options.go +++ b/options.go @@ -43,11 +43,11 @@ type Options struct { ReadTimeout time.Duration // Timeout for socket writes. If reached, commands will fail // with a timeout instead of blocking. - // Default is 3 seconds. + // Default is ReadTimeout. WriteTimeout time.Duration // Maximum number of socket connections. - // Default is 10 connections. + // Default is 100 connections. PoolSize int // Amount of time client waits for connection if all connections // are busy before returning an error. From 6499563e0756905a87eb135757acfccf5caf1e80 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 17 Apr 2017 15:43:58 +0300 Subject: [PATCH 0321/1746] PubSub conns don't share connection pool limit --- export_test.go | 5 +- internal/pool/bench_test.go | 4 +- internal/pool/pool.go | 66 +++++++++-------- internal/pool/pool_single.go | 10 ++- internal/pool/pool_sticky.go | 24 ++++--- internal/pool/pool_test.go | 7 +- options.go | 2 +- pool_test.go | 12 ---- pubsub.go | 133 ++++++++++++++++++++++++----------- pubsub_test.go | 11 +-- race_test.go | 29 -------- redis.go | 7 +- redis_test.go | 13 +++- sentinel.go | 8 +-- 14 files changed, 179 insertions(+), 152 deletions(-) diff --git a/export_test.go b/export_test.go index d043b0df80..b88e41be9e 100644 --- a/export_test.go +++ b/export_test.go @@ -1,6 +1,7 @@ package redis import ( + "net" "time" "github.com/go-redis/redis/internal/pool" @@ -10,8 +11,8 @@ func (c *baseClient) Pool() pool.Pooler { return c.connPool } -func (c *PubSub) Pool() pool.Pooler { - return c.base.connPool +func (c *PubSub) SetNetConn(netConn net.Conn) { + c.cn = pool.NewConn(netConn) } func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) { diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index 610e12c7c7..5c021693e2 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -1,7 +1,6 @@ package pool_test import ( - "errors" "testing" "time" @@ -40,7 +39,6 @@ func BenchmarkPoolGetPut1000Conns(b *testing.B) { func benchmarkPoolGetRemove(b *testing.B, poolSize int) { connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) - removeReason := errors.New("benchmark") b.ResetTimer() @@ -50,7 +48,7 @@ func benchmarkPoolGetRemove(b *testing.B, poolSize int) { if err != nil { b.Fatal(err) } - if err := connPool.Remove(cn, removeReason); err != nil { + if err := connPool.Remove(cn); err != nil { b.Fatal(err) } } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 45743fed70..da8337a4f7 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -2,7 +2,6 @@ package pool import ( "errors" - "fmt" "net" "sync" "sync/atomic" @@ -11,11 +10,8 @@ import ( "github.com/go-redis/redis/internal" ) -var ( - ErrClosed = errors.New("redis: client is closed") - ErrPoolTimeout = errors.New("redis: connection pool timeout") - errConnStale = errors.New("connection is stale") -) +var ErrClosed = errors.New("redis: client is closed") +var ErrPoolTimeout = errors.New("redis: connection pool timeout") var timers = sync.Pool{ New: func() interface{} { @@ -36,12 +32,17 @@ type Stats struct { } type Pooler interface { + NewConn() (*Conn, error) + CloseConn(*Conn) error + Get() (*Conn, bool, error) Put(*Conn) error - Remove(*Conn, error) error + Remove(*Conn) error + Len() int FreeLen() int Stats() *Stats + Close() error } @@ -87,11 +88,21 @@ func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout, idleCheckF } func (p *ConnPool) NewConn() (*Conn, error) { + if p.closed() { + return nil, ErrClosed + } + netConn, err := p.dial() if err != nil { return nil, err } - return NewConn(netConn), nil + + cn := NewConn(netConn) + p.connsMu.Lock() + p.conns = append(p.conns, cn) + p.connsMu.Unlock() + + return cn, nil } func (p *ConnPool) PopFree() *Conn { @@ -164,7 +175,7 @@ func (p *ConnPool) Get() (*Conn, bool, error) { } if cn.IsStale(p.idleTimeout) { - p.remove(cn, errConnStale) + p.CloseConn(cn) continue } @@ -178,18 +189,13 @@ func (p *ConnPool) Get() (*Conn, bool, error) { return nil, false, err } - p.connsMu.Lock() - p.conns = append(p.conns, newcn) - p.connsMu.Unlock() - return newcn, true, nil } func (p *ConnPool) Put(cn *Conn) error { if data := cn.Rd.PeekBuffered(); data != nil { - err := fmt.Errorf("connection has unread data: %q", data) - internal.Logf(err.Error()) - return p.Remove(cn, err) + internal.Logf("connection has unread data: %q", data) + return p.Remove(cn) } p.freeConnsMu.Lock() p.freeConns = append(p.freeConns, cn) @@ -198,15 +204,13 @@ func (p *ConnPool) Put(cn *Conn) error { return nil } -func (p *ConnPool) Remove(cn *Conn, reason error) error { - p.remove(cn, reason) +func (p *ConnPool) Remove(cn *Conn) error { + _ = p.CloseConn(cn) <-p.queue return nil } -func (p *ConnPool) remove(cn *Conn, reason error) { - _ = p.closeConn(cn, reason) - +func (p *ConnPool) CloseConn(cn *Conn) error { p.connsMu.Lock() for i, c := range p.conns { if c == cn { @@ -215,6 +219,15 @@ func (p *ConnPool) remove(cn *Conn, reason error) { } } p.connsMu.Unlock() + + return p.closeConn(cn) +} + +func (p *ConnPool) closeConn(cn *Conn) error { + if p.OnClose != nil { + _ = p.OnClose(cn) + } + return cn.Close() } // Len returns total number of connections. @@ -258,7 +271,7 @@ func (p *ConnPool) Close() error { if cn == nil { continue } - if err := p.closeConn(cn, ErrClosed); err != nil && firstErr == nil { + if err := p.closeConn(cn); err != nil && firstErr == nil { firstErr = err } } @@ -272,13 +285,6 @@ func (p *ConnPool) Close() error { return firstErr } -func (p *ConnPool) closeConn(cn *Conn, reason error) error { - if p.OnClose != nil { - _ = p.OnClose(cn) - } - return cn.Close() -} - func (p *ConnPool) reapStaleConn() bool { if len(p.freeConns) == 0 { return false @@ -289,7 +295,7 @@ func (p *ConnPool) reapStaleConn() bool { return false } - p.remove(cn, errConnStale) + p.CloseConn(cn) p.freeConns = append(p.freeConns[:0], p.freeConns[1:]...) return true diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index 22eaba9d40..ff91279b36 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -12,6 +12,14 @@ func NewSingleConnPool(cn *Conn) *SingleConnPool { } } +func (p *SingleConnPool) NewConn() (*Conn, error) { + panic("not implemented") +} + +func (p *SingleConnPool) CloseConn(*Conn) error { + panic("not implemented") +} + func (p *SingleConnPool) Get() (*Conn, bool, error) { return p.cn, false, nil } @@ -23,7 +31,7 @@ func (p *SingleConnPool) Put(cn *Conn) error { return nil } -func (p *SingleConnPool) Remove(cn *Conn, _ error) error { +func (p *SingleConnPool) Remove(cn *Conn) error { if p.cn != cn { panic("p.cn != cn") } diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 7426cd260c..17f163858b 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -1,9 +1,6 @@ package pool -import ( - "errors" - "sync" -) +import "sync" type StickyConnPool struct { pool *ConnPool @@ -23,6 +20,14 @@ func NewStickyConnPool(pool *ConnPool, reusable bool) *StickyConnPool { } } +func (p *StickyConnPool) NewConn() (*Conn, error) { + panic("not implemented") +} + +func (p *StickyConnPool) CloseConn(*Conn) error { + panic("not implemented") +} + func (p *StickyConnPool) Get() (*Conn, bool, error) { p.mu.Lock() defer p.mu.Unlock() @@ -58,20 +63,20 @@ func (p *StickyConnPool) Put(cn *Conn) error { return nil } -func (p *StickyConnPool) removeUpstream(reason error) error { - err := p.pool.Remove(p.cn, reason) +func (p *StickyConnPool) removeUpstream() error { + err := p.pool.Remove(p.cn) p.cn = nil return err } -func (p *StickyConnPool) Remove(cn *Conn, reason error) error { +func (p *StickyConnPool) Remove(cn *Conn) error { p.mu.Lock() defer p.mu.Unlock() if p.closed { return nil } - return p.removeUpstream(reason) + return p.removeUpstream() } func (p *StickyConnPool) Len() int { @@ -111,8 +116,7 @@ func (p *StickyConnPool) Close() error { if p.reusable { err = p.putUpstream() } else { - reason := errors.New("redis: unreusable sticky connection") - err = p.removeUpstream(reason) + err = p.removeUpstream() } } return err diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index c2983dd06c..c8fbeb9b88 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -1,7 +1,6 @@ package pool_test import ( - "errors" "testing" "time" @@ -59,7 +58,7 @@ var _ = Describe("ConnPool", func() { // ok } - err = connPool.Remove(cn, errors.New("test")) + err = connPool.Remove(cn) Expect(err).NotTo(HaveOccurred()) // Check that Ping is unblocked. @@ -169,7 +168,7 @@ var _ = Describe("conns reaper", func() { Expect(connPool.Len()).To(Equal(4)) Expect(connPool.FreeLen()).To(Equal(0)) - err = connPool.Remove(cn, errors.New("test")) + err = connPool.Remove(cn) Expect(err).NotTo(HaveOccurred()) Expect(connPool.Len()).To(Equal(3)) @@ -219,7 +218,7 @@ var _ = Describe("race", func() { cn, _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) if err == nil { - Expect(connPool.Remove(cn, errors.New("test"))).NotTo(HaveOccurred()) + Expect(connPool.Remove(cn)).NotTo(HaveOccurred()) } } }) diff --git a/options.go b/options.go index 03c64c784a..8c30a67c4b 100644 --- a/options.go +++ b/options.go @@ -84,7 +84,7 @@ func (opt *Options) init() { } } if opt.PoolSize == 0 { - opt.PoolSize = 100 + opt.PoolSize = 10 } if opt.DialTimeout == 0 { opt.DialTimeout = 5 * time.Second diff --git a/pool_test.go b/pool_test.go index c6731e4866..5363c4008a 100644 --- a/pool_test.go +++ b/pool_test.go @@ -77,18 +77,6 @@ var _ = Describe("pool", func() { Expect(pool.Len()).To(Equal(pool.FreeLen())) }) - It("respects max size on pubsub", func() { - connPool := client.Pool() - - perform(1000, func(id int) { - pubsub := client.Subscribe("test") - Expect(pubsub.Close()).NotTo(HaveOccurred()) - }) - - Expect(connPool.Len()).To(Equal(connPool.FreeLen())) - Expect(connPool.Len()).To(BeNumerically("<=", 10)) - }) - It("removes broken connections", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) diff --git a/pubsub.go b/pubsub.go index 8705a137c4..497b27cc62 100644 --- a/pubsub.go +++ b/pubsub.go @@ -3,6 +3,7 @@ package redis import ( "fmt" "net" + "sync" "time" "github.com/go-redis/redis/internal" @@ -14,25 +15,72 @@ import ( // multiple goroutines. type PubSub struct { base baseClient - cmd *Cmd + + mu sync.Mutex + cn *pool.Conn + closed bool + + cmd *Cmd channels []string patterns []string } -func (c *PubSub) conn() (*pool.Conn, bool, error) { - cn, isNew, err := c.base.conn() +func (c *PubSub) conn() (*pool.Conn, error) { + cn, isNew, err := c._conn() if err != nil { - return nil, false, err + return nil, err } + if isNew { c.resubscribe() } - return cn, isNew, nil + + return cn, nil +} + +func (c *PubSub) resubscribe() { + if len(c.channels) > 0 { + if err := c.subscribe("subscribe", c.channels...); err != nil { + internal.Logf("Subscribe failed: %s", err) + } + } + if len(c.patterns) > 0 { + if err := c.subscribe("psubscribe", c.patterns...); err != nil { + internal.Logf("PSubscribe failed: %s", err) + } + } +} + +func (c *PubSub) _conn() (*pool.Conn, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return nil, false, pool.ErrClosed + } + + if c.cn != nil { + return c.cn, false, nil + } + + cn, err := c.base.connPool.NewConn() + if err != nil { + return nil, false, err + } + c.cn = cn + + return cn, true, nil } func (c *PubSub) putConn(cn *pool.Conn, err error) { - c.base.putConn(cn, err, true) + if internal.IsBadConn(err, true) { + c.mu.Lock() + if c.cn == cn { + _ = c.closeConn() + } + c.mu.Unlock() + } } func (c *PubSub) subscribe(redisCmd string, channels ...string) error { @@ -43,7 +91,7 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { } cmd := NewSliceCmd(args...) - cn, _, err := c.conn() + cn, err := c.conn() if err != nil { return err } @@ -56,14 +104,14 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { // Subscribes the client to the specified channels. func (c *PubSub) Subscribe(channels ...string) error { - err := c.subscribe("SUBSCRIBE", channels...) + err := c.subscribe("subscribe", channels...) c.channels = appendIfNotExists(c.channels, channels...) return err } // Subscribes the client to the given patterns. func (c *PubSub) PSubscribe(patterns ...string) error { - err := c.subscribe("PSUBSCRIBE", patterns...) + err := c.subscribe("psubscribe", patterns...) c.patterns = appendIfNotExists(c.patterns, patterns...) return err } @@ -71,7 +119,7 @@ func (c *PubSub) PSubscribe(patterns ...string) error { // Unsubscribes the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { - err := c.subscribe("UNSUBSCRIBE", channels...) + err := c.subscribe("unsubscribe", channels...) c.channels = remove(c.channels, channels...) return err } @@ -79,23 +127,41 @@ func (c *PubSub) Unsubscribe(channels ...string) error { // Unsubscribes the client from the given patterns, or from all of // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { - err := c.subscribe("PUNSUBSCRIBE", patterns...) + err := c.subscribe("punsubscribe", patterns...) c.patterns = remove(c.patterns, patterns...) return err } func (c *PubSub) Close() error { - return c.base.Close() + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return pool.ErrClosed + } + c.closed = true + + if c.cn != nil { + _ = c.closeConn() + } + + return nil +} + +func (c *PubSub) closeConn() error { + err := c.base.connPool.CloseConn(c.cn) + c.cn = nil + return err } func (c *PubSub) Ping(payload ...string) error { - args := []interface{}{"PING"} + args := []interface{}{"ping"} if len(payload) == 1 { args = append(args, payload[0]) } cmd := NewCmd(args...) - cn, _, err := c.conn() + cn, err := c.conn() if err != nil { return err } @@ -188,7 +254,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { c.cmd = NewCmd() } - cn, _, err := c.conn() + cn, err := c.conn() if err != nil { return nil, err } @@ -259,19 +325,6 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { } } -func (c *PubSub) resubscribe() { - if len(c.channels) > 0 { - if err := c.Subscribe(c.channels...); err != nil { - internal.Logf("Subscribe failed: %s", err) - } - } - if len(c.patterns) > 0 { - if err := c.PSubscribe(c.patterns...); err != nil { - internal.Logf("PSubscribe failed: %s", err) - } - } -} - // Channel returns a channel for concurrently receiving messages. // The channel is closed with PubSub. func (c *PubSub) Channel() <-chan *Message { @@ -292,30 +345,30 @@ func (c *PubSub) Channel() <-chan *Message { return ch } -func remove(ss []string, es ...string) []string { - if len(es) == 0 { - return ss[:0] - } +func appendIfNotExists(ss []string, es ...string) []string { +loop: for _, e := range es { - for i, s := range ss { + for _, s := range ss { if s == e { - ss = append(ss[:i], ss[i+1:]...) - break + continue loop } } + ss = append(ss, e) } return ss } -func appendIfNotExists(ss []string, es ...string) []string { -loop: +func remove(ss []string, es ...string) []string { + if len(es) == 0 { + return ss[:0] + } for _, e := range es { - for _, s := range ss { + for i, s := range ss { if s == e { - continue loop + ss = append(ss[:i], ss[i+1:]...) + break } } - ss = append(ss, e) } return ss } diff --git a/pubsub_test.go b/pubsub_test.go index 0164805ab4..b17ca7ad47 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -274,18 +274,15 @@ var _ = Describe("PubSub", func() { Eventually(done).Should(Receive()) stats := client.PoolStats() - Expect(stats.Requests).To(Equal(uint32(3))) + Expect(stats.Requests).To(Equal(uint32(2))) Expect(stats.Hits).To(Equal(uint32(1))) }) expectReceiveMessageOnError := func(pubsub *redis.PubSub) { - cn, _, err := pubsub.Pool().Get() - Expect(err).NotTo(HaveOccurred()) - cn.SetNetConn(&badConn{ + pubsub.SetNetConn(&badConn{ readErr: io.EOF, writeErr: io.EOF, }) - pubsub.Pool().Put(cn) done := make(chan bool, 1) go func() { @@ -305,10 +302,6 @@ var _ = Describe("PubSub", func() { Expect(msg.Payload).To(Equal("hello")) Eventually(done).Should(Receive()) - - stats := client.PoolStats() - Expect(stats.Requests).To(Equal(uint32(4))) - Expect(stats.Hits).To(Equal(uint32(1))) } It("Subscribe should reconnect on ReceiveMessage error", func() { diff --git a/race_test.go b/race_test.go index 3af7226f03..0ec6a140dc 100644 --- a/race_test.go +++ b/race_test.go @@ -136,35 +136,6 @@ var _ = Describe("races", func() { }) }) - It("should PubSub", func() { - connPool := client.Pool() - - perform(C, func(id int) { - for i := 0; i < N; i++ { - pubsub := client.Subscribe(fmt.Sprintf("mychannel%d", id)) - - go func() { - defer GinkgoRecover() - - time.Sleep(time.Millisecond) - err := pubsub.Close() - Expect(err).NotTo(HaveOccurred()) - }() - - _, err := pubsub.ReceiveMessage() - Expect(err.Error()).To(ContainSubstring("closed")) - - val := "echo" + strconv.Itoa(i) - echo, err := client.Echo(val).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(echo).To(Equal(val)) - } - }) - - Expect(connPool.Len()).To(Equal(connPool.FreeLen())) - Expect(connPool.Len()).To(BeNumerically("<=", 10)) - }) - It("should select db", func() { err := client.Set("db", 1, 0).Err() Expect(err).NotTo(HaveOccurred()) diff --git a/redis.go b/redis.go index 7fbb7fd38d..873d58056e 100644 --- a/redis.go +++ b/redis.go @@ -31,9 +31,10 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { if err != nil { return nil, false, err } + if !cn.Inited { if err := c.initConn(cn); err != nil { - _ = c.connPool.Remove(cn, err) + _ = c.connPool.Remove(cn) return nil, false, err } } @@ -42,7 +43,7 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { if internal.IsBadConn(err, allowTimeout) { - _ = c.connPool.Remove(cn, err) + _ = c.connPool.Remove(cn) return false } @@ -353,7 +354,7 @@ func (c *Client) pubSub() *PubSub { return &PubSub{ base: baseClient{ opt: c.opt, - connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), + connPool: c.connPool, }, } } diff --git a/redis_test.go b/redis_test.go index f044722270..2847963f43 100644 --- a/redis_test.go +++ b/redis_test.go @@ -95,7 +95,7 @@ var _ = Describe("Client", func() { Expect(client.Close()).NotTo(HaveOccurred()) _, err := pubsub.Receive() - Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("redis: client is closed")) Expect(pubsub.Close()).NotTo(HaveOccurred()) }) @@ -217,6 +217,7 @@ var _ = Describe("Client", func() { }) var _ = Describe("Client timeout", func() { + var opt *redis.Options var client *redis.Client AfterEach(func() { @@ -240,7 +241,13 @@ var _ = Describe("Client timeout", func() { }) It("Subscribe timeouts", func() { + if opt.WriteTimeout == 0 { + return + } + pubsub := client.Subscribe() + defer pubsub.Close() + err := pubsub.Subscribe("_") Expect(err).To(HaveOccurred()) Expect(err.(net.Error).Timeout()).To(BeTrue()) @@ -269,7 +276,7 @@ var _ = Describe("Client timeout", func() { Context("read timeout", func() { BeforeEach(func() { - opt := redisOptions() + opt = redisOptions() opt.ReadTimeout = time.Nanosecond opt.WriteTimeout = -1 client = redis.NewClient(opt) @@ -280,7 +287,7 @@ var _ = Describe("Client timeout", func() { Context("write timeout", func() { BeforeEach(func() { - opt := redisOptions() + opt = redisOptions() opt.ReadTimeout = -1 opt.WriteTimeout = time.Nanosecond client = redis.NewClient(opt) diff --git a/sentinel.go b/sentinel.go index 8070b46424..799f530fca 100644 --- a/sentinel.go +++ b/sentinel.go @@ -2,7 +2,6 @@ package redis import ( "errors" - "fmt" "net" "strings" "sync" @@ -111,7 +110,7 @@ func (c *sentinelClient) PubSub() *PubSub { return &PubSub{ base: baseClient{ opt: c.opt, - connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), false), + connPool: c.connPool, }, } } @@ -268,12 +267,11 @@ func (d *sentinelFailover) closeOldConns(newMaster string) { break } if cn.RemoteAddr().String() != newMaster { - err := fmt.Errorf( + internal.Logf( "sentinel: closing connection to the old master %s", cn.RemoteAddr(), ) - internal.Logf(err.Error()) - d.pool.Remove(cn, err) + d.pool.Remove(cn) } else { cnsToPut = append(cnsToPut, cn) } From 191f839e81e7a2885c7f91b5799be1a03d9cab97 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 17 Apr 2017 16:05:01 +0300 Subject: [PATCH 0322/1746] Fix race between Subscribe and resubscribe --- options.go | 2 +- pubsub.go | 52 +++++++++++++++++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/options.go b/options.go index 8c30a67c4b..d2aefb4755 100644 --- a/options.go +++ b/options.go @@ -47,7 +47,7 @@ type Options struct { WriteTimeout time.Duration // Maximum number of socket connections. - // Default is 100 connections. + // Default is 10 connections. PoolSize int // Amount of time client waits for connection if all connections // are busy before returning an error. diff --git a/pubsub.go b/pubsub.go index 497b27cc62..a58155684e 100644 --- a/pubsub.go +++ b/pubsub.go @@ -22,6 +22,7 @@ type PubSub struct { cmd *Cmd + subMu sync.Mutex channels []string patterns []string } @@ -33,23 +34,32 @@ func (c *PubSub) conn() (*pool.Conn, error) { } if isNew { - c.resubscribe() + if err := c.resubscribe(); err != nil { + internal.Logf("resubscribe failed: %s", err) + } } return cn, nil } -func (c *PubSub) resubscribe() { - if len(c.channels) > 0 { - if err := c.subscribe("subscribe", c.channels...); err != nil { - internal.Logf("Subscribe failed: %s", err) +func (c *PubSub) resubscribe() error { + c.subMu.Lock() + channels := c.channels + patterns := c.patterns + c.subMu.Unlock() + + var firstErr error + if len(channels) > 0 { + if err := c.subscribe("subscribe", channels...); err != nil && firstErr == nil { + firstErr = err } } - if len(c.patterns) > 0 { - if err := c.subscribe("psubscribe", c.patterns...); err != nil { - internal.Logf("PSubscribe failed: %s", err) + if len(patterns) > 0 { + if err := c.subscribe("psubscribe", patterns...); err != nil && firstErr == nil { + firstErr = err } } + return firstErr } func (c *PubSub) _conn() (*pool.Conn, bool, error) { @@ -91,11 +101,15 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { } cmd := NewSliceCmd(args...) - cn, err := c.conn() + cn, isNew, err := c._conn() if err != nil { return err } + if isNew { + return c.resubscribe() + } + cn.SetWriteTimeout(c.base.opt.WriteTimeout) err = writeCmd(cn, cmd) c.putConn(cn, err) @@ -104,32 +118,36 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { // Subscribes the client to the specified channels. func (c *PubSub) Subscribe(channels ...string) error { - err := c.subscribe("subscribe", channels...) + c.subMu.Lock() c.channels = appendIfNotExists(c.channels, channels...) - return err + c.subMu.Unlock() + return c.subscribe("subscribe", channels...) } // Subscribes the client to the given patterns. func (c *PubSub) PSubscribe(patterns ...string) error { - err := c.subscribe("psubscribe", patterns...) + c.subMu.Lock() c.patterns = appendIfNotExists(c.patterns, patterns...) - return err + c.subMu.Unlock() + return c.subscribe("psubscribe", patterns...) } // Unsubscribes the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { - err := c.subscribe("unsubscribe", channels...) + c.subMu.Lock() c.channels = remove(c.channels, channels...) - return err + c.subMu.Unlock() + return c.subscribe("unsubscribe", channels...) } // Unsubscribes the client from the given patterns, or from all of // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { - err := c.subscribe("punsubscribe", patterns...) + c.subMu.Lock() c.patterns = remove(c.patterns, patterns...) - return err + c.subMu.Unlock() + return c.subscribe("punsubscribe", patterns...) } func (c *PubSub) Close() error { From 7646d48e29f22789b6a541c3bce5f5f0c97f368b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 18 Apr 2017 13:12:38 +0300 Subject: [PATCH 0323/1746] Initialize PubSub connection --- pubsub.go | 9 ++++++++- redis.go | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pubsub.go b/pubsub.go index a58155684e..e47978c265 100644 --- a/pubsub.go +++ b/pubsub.go @@ -78,8 +78,15 @@ func (c *PubSub) _conn() (*pool.Conn, bool, error) { if err != nil { return nil, false, err } - c.cn = cn + if !cn.Inited { + if err := c.base.initConn(cn); err != nil { + _ = c.base.connPool.Remove(cn) + return nil, false, err + } + } + + c.cn = cn return cn, true, nil } diff --git a/redis.go b/redis.go index 873d58056e..ecf1fc0aff 100644 --- a/redis.go +++ b/redis.go @@ -38,6 +38,7 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { return nil, false, err } } + return cn, isNew, nil } From 8c21ec00067496e12ca2b6f1b08779662b242835 Mon Sep 17 00:00:00 2001 From: Yuval Pavel Zholkover Date: Fri, 21 Apr 2017 15:50:26 +0300 Subject: [PATCH 0324/1746] Fix typo in ClusterClient.TxPipelined() It should be using ClusterClient.TxPipeline() and not ClusterClient.Pipeline(). --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 9bc650659c..223d2522cd 100644 --- a/cluster.go +++ b/cluster.go @@ -807,7 +807,7 @@ func (c *ClusterClient) TxPipeline() *Pipeline { } func (c *ClusterClient) TxPipelined(fn func(*Pipeline) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) + return c.TxPipeline().pipelined(fn) } func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { From 9ebd89772add2db0c136b8a634739de553984841 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 24 Apr 2017 12:43:15 +0300 Subject: [PATCH 0325/1746] Rework PubSub conn management --- cluster.go | 4 +- pubsub.go | 144 +++++++++++++++++++++++++---------------------------- redis.go | 10 ++-- ring.go | 2 +- 4 files changed, 76 insertions(+), 84 deletions(-) diff --git a/cluster.go b/cluster.go index 223d2522cd..43cbf64346 100644 --- a/cluster.go +++ b/cluster.go @@ -704,7 +704,7 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { } err = c.pipelineProcessCmds(cn, cmds, failedCmds) - node.Client.putConn(cn, err, false) + node.Client.putConn(cn, err) } if len(failedCmds) == 0 { @@ -840,7 +840,7 @@ func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { } err = c.txPipelineProcessCmds(node, cn, cmds, failedCmds) - node.Client.putConn(cn, err, false) + node.Client.putConn(cn, err) } if len(failedCmds) == 0 { diff --git a/pubsub.go b/pubsub.go index e47978c265..3680323a05 100644 --- a/pubsub.go +++ b/pubsub.go @@ -20,87 +20,112 @@ type PubSub struct { cn *pool.Conn closed bool - cmd *Cmd - subMu sync.Mutex channels []string patterns []string + + cmd *Cmd } -func (c *PubSub) conn() (*pool.Conn, error) { - cn, isNew, err := c._conn() +func (c *PubSub) conn() (*pool.Conn, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return nil, false, pool.ErrClosed + } + + if c.cn != nil { + return c.cn, false, nil + } + + cn, err := c.base.connPool.NewConn() if err != nil { - return nil, err + return nil, false, err } - if isNew { - if err := c.resubscribe(); err != nil { - internal.Logf("resubscribe failed: %s", err) + if !cn.Inited { + if err := c.base.initConn(cn); err != nil { + _ = c.base.connPool.Remove(cn) + return nil, false, err } } - return cn, nil + if err := c.resubscribe(cn); err != nil { + return nil, false, err + } + + c.cn = cn + return cn, true, nil } -func (c *PubSub) resubscribe() error { +func (c *PubSub) resubscribe(cn *pool.Conn) error { c.subMu.Lock() - channels := c.channels - patterns := c.patterns - c.subMu.Unlock() + defer c.subMu.Unlock() var firstErr error - if len(channels) > 0 { - if err := c.subscribe("subscribe", channels...); err != nil && firstErr == nil { + if len(c.channels) > 0 { + if err := c._subscribe(cn, "subscribe", c.channels...); err != nil && firstErr == nil { firstErr = err } } - if len(patterns) > 0 { - if err := c.subscribe("psubscribe", patterns...); err != nil && firstErr == nil { + if len(c.patterns) > 0 { + if err := c._subscribe(cn, "psubscribe", c.patterns...); err != nil && firstErr == nil { firstErr = err } } return firstErr } -func (c *PubSub) _conn() (*pool.Conn, bool, error) { +func (c *PubSub) putConn(cn *pool.Conn, err error) { + if !internal.IsBadConn(err, true) { + return + } + + c.mu.Lock() + if c.cn == cn { + _ = c.closeConn() + } + c.mu.Unlock() +} + +func (c *PubSub) closeConn() error { + err := c.base.connPool.CloseConn(c.cn) + c.cn = nil + return err +} + +func (c *PubSub) Close() error { c.mu.Lock() defer c.mu.Unlock() if c.closed { - return nil, false, pool.ErrClosed + return pool.ErrClosed } + c.closed = true if c.cn != nil { - return c.cn, false, nil + return c.closeConn() } + return nil +} - cn, err := c.base.connPool.NewConn() +func (c *PubSub) subscribe(redisCmd string, channels ...string) error { + cn, isNew, err := c.conn() if err != nil { - return nil, false, err + return err } - if !cn.Inited { - if err := c.base.initConn(cn); err != nil { - _ = c.base.connPool.Remove(cn) - return nil, false, err - } + if isNew { + return nil } - c.cn = cn - return cn, true, nil -} - -func (c *PubSub) putConn(cn *pool.Conn, err error) { - if internal.IsBadConn(err, true) { - c.mu.Lock() - if c.cn == cn { - _ = c.closeConn() - } - c.mu.Unlock() - } + err = c._subscribe(cn, redisCmd, channels...) + c.putConn(cn, err) + return err } -func (c *PubSub) subscribe(redisCmd string, channels ...string) error { +func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) error { args := make([]interface{}, 1+len(channels)) args[0] = redisCmd for i, channel := range channels { @@ -108,19 +133,8 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { } cmd := NewSliceCmd(args...) - cn, isNew, err := c._conn() - if err != nil { - return err - } - - if isNew { - return c.resubscribe() - } - cn.SetWriteTimeout(c.base.opt.WriteTimeout) - err = writeCmd(cn, cmd) - c.putConn(cn, err) - return err + return writeCmd(cn, cmd) } // Subscribes the client to the specified channels. @@ -157,28 +171,6 @@ func (c *PubSub) PUnsubscribe(patterns ...string) error { return c.subscribe("punsubscribe", patterns...) } -func (c *PubSub) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.closed { - return pool.ErrClosed - } - c.closed = true - - if c.cn != nil { - _ = c.closeConn() - } - - return nil -} - -func (c *PubSub) closeConn() error { - err := c.base.connPool.CloseConn(c.cn) - c.cn = nil - return err -} - func (c *PubSub) Ping(payload ...string) error { args := []interface{}{"ping"} if len(payload) == 1 { @@ -186,7 +178,7 @@ func (c *PubSub) Ping(payload ...string) error { } cmd := NewCmd(args...) - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { return err } @@ -279,7 +271,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { c.cmd = NewCmd() } - cn, err := c.conn() + cn, _, err := c.conn() if err != nil { return nil, err } diff --git a/redis.go b/redis.go index ecf1fc0aff..b71b9fc608 100644 --- a/redis.go +++ b/redis.go @@ -42,8 +42,8 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { return cn, isNew, nil } -func (c *baseClient) putConn(cn *pool.Conn, err error, allowTimeout bool) bool { - if internal.IsBadConn(err, allowTimeout) { +func (c *baseClient) putConn(cn *pool.Conn, err error) bool { + if internal.IsBadConn(err, false) { _ = c.connPool.Remove(cn) return false } @@ -104,7 +104,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmd); err != nil { - c.putConn(cn, err, false) + c.putConn(cn, err) cmd.setErr(err) if err != nil && internal.IsRetryableError(err) { continue @@ -114,7 +114,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { cn.SetReadTimeout(c.cmdTimeout(cmd)) err = cmd.readReply(cn) - c.putConn(cn, err, false) + c.putConn(cn, err) if err != nil && internal.IsRetryableError(err) { continue } @@ -167,7 +167,7 @@ func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { } canRetry, err := p(cn, cmds) - c.putConn(cn, err, false) + c.putConn(cn, err) if err == nil { return nil } diff --git a/ring.go b/ring.go index 69715b6b78..d13a33b709 100644 --- a/ring.go +++ b/ring.go @@ -428,7 +428,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { } canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds) - shard.Client.putConn(cn, err, false) + shard.Client.putConn(cn, err) if err == nil { continue } From 59d94967b651d1824a69030912c3b8711da19583 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 24 Apr 2017 14:00:28 +0300 Subject: [PATCH 0326/1746] Fix conn management in PubSub --- pubsub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub.go b/pubsub.go index 3680323a05..6393d8ae71 100644 --- a/pubsub.go +++ b/pubsub.go @@ -46,7 +46,7 @@ func (c *PubSub) conn() (*pool.Conn, bool, error) { if !cn.Inited { if err := c.base.initConn(cn); err != nil { - _ = c.base.connPool.Remove(cn) + _ = c.base.connPool.CloseConn(cn) return nil, false, err } } From 13b6f3ffb7d990e6774067abedeac598a50f8f20 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 24 Apr 2017 14:00:28 +0300 Subject: [PATCH 0327/1746] Fix conn management in PubSub --- pubsub.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pubsub.go b/pubsub.go index 3680323a05..b9d5c60b60 100644 --- a/pubsub.go +++ b/pubsub.go @@ -46,12 +46,13 @@ func (c *PubSub) conn() (*pool.Conn, bool, error) { if !cn.Inited { if err := c.base.initConn(cn); err != nil { - _ = c.base.connPool.Remove(cn) + _ = c.base.connPool.CloseConn(cn) return nil, false, err } } if err := c.resubscribe(cn); err != nil { + _ = c.base.connPool.CloseConn(cn) return nil, false, err } From bd29077241a6da5222a555bc566ac51fedd68222 Mon Sep 17 00:00:00 2001 From: Nykolas Laurentino de Lima Date: Thu, 27 Apr 2017 14:24:25 -0300 Subject: [PATCH 0328/1746] Remove unnecessary code Remove unnecessary if and return the value directly --- command.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/command.go b/command.go index 543aeac18e..33c11aafe2 100644 --- a/command.go +++ b/command.go @@ -111,10 +111,7 @@ type baseCmd struct { } func (cmd *baseCmd) Err() error { - if cmd.err != nil { - return cmd.err - } - return nil + return cmd.err } func (cmd *baseCmd) args() []interface{} { From 6fca4d5ad05fdb809fb3ffafcb2b30c6be8b41f7 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Mon, 1 May 2017 12:42:58 -0300 Subject: [PATCH 0329/1746] pipeline now has its own interface "Pipelineable" --- bench_test.go | 2 +- cluster.go | 8 ++++---- cluster_test.go | 14 +++++++------- commands.go | 13 +++++++++++-- commands_test.go | 2 +- example_test.go | 6 +++--- pipeline.go | 21 ++++++++++++++++++++- pipeline_test.go | 4 ++-- pool_test.go | 2 +- race_test.go | 2 +- redis.go | 8 ++++---- redis_test.go | 6 +++--- ring.go | 4 ++-- ring_test.go | 6 +++--- tx.go | 4 ++-- tx_test.go | 10 +++++----- 16 files changed, 70 insertions(+), 42 deletions(-) diff --git a/bench_test.go b/bench_test.go index 021dba96b4..cba9ff5bac 100644 --- a/bench_test.go +++ b/bench_test.go @@ -188,7 +188,7 @@ func BenchmarkPipeline(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { pipe.Set("key", "hello", 0) pipe.Expire("key", time.Second) return nil diff --git a/cluster.go b/cluster.go index 43cbf64346..5bb1b5adec 100644 --- a/cluster.go +++ b/cluster.go @@ -674,7 +674,7 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { } } -func (c *ClusterClient) Pipeline() *Pipeline { +func (c *ClusterClient) Pipeline() Pipelineable { pipe := Pipeline{ exec: c.pipelineExec, } @@ -683,7 +683,7 @@ func (c *ClusterClient) Pipeline() *Pipeline { return &pipe } -func (c *ClusterClient) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *ClusterClient) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } @@ -797,7 +797,7 @@ func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]C } // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. -func (c *ClusterClient) TxPipeline() *Pipeline { +func (c *ClusterClient) TxPipeline() Pipelineable { pipe := Pipeline{ exec: c.txPipelineExec, } @@ -806,7 +806,7 @@ func (c *ClusterClient) TxPipeline() *Pipeline { return &pipe } -func (c *ClusterClient) TxPipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *ClusterClient) TxPipelined(fn func(Pipelineable) error) ([]Cmder, error) { return c.TxPipeline().pipelined(fn) } diff --git a/cluster_test.go b/cluster_test.go index d9db8975e9..c68ff92eea 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -347,7 +347,7 @@ var _ = Describe("ClusterClient", func() { return err } - _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err = tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) @@ -449,7 +449,7 @@ var _ = Describe("ClusterClient", func() { Describe("Pipeline", func() { BeforeEach(func() { - pipe = client.Pipeline() + pipe = client.Pipeline().(*redis.Pipeline) }) AfterEach(func() { @@ -461,7 +461,7 @@ var _ = Describe("ClusterClient", func() { Describe("TxPipeline", func() { BeforeEach(func() { - pipe = client.TxPipeline() + pipe = client.TxPipeline().(*redis.Pipeline) }) AfterEach(func() { @@ -544,7 +544,7 @@ var _ = Describe("ClusterClient without nodes", func() { }) It("pipeline returns an error", func() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) @@ -571,7 +571,7 @@ var _ = Describe("ClusterClient without valid nodes", func() { }) It("pipeline returns an error", func() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) @@ -594,7 +594,7 @@ var _ = Describe("ClusterClient timeout", func() { }) It("Pipeline timeouts", func() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) @@ -612,7 +612,7 @@ var _ = Describe("ClusterClient timeout", func() { It("Tx Pipeline timeouts", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) diff --git a/commands.go b/commands.go index fb0e9105fb..d1efd21424 100644 --- a/commands.go +++ b/commands.go @@ -39,8 +39,8 @@ func formatSec(dur time.Duration) int64 { } type Cmdable interface { - Pipeline() *Pipeline - Pipelined(fn func(*Pipeline) error) ([]Cmder, error) + Pipeline() Pipelineable + Pipelined(fn func(Pipelineable) error) ([]Cmder, error) Echo(message interface{}) *StringCmd Ping() *StatusCmd @@ -237,6 +237,15 @@ type Cmdable interface { Command() *CommandsInfoCmd } +type StatefulCmdable interface { + Auth(password string) *StatusCmd + Select(index int) *StatusCmd + ClientSetName(name string) *BoolCmd + ClientGetName() *StringCmd + ReadOnly() *StatusCmd + ReadWrite() *StatusCmd +} + var _ Cmdable = (*Client)(nil) var _ Cmdable = (*Tx)(nil) var _ Cmdable = (*Ring)(nil) diff --git a/commands_test.go b/commands_test.go index a123133413..b9b601abf7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -27,7 +27,7 @@ var _ = Describe("Commands", func() { Describe("server", func() { It("should Auth", func() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { pipe.Auth("password") return nil }) diff --git a/example_test.go b/example_test.go index 6c3d6bdb57..098f0aec34 100644 --- a/example_test.go +++ b/example_test.go @@ -159,7 +159,7 @@ func ExampleClient_Scan() { func ExampleClient_Pipelined() { var incr *redis.IntCmd - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { incr = pipe.Incr("pipelined_counter") pipe.Expire("pipelined_counter", time.Hour) return nil @@ -187,7 +187,7 @@ func ExampleClient_Pipeline() { func ExampleClient_TxPipelined() { var incr *redis.IntCmd - _, err := client.TxPipelined(func(pipe *redis.Pipeline) error { + _, err := client.TxPipelined(func(pipe redis.Pipelineable) error { incr = pipe.Incr("tx_pipelined_counter") pipe.Expire("tx_pipelined_counter", time.Hour) return nil @@ -226,7 +226,7 @@ func ExampleClient_Watch() { return err } - _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err = tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) diff --git a/pipeline.go b/pipeline.go index 9e3ba0e683..cb441dc307 100644 --- a/pipeline.go +++ b/pipeline.go @@ -9,6 +9,17 @@ import ( type pipelineExecer func([]Cmder) error +type Pipelineable interface { + Cmdable + StatefulCmdable + Process(cmd Cmder) error + Close() error + Discard() error + discard() error + Exec() ([]Cmder, error) + pipelined(fn func(Pipelineable) error) ([]Cmder, error) +} + // Pipeline implements pipelining as described in // http://redis.io/topics/pipelining. It's safe for concurrent use // by multiple goroutines. @@ -78,7 +89,7 @@ func (c *Pipeline) Exec() ([]Cmder, error) { return cmds, c.exec(cmds) } -func (c *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *Pipeline) pipelined(fn func(Pipelineable) error) ([]Cmder, error) { if err := fn(c); err != nil { return nil, err } @@ -86,3 +97,11 @@ func (c *Pipeline) pipelined(fn func(*Pipeline) error) ([]Cmder, error) { _ = c.Close() return cmds, err } + +func (c *Pipeline) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { + return c.pipelined(fn) +} + +func (c *Pipeline) Pipeline() Pipelineable { + return c +} diff --git a/pipeline_test.go b/pipeline_test.go index 563bb66bdd..7b79dfe8dc 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -22,7 +22,7 @@ var _ = Describe("pipelining", func() { It("supports block style", func() { var get *redis.StringCmd - cmds, err := client.Pipelined(func(pipe *redis.Pipeline) error { + cmds, err := client.Pipelined(func(pipe redis.Pipelineable) error { get = pipe.Get("foo") return nil }) @@ -63,7 +63,7 @@ var _ = Describe("pipelining", func() { Describe("Pipeline", func() { BeforeEach(func() { - pipe = client.Pipeline() + pipe = client.Pipeline().(*redis.Pipeline) }) assertPipeline() diff --git a/pool_test.go b/pool_test.go index 5363c4008a..274b90b00c 100644 --- a/pool_test.go +++ b/pool_test.go @@ -39,7 +39,7 @@ var _ = Describe("pool", func() { var ping *redis.StatusCmd err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { ping = pipe.Ping() return nil }) diff --git a/race_test.go b/race_test.go index 0ec6a140dc..2198a34ba1 100644 --- a/race_test.go +++ b/race_test.go @@ -193,7 +193,7 @@ var _ = Describe("races", func() { num, err := strconv.ParseInt(val, 10, 64) Expect(err).NotTo(HaveOccurred()) - cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Set("key", strconv.FormatInt(num+1, 10), 0) return nil }) diff --git a/redis.go b/redis.go index b71b9fc608..a84a98f37e 100644 --- a/redis.go +++ b/redis.go @@ -61,7 +61,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { // Temp client for Auth and Select. client := newClient(c.opt, pool.NewSingleConnPool(cn)) - _, err := client.Pipelined(func(pipe *Pipeline) error { + _, err := client.Pipelined(func(pipe Pipelineable) error { if c.opt.Password != "" { pipe.Auth(c.opt.Password) } @@ -324,11 +324,11 @@ func (c *Client) PoolStats() *PoolStats { } } -func (c *Client) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *Client) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } -func (c *Client) Pipeline() *Pipeline { +func (c *Client) Pipeline() Pipelineable { pipe := Pipeline{ exec: c.pipelineExecer(c.pipelineProcessCmds), } @@ -337,7 +337,7 @@ func (c *Client) Pipeline() *Pipeline { return &pipe } -func (c *Client) TxPipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *Client) TxPipelined(fn func(Pipelineable) error) ([]Cmder, error) { return c.TxPipeline().pipelined(fn) } diff --git a/redis_test.go b/redis_test.go index 2847963f43..f3d2e7cef9 100644 --- a/redis_test.go +++ b/redis_test.go @@ -68,7 +68,7 @@ var _ = Describe("Client", func() { It("should close Tx without closing the client", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) @@ -232,7 +232,7 @@ var _ = Describe("Client timeout", func() { }) It("Pipeline timeouts", func() { - _, err := client.Pipelined(func(pipe *redis.Pipeline) error { + _, err := client.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) @@ -263,7 +263,7 @@ var _ = Describe("Client timeout", func() { It("Tx Pipeline timeouts", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) diff --git a/ring.go b/ring.go index d13a33b709..df6649ef28 100644 --- a/ring.go +++ b/ring.go @@ -381,7 +381,7 @@ func (c *Ring) Close() error { return firstErr } -func (c *Ring) Pipeline() *Pipeline { +func (c *Ring) Pipeline() Pipelineable { pipe := Pipeline{ exec: c.pipelineExec, } @@ -390,7 +390,7 @@ func (c *Ring) Pipeline() *Pipeline { return &pipe } -func (c *Ring) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *Ring) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } diff --git a/ring_test.go b/ring_test.go index 21adab2baa..deb6e35d61 100644 --- a/ring_test.go +++ b/ring_test.go @@ -137,7 +137,7 @@ var _ = Describe("Redis Ring", func() { keys = append(keys, string(key)) } - _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { + _, err := ring.Pipelined(func(pipe redis.Pipelineable) error { for _, key := range keys { pipe.Set(key, "value", 0).Err() } @@ -153,7 +153,7 @@ var _ = Describe("Redis Ring", func() { }) It("supports hash tags", func() { - _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { + _, err := ring.Pipelined(func(pipe redis.Pipelineable) error { for i := 0; i < 100; i++ { pipe.Set(fmt.Sprintf("key%d{tag}", i), "value", 0).Err() } @@ -184,7 +184,7 @@ var _ = Describe("empty Redis Ring", func() { }) It("pipeline returns an error", func() { - _, err := ring.Pipelined(func(pipe *redis.Pipeline) error { + _, err := ring.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) diff --git a/tx.go b/tx.go index 762bddc90d..a0b642c087 100644 --- a/tx.go +++ b/tx.go @@ -76,7 +76,7 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd { return cmd } -func (c *Tx) Pipeline() *Pipeline { +func (c *Tx) Pipeline() Pipelineable { pipe := Pipeline{ exec: c.pipelineExecer(c.txPipelineProcessCmds), } @@ -94,6 +94,6 @@ func (c *Tx) Pipeline() *Pipeline { // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. -func (c *Tx) Pipelined(fn func(*Pipeline) error) ([]Cmder, error) { +func (c *Tx) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } diff --git a/tx_test.go b/tx_test.go index 583ea05752..127ceeeae7 100644 --- a/tx_test.go +++ b/tx_test.go @@ -33,7 +33,7 @@ var _ = Describe("Tx", func() { return err } - _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err = tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) @@ -65,7 +65,7 @@ var _ = Describe("Tx", func() { It("should discard", func() { err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Set("key1", "hello1", 0) pipe.Discard() pipe.Set("key2", "hello2", 0) @@ -88,7 +88,7 @@ var _ = Describe("Tx", func() { It("returns an error when there are no commands", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(*redis.Pipeline) error { return nil }) + _, err := tx.Pipelined(func(redis.Pipelineable) error { return nil }) return err }) Expect(err).To(MatchError("redis: pipeline is empty")) @@ -102,7 +102,7 @@ var _ = Describe("Tx", func() { const N = 20000 err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { for i := 0; i < N; i++ { pipe.Incr("key") } @@ -133,7 +133,7 @@ var _ = Describe("Tx", func() { do := func() error { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe *redis.Pipeline) error { + _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { pipe.Ping() return nil }) From eeb4d091986ee12fc98796e7d7c73348aa2d8871 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Tue, 2 May 2017 12:00:53 -0300 Subject: [PATCH 0330/1746] rename Pipelineable to Pipeliner --- bench_test.go | 2 +- cluster.go | 8 ++++---- cluster_test.go | 10 +++++----- commands.go | 4 ++-- commands_test.go | 2 +- example_test.go | 6 +++--- pipeline.go | 10 +++++----- pipeline_test.go | 2 +- pool_test.go | 2 +- race_test.go | 2 +- redis.go | 8 ++++---- redis_test.go | 6 +++--- ring.go | 4 ++-- ring_test.go | 6 +++--- tx.go | 4 ++-- tx_test.go | 10 +++++----- 16 files changed, 43 insertions(+), 43 deletions(-) diff --git a/bench_test.go b/bench_test.go index cba9ff5bac..855140669f 100644 --- a/bench_test.go +++ b/bench_test.go @@ -188,7 +188,7 @@ func BenchmarkPipeline(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Set("key", "hello", 0) pipe.Expire("key", time.Second) return nil diff --git a/cluster.go b/cluster.go index 5bb1b5adec..5c2e59ffd6 100644 --- a/cluster.go +++ b/cluster.go @@ -674,7 +674,7 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { } } -func (c *ClusterClient) Pipeline() Pipelineable { +func (c *ClusterClient) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExec, } @@ -683,7 +683,7 @@ func (c *ClusterClient) Pipeline() Pipelineable { return &pipe } -func (c *ClusterClient) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *ClusterClient) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } @@ -797,7 +797,7 @@ func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]C } // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. -func (c *ClusterClient) TxPipeline() Pipelineable { +func (c *ClusterClient) TxPipeline() Pipeliner { pipe := Pipeline{ exec: c.txPipelineExec, } @@ -806,7 +806,7 @@ func (c *ClusterClient) TxPipeline() Pipelineable { return &pipe } -func (c *ClusterClient) TxPipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().pipelined(fn) } diff --git a/cluster_test.go b/cluster_test.go index c68ff92eea..1e55be6e2d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -347,7 +347,7 @@ var _ = Describe("ClusterClient", func() { return err } - _, err = tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err = tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) @@ -544,7 +544,7 @@ var _ = Describe("ClusterClient without nodes", func() { }) It("pipeline returns an error", func() { - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) @@ -571,7 +571,7 @@ var _ = Describe("ClusterClient without valid nodes", func() { }) It("pipeline returns an error", func() { - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) @@ -594,7 +594,7 @@ var _ = Describe("ClusterClient timeout", func() { }) It("Pipeline timeouts", func() { - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) @@ -612,7 +612,7 @@ var _ = Describe("ClusterClient timeout", func() { It("Tx Pipeline timeouts", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err := tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) diff --git a/commands.go b/commands.go index d1efd21424..a0984321bd 100644 --- a/commands.go +++ b/commands.go @@ -39,8 +39,8 @@ func formatSec(dur time.Duration) int64 { } type Cmdable interface { - Pipeline() Pipelineable - Pipelined(fn func(Pipelineable) error) ([]Cmder, error) + Pipeline() Pipeliner + Pipelined(fn func(Pipeliner) error) ([]Cmder, error) Echo(message interface{}) *StringCmd Ping() *StatusCmd diff --git a/commands_test.go b/commands_test.go index b9b601abf7..3a0240dd47 100644 --- a/commands_test.go +++ b/commands_test.go @@ -27,7 +27,7 @@ var _ = Describe("Commands", func() { Describe("server", func() { It("should Auth", func() { - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Auth("password") return nil }) diff --git a/example_test.go b/example_test.go index 098f0aec34..6453fd1592 100644 --- a/example_test.go +++ b/example_test.go @@ -159,7 +159,7 @@ func ExampleClient_Scan() { func ExampleClient_Pipelined() { var incr *redis.IntCmd - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { incr = pipe.Incr("pipelined_counter") pipe.Expire("pipelined_counter", time.Hour) return nil @@ -187,7 +187,7 @@ func ExampleClient_Pipeline() { func ExampleClient_TxPipelined() { var incr *redis.IntCmd - _, err := client.TxPipelined(func(pipe redis.Pipelineable) error { + _, err := client.TxPipelined(func(pipe redis.Pipeliner) error { incr = pipe.Incr("tx_pipelined_counter") pipe.Expire("tx_pipelined_counter", time.Hour) return nil @@ -226,7 +226,7 @@ func ExampleClient_Watch() { return err } - _, err = tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err = tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) diff --git a/pipeline.go b/pipeline.go index cb441dc307..f1caf152a4 100644 --- a/pipeline.go +++ b/pipeline.go @@ -9,7 +9,7 @@ import ( type pipelineExecer func([]Cmder) error -type Pipelineable interface { +type Pipeliner interface { Cmdable StatefulCmdable Process(cmd Cmder) error @@ -17,7 +17,7 @@ type Pipelineable interface { Discard() error discard() error Exec() ([]Cmder, error) - pipelined(fn func(Pipelineable) error) ([]Cmder, error) + pipelined(fn func(Pipeliner) error) ([]Cmder, error) } // Pipeline implements pipelining as described in @@ -89,7 +89,7 @@ func (c *Pipeline) Exec() ([]Cmder, error) { return cmds, c.exec(cmds) } -func (c *Pipeline) pipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *Pipeline) pipelined(fn func(Pipeliner) error) ([]Cmder, error) { if err := fn(c); err != nil { return nil, err } @@ -98,10 +98,10 @@ func (c *Pipeline) pipelined(fn func(Pipelineable) error) ([]Cmder, error) { return cmds, err } -func (c *Pipeline) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *Pipeline) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.pipelined(fn) } -func (c *Pipeline) Pipeline() Pipelineable { +func (c *Pipeline) Pipeline() Pipeliner { return c } diff --git a/pipeline_test.go b/pipeline_test.go index 7b79dfe8dc..706ff59a14 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -22,7 +22,7 @@ var _ = Describe("pipelining", func() { It("supports block style", func() { var get *redis.StringCmd - cmds, err := client.Pipelined(func(pipe redis.Pipelineable) error { + cmds, err := client.Pipelined(func(pipe redis.Pipeliner) error { get = pipe.Get("foo") return nil }) diff --git a/pool_test.go b/pool_test.go index 274b90b00c..9d33559b87 100644 --- a/pool_test.go +++ b/pool_test.go @@ -39,7 +39,7 @@ var _ = Describe("pool", func() { var ping *redis.StatusCmd err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipeliner) error { ping = pipe.Ping() return nil }) diff --git a/race_test.go b/race_test.go index 2198a34ba1..439e5e2926 100644 --- a/race_test.go +++ b/race_test.go @@ -193,7 +193,7 @@ var _ = Describe("races", func() { num, err := strconv.ParseInt(val, 10, 64) Expect(err).NotTo(HaveOccurred()) - cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Set("key", strconv.FormatInt(num+1, 10), 0) return nil }) diff --git a/redis.go b/redis.go index a84a98f37e..e81100c434 100644 --- a/redis.go +++ b/redis.go @@ -61,7 +61,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { // Temp client for Auth and Select. client := newClient(c.opt, pool.NewSingleConnPool(cn)) - _, err := client.Pipelined(func(pipe Pipelineable) error { + _, err := client.Pipelined(func(pipe Pipeliner) error { if c.opt.Password != "" { pipe.Auth(c.opt.Password) } @@ -324,11 +324,11 @@ func (c *Client) PoolStats() *PoolStats { } } -func (c *Client) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *Client) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } -func (c *Client) Pipeline() Pipelineable { +func (c *Client) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExecer(c.pipelineProcessCmds), } @@ -337,7 +337,7 @@ func (c *Client) Pipeline() Pipelineable { return &pipe } -func (c *Client) TxPipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *Client) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().pipelined(fn) } diff --git a/redis_test.go b/redis_test.go index f3d2e7cef9..a27e3bc14a 100644 --- a/redis_test.go +++ b/redis_test.go @@ -68,7 +68,7 @@ var _ = Describe("Client", func() { It("should close Tx without closing the client", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err := tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) @@ -232,7 +232,7 @@ var _ = Describe("Client timeout", func() { }) It("Pipeline timeouts", func() { - _, err := client.Pipelined(func(pipe redis.Pipelineable) error { + _, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) @@ -263,7 +263,7 @@ var _ = Describe("Client timeout", func() { It("Tx Pipeline timeouts", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err := tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) diff --git a/ring.go b/ring.go index df6649ef28..c190b2845f 100644 --- a/ring.go +++ b/ring.go @@ -381,7 +381,7 @@ func (c *Ring) Close() error { return firstErr } -func (c *Ring) Pipeline() Pipelineable { +func (c *Ring) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExec, } @@ -390,7 +390,7 @@ func (c *Ring) Pipeline() Pipelineable { return &pipe } -func (c *Ring) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } diff --git a/ring_test.go b/ring_test.go index deb6e35d61..a61ee96a04 100644 --- a/ring_test.go +++ b/ring_test.go @@ -137,7 +137,7 @@ var _ = Describe("Redis Ring", func() { keys = append(keys, string(key)) } - _, err := ring.Pipelined(func(pipe redis.Pipelineable) error { + _, err := ring.Pipelined(func(pipe redis.Pipeliner) error { for _, key := range keys { pipe.Set(key, "value", 0).Err() } @@ -153,7 +153,7 @@ var _ = Describe("Redis Ring", func() { }) It("supports hash tags", func() { - _, err := ring.Pipelined(func(pipe redis.Pipelineable) error { + _, err := ring.Pipelined(func(pipe redis.Pipeliner) error { for i := 0; i < 100; i++ { pipe.Set(fmt.Sprintf("key%d{tag}", i), "value", 0).Err() } @@ -184,7 +184,7 @@ var _ = Describe("empty Redis Ring", func() { }) It("pipeline returns an error", func() { - _, err := ring.Pipelined(func(pipe redis.Pipelineable) error { + _, err := ring.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) diff --git a/tx.go b/tx.go index a0b642c087..21c5c70f5e 100644 --- a/tx.go +++ b/tx.go @@ -76,7 +76,7 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd { return cmd } -func (c *Tx) Pipeline() Pipelineable { +func (c *Tx) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExecer(c.txPipelineProcessCmds), } @@ -94,6 +94,6 @@ func (c *Tx) Pipeline() Pipelineable { // Exec always returns list of commands. If transaction fails // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. -func (c *Tx) Pipelined(fn func(Pipelineable) error) ([]Cmder, error) { +func (c *Tx) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } diff --git a/tx_test.go b/tx_test.go index 127ceeeae7..83b12e1f04 100644 --- a/tx_test.go +++ b/tx_test.go @@ -33,7 +33,7 @@ var _ = Describe("Tx", func() { return err } - _, err = tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err = tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Set(key, strconv.FormatInt(n+1, 10), 0) return nil }) @@ -65,7 +65,7 @@ var _ = Describe("Tx", func() { It("should discard", func() { err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Set("key1", "hello1", 0) pipe.Discard() pipe.Set("key2", "hello2", 0) @@ -88,7 +88,7 @@ var _ = Describe("Tx", func() { It("returns an error when there are no commands", func() { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(redis.Pipelineable) error { return nil }) + _, err := tx.Pipelined(func(redis.Pipeliner) error { return nil }) return err }) Expect(err).To(MatchError("redis: pipeline is empty")) @@ -102,7 +102,7 @@ var _ = Describe("Tx", func() { const N = 20000 err := client.Watch(func(tx *redis.Tx) error { - cmds, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + cmds, err := tx.Pipelined(func(pipe redis.Pipeliner) error { for i := 0; i < N; i++ { pipe.Incr("key") } @@ -133,7 +133,7 @@ var _ = Describe("Tx", func() { do := func() error { err := client.Watch(func(tx *redis.Tx) error { - _, err := tx.Pipelined(func(pipe redis.Pipelineable) error { + _, err := tx.Pipelined(func(pipe redis.Pipeliner) error { pipe.Ping() return nil }) From 3f43c7448baedb238ad5304a303d1c431a13e224 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Tue, 2 May 2017 12:41:42 -0300 Subject: [PATCH 0331/1746] add runtime type assertion --- pipeline.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipeline.go b/pipeline.go index f1caf152a4..977f5eb3dc 100644 --- a/pipeline.go +++ b/pipeline.go @@ -20,6 +20,8 @@ type Pipeliner interface { pipelined(fn func(Pipeliner) error) ([]Cmder, error) } +var _ Pipeliner = (*Pipeline)(nil) + // Pipeline implements pipelining as described in // http://redis.io/topics/pipelining. It's safe for concurrent use // by multiple goroutines. From 7379f211a4dde9bff6a9bf5b5cb962c2e78282d1 Mon Sep 17 00:00:00 2001 From: Felipe Cavalcanti Date: Wed, 3 May 2017 10:48:12 -0300 Subject: [PATCH 0332/1746] Client TxPipeline method should also return a Pipeliner --- pipeline_test.go | 2 +- redis.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline_test.go b/pipeline_test.go index 706ff59a14..3f69e0c715 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -71,7 +71,7 @@ var _ = Describe("pipelining", func() { Describe("TxPipeline", func() { BeforeEach(func() { - pipe = client.TxPipeline() + pipe = client.TxPipeline().(*redis.Pipeline) }) assertPipeline() diff --git a/redis.go b/redis.go index e81100c434..ca88df0d16 100644 --- a/redis.go +++ b/redis.go @@ -342,7 +342,7 @@ func (c *Client) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { } // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. -func (c *Client) TxPipeline() *Pipeline { +func (c *Client) TxPipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExecer(c.txPipelineProcessCmds), } From 6307b9503901a1b01e73c03f7cecb74bef18b45c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 9 May 2017 12:44:36 +0300 Subject: [PATCH 0333/1746] Export Cmder.Name --- cluster.go | 2 +- command.go | 8 ++++---- ring.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cluster.go b/cluster.go index 5c2e59ffd6..99d8d7da61 100644 --- a/cluster.go +++ b/cluster.go @@ -392,7 +392,7 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl return 0, node, err } - cmdInfo := c.cmds[cmd.name()] + cmdInfo := c.cmds[cmd.Name()] firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) slot := hashtag.Slot(firstKey) diff --git a/command.go b/command.go index 33c11aafe2..361661adfb 100644 --- a/command.go +++ b/command.go @@ -33,7 +33,7 @@ var ( type Cmder interface { args() []interface{} arg(int) string - name() string + Name() string readReply(*pool.Conn) error setErr(error) @@ -84,7 +84,7 @@ func cmdString(cmd Cmder, val interface{}) string { } func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { - switch cmd.name() { + switch cmd.Name() { case "eval", "evalsha": if cmd.arg(2) != "0" { return 3 @@ -95,7 +95,7 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { return 1 } if info == nil { - internal.Logf("info for cmd=%s not found", cmd.name()) + internal.Logf("info for cmd=%s not found", cmd.Name()) return -1 } return int(info.FirstKeyPos) @@ -126,7 +126,7 @@ func (cmd *baseCmd) arg(pos int) string { return s } -func (cmd *baseCmd) name() string { +func (cmd *baseCmd) Name() string { if len(cmd._args) > 0 { // Cmd name must be lower cased. s := internal.ToLower(cmd.arg(0)) diff --git a/ring.go b/ring.go index c190b2845f..270a81f978 100644 --- a/ring.go +++ b/ring.go @@ -298,7 +298,7 @@ func (c *Ring) shardByName(name string) (*ringShard, error) { } func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { - cmdInfo := c.cmdInfo(cmd.name()) + cmdInfo := c.cmdInfo(cmd.Name()) firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) return c.shardByKey(firstKey) } @@ -397,7 +397,7 @@ func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { - cmdInfo := c.cmdInfo(cmd.name()) + cmdInfo := c.cmdInfo(cmd.Name()) name := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) if name != "" { name = c.hash.Get(hashtag.Key(name)) From 18c1db946601fba100ad23e98eb8c04055a2a6cd Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 11 May 2017 17:02:26 +0300 Subject: [PATCH 0334/1746] Doc --- pubsub.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pubsub.go b/pubsub.go index b9d5c60b60..19016b9d7e 100644 --- a/pubsub.go +++ b/pubsub.go @@ -13,6 +13,9 @@ import ( // PubSub implements Pub/Sub commands as described in // http://redis.io/topics/pubsub. It's NOT safe for concurrent use by // multiple goroutines. +// +// PubSub automatically resubscribes to the channels and patterns +// when Redis becomes unavailable. type PubSub struct { base baseClient @@ -138,7 +141,8 @@ func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) return writeCmd(cn, cmd) } -// Subscribes the client to the specified channels. +// Subscribes the client to the specified channels. It returns +// empty subscription if there are no channels. func (c *PubSub) Subscribe(channels ...string) error { c.subMu.Lock() c.channels = appendIfNotExists(c.channels, channels...) @@ -146,7 +150,8 @@ func (c *PubSub) Subscribe(channels ...string) error { return c.subscribe("subscribe", channels...) } -// Subscribes the client to the given patterns. +// Subscribes the client to the given patterns. It returns +// empty subscription if there are no patterns. func (c *PubSub) PSubscribe(patterns ...string) error { c.subMu.Lock() c.patterns = appendIfNotExists(c.patterns, patterns...) From e3550dd65c64587cb533914708343a0be4c01139 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 24 May 2017 15:24:47 +0300 Subject: [PATCH 0335/1746] Add Script.Hash --- script.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script.go b/script.go index 133117b20d..74135f5a5c 100644 --- a/script.go +++ b/script.go @@ -31,6 +31,10 @@ func NewScript(src string) *Script { } } +func (s *Script) Hash() string { + return s.hash +} + func (s *Script) Load(c scripter) *StringCmd { return c.ScriptLoad(s.src) } From 368f0ea0bac726049f4a81aebfee0a76c91c50a7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 24 May 2017 15:53:41 +0300 Subject: [PATCH 0336/1746] Cleanup tests --- commands_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/commands_test.go b/commands_test.go index 3a0240dd47..7e5d4010f8 100644 --- a/commands_test.go +++ b/commands_test.go @@ -114,13 +114,12 @@ var _ = Describe("Commands", func() { Expect(get.Err()).NotTo(HaveOccurred()) Expect(get.Val()).To(Equal("theclientname")) - }) It("should ConfigGet", func() { - r := client.ConfigGet("*") - Expect(r.Err()).NotTo(HaveOccurred()) - Expect(r.Val()).NotTo(BeEmpty()) + val, err := client.ConfigGet("*").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).NotTo(BeEmpty()) }) It("should ConfigResetStat", func() { From 7e8890b64422ad56e3c609a7b2afe54a6126f03a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 25 May 2017 13:38:04 +0300 Subject: [PATCH 0337/1746] Embed Cmdable into StatefulCmdable --- cluster.go | 8 +++----- commands.go | 12 +++++++++++- pipeline.go | 2 -- redis.go | 10 ++++------ ring.go | 5 ++--- sentinel.go | 2 +- tx.go | 7 ++----- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cluster.go b/cluster.go index 99d8d7da61..6a4bbe8f17 100644 --- a/cluster.go +++ b/cluster.go @@ -349,7 +349,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt: opt, nodes: newClusterNodes(opt), } - c.cmdable.process = c.Process + c.setProcessor(c.Process) // Add initial nodes. for _, addr := range opt.Addrs { @@ -678,8 +678,7 @@ func (c *ClusterClient) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExec, } - pipe.cmdable.process = pipe.Process - pipe.statefulCmdable.process = pipe.Process + pipe.setProcessor(pipe.Process) return &pipe } @@ -801,8 +800,7 @@ func (c *ClusterClient) TxPipeline() Pipeliner { pipe := Pipeline{ exec: c.txPipelineExec, } - pipe.cmdable.process = pipe.Process - pipe.statefulCmdable.process = pipe.Process + pipe.setProcessor(pipe.Process) return &pipe } diff --git a/commands.go b/commands.go index a0984321bd..51dcf941fa 100644 --- a/commands.go +++ b/commands.go @@ -238,6 +238,7 @@ type Cmdable interface { } type StatefulCmdable interface { + Cmdable Auth(password string) *StatusCmd Select(index int) *StatusCmd ClientSetName(name string) *BoolCmd @@ -255,10 +256,20 @@ type cmdable struct { process func(cmd Cmder) error } +func (c *cmdable) setProcessor(fn func(Cmder) error) { + c.process = fn +} + type statefulCmdable struct { + cmdable process func(cmd Cmder) error } +func (c *statefulCmdable) setProcessor(fn func(Cmder) error) { + c.process = fn + c.cmdable.setProcessor(fn) +} + //------------------------------------------------------------------------------ func (c *statefulCmdable) Auth(password string) *StatusCmd { @@ -280,7 +291,6 @@ func (c *cmdable) Ping() *StatusCmd { } func (c *cmdable) Wait(numSlaves int, timeout time.Duration) *IntCmd { - cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Millisecond)) c.process(cmd) return cmd diff --git a/pipeline.go b/pipeline.go index 977f5eb3dc..de99f12459 100644 --- a/pipeline.go +++ b/pipeline.go @@ -10,7 +10,6 @@ import ( type pipelineExecer func([]Cmder) error type Pipeliner interface { - Cmdable StatefulCmdable Process(cmd Cmder) error Close() error @@ -26,7 +25,6 @@ var _ Pipeliner = (*Pipeline)(nil) // http://redis.io/topics/pipelining. It's safe for concurrent use // by multiple goroutines. type Pipeline struct { - cmdable statefulCmdable exec pipelineExecer diff --git a/redis.go b/redis.go index ca88df0d16..89f985ee7d 100644 --- a/redis.go +++ b/redis.go @@ -294,7 +294,7 @@ func newClient(opt *Options, pool pool.Pooler) *Client { connPool: pool, }, } - client.cmdable.process = client.Process + client.setProcessor(client.Process) return &client } @@ -307,7 +307,7 @@ func NewClient(opt *Options) *Client { func (c *Client) copy() *Client { c2 := new(Client) *c2 = *c - c2.cmdable.process = c2.Process + c2.setProcessor(c2.Process) return c2 } @@ -332,8 +332,7 @@ func (c *Client) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExecer(c.pipelineProcessCmds), } - pipe.cmdable.process = pipe.Process - pipe.statefulCmdable.process = pipe.Process + pipe.setProcessor(pipe.Process) return &pipe } @@ -346,8 +345,7 @@ func (c *Client) TxPipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExecer(c.txPipelineProcessCmds), } - pipe.cmdable.process = pipe.Process - pipe.statefulCmdable.process = pipe.Process + pipe.setProcessor(pipe.Process) return &pipe } diff --git a/ring.go b/ring.go index 270a81f978..9c57430d02 100644 --- a/ring.go +++ b/ring.go @@ -148,7 +148,7 @@ func NewRing(opt *RingOptions) *Ring { cmdsInfoOnce: new(sync.Once), } - ring.cmdable.process = ring.Process + ring.setProcessor(ring.Process) for name, addr := range opt.Addrs { clopt := opt.clientOptions() clopt.Addr = addr @@ -385,8 +385,7 @@ func (c *Ring) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExec, } - pipe.cmdable.process = pipe.Process - pipe.statefulCmdable.process = pipe.Process + pipe.setProcessor(pipe.Process) return &pipe } diff --git a/sentinel.go b/sentinel.go index 799f530fca..da3a4312bb 100644 --- a/sentinel.go +++ b/sentinel.go @@ -82,7 +82,7 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { }, }, } - client.cmdable.process = client.Process + client.setProcessor(client.Process) return &client } diff --git a/tx.go b/tx.go index 21c5c70f5e..5ef89619ba 100644 --- a/tx.go +++ b/tx.go @@ -13,7 +13,6 @@ const TxFailedErr = internal.RedisError("redis: transaction failed") // by multiple goroutines, because Exec resets list of watched keys. // If you don't need WATCH it is better to use Pipeline. type Tx struct { - cmdable statefulCmdable baseClient } @@ -25,8 +24,7 @@ func (c *Client) newTx() *Tx { connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true), }, } - tx.cmdable.process = tx.Process - tx.statefulCmdable.process = tx.Process + tx.setProcessor(tx.Process) return &tx } @@ -80,8 +78,7 @@ func (c *Tx) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.pipelineExecer(c.txPipelineProcessCmds), } - pipe.cmdable.process = pipe.Process - pipe.statefulCmdable.process = pipe.Process + pipe.setProcessor(pipe.Process) return &pipe } From 406e882c43266d58b7c8cb38b0e7085c59506952 Mon Sep 17 00:00:00 2001 From: Jonathan Chan Date: Thu, 25 May 2017 01:08:44 -0400 Subject: [PATCH 0338/1746] Added backoff retry --- internal/errors.go | 2 +- internal/internal.go | 23 +++++++++++++++++++++ internal/internal_test.go | 17 ++++++++++++++++ options.go | 24 ++++++++++++++++------ redis.go | 9 ++++++++- redis_test.go | 42 +++++++++++++++++++++++++++++++++++++++ testdata/redis.conf | 1 + 7 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 internal/internal.go create mode 100644 internal/internal_test.go diff --git a/internal/errors.go b/internal/errors.go index 67b29aec3e..c93e008182 100644 --- a/internal/errors.go +++ b/internal/errors.go @@ -13,7 +13,7 @@ type RedisError string func (e RedisError) Error() string { return string(e) } func IsRetryableError(err error) bool { - return IsNetworkError(err) + return IsNetworkError(err) || err.Error() == "ERR max number of clients reached" } func IsInternalError(err error) bool { diff --git a/internal/internal.go b/internal/internal.go new file mode 100644 index 0000000000..fb4efa5f04 --- /dev/null +++ b/internal/internal.go @@ -0,0 +1,23 @@ +package internal + +import ( + "math/rand" + "time" +) + +const retryBackoff = 8 * time.Millisecond + +// Retry backoff with jitter sleep to prevent overloaded conditions during intervals +// https://www.awsarchitectureblog.com/2015/03/backoff.html +func RetryBackoff(retry int, maxRetryBackoff time.Duration) time.Duration { + if retry < 0 { + retry = 0 + } + + backoff := retryBackoff << uint(retry) + if backoff > maxRetryBackoff { + backoff = maxRetryBackoff + } + + return time.Duration(rand.Int63n(int64(backoff))) +} diff --git a/internal/internal_test.go b/internal/internal_test.go new file mode 100644 index 0000000000..5c7000e1ea --- /dev/null +++ b/internal/internal_test.go @@ -0,0 +1,17 @@ +package internal + +import ( + "testing" + . "github.com/onsi/gomega" + "time" +) + +func TestRetryBackoff(t *testing.T) { + RegisterTestingT(t) + + for i := -1; i<= 8; i++ { + backoff := RetryBackoff(i, 512*time.Millisecond) + Expect(backoff >= 0).To(BeTrue()) + Expect(backoff <= 512*time.Millisecond).To(BeTrue()) + } +} diff --git a/options.go b/options.go index d2aefb4755..7278845da4 100644 --- a/options.go +++ b/options.go @@ -34,6 +34,10 @@ type Options struct { // Default is to not retry failed commands. MaxRetries int + // Retry using exponential backoff wait algorithm between each retry + // Default is 512 milliseconds; set to -1 to disable any backoff sleep + MaxRetryBackoff time.Duration + // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration @@ -89,15 +93,17 @@ func (opt *Options) init() { if opt.DialTimeout == 0 { opt.DialTimeout = 5 * time.Second } - if opt.ReadTimeout == 0 { - opt.ReadTimeout = 3 * time.Second - } else if opt.ReadTimeout == -1 { + switch opt.ReadTimeout { + case -1: opt.ReadTimeout = 0 + case 0: + opt.ReadTimeout = 3 * time.Second } - if opt.WriteTimeout == 0 { - opt.WriteTimeout = opt.ReadTimeout - } else if opt.WriteTimeout == -1 { + switch opt.WriteTimeout { + case -1: opt.WriteTimeout = 0 + case 0: + opt.WriteTimeout = opt.ReadTimeout } if opt.PoolTimeout == 0 { opt.PoolTimeout = opt.ReadTimeout + time.Second @@ -108,6 +114,12 @@ func (opt *Options) init() { if opt.IdleCheckFrequency == 0 { opt.IdleCheckFrequency = time.Minute } + switch opt.MaxRetryBackoff { + case -1: + opt.MaxRetryBackoff = 0 + case 0: + opt.MaxRetryBackoff = 512 * time.Millisecond + } } // ParseURL parses a redis URL into options that can be used to connect to redis diff --git a/redis.go b/redis.go index ca88df0d16..3a83d10047 100644 --- a/redis.go +++ b/redis.go @@ -96,9 +96,16 @@ func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func( func (c *baseClient) defaultProcess(cmd Cmder) error { for i := 0; i <= c.opt.MaxRetries; i++ { + if i > 0 { + time.Sleep(internal.RetryBackoff(i, c.opt.MaxRetryBackoff)) + } + cn, _, err := c.conn() if err != nil { cmd.setErr(err) + if internal.IsRetryableError(err) { + continue + } return err } @@ -106,7 +113,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { if err := writeCmd(cn, cmd); err != nil { c.putConn(cn, err) cmd.setErr(err) - if err != nil && internal.IsRetryableError(err) { + if internal.IsRetryableError(err) { continue } return err diff --git a/redis_test.go b/redis_test.go index a27e3bc14a..fde10e3733 100644 --- a/redis_test.go +++ b/redis_test.go @@ -156,6 +156,48 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) }) + It("should retry with backoff", func() { + Expect(client.Close()).NotTo(HaveOccurred()) + + // use up all the available connections to force a fail + connectionHogClient := redis.NewClient(&redis.Options{ + Addr: redisAddr, + MaxRetries: 1, + }) + defer connectionHogClient.Close() + + for i := 0; i <= 1002; i++ { + connectionHogClient.Pool().NewConn() + } + + clientNoRetry := redis.NewClient(&redis.Options{ + Addr: redisAddr, + PoolSize: 1, + MaxRetryBackoff: -1, + }) + defer clientNoRetry.Close() + + clientRetry := redis.NewClient(&redis.Options{ + Addr: redisAddr, + MaxRetries: 5, + PoolSize: 1, + MaxRetryBackoff: 128 * time.Millisecond, + }) + defer clientRetry.Close() + + startNoRetry := time.Now() + err := clientNoRetry.Ping().Err() + Expect(err).To(HaveOccurred()) + elapseNoRetry := time.Since(startNoRetry) + + startRetry := time.Now() + err = clientRetry.Ping().Err() + Expect(err).To(HaveOccurred()) + elapseRetry := time.Since(startRetry) + + Expect(elapseRetry > elapseNoRetry).To(BeTrue()) + }) + It("should update conn.UsedAt on read/write", func() { cn, _, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) diff --git a/testdata/redis.conf b/testdata/redis.conf index 016fa0a715..235b2954aa 100644 --- a/testdata/redis.conf +++ b/testdata/redis.conf @@ -7,3 +7,4 @@ save "" appendonly yes cluster-config-file nodes.conf cluster-node-timeout 30000 +maxclients 1001 \ No newline at end of file From 4a3a3006655b3e92d89cdf45c339b1b1d9fb2f66 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 25 May 2017 14:16:39 +0300 Subject: [PATCH 0339/1746] Add Options.OnConnect --- cluster.go | 5 +++- commands.go | 4 +-- options.go | 3 +++ redis.go | 72 +++++++++++++++++++++++++++++++++++++++++++-------- redis_test.go | 23 ++++++++++++++++ ring.go | 4 +++ sentinel.go | 4 +++ 7 files changed, 101 insertions(+), 14 deletions(-) diff --git a/cluster.go b/cluster.go index 6a4bbe8f17..e3c5832fcc 100644 --- a/cluster.go +++ b/cluster.go @@ -35,6 +35,8 @@ type ClusterOptions struct { // Following options are copied from Options struct. + OnConnect func(*Conn) error + MaxRetries int Password string @@ -65,6 +67,8 @@ func (opt *ClusterOptions) clientOptions() *Options { const disableIdleCheck = -1 return &Options{ + OnConnect: opt.OnConnect, + MaxRetries: opt.MaxRetries, Password: opt.Password, ReadOnly: opt.ReadOnly, @@ -77,7 +81,6 @@ func (opt *ClusterOptions) clientOptions() *Options { PoolTimeout: opt.PoolTimeout, IdleTimeout: opt.IdleTimeout, - // IdleCheckFrequency is not copied to disable reaper IdleCheckFrequency: disableIdleCheck, } } diff --git a/commands.go b/commands.go index 51dcf941fa..3956cf74d8 100644 --- a/commands.go +++ b/commands.go @@ -42,6 +42,7 @@ type Cmdable interface { Pipeline() Pipeliner Pipelined(fn func(Pipeliner) error) ([]Cmder, error) + ClientGetName() *StringCmd Echo(message interface{}) *StringCmd Ping() *StatusCmd Quit() *StatusCmd @@ -242,7 +243,6 @@ type StatefulCmdable interface { Auth(password string) *StatusCmd Select(index int) *StatusCmd ClientSetName(name string) *BoolCmd - ClientGetName() *StringCmd ReadOnly() *StatusCmd ReadWrite() *StatusCmd } @@ -1649,7 +1649,7 @@ func (c *statefulCmdable) ClientSetName(name string) *BoolCmd { } // ClientGetName returns the name of the connection. -func (c *statefulCmdable) ClientGetName() *StringCmd { +func (c *cmdable) ClientGetName() *StringCmd { cmd := NewStringCmd("client", "getname") c.process(cmd) return cmd diff --git a/options.go b/options.go index d2aefb4755..1695c0b84c 100644 --- a/options.go +++ b/options.go @@ -24,6 +24,9 @@ type Options struct { // Network and Addr options. Dialer func() (net.Conn, error) + // Hook that is called when new connection is established. + OnConnect func(*Conn) error + // Optional password. Must match the password specified in the // requirepass server configuration option. Password string diff --git a/redis.go b/redis.go index 89f985ee7d..303877fcdb 100644 --- a/redis.go +++ b/redis.go @@ -21,11 +21,6 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB) } -// Options returns read-only Options that were used to create the client. -func (c *baseClient) Options() *Options { - return c.opt -} - func (c *baseClient) conn() (*pool.Conn, bool, error) { cn, isNew, err := c.connPool.Get() if err != nil { @@ -55,13 +50,23 @@ func (c *baseClient) putConn(cn *pool.Conn, err error) bool { func (c *baseClient) initConn(cn *pool.Conn) error { cn.Inited = true - if c.opt.Password == "" && c.opt.DB == 0 && !c.opt.ReadOnly { + if c.opt.Password == "" && + c.opt.DB == 0 && + !c.opt.ReadOnly && + c.opt.OnConnect == nil { return nil } - // Temp client for Auth and Select. - client := newClient(c.opt, pool.NewSingleConnPool(cn)) - _, err := client.Pipelined(func(pipe Pipeliner) error { + // Temp client to initialize connection. + conn := &Conn{ + baseClient: baseClient{ + opt: c.opt, + connPool: pool.NewSingleConnPool(cn), + }, + } + conn.setProcessor(conn.Process) + + _, err := conn.Pipelined(func(pipe Pipeliner) error { if c.opt.Password != "" { pipe.Auth(c.opt.Password) } @@ -76,7 +81,14 @@ func (c *baseClient) initConn(cn *pool.Conn) error { return nil }) - return err + if err != nil { + return err + } + + if c.opt.OnConnect != nil { + return c.opt.OnConnect(conn) + } + return nil } func (c *baseClient) Process(cmd Cmder) error { @@ -182,7 +194,7 @@ func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { } } -func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { +func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) { cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) @@ -311,6 +323,11 @@ func (c *Client) copy() *Client { return c2 } +// Options returns read-only Options that were used to create the client. +func (c *Client) Options() *Options { + return c.opt +} + // PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { s := c.connPool.Stats() @@ -375,3 +392,36 @@ func (c *Client) PSubscribe(channels ...string) *PubSub { } return pubsub } + +//------------------------------------------------------------------------------ + +// Conn is like Client, but its pool contains single connection. +type Conn struct { + baseClient + statefulCmdable +} + +func (c *Conn) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { + return c.Pipeline().pipelined(fn) +} + +func (c *Conn) Pipeline() Pipeliner { + pipe := Pipeline{ + exec: c.pipelineExecer(c.pipelineProcessCmds), + } + pipe.setProcessor(pipe.Process) + return &pipe +} + +func (c *Conn) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { + return c.TxPipeline().pipelined(fn) +} + +// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. +func (c *Conn) TxPipeline() Pipeliner { + pipe := Pipeline{ + exec: c.pipelineExecer(c.txPipelineProcessCmds), + } + pipe.setProcessor(pipe.Process) + return &pipe +} diff --git a/redis_test.go b/redis_test.go index a27e3bc14a..407d378406 100644 --- a/redis_test.go +++ b/redis_test.go @@ -296,3 +296,26 @@ var _ = Describe("Client timeout", func() { testTimeout() }) }) + +var _ = Describe("Client OnConnect", func() { + var client *redis.Client + + BeforeEach(func() { + opt := redisOptions() + opt.OnConnect = func(cn *redis.Conn) error { + return cn.ClientSetName("on_connect").Err() + } + + client = redis.NewClient(opt) + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + It("calls OnConnect", func() { + name, err := client.ClientGetName().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(name).To(Equal("on_connect")) + }) +}) diff --git a/ring.go b/ring.go index 9c57430d02..a9666bc72f 100644 --- a/ring.go +++ b/ring.go @@ -29,6 +29,8 @@ type RingOptions struct { // Following options are copied from Options struct. + OnConnect func(*Conn) error + DB int Password string @@ -52,6 +54,8 @@ func (opt *RingOptions) init() { func (opt *RingOptions) clientOptions() *Options { return &Options{ + OnConnect: opt.OnConnect, + DB: opt.DB, Password: opt.Password, diff --git a/sentinel.go b/sentinel.go index da3a4312bb..b28c3706ef 100644 --- a/sentinel.go +++ b/sentinel.go @@ -23,6 +23,8 @@ type FailoverOptions struct { // Following options are copied from Options struct. + OnConnect func(*Conn) error + Password string DB int @@ -42,6 +44,8 @@ func (opt *FailoverOptions) options() *Options { return &Options{ Addr: "FailoverClient", + OnConnect: opt.OnConnect, + DB: opt.DB, Password: opt.Password, From f60bce9166507f1526f128b57841992612814274 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 30 May 2017 15:45:36 +0300 Subject: [PATCH 0340/1746] Don't return an error when pipeline is empty --- pipeline.go | 3 +-- pipeline_test.go | 9 +++++---- redis_test.go | 1 + tx_test.go | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pipeline.go b/pipeline.go index de99f12459..b66c0597f5 100644 --- a/pipeline.go +++ b/pipeline.go @@ -1,7 +1,6 @@ package redis import ( - "errors" "sync" "github.com/go-redis/redis/internal/pool" @@ -80,7 +79,7 @@ func (c *Pipeline) Exec() ([]Cmder, error) { } if len(c.cmds) == 0 { - return nil, errors.New("redis: pipeline is empty") + return nil, nil } cmds := c.cmds diff --git a/pipeline_test.go b/pipeline_test.go index 3f69e0c715..5dc5f94af5 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -34,16 +34,17 @@ var _ = Describe("pipelining", func() { }) assertPipeline := func() { - It("returns an error when there are no commands", func() { + It("returns no errors when there are no commands", func() { _, err := pipe.Exec() - Expect(err).To(MatchError("redis: pipeline is empty")) + Expect(err).NotTo(HaveOccurred()) }) It("discards queued commands", func() { pipe.Get("key") pipe.Discard() - _, err := pipe.Exec() - Expect(err).To(MatchError("redis: pipeline is empty")) + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(BeNil()) }) It("handles val/err", func() { diff --git a/redis_test.go b/redis_test.go index 79d727a0f3..972b67be9b 100644 --- a/redis_test.go +++ b/redis_test.go @@ -344,6 +344,7 @@ var _ = Describe("Client OnConnect", func() { BeforeEach(func() { opt := redisOptions() + opt.DB = 0 opt.OnConnect = func(cn *redis.Conn) error { return cn.ClientSetName("on_connect").Err() } diff --git a/tx_test.go b/tx_test.go index 83b12e1f04..8d9cc036e5 100644 --- a/tx_test.go +++ b/tx_test.go @@ -86,12 +86,12 @@ var _ = Describe("Tx", func() { Expect(get.Val()).To(Equal("hello2")) }) - It("returns an error when there are no commands", func() { + It("returns no error when there are no commands", func() { err := client.Watch(func(tx *redis.Tx) error { _, err := tx.Pipelined(func(redis.Pipeliner) error { return nil }) return err }) - Expect(err).To(MatchError("redis: pipeline is empty")) + Expect(err).NotTo(HaveOccurred()) v, err := client.Ping().Result() Expect(err).NotTo(HaveOccurred()) From 3802e09b4279289fad5a6b90f1abeabc60b62d71 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 1 Jun 2017 17:49:27 +0300 Subject: [PATCH 0341/1746] Simplify doc wording --- options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index 9ccdb5928d..4e1c9d0d90 100644 --- a/options.go +++ b/options.go @@ -37,8 +37,8 @@ type Options struct { // Default is to not retry failed commands. MaxRetries int - // Retry using exponential backoff wait algorithm between each retry - // Default is 512 milliseconds; set to -1 to disable any backoff sleep + // Maximum backoff between each retry. + // Default is 512 seconds; -1 disables backoff. MaxRetryBackoff time.Duration // Dial timeout for establishing new connections. From f29951c8990f0d31d1bacc27393ac88e3edf7e21 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 2 Jun 2017 16:19:43 +0300 Subject: [PATCH 0342/1746] Speedup ScanSlice --- internal/proto/scan.go | 2 +- internal/safe.go | 4 ++++ internal/unsafe.go | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/proto/scan.go b/internal/proto/scan.go index a73a369d5f..3ab40b94f2 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -122,7 +122,7 @@ func ScanSlice(data []string, slice interface{}) error { for i, s := range data { elem := internal.SliceNextElem(v) - if err := Scan([]byte(s), elem.Addr().Interface()); err != nil { + if err := Scan(internal.StringToBytes(s), elem.Addr().Interface()); err != nil { return fmt.Errorf("redis: ScanSlice(index=%d value=%q) failed: %s", i, s, err) } } diff --git a/internal/safe.go b/internal/safe.go index dc5f4cc8a4..870fe541f0 100644 --- a/internal/safe.go +++ b/internal/safe.go @@ -5,3 +5,7 @@ package internal func BytesToString(b []byte) string { return string(b) } + +func StringToBytes(s string) []byte { + return []byte(s) +} diff --git a/internal/unsafe.go b/internal/unsafe.go index 94e4a9d35d..c18b25c17a 100644 --- a/internal/unsafe.go +++ b/internal/unsafe.go @@ -9,6 +9,19 @@ import ( func BytesToString(b []byte) string { bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - strHeader := reflect.StringHeader{bytesHeader.Data, bytesHeader.Len} + strHeader := reflect.StringHeader{ + Data: bytesHeader.Data, + Len: bytesHeader.Len, + } return *(*string)(unsafe.Pointer(&strHeader)) } + +func StringToBytes(s string) []byte { + sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) + bh := reflect.SliceHeader{ + Data: sh.Data, + Len: sh.Len, + Cap: sh.Len, + } + return *(*[]byte)(unsafe.Pointer(&bh)) +} From 2dbe5a3d990d0d414ae710615ef05b05961a351f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 9 Jun 2017 13:55:45 +0300 Subject: [PATCH 0343/1746] Add ParseURL example --- example_test.go | 17 +++++++++++++++++ options.go | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/example_test.go b/example_test.go index 6453fd1592..fbb92c1747 100644 --- a/example_test.go +++ b/example_test.go @@ -35,6 +35,23 @@ func ExampleNewClient() { // Output: PONG } +func ExampleParseURL() { + opt, err := redis.ParseURL("redis://:qwerty@localhost:6379/1") + if err != nil { + panic(err) + } + fmt.Println("addr is", opt.Addr) + fmt.Println("db is", opt.DB) + fmt.Println("password is", opt.Password) + + // Create client as usually. + _ = redis.NewClient(opt) + + // Output: addr is localhost:6379 + // db is 1 + // password is qwerty +} + func ExampleNewFailoverClient() { // See http://redis.io/topics/sentinel for instructions how to // setup Redis Sentinel. diff --git a/options.go b/options.go index 4e1c9d0d90..dfda7df307 100644 --- a/options.go +++ b/options.go @@ -125,7 +125,7 @@ func (opt *Options) init() { } } -// ParseURL parses a redis URL into options that can be used to connect to redis +// ParseURL parses an URL into Options that can be used to connect to Redis. func ParseURL(redisURL string) (*Options, error) { o := &Options{Network: "tcp"} u, err := url.Parse(redisURL) From 5132e15c9341645161a2a67613de96a134480ee3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 17 Jun 2017 12:34:39 +0300 Subject: [PATCH 0344/1746] Fix cmd info race. Fixes #578 --- .travis.yml | 1 + cluster.go | 38 +++++++++++++++++++++++++++----------- ring.go | 18 +++++++++++------- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8e0d652ee..7df76b9c7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,5 +16,6 @@ matrix: - go: tip install: + - go get go4.org/syncutil - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega diff --git a/cluster.go b/cluster.go index e3c5832fcc..51e94a7529 100644 --- a/cluster.go +++ b/cluster.go @@ -7,6 +7,8 @@ import ( "sync/atomic" "time" + "go4.org/syncutil" + "github.com/go-redis/redis/internal" "github.com/go-redis/redis/internal/hashtag" "github.com/go-redis/redis/internal/pool" @@ -335,10 +337,12 @@ type ClusterClient struct { cmdable opt *ClusterOptions - cmds map[string]*CommandInfo nodes *clusterNodes _state atomic.Value + cmdsInfoOnce syncutil.Once + cmdsInfo map[string]*CommandInfo + // Reports where slots reloading is in progress. reloading uint32 } @@ -389,13 +393,34 @@ func (c *ClusterClient) state() *clusterState { return nil } +func (c *ClusterClient) cmdInfo(name string) *CommandInfo { + err := c.cmdsInfoOnce.Do(func() error { + node, err := c.nodes.Random() + if err != nil { + return err + } + + cmdsInfo, err := node.Client.Command().Result() + if err != nil { + return err + } + + c.cmdsInfo = cmdsInfo + return nil + }) + if err != nil { + return nil + } + return c.cmdsInfo[name] +} + func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { if state == nil { node, err := c.nodes.Random() return 0, node, err } - cmdInfo := c.cmds[cmd.Name()] + cmdInfo := c.cmdInfo(cmd.Name()) firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) slot := hashtag.Slot(firstKey) @@ -631,15 +656,6 @@ func (c *ClusterClient) reloadSlots() (*clusterState, error) { return nil, err } - // TODO: fix race - if c.cmds == nil { - cmds, err := node.Client.Command().Result() - if err != nil { - return nil, err - } - c.cmds = cmds - } - slots, err := node.Client.ClusterSlots().Result() if err != nil { return nil, err diff --git a/ring.go b/ring.go index a9666bc72f..5d94519032 100644 --- a/ring.go +++ b/ring.go @@ -9,6 +9,8 @@ import ( "sync/atomic" "time" + "go4.org/syncutil" + "github.com/go-redis/redis/internal" "github.com/go-redis/redis/internal/consistenthash" "github.com/go-redis/redis/internal/hashtag" @@ -134,7 +136,7 @@ type Ring struct { hash *consistenthash.Map shards map[string]*ringShard - cmdsInfoOnce *sync.Once + cmdsInfoOnce syncutil.Once cmdsInfo map[string]*CommandInfo closed bool @@ -149,8 +151,6 @@ func NewRing(opt *RingOptions) *Ring { hash: consistenthash.New(nreplicas, nil), shards: make(map[string]*ringShard), - - cmdsInfoOnce: new(sync.Once), } ring.setProcessor(ring.Process) for name, addr := range opt.Addrs { @@ -242,17 +242,21 @@ func (c *Ring) ForEachShard(fn func(client *Client) error) error { } func (c *Ring) cmdInfo(name string) *CommandInfo { - c.cmdsInfoOnce.Do(func() { + err := c.cmdsInfoOnce.Do(func() error { + var firstErr error for _, shard := range c.shards { cmdsInfo, err := shard.Client.Command().Result() if err == nil { c.cmdsInfo = cmdsInfo - return + return nil + } + if firstErr == nil { + firstErr = err } } - c.cmdsInfoOnce = &sync.Once{} + return firstErr }) - if c.cmdsInfo == nil { + if err != nil { return nil } return c.cmdsInfo[name] From 167410bcd747d34243efbf53332a7c00b6bd9377 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 17 Jun 2017 12:43:19 +0300 Subject: [PATCH 0345/1746] Improve comment --- redis.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redis.go b/redis.go index 8d5d756373..9812daf660 100644 --- a/redis.go +++ b/redis.go @@ -383,6 +383,7 @@ func (c *Client) pubSub() *PubSub { } // Subscribe subscribes the client to the specified channels. +// Channels can be omitted to create empty subscription. func (c *Client) Subscribe(channels ...string) *PubSub { pubsub := c.pubSub() if len(channels) > 0 { @@ -392,6 +393,7 @@ func (c *Client) Subscribe(channels ...string) *PubSub { } // PSubscribe subscribes the client to the given patterns. +// Patterns can be omitted to create empty subscription. func (c *Client) PSubscribe(channels ...string) *PubSub { pubsub := c.pubSub() if len(channels) > 0 { From f1ed2ad288902494728179ad178e4c7925ba4aee Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 17 Jun 2017 12:53:16 +0300 Subject: [PATCH 0346/1746] Add FlushDBAsync and FlushAllAsync --- bench_test.go | 2 +- cluster_test.go | 8 ++++---- command_test.go | 2 +- commands.go | 23 ++++++++++++++++++++++- commands_test.go | 4 ++-- example_test.go | 4 ++-- iterator_test.go | 2 +- pipeline_test.go | 2 +- pool_test.go | 2 +- pubsub_test.go | 2 +- race_test.go | 2 +- redis_test.go | 6 +++--- ring_test.go | 2 +- sentinel_test.go | 2 +- tx_test.go | 2 +- 15 files changed, 43 insertions(+), 22 deletions(-) diff --git a/bench_test.go b/bench_test.go index 855140669f..f6b75c72a2 100644 --- a/bench_test.go +++ b/bench_test.go @@ -16,7 +16,7 @@ func benchmarkRedisClient(poolSize int) *redis.Client { WriteTimeout: time.Second, PoolSize: poolSize, }) - if err := client.FlushDb().Err(); err != nil { + if err := client.FlushDB().Err(); err != nil { panic(err) } return client diff --git a/cluster_test.go b/cluster_test.go index 1e55be6e2d..3a69255a48 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -478,7 +478,7 @@ var _ = Describe("ClusterClient", func() { } err := client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDb().Err() + return master.FlushDB().Err() }) Expect(err).NotTo(HaveOccurred()) @@ -496,7 +496,7 @@ var _ = Describe("ClusterClient", func() { client = cluster.clusterClient(opt) _ = client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDb().Err() + return master.FlushDB().Err() }) }) @@ -514,12 +514,12 @@ var _ = Describe("ClusterClient", func() { client = cluster.clusterClient(opt) _ = client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDb().Err() + return master.FlushDB().Err() }) }) AfterEach(func() { - client.FlushDb() + client.FlushDB() Expect(client.Close()).NotTo(HaveOccurred()) }) diff --git a/command_test.go b/command_test.go index 920f14b7c4..e42375eda9 100644 --- a/command_test.go +++ b/command_test.go @@ -12,7 +12,7 @@ var _ = Describe("Cmd", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/commands.go b/commands.go index 3956cf74d8..4ea78777c0 100644 --- a/commands.go +++ b/commands.go @@ -192,7 +192,9 @@ type Cmdable interface { ConfigSet(parameter, value string) *StatusCmd DbSize() *IntCmd FlushAll() *StatusCmd - FlushDb() *StatusCmd + FlushAllAsync() *StatusCmd + FlushDB() *StatusCmd + FlushDBAsync() *StatusCmd Info(section ...string) *StringCmd LastSave() *IntCmd Save() *StatusCmd @@ -1685,12 +1687,31 @@ func (c *cmdable) FlushAll() *StatusCmd { return cmd } +func (c *cmdable) FlushAllAsync() *StatusCmd { + cmd := NewStatusCmd("flushall", "async") + c.process(cmd) + return cmd +} + +// Deprecated. Use FlushDB instead. func (c *cmdable) FlushDb() *StatusCmd { cmd := NewStatusCmd("flushdb") c.process(cmd) return cmd } +func (c *cmdable) FlushDB() *StatusCmd { + cmd := NewStatusCmd("flushdb") + c.process(cmd) + return cmd +} + +func (c *cmdable) FlushDBAsync() *StatusCmd { + cmd := NewStatusCmd("flushdb", "async") + c.process(cmd) + return cmd +} + func (c *cmdable) Info(section ...string) *StringCmd { args := []interface{}{"info"} if len(section) > 0 { diff --git a/commands_test.go b/commands_test.go index 7e5d4010f8..64a50b50d2 100644 --- a/commands_test.go +++ b/commands_test.go @@ -17,7 +17,7 @@ var _ = Describe("Commands", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -352,7 +352,7 @@ var _ = Describe("Commands", func() { pipe := client.Pipeline() pipe.Select(2) get = pipe.Get("key") - pipe.FlushDb() + pipe.FlushDB() _, err := pipe.Exec() Expect(err).NotTo(HaveOccurred()) diff --git a/example_test.go b/example_test.go index fbb92c1747..94090416c0 100644 --- a/example_test.go +++ b/example_test.go @@ -20,7 +20,7 @@ func init() { PoolSize: 10, PoolTimeout: 30 * time.Second, }) - client.FlushDb() + client.FlushDB() } func ExampleNewClient() { @@ -147,7 +147,7 @@ func ExampleClient_BLPop() { } func ExampleClient_Scan() { - client.FlushDb() + client.FlushDB() for i := 0; i < 33; i++ { err := client.Set(fmt.Sprintf("key%d", i), "value", 0).Err() if err != nil { diff --git a/iterator_test.go b/iterator_test.go index 954165cc6f..a2e6238131 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -45,7 +45,7 @@ var _ = Describe("ScanIterator", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/pipeline_test.go b/pipeline_test.go index 5dc5f94af5..11896c6bb3 100644 --- a/pipeline_test.go +++ b/pipeline_test.go @@ -13,7 +13,7 @@ var _ = Describe("pipelining", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/pool_test.go b/pool_test.go index 9d33559b87..34a548a639 100644 --- a/pool_test.go +++ b/pool_test.go @@ -14,7 +14,7 @@ var _ = Describe("pool", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/pubsub_test.go b/pubsub_test.go index b17ca7ad47..d03270c318 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -17,7 +17,7 @@ var _ = Describe("PubSub", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/race_test.go b/race_test.go index 439e5e2926..5bcb0768e2 100644 --- a/race_test.go +++ b/race_test.go @@ -20,7 +20,7 @@ var _ = Describe("races", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).To(BeNil()) + Expect(client.FlushDB().Err()).To(BeNil()) C, N = 10, 1000 if testing.Short() { diff --git a/redis_test.go b/redis_test.go index 972b67be9b..49d3fb3295 100644 --- a/redis_test.go +++ b/redis_test.go @@ -16,7 +16,7 @@ var _ = Describe("Client", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -111,7 +111,7 @@ var _ = Describe("Client", func() { Addr: redisAddr, DB: 2, }) - Expect(db2.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(db2.FlushDB().Err()).NotTo(HaveOccurred()) Expect(db2.Get("db").Err()).To(Equal(redis.Nil)) Expect(db2.Set("db", 2, 0).Err()).NotTo(HaveOccurred()) @@ -121,7 +121,7 @@ var _ = Describe("Client", func() { Expect(client.Get("db").Err()).To(Equal(redis.Nil)) - Expect(db2.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(db2.FlushDB().Err()).NotTo(HaveOccurred()) Expect(db2.Close()).NotTo(HaveOccurred()) }) diff --git a/ring_test.go b/ring_test.go index a61ee96a04..0cad4298bc 100644 --- a/ring_test.go +++ b/ring_test.go @@ -29,7 +29,7 @@ var _ = Describe("Redis Ring", func() { ring = redis.NewRing(opt) err := ring.ForEachShard(func(cl *redis.Client) error { - return cl.FlushDb().Err() + return cl.FlushDB().Err() }) Expect(err).NotTo(HaveOccurred()) }) diff --git a/sentinel_test.go b/sentinel_test.go index 04bbabd1e8..f1f580f30a 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -15,7 +15,7 @@ var _ = Describe("Sentinel", func() { MasterName: sentinelName, SentinelAddrs: []string{":" + sentinelPort}, }) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/tx_test.go b/tx_test.go index 8d9cc036e5..de597ff064 100644 --- a/tx_test.go +++ b/tx_test.go @@ -15,7 +15,7 @@ var _ = Describe("Tx", func() { BeforeEach(func() { client = redis.NewClient(redisOptions()) - Expect(client.FlushDb().Err()).NotTo(HaveOccurred()) + Expect(client.FlushDB().Err()).NotTo(HaveOccurred()) }) AfterEach(func() { From 9b8cd3e5a7c06ea8c70938a2c68fefe4d8a9d890 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 17 Jun 2017 13:01:40 +0300 Subject: [PATCH 0347/1746] readme: add godoc badge --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 99ca9b6b81..f3c61795e8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# Redis client for Golang [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) +# Redis client for Golang + +[![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) +[![GoDoc](https://godoc.org/github.com/go-redis/redis?status.svg)](https://godoc.org/github.com/go-redis/redis) Supports: From 9acf745fafce8e547ea3a83c192e7e70fa1c6e91 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 28 Jun 2017 18:20:26 +0300 Subject: [PATCH 0348/1746] Scale pool size with number of cores since Redis connections are cheap --- options.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index dfda7df307..d3ea254bb4 100644 --- a/options.go +++ b/options.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/url" + "runtime" "strconv" "strings" "time" @@ -54,7 +55,7 @@ type Options struct { WriteTimeout time.Duration // Maximum number of socket connections. - // Default is 10 connections. + // Default is 10 connections per every CPU as reported by runtime.NumCPU. PoolSize int // Amount of time client waits for connection if all connections // are busy before returning an error. @@ -91,7 +92,7 @@ func (opt *Options) init() { } } if opt.PoolSize == 0 { - opt.PoolSize = 10 + opt.PoolSize = 10 * runtime.NumCPU() } if opt.DialTimeout == 0 { opt.DialTimeout = 5 * time.Second From 0a965c5d706923057cd6849622b74d9a428aa398 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 29 Jun 2017 12:54:49 +0300 Subject: [PATCH 0349/1746] pool: add fast path --- internal/pool/pool.go | 48 +++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index da8337a4f7..88a252a792 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -106,19 +106,23 @@ func (p *ConnPool) NewConn() (*Conn, error) { } func (p *ConnPool) PopFree() *Conn { - timer := timers.Get().(*time.Timer) - timer.Reset(p.poolTimeout) - select { case p.queue <- struct{}{}: - if !timer.Stop() { - <-timer.C + default: + timer := timers.Get().(*time.Timer) + timer.Reset(p.poolTimeout) + + select { + case p.queue <- struct{}{}: + if !timer.Stop() { + <-timer.C + } + timers.Put(timer) + case <-timer.C: + timers.Put(timer) + atomic.AddUint32(&p.stats.Timeouts, 1) + return nil } - timers.Put(timer) - case <-timer.C: - timers.Put(timer) - atomic.AddUint32(&p.stats.Timeouts, 1) - return nil } p.freeConnsMu.Lock() @@ -150,19 +154,23 @@ func (p *ConnPool) Get() (*Conn, bool, error) { atomic.AddUint32(&p.stats.Requests, 1) - timer := timers.Get().(*time.Timer) - timer.Reset(p.poolTimeout) - select { case p.queue <- struct{}{}: - if !timer.Stop() { - <-timer.C + default: + timer := timers.Get().(*time.Timer) + timer.Reset(p.poolTimeout) + + select { + case p.queue <- struct{}{}: + if !timer.Stop() { + <-timer.C + } + timers.Put(timer) + case <-timer.C: + timers.Put(timer) + atomic.AddUint32(&p.stats.Timeouts, 1) + return nil, false, ErrPoolTimeout } - timers.Put(timer) - case <-timer.C: - timers.Put(timer) - atomic.AddUint32(&p.stats.Timeouts, 1) - return nil, false, ErrPoolTimeout } for { From 9cf5f251bedc81fc63a63e5e32440ec988ab201b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 29 Jun 2017 14:26:52 +0300 Subject: [PATCH 0350/1746] Gracefully handle situation when Redis Server is down --- internal/pool/bench_test.go | 16 ++++++- internal/pool/pool.go | 85 ++++++++++++++++++++++++++----------- internal/pool/pool_test.go | 47 ++++++++++++++------ options.go | 14 +++--- 4 files changed, 115 insertions(+), 47 deletions(-) diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index 5c021693e2..e0bb524466 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -8,7 +8,13 @@ import ( ) func benchmarkPoolGetPut(b *testing.B, poolSize int) { - connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) + connPool := pool.NewConnPool(&pool.Options{ + Dialer: dummyDialer, + PoolSize: poolSize, + PoolTimeout: time.Second, + IdleTimeout: time.Hour, + IdleCheckFrequency: time.Hour, + }) b.ResetTimer() @@ -38,7 +44,13 @@ func BenchmarkPoolGetPut1000Conns(b *testing.B) { } func benchmarkPoolGetRemove(b *testing.B, poolSize int) { - connPool := pool.NewConnPool(dummyDialer, poolSize, time.Second, time.Hour, time.Hour) + connPool := pool.NewConnPool(&pool.Options{ + Dialer: dummyDialer, + PoolSize: poolSize, + PoolTimeout: time.Second, + IdleTimeout: time.Hour, + IdleCheckFrequency: time.Hour, + }) b.ResetTimer() diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 88a252a792..bef000ea61 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -46,14 +46,21 @@ type Pooler interface { Close() error } -type dialer func() (net.Conn, error) +type Options struct { + Dialer func() (net.Conn, error) + OnClose func(*Conn) error + + PoolSize int + PoolTimeout time.Duration + IdleTimeout time.Duration + IdleCheckFrequency time.Duration +} type ConnPool struct { - dial dialer - OnClose func(*Conn) error + opt *Options - poolTimeout time.Duration - idleTimeout time.Duration + dialErrorsNum uint32 // atomic + _lastDialError atomic.Value queue chan struct{} @@ -65,24 +72,21 @@ type ConnPool struct { stats Stats - _closed int32 // atomic + _closed uint32 // atomic } var _ Pooler = (*ConnPool)(nil) -func NewConnPool(dial dialer, poolSize int, poolTimeout, idleTimeout, idleCheckFrequency time.Duration) *ConnPool { +func NewConnPool(opt *Options) *ConnPool { p := &ConnPool{ - dial: dial, - - poolTimeout: poolTimeout, - idleTimeout: idleTimeout, + opt: opt, - queue: make(chan struct{}, poolSize), - conns: make([]*Conn, 0, poolSize), - freeConns: make([]*Conn, 0, poolSize), + queue: make(chan struct{}, opt.PoolSize), + conns: make([]*Conn, 0, opt.PoolSize), + freeConns: make([]*Conn, 0, opt.PoolSize), } - if idleTimeout > 0 && idleCheckFrequency > 0 { - go p.reaper(idleCheckFrequency) + if opt.IdleTimeout > 0 && opt.IdleCheckFrequency > 0 { + go p.reaper(opt.IdleCheckFrequency) } return p } @@ -92,8 +96,16 @@ func (p *ConnPool) NewConn() (*Conn, error) { return nil, ErrClosed } - netConn, err := p.dial() + if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.opt.PoolSize) { + return nil, p.lastDialError() + } + + netConn, err := p.opt.Dialer() if err != nil { + p.setLastDialError(err) + if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.opt.PoolSize) { + go p.tryDial() + } return nil, err } @@ -105,12 +117,35 @@ func (p *ConnPool) NewConn() (*Conn, error) { return cn, nil } +func (p *ConnPool) tryDial() { + for { + conn, err := p.opt.Dialer() + if err != nil { + p.setLastDialError(err) + time.Sleep(time.Second) + continue + } + + atomic.StoreUint32(&p.dialErrorsNum, 0) + _ = conn.Close() + return + } +} + +func (p *ConnPool) setLastDialError(err error) { + p._lastDialError.Store(err) +} + +func (p *ConnPool) lastDialError() error { + return p._lastDialError.Load().(error) +} + func (p *ConnPool) PopFree() *Conn { select { case p.queue <- struct{}{}: default: timer := timers.Get().(*time.Timer) - timer.Reset(p.poolTimeout) + timer.Reset(p.opt.PoolTimeout) select { case p.queue <- struct{}{}: @@ -158,7 +193,7 @@ func (p *ConnPool) Get() (*Conn, bool, error) { case p.queue <- struct{}{}: default: timer := timers.Get().(*time.Timer) - timer.Reset(p.poolTimeout) + timer.Reset(p.opt.PoolTimeout) select { case p.queue <- struct{}{}: @@ -182,7 +217,7 @@ func (p *ConnPool) Get() (*Conn, bool, error) { break } - if cn.IsStale(p.idleTimeout) { + if cn.IsStale(p.opt.IdleTimeout) { p.CloseConn(cn) continue } @@ -232,8 +267,8 @@ func (p *ConnPool) CloseConn(cn *Conn) error { } func (p *ConnPool) closeConn(cn *Conn) error { - if p.OnClose != nil { - _ = p.OnClose(cn) + if p.opt.OnClose != nil { + _ = p.opt.OnClose(cn) } return cn.Close() } @@ -265,11 +300,11 @@ func (p *ConnPool) Stats() *Stats { } func (p *ConnPool) closed() bool { - return atomic.LoadInt32(&p._closed) == 1 + return atomic.LoadUint32(&p._closed) == 1 } func (p *ConnPool) Close() error { - if !atomic.CompareAndSwapInt32(&p._closed, 0, 1) { + if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) { return ErrClosed } @@ -299,7 +334,7 @@ func (p *ConnPool) reapStaleConn() bool { } cn := p.freeConns[0] - if !cn.IsStale(p.idleTimeout) { + if !cn.IsStale(p.opt.IdleTimeout) { return false } diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index c8fbeb9b88..f86327a4c5 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -14,8 +14,13 @@ var _ = Describe("ConnPool", func() { var connPool *pool.ConnPool BeforeEach(func() { - connPool = pool.NewConnPool( - dummyDialer, 10, time.Hour, time.Millisecond, time.Millisecond) + connPool = pool.NewConnPool(&pool.Options{ + Dialer: dummyDialer, + PoolSize: 10, + PoolTimeout: time.Hour, + IdleTimeout: time.Millisecond, + IdleCheckFrequency: time.Millisecond, + }) }) AfterEach(func() { @@ -83,16 +88,21 @@ var _ = Describe("conns reaper", func() { var conns, idleConns, closedConns []*pool.Conn BeforeEach(func() { - connPool = pool.NewConnPool( - dummyDialer, 10, time.Second, idleTimeout, time.Hour) - + conns = nil closedConns = nil - connPool.OnClose = func(cn *pool.Conn) error { - closedConns = append(closedConns, cn) - return nil - } - conns = nil + connPool = pool.NewConnPool(&pool.Options{ + Dialer: dummyDialer, + PoolSize: 10, + PoolTimeout: time.Second, + IdleTimeout: idleTimeout, + IdleCheckFrequency: time.Hour, + + OnClose: func(cn *pool.Conn) error { + closedConns = append(closedConns, cn) + return nil + }, + }) // add stale connections idleConns = nil @@ -202,8 +212,13 @@ var _ = Describe("race", func() { }) It("does not happen on Get, Put, and Remove", func() { - connPool = pool.NewConnPool( - dummyDialer, 10, time.Minute, time.Millisecond, time.Millisecond) + connPool = pool.NewConnPool(&pool.Options{ + Dialer: dummyDialer, + PoolSize: 10, + PoolTimeout: time.Minute, + IdleTimeout: time.Millisecond, + IdleCheckFrequency: time.Millisecond, + }) perform(C, func(id int) { for i := 0; i < N; i++ { @@ -226,7 +241,13 @@ var _ = Describe("race", func() { It("does not happen on Get and PopFree", func() { connPool = pool.NewConnPool( - dummyDialer, 10, time.Minute, time.Second, time.Millisecond) + &pool.Options{ + Dialer: dummyDialer, + PoolSize: 10, + PoolTimeout: time.Minute, + IdleTimeout: time.Second, + IdleCheckFrequency: time.Millisecond, + }) perform(C, func(id int) { for i := 0; i < N; i++ { diff --git a/options.go b/options.go index d3ea254bb4..cd6fa981a3 100644 --- a/options.go +++ b/options.go @@ -181,13 +181,13 @@ func ParseURL(redisURL string) (*Options, error) { } func newConnPool(opt *Options) *pool.ConnPool { - return pool.NewConnPool( - opt.Dialer, - opt.PoolSize, - opt.PoolTimeout, - opt.IdleTimeout, - opt.IdleCheckFrequency, - ) + return pool.NewConnPool(&pool.Options{ + Dialer: opt.Dialer, + PoolSize: opt.PoolSize, + PoolTimeout: opt.PoolTimeout, + IdleTimeout: opt.IdleTimeout, + IdleCheckFrequency: opt.IdleCheckFrequency, + }) } // PoolStats contains pool state information and accumulated stats. From fbc8000fd1719c35313f64d464ea693326fa777e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 29 Jun 2017 16:53:49 +0300 Subject: [PATCH 0351/1746] Resent client pool when sentinel switches master --- internal/pool/pool.go | 71 +++++++++++----------------- main_test.go | 5 ++ pubsub.go | 81 +++++++++++++++----------------- redis.go | 5 +- sentinel.go | 106 ++++++++++++++++++++---------------------- sentinel_test.go | 17 ++++++- 6 files changed, 139 insertions(+), 146 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index bef000ea61..a4e650847f 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -140,47 +140,6 @@ func (p *ConnPool) lastDialError() error { return p._lastDialError.Load().(error) } -func (p *ConnPool) PopFree() *Conn { - select { - case p.queue <- struct{}{}: - default: - timer := timers.Get().(*time.Timer) - timer.Reset(p.opt.PoolTimeout) - - select { - case p.queue <- struct{}{}: - if !timer.Stop() { - <-timer.C - } - timers.Put(timer) - case <-timer.C: - timers.Put(timer) - atomic.AddUint32(&p.stats.Timeouts, 1) - return nil - } - } - - p.freeConnsMu.Lock() - cn := p.popFree() - p.freeConnsMu.Unlock() - - if cn == nil { - <-p.queue - } - return cn -} - -func (p *ConnPool) popFree() *Conn { - if len(p.freeConns) == 0 { - return nil - } - - idx := len(p.freeConns) - 1 - cn := p.freeConns[idx] - p.freeConns = p.freeConns[:idx] - return cn -} - // Get returns existed connection from the pool or creates a new one. func (p *ConnPool) Get() (*Conn, bool, error) { if p.closed() { @@ -235,6 +194,17 @@ func (p *ConnPool) Get() (*Conn, bool, error) { return newcn, true, nil } +func (p *ConnPool) popFree() *Conn { + if len(p.freeConns) == 0 { + return nil + } + + idx := len(p.freeConns) - 1 + cn := p.freeConns[idx] + p.freeConns = p.freeConns[:idx] + return cn +} + func (p *ConnPool) Put(cn *Conn) error { if data := cn.Rd.PeekBuffered(); data != nil { internal.Logf("connection has unread data: %q", data) @@ -303,17 +273,28 @@ func (p *ConnPool) closed() bool { return atomic.LoadUint32(&p._closed) == 1 } +func (p *ConnPool) Filter(fn func(*Conn) bool) error { + var firstErr error + p.connsMu.Lock() + for _, cn := range p.conns { + if fn(cn) { + if err := p.closeConn(cn); err != nil && firstErr == nil { + firstErr = err + } + } + } + p.connsMu.Unlock() + return firstErr +} + func (p *ConnPool) Close() error { if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) { return ErrClosed } - p.connsMu.Lock() var firstErr error + p.connsMu.Lock() for _, cn := range p.conns { - if cn == nil { - continue - } if err := p.closeConn(cn); err != nil && firstErr == nil { firstErr = err } diff --git a/main_test.go b/main_test.go index 7c5a6a969f..64f25d9932 100644 --- a/main_test.go +++ b/main_test.go @@ -3,6 +3,7 @@ package redis_test import ( "errors" "fmt" + "log" "net" "os" "os/exec" @@ -50,6 +51,10 @@ var cluster = &clusterScenario{ clients: make(map[string]*redis.Client, 6), } +func init() { + redis.SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) +} + var _ = BeforeSuite(func() { var err error diff --git a/pubsub.go b/pubsub.go index 19016b9d7e..7eba98b0a5 100644 --- a/pubsub.go +++ b/pubsub.go @@ -19,13 +19,11 @@ import ( type PubSub struct { base baseClient - mu sync.Mutex - cn *pool.Conn - closed bool - - subMu sync.Mutex + mu sync.Mutex + cn *pool.Conn channels []string patterns []string + closed bool cmd *Cmd } @@ -64,9 +62,6 @@ func (c *PubSub) conn() (*pool.Conn, bool, error) { } func (c *PubSub) resubscribe(cn *pool.Conn) error { - c.subMu.Lock() - defer c.subMu.Unlock() - var firstErr error if len(c.channels) > 0 { if err := c._subscribe(cn, "subscribe", c.channels...); err != nil && firstErr == nil { @@ -81,6 +76,18 @@ func (c *PubSub) resubscribe(cn *pool.Conn) error { return firstErr } +func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) error { + args := make([]interface{}, 1+len(channels)) + args[0] = redisCmd + for i, channel := range channels { + args[1+i] = channel + } + cmd := NewSliceCmd(args...) + + cn.SetWriteTimeout(c.base.opt.WriteTimeout) + return writeCmd(cn, cmd) +} + func (c *PubSub) putConn(cn *pool.Conn, err error) { if !internal.IsBadConn(err, true) { return @@ -114,69 +121,57 @@ func (c *PubSub) Close() error { return nil } -func (c *PubSub) subscribe(redisCmd string, channels ...string) error { - cn, isNew, err := c.conn() - if err != nil { - return err - } - - if isNew { - return nil - } - - err = c._subscribe(cn, redisCmd, channels...) - c.putConn(cn, err) - return err -} - -func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) error { - args := make([]interface{}, 1+len(channels)) - args[0] = redisCmd - for i, channel := range channels { - args[1+i] = channel - } - cmd := NewSliceCmd(args...) - - cn.SetWriteTimeout(c.base.opt.WriteTimeout) - return writeCmd(cn, cmd) -} - // Subscribes the client to the specified channels. It returns // empty subscription if there are no channels. func (c *PubSub) Subscribe(channels ...string) error { - c.subMu.Lock() + c.mu.Lock() c.channels = appendIfNotExists(c.channels, channels...) - c.subMu.Unlock() + c.mu.Unlock() return c.subscribe("subscribe", channels...) } // Subscribes the client to the given patterns. It returns // empty subscription if there are no patterns. func (c *PubSub) PSubscribe(patterns ...string) error { - c.subMu.Lock() + c.mu.Lock() c.patterns = appendIfNotExists(c.patterns, patterns...) - c.subMu.Unlock() + c.mu.Unlock() return c.subscribe("psubscribe", patterns...) } // Unsubscribes the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { - c.subMu.Lock() + c.mu.Lock() c.channels = remove(c.channels, channels...) - c.subMu.Unlock() + c.mu.Unlock() return c.subscribe("unsubscribe", channels...) } // Unsubscribes the client from the given patterns, or from all of // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { - c.subMu.Lock() + c.mu.Lock() c.patterns = remove(c.patterns, patterns...) - c.subMu.Unlock() + c.mu.Unlock() return c.subscribe("punsubscribe", patterns...) } +func (c *PubSub) subscribe(redisCmd string, channels ...string) error { + cn, isNew, err := c.conn() + if err != nil { + return err + } + + if isNew { + return nil + } + + err = c._subscribe(cn, redisCmd, channels...) + c.putConn(cn, err) + return err +} + func (c *PubSub) Ping(payload ...string) error { args := []interface{}{"ping"} if len(payload) == 1 { diff --git a/redis.go b/redis.go index 9812daf660..1fc25e1db4 100644 --- a/redis.go +++ b/redis.go @@ -387,7 +387,10 @@ func (c *Client) pubSub() *PubSub { func (c *Client) Subscribe(channels ...string) *PubSub { pubsub := c.pubSub() if len(channels) > 0 { - _ = pubsub.Subscribe(channels...) + err := pubsub.Subscribe(channels...) + if err != nil { + panic(err) + } } return pubsub } diff --git a/sentinel.go b/sentinel.go index b28c3706ef..ed6e7ffb36 100644 --- a/sentinel.go +++ b/sentinel.go @@ -132,7 +132,6 @@ func (c *sentinelClient) Sentinels(name string) *SliceCmd { } type sentinelFailover struct { - masterName string sentinelAddrs []string opt *Options @@ -140,8 +139,10 @@ type sentinelFailover struct { pool *pool.ConnPool poolOnce sync.Once - mu sync.RWMutex - sentinel *sentinelClient + mu sync.RWMutex + masterName string + _masterAddr string + sentinel *sentinelClient } func (d *sentinelFailover) Close() error { @@ -168,17 +169,30 @@ func (d *sentinelFailover) MasterAddr() (string, error) { d.mu.Lock() defer d.mu.Unlock() + addr, err := d.masterAddr() + if err != nil { + return "", err + } + + if d._masterAddr != addr { + d.switchMaster(addr) + } + + return addr, nil +} + +func (d *sentinelFailover) masterAddr() (string, error) { // Try last working sentinel. if d.sentinel != nil { addr, err := d.sentinel.GetMasterAddrByName(d.masterName).Result() - if err != nil { - internal.Logf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) - d._resetSentinel() - } else { + if err == nil { addr := net.JoinHostPort(addr[0], addr[1]) - internal.Logf("sentinel: %q addr is %s", d.masterName, addr) + internal.Logf("sentinel: master=%q addr=%q", d.masterName, addr) return addr, nil } + + internal.Logf("sentinel: GetMasterAddrByName name=%q failed: %s", d.masterName, err) + d._resetSentinel() } for i, sentinelAddr := range d.sentinelAddrs { @@ -193,25 +207,36 @@ func (d *sentinelFailover) MasterAddr() (string, error) { PoolTimeout: d.opt.PoolTimeout, IdleTimeout: d.opt.IdleTimeout, }) + masterAddr, err := sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - internal.Logf("sentinel: GetMasterAddrByName %q failed: %s", d.masterName, err) + internal.Logf("sentinel: GetMasterAddrByName master=%q failed: %s", d.masterName, err) sentinel.Close() continue } // Push working sentinel to the top. d.sentinelAddrs[0], d.sentinelAddrs[i] = d.sentinelAddrs[i], d.sentinelAddrs[0] - d.setSentinel(sentinel) + addr := net.JoinHostPort(masterAddr[0], masterAddr[1]) - internal.Logf("sentinel: %q addr is %s", d.masterName, addr) return addr, nil } return "", errors.New("redis: all sentinels are unreachable") } +func (d *sentinelFailover) switchMaster(masterAddr string) { + internal.Logf( + "sentinel: new master=%q addr=%q", + d.masterName, masterAddr, + ) + _ = d.Pool().Filter(func(cn *pool.Conn) bool { + return cn.RemoteAddr().String() != masterAddr + }) + d._masterAddr = masterAddr +} + func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { d.discoverSentinels(sentinel) d.sentinel = sentinel @@ -219,25 +244,25 @@ func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { } func (d *sentinelFailover) resetSentinel() error { + var err error d.mu.Lock() - err := d._resetSentinel() + if d.sentinel != nil { + err = d._resetSentinel() + } d.mu.Unlock() return err } func (d *sentinelFailover) _resetSentinel() error { - var err error - if d.sentinel != nil { - err = d.sentinel.Close() - d.sentinel = nil - } + err := d.sentinel.Close() + d.sentinel = nil return err } func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinels, err := sentinel.Sentinels(d.masterName).Result() if err != nil { - internal.Logf("sentinel: Sentinels %q failed: %s", d.masterName, err) + internal.Logf("sentinel: Sentinels master=%q failed: %s", d.masterName, err) return } for _, sentinel := range sentinels { @@ -248,8 +273,8 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { sentinelAddr := vals[i+1].(string) if !contains(d.sentinelAddrs, sentinelAddr) { internal.Logf( - "sentinel: discovered new %q sentinel: %s", - d.masterName, sentinelAddr, + "sentinel: discovered new sentinel=%q for master=%q", + sentinelAddr, d.masterName, ) d.sentinelAddrs = append(d.sentinelAddrs, sentinelAddr) } @@ -258,34 +283,6 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { } } -// closeOldConns closes connections to the old master after failover switch. -func (d *sentinelFailover) closeOldConns(newMaster string) { - // Good connections that should be put back to the pool. They - // can't be put immediately, because pool.PopFree will return them - // again on next iteration. - cnsToPut := make([]*pool.Conn, 0) - - for { - cn := d.pool.PopFree() - if cn == nil { - break - } - if cn.RemoteAddr().String() != newMaster { - internal.Logf( - "sentinel: closing connection to the old master %s", - cn.RemoteAddr(), - ) - d.pool.Remove(cn) - } else { - cnsToPut = append(cnsToPut, cn) - } - } - - for _, cn := range cnsToPut { - d.pool.Put(cn) - } -} - func (d *sentinelFailover) listen(sentinel *sentinelClient) { var pubsub *PubSub for { @@ -312,17 +309,16 @@ func (d *sentinelFailover) listen(sentinel *sentinelClient) { case "+switch-master": parts := strings.Split(msg.Payload, " ") if parts[0] != d.masterName { - internal.Logf("sentinel: ignore new %s addr", parts[0]) + internal.Logf("sentinel: ignore addr for master=%q", parts[0]) continue } - addr := net.JoinHostPort(parts[3], parts[4]) - internal.Logf( - "sentinel: new %q addr is %s", - d.masterName, addr, - ) - d.closeOldConns(addr) + d.mu.Lock() + if d._masterAddr != addr { + d.switchMaster(addr) + } + d.mu.Unlock() } } } diff --git a/sentinel_test.go b/sentinel_test.go index f1f580f30a..c67713cd01 100644 --- a/sentinel_test.go +++ b/sentinel_test.go @@ -23,15 +23,19 @@ var _ = Describe("Sentinel", func() { }) It("should facilitate failover", func() { - // Set value on master, verify + // Set value on master. err := client.Set("foo", "master", 0).Err() Expect(err).NotTo(HaveOccurred()) + // Verify. val, err := sentinelMaster.Get("foo").Result() Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("master")) - // Wait until replicated + // Create subscription. + ch := client.Subscribe("foo").Channel() + + // Wait until replicated. Eventually(func() string { return sentinelSlave1.Get("foo").Val() }, "1s", "100ms").Should(Equal("master")) @@ -59,6 +63,15 @@ var _ = Describe("Sentinel", func() { Eventually(func() error { return client.Get("foo").Err() }, "5s", "100ms").ShouldNot(HaveOccurred()) + + // Publish message to check if subscription is renewed. + err = client.Publish("foo", "hello").Err() + Expect(err).NotTo(HaveOccurred()) + + var msg *redis.Message + Eventually(ch).Should(Receive(&msg)) + Expect(msg.Channel).To(Equal("foo")) + Expect(msg.Payload).To(Equal("hello")) }) It("supports DB selection", func() { From 0d94a7bc885f25fba5a2fb2e8611477ceeb476e8 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 29 Jun 2017 17:05:08 +0300 Subject: [PATCH 0352/1746] Fix race in PubSub --- internal/pool/pool_test.go | 26 ----------------------- main_test.go | 3 +-- pubsub.go | 42 +++++++++++++++++++++----------------- redis.go | 5 +---- 4 files changed, 25 insertions(+), 51 deletions(-) diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index f86327a4c5..68c9a1bef8 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -238,30 +238,4 @@ var _ = Describe("race", func() { } }) }) - - It("does not happen on Get and PopFree", func() { - connPool = pool.NewConnPool( - &pool.Options{ - Dialer: dummyDialer, - PoolSize: 10, - PoolTimeout: time.Minute, - IdleTimeout: time.Second, - IdleCheckFrequency: time.Millisecond, - }) - - perform(C, func(id int) { - for i := 0; i < N; i++ { - cn, _, err := connPool.Get() - Expect(err).NotTo(HaveOccurred()) - if err == nil { - Expect(connPool.Put(cn)).NotTo(HaveOccurred()) - } - - cn = connPool.PopFree() - if cn != nil { - Expect(connPool.Put(cn)).NotTo(HaveOccurred()) - } - } - }) - }) }) diff --git a/main_test.go b/main_test.go index 64f25d9932..30f09c618b 100644 --- a/main_test.go +++ b/main_test.go @@ -3,7 +3,6 @@ package redis_test import ( "errors" "fmt" - "log" "net" "os" "os/exec" @@ -52,7 +51,7 @@ var cluster = &clusterScenario{ } func init() { - redis.SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) + //redis.SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) } var _ = BeforeSuite(func() { diff --git a/pubsub.go b/pubsub.go index 7eba98b0a5..4872b4e88a 100644 --- a/pubsub.go +++ b/pubsub.go @@ -28,37 +28,41 @@ type PubSub struct { cmd *Cmd } -func (c *PubSub) conn() (*pool.Conn, bool, error) { +func (c *PubSub) conn() (*pool.Conn, error) { c.mu.Lock() - defer c.mu.Unlock() + cn, err := c._conn() + c.mu.Unlock() + return cn, err +} +func (c *PubSub) _conn() (*pool.Conn, error) { if c.closed { - return nil, false, pool.ErrClosed + return nil, pool.ErrClosed } if c.cn != nil { - return c.cn, false, nil + return c.cn, nil } cn, err := c.base.connPool.NewConn() if err != nil { - return nil, false, err + return nil, err } if !cn.Inited { if err := c.base.initConn(cn); err != nil { _ = c.base.connPool.CloseConn(cn) - return nil, false, err + return nil, err } } if err := c.resubscribe(cn); err != nil { _ = c.base.connPool.CloseConn(cn) - return nil, false, err + return nil, err } c.cn = cn - return cn, true, nil + return cn, nil } func (c *PubSub) resubscribe(cn *pool.Conn) error { @@ -125,48 +129,48 @@ func (c *PubSub) Close() error { // empty subscription if there are no channels. func (c *PubSub) Subscribe(channels ...string) error { c.mu.Lock() + err := c.subscribe("subscribe", channels...) c.channels = appendIfNotExists(c.channels, channels...) c.mu.Unlock() - return c.subscribe("subscribe", channels...) + return err } // Subscribes the client to the given patterns. It returns // empty subscription if there are no patterns. func (c *PubSub) PSubscribe(patterns ...string) error { c.mu.Lock() + err := c.subscribe("psubscribe", patterns...) c.patterns = appendIfNotExists(c.patterns, patterns...) c.mu.Unlock() - return c.subscribe("psubscribe", patterns...) + return err } // Unsubscribes the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { c.mu.Lock() + err := c.subscribe("unsubscribe", channels...) c.channels = remove(c.channels, channels...) c.mu.Unlock() - return c.subscribe("unsubscribe", channels...) + return err } // Unsubscribes the client from the given patterns, or from all of // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { c.mu.Lock() + err := c.subscribe("punsubscribe", patterns...) c.patterns = remove(c.patterns, patterns...) c.mu.Unlock() - return c.subscribe("punsubscribe", patterns...) + return err } func (c *PubSub) subscribe(redisCmd string, channels ...string) error { - cn, isNew, err := c.conn() + cn, err := c._conn() if err != nil { return err } - if isNew { - return nil - } - err = c._subscribe(cn, redisCmd, channels...) c.putConn(cn, err) return err @@ -179,7 +183,7 @@ func (c *PubSub) Ping(payload ...string) error { } cmd := NewCmd(args...) - cn, _, err := c.conn() + cn, err := c.conn() if err != nil { return err } @@ -272,7 +276,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { c.cmd = NewCmd() } - cn, _, err := c.conn() + cn, err := c.conn() if err != nil { return nil, err } diff --git a/redis.go b/redis.go index 1fc25e1db4..9812daf660 100644 --- a/redis.go +++ b/redis.go @@ -387,10 +387,7 @@ func (c *Client) pubSub() *PubSub { func (c *Client) Subscribe(channels ...string) *PubSub { pubsub := c.pubSub() if len(channels) > 0 { - err := pubsub.Subscribe(channels...) - if err != nil { - panic(err) - } + _ = pubsub.Subscribe(channels...) } return pubsub } From 94ea195dc19232dbf5499b479901a7bb0989afa0 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Thu, 29 Jun 2017 22:43:19 +0100 Subject: [PATCH 0353/1746] Use node address instead of relying on loopback reported by redis --- cluster.go | 27 ++++++++++++++++++++++++--- commands_test.go | 4 ++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index 51e94a7529..62dfe4a5a1 100644 --- a/cluster.go +++ b/cluster.go @@ -3,6 +3,7 @@ package redis import ( "fmt" "math/rand" + "net" "sync" "sync/atomic" "time" @@ -244,16 +245,22 @@ type clusterState struct { slots [][]*clusterNode } -func newClusterState(nodes *clusterNodes, slots []ClusterSlot) (*clusterState, error) { +func newClusterState(nodes *clusterNodes, slots []ClusterSlot, origin string) (*clusterState, error) { c := clusterState{ nodes: nodes, slots: make([][]*clusterNode, hashtag.SlotNumber), } + isLoopbackOrigin := isLoopbackAddr(origin) for _, slot := range slots { var nodes []*clusterNode for _, slotNode := range slot.Nodes { - node, err := c.nodes.Get(slotNode.Addr) + addr := slotNode.Addr + if !isLoopbackOrigin && isLoopbackAddr(addr) { + addr = origin + } + + node, err := c.nodes.Get(addr) if err != nil { return nil, err } @@ -661,7 +668,7 @@ func (c *ClusterClient) reloadSlots() (*clusterState, error) { return nil, err } - return newClusterState(c.nodes, slots) + return newClusterState(c.nodes, slots, node.Client.opt.Addr) } // reaper closes idle connections to the cluster. @@ -960,3 +967,17 @@ func (c *ClusterClient) txPipelineReadQueued( return firstErr } + +func isLoopbackAddr(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + + ip := net.ParseIP(host) + if ip == nil { + return false + } + + return ip.IsLoopback() +} diff --git a/commands_test.go b/commands_test.go index 64a50b50d2..e8cdb205e9 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2888,12 +2888,12 @@ var _ = Describe("Commands", func() { It("returns map of commands", func() { cmds, err := client.Command().Result() Expect(err).NotTo(HaveOccurred()) - Expect(len(cmds)).To(BeNumerically("~", 173, 5)) + Expect(len(cmds)).To(BeNumerically("~", 180, 10)) cmd := cmds["mget"] Expect(cmd.Name).To(Equal("mget")) Expect(cmd.Arity).To(Equal(int8(-2))) - Expect(cmd.Flags).To(Equal([]string{"readonly"})) + Expect(cmd.Flags).To(ContainElement("readonly")) Expect(cmd.FirstKeyPos).To(Equal(int8(1))) Expect(cmd.LastKeyPos).To(Equal(int8(-1))) Expect(cmd.StepCount).To(Equal(int8(1))) From 9dbcc5ae805ff5af66bdd30076456862408fded1 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 1 Jul 2017 12:51:06 +0300 Subject: [PATCH 0354/1746] Vendor syncutil.Once --- .travis.yml | 1 - cluster.go | 4 +--- internal/once.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ ring.go | 4 +--- 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 internal/once.go diff --git a/.travis.yml b/.travis.yml index 7df76b9c7e..f8e0d652ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,5 @@ matrix: - go: tip install: - - go get go4.org/syncutil - go get github.com/onsi/ginkgo - go get github.com/onsi/gomega diff --git a/cluster.go b/cluster.go index 62dfe4a5a1..f758b01b9d 100644 --- a/cluster.go +++ b/cluster.go @@ -8,8 +8,6 @@ import ( "sync/atomic" "time" - "go4.org/syncutil" - "github.com/go-redis/redis/internal" "github.com/go-redis/redis/internal/hashtag" "github.com/go-redis/redis/internal/pool" @@ -347,7 +345,7 @@ type ClusterClient struct { nodes *clusterNodes _state atomic.Value - cmdsInfoOnce syncutil.Once + cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo // Reports where slots reloading is in progress. diff --git a/internal/once.go b/internal/once.go new file mode 100644 index 0000000000..64f46272ae --- /dev/null +++ b/internal/once.go @@ -0,0 +1,60 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "sync" + "sync/atomic" +) + +// A Once will perform a successful action exactly once. +// +// Unlike a sync.Once, this Once's func returns an error +// and is re-armed on failure. +type Once struct { + m sync.Mutex + done uint32 +} + +// Do calls the function f if and only if Do has not been invoked +// without error for this instance of Once. In other words, given +// var once Once +// if once.Do(f) is called multiple times, only the first call will +// invoke f, even if f has a different value in each invocation unless +// f returns an error. A new instance of Once is required for each +// function to execute. +// +// Do is intended for initialization that must be run exactly once. Since f +// is niladic, it may be necessary to use a function literal to capture the +// arguments to a function to be invoked by Do: +// err := config.once.Do(func() error { return config.init(filename) }) +func (o *Once) Do(f func() error) error { + if atomic.LoadUint32(&o.done) == 1 { + return nil + } + // Slow-path. + o.m.Lock() + defer o.m.Unlock() + var err error + if o.done == 0 { + err = f() + if err == nil { + atomic.StoreUint32(&o.done, 1) + } + } + return err +} diff --git a/ring.go b/ring.go index 5d94519032..be92510964 100644 --- a/ring.go +++ b/ring.go @@ -9,8 +9,6 @@ import ( "sync/atomic" "time" - "go4.org/syncutil" - "github.com/go-redis/redis/internal" "github.com/go-redis/redis/internal/consistenthash" "github.com/go-redis/redis/internal/hashtag" @@ -136,7 +134,7 @@ type Ring struct { hash *consistenthash.Map shards map[string]*ringShard - cmdsInfoOnce syncutil.Once + cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo closed bool From 55da68487fef554f4dddce83df20916e3558b495 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 1 Jul 2017 13:22:39 +0300 Subject: [PATCH 0355/1746] Fix PubSub example. Fixes #575 --- example_test.go | 30 ++++++++++++++++++------------ pubsub_test.go | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/example_test.go b/example_test.go index 94090416c0..319ea0ca2d 100644 --- a/example_test.go +++ b/example_test.go @@ -278,7 +278,14 @@ func ExamplePubSub() { pubsub := client.Subscribe("mychannel1") defer pubsub.Close() - err := client.Publish("mychannel1", "hello").Err() + // Wait for subscription to be created before publishing message. + subscr, err := pubsub.ReceiveTimeout(time.Second) + if err != nil { + panic(err) + } + fmt.Println(subscr) + + err = client.Publish("mychannel1", "hello").Err() if err != nil { panic(err) } @@ -289,22 +296,17 @@ func ExamplePubSub() { } fmt.Println(msg.Channel, msg.Payload) - // Output: mychannel1 hello + // Output: subscribe: mychannel1 + // mychannel1 hello } func ExamplePubSub_Receive() { pubsub := client.Subscribe("mychannel2") defer pubsub.Close() - n, err := client.Publish("mychannel2", "hello").Result() - if err != nil { - panic(err) - } - fmt.Println(n, "clients received message") - for i := 0; i < 2; i++ { // ReceiveTimeout is a low level API. Use ReceiveMessage instead. - msgi, err := pubsub.ReceiveTimeout(5 * time.Second) + msgi, err := pubsub.ReceiveTimeout(time.Second) if err != nil { break } @@ -312,15 +314,19 @@ func ExamplePubSub_Receive() { switch msg := msgi.(type) { case *redis.Subscription: fmt.Println("subscribed to", msg.Channel) + + _, err := client.Publish("mychannel2", "hello").Result() + if err != nil { + panic(err) + } case *redis.Message: fmt.Println("received", msg.Payload, "from", msg.Channel) default: - panic(fmt.Errorf("unknown message: %#v", msgi)) + panic("unreached") } } - // Output: 1 clients received message - // subscribed to mychannel2 + // sent message to 1 client // received hello from mychannel2 } diff --git a/pubsub_test.go b/pubsub_test.go index d03270c318..e8589f4614 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -230,7 +230,15 @@ var _ = Describe("PubSub", func() { pubsub := client.Subscribe("mychannel") defer pubsub.Close() - err := client.Publish("mychannel", "hello").Err() + subscr, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(subscr).To(Equal(&redis.Subscription{ + Kind: "subscribe", + Channel: "mychannel", + Count: 1, + })) + + err = client.Publish("mychannel", "hello").Err() Expect(err).NotTo(HaveOccurred()) err = client.Publish("mychannel", "world").Err() @@ -253,6 +261,14 @@ var _ = Describe("PubSub", func() { pubsub := client.Subscribe("mychannel") defer pubsub.Close() + subscr, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(subscr).To(Equal(&redis.Subscription{ + Kind: "subscribe", + Channel: "mychannel", + Count: 1, + })) + done := make(chan bool, 1) go func() { defer GinkgoRecover() @@ -308,6 +324,14 @@ var _ = Describe("PubSub", func() { pubsub := client.Subscribe("mychannel") defer pubsub.Close() + subscr, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(subscr).To(Equal(&redis.Subscription{ + Kind: "subscribe", + Channel: "mychannel", + Count: 1, + })) + expectReceiveMessageOnError(pubsub) }) @@ -315,6 +339,14 @@ var _ = Describe("PubSub", func() { pubsub := client.PSubscribe("mychannel") defer pubsub.Close() + subscr, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(subscr).To(Equal(&redis.Subscription{ + Kind: "psubscribe", + Channel: "mychannel", + Count: 1, + })) + expectReceiveMessageOnError(pubsub) }) @@ -356,9 +388,12 @@ var _ = Describe("PubSub", func() { defer GinkgoRecover() time.Sleep(2 * timeout) + err := pubsub.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) + time.Sleep(timeout) + err = client.Publish("mychannel", "hello").Err() Expect(err).NotTo(HaveOccurred()) }() From 6060f097e1dd91aac7569de37bf49c8a5d4c2043 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Jul 2017 10:07:20 +0300 Subject: [PATCH 0356/1746] Add PubSub support to Cluster client --- cluster.go | 60 ++++++++++++++++++++++++++++++++++++++++++++----- cluster_test.go | 22 ++++++++++++++++++ pubsub.go | 34 +++++++++++++--------------- pubsub_test.go | 6 ++--- redis.go | 38 ++++++++++++++++++++++--------- ring.go | 4 ++-- sentinel.go | 24 +++++++++++--------- 7 files changed, 138 insertions(+), 50 deletions(-) diff --git a/cluster.go b/cluster.go index f758b01b9d..2641b16e43 100644 --- a/cluster.go +++ b/cluster.go @@ -327,7 +327,7 @@ func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { } func (c *clusterState) slotNodes(slot int) []*clusterNode { - if slot < len(c.slots) { + if slot >= 0 && slot < len(c.slots) { return c.slots[slot] } return nil @@ -720,14 +720,14 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { - cn, _, err := node.Client.conn() + cn, _, err := node.Client.getConn() if err != nil { setCmdsErr(cmds, err) continue } err = c.pipelineProcessCmds(cn, cmds, failedCmds) - node.Client.putConn(cn, err) + node.Client.releaseConn(cn, err) } if len(failedCmds) == 0 { @@ -855,14 +855,14 @@ func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { - cn, _, err := node.Client.conn() + cn, _, err := node.Client.getConn() if err != nil { setCmdsErr(cmds, err) continue } err = c.txPipelineProcessCmds(node, cn, cmds, failedCmds) - node.Client.putConn(cn, err) + node.Client.releaseConn(cn, err) } if len(failedCmds) == 0 { @@ -966,6 +966,56 @@ func (c *ClusterClient) txPipelineReadQueued( return firstErr } +func (c *ClusterClient) pubSub(channels []string) *PubSub { + opt := c.opt.clientOptions() + + var node *clusterNode + return &PubSub{ + opt: opt, + + newConn: func(channels []string) (*pool.Conn, error) { + if node == nil { + var slot int + if len(channels) > 0 { + slot = hashtag.Slot(channels[0]) + } else { + slot = -1 + } + + masterNode, err := c.state().slotMasterNode(slot) + if err != nil { + return nil, err + } + node = masterNode + } + return node.Client.newConn() + }, + closeConn: func(cn *pool.Conn) error { + return node.Client.connPool.CloseConn(cn) + }, + } +} + +// Subscribe subscribes the client to the specified channels. +// Channels can be omitted to create empty subscription. +func (c *ClusterClient) Subscribe(channels ...string) *PubSub { + pubsub := c.pubSub(channels) + if len(channels) > 0 { + _ = pubsub.Subscribe(channels...) + } + return pubsub +} + +// PSubscribe subscribes the client to the given patterns. +// Patterns can be omitted to create empty subscription. +func (c *ClusterClient) PSubscribe(channels ...string) *PubSub { + pubsub := c.pubSub(channels) + if len(channels) > 0 { + _ = pubsub.PSubscribe(channels...) + } + return pubsub +} + func isLoopbackAddr(addr string) bool { host, _, err := net.SplitHostPort(addr) if err != nil { diff --git a/cluster_test.go b/cluster_test.go index 3a69255a48..1dc7229501 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -472,6 +472,28 @@ var _ = Describe("ClusterClient", func() { }) }) + It("supports PubSub", func() { + pubsub := client.Subscribe("mychannel") + defer pubsub.Close() + + msgi, err := pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + subscr := msgi.(*redis.Subscription) + Expect(subscr.Kind).To(Equal("subscribe")) + Expect(subscr.Channel).To(Equal("mychannel")) + Expect(subscr.Count).To(Equal(1)) + + n, err := client.Publish("mychannel", "hello").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + msgi, err = pubsub.ReceiveTimeout(time.Second) + Expect(err).NotTo(HaveOccurred()) + msg := msgi.(*redis.Message) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("hello")) + }) + It("calls fn for every master node", func() { for i := 0; i < 10; i++ { Expect(client.Set(strconv.Itoa(i), "", 0).Err()).NotTo(HaveOccurred()) diff --git a/pubsub.go b/pubsub.go index 4872b4e88a..74ac51c108 100644 --- a/pubsub.go +++ b/pubsub.go @@ -17,7 +17,10 @@ import ( // PubSub automatically resubscribes to the channels and patterns // when Redis becomes unavailable. type PubSub struct { - base baseClient + opt *Options + + newConn func([]string) (*pool.Conn, error) + closeConn func(*pool.Conn) error mu sync.Mutex cn *pool.Conn @@ -30,12 +33,12 @@ type PubSub struct { func (c *PubSub) conn() (*pool.Conn, error) { c.mu.Lock() - cn, err := c._conn() + cn, err := c._conn(nil) c.mu.Unlock() return cn, err } -func (c *PubSub) _conn() (*pool.Conn, error) { +func (c *PubSub) _conn(channels []string) (*pool.Conn, error) { if c.closed { return nil, pool.ErrClosed } @@ -44,20 +47,13 @@ func (c *PubSub) _conn() (*pool.Conn, error) { return c.cn, nil } - cn, err := c.base.connPool.NewConn() + cn, err := c.newConn(channels) if err != nil { return nil, err } - if !cn.Inited { - if err := c.base.initConn(cn); err != nil { - _ = c.base.connPool.CloseConn(cn) - return nil, err - } - } - if err := c.resubscribe(cn); err != nil { - _ = c.base.connPool.CloseConn(cn) + _ = c.closeConn(cn) return nil, err } @@ -88,7 +84,7 @@ func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) } cmd := NewSliceCmd(args...) - cn.SetWriteTimeout(c.base.opt.WriteTimeout) + cn.SetWriteTimeout(c.opt.WriteTimeout) return writeCmd(cn, cmd) } @@ -99,13 +95,13 @@ func (c *PubSub) putConn(cn *pool.Conn, err error) { c.mu.Lock() if c.cn == cn { - _ = c.closeConn() + _ = c.releaseConn() } c.mu.Unlock() } -func (c *PubSub) closeConn() error { - err := c.base.connPool.CloseConn(c.cn) +func (c *PubSub) releaseConn() error { + err := c.closeConn(c.cn) c.cn = nil return err } @@ -120,7 +116,7 @@ func (c *PubSub) Close() error { c.closed = true if c.cn != nil { - return c.closeConn() + return c.releaseConn() } return nil } @@ -166,7 +162,7 @@ func (c *PubSub) PUnsubscribe(patterns ...string) error { } func (c *PubSub) subscribe(redisCmd string, channels ...string) error { - cn, err := c._conn() + cn, err := c._conn(channels) if err != nil { return err } @@ -188,7 +184,7 @@ func (c *PubSub) Ping(payload ...string) error { return err } - cn.SetWriteTimeout(c.base.opt.WriteTimeout) + cn.SetWriteTimeout(c.opt.WriteTimeout) err = writeCmd(cn, cmd) c.putConn(cn, err) return err diff --git a/pubsub_test.go b/pubsub_test.go index e8589f4614..3cb9627b53 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -159,9 +159,9 @@ var _ = Describe("PubSub", func() { { msgi, err := pubsub.ReceiveTimeout(time.Second) Expect(err).NotTo(HaveOccurred()) - subscr := msgi.(*redis.Message) - Expect(subscr.Channel).To(Equal("mychannel")) - Expect(subscr.Payload).To(Equal("hello")) + msg := msgi.(*redis.Message) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal("hello")) } { diff --git a/redis.go b/redis.go index 9812daf660..1a2ecb0b99 100644 --- a/redis.go +++ b/redis.go @@ -21,7 +21,23 @@ func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB) } -func (c *baseClient) conn() (*pool.Conn, bool, error) { +func (c *baseClient) newConn() (*pool.Conn, error) { + cn, err := c.connPool.NewConn() + if err != nil { + return nil, err + } + + if !cn.Inited { + if err := c.initConn(cn); err != nil { + _ = c.connPool.CloseConn(cn) + return nil, err + } + } + + return cn, nil +} + +func (c *baseClient) getConn() (*pool.Conn, bool, error) { cn, isNew, err := c.connPool.Get() if err != nil { return nil, false, err @@ -37,7 +53,7 @@ func (c *baseClient) conn() (*pool.Conn, bool, error) { return cn, isNew, nil } -func (c *baseClient) putConn(cn *pool.Conn, err error) bool { +func (c *baseClient) releaseConn(cn *pool.Conn, err error) bool { if internal.IsBadConn(err, false) { _ = c.connPool.Remove(cn) return false @@ -112,7 +128,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { time.Sleep(internal.RetryBackoff(i, c.opt.MaxRetryBackoff)) } - cn, _, err := c.conn() + cn, _, err := c.getConn() if err != nil { cmd.setErr(err) if internal.IsRetryableError(err) { @@ -123,7 +139,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmd); err != nil { - c.putConn(cn, err) + c.releaseConn(cn, err) cmd.setErr(err) if internal.IsRetryableError(err) { continue @@ -133,7 +149,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { cn.SetReadTimeout(c.cmdTimeout(cmd)) err = cmd.readReply(cn) - c.putConn(cn, err) + c.releaseConn(cn, err) if err != nil && internal.IsRetryableError(err) { continue } @@ -179,14 +195,14 @@ func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { return func(cmds []Cmder) error { var firstErr error for i := 0; i <= c.opt.MaxRetries; i++ { - cn, _, err := c.conn() + cn, _, err := c.getConn() if err != nil { setCmdsErr(cmds, err) return err } canRetry, err := p(cn, cmds) - c.putConn(cn, err) + c.releaseConn(cn, err) if err == nil { return nil } @@ -375,10 +391,12 @@ func (c *Client) TxPipeline() Pipeliner { func (c *Client) pubSub() *PubSub { return &PubSub{ - base: baseClient{ - opt: c.opt, - connPool: c.connPool, + opt: c.opt, + + newConn: func(channels []string) (*pool.Conn, error) { + return c.newConn() }, + closeConn: c.connPool.CloseConn, } } diff --git a/ring.go b/ring.go index be92510964..72d52bf755 100644 --- a/ring.go +++ b/ring.go @@ -423,7 +423,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { continue } - cn, _, err := shard.Client.conn() + cn, _, err := shard.Client.getConn() if err != nil { setCmdsErr(cmds, err) if firstErr == nil { @@ -433,7 +433,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { } canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds) - shard.Client.putConn(cn, err) + shard.Client.releaseConn(cn, err) if err == nil { continue } diff --git a/sentinel.go b/sentinel.go index ed6e7ffb36..3bfdb4a3f1 100644 --- a/sentinel.go +++ b/sentinel.go @@ -112,10 +112,12 @@ func newSentinel(opt *Options) *sentinelClient { func (c *sentinelClient) PubSub() *PubSub { return &PubSub{ - base: baseClient{ - opt: c.opt, - connPool: c.connPool, + opt: c.opt, + + newConn: func(channels []string) (*pool.Conn, error) { + return c.newConn() }, + closeConn: c.connPool.CloseConn, } } @@ -149,14 +151,6 @@ func (d *sentinelFailover) Close() error { return d.resetSentinel() } -func (d *sentinelFailover) dial() (net.Conn, error) { - addr, err := d.MasterAddr() - if err != nil { - return nil, err - } - return net.DialTimeout("tcp", addr, d.opt.DialTimeout) -} - func (d *sentinelFailover) Pool() *pool.ConnPool { d.poolOnce.Do(func() { d.opt.Dialer = d.dial @@ -165,6 +159,14 @@ func (d *sentinelFailover) Pool() *pool.ConnPool { return d.pool } +func (d *sentinelFailover) dial() (net.Conn, error) { + addr, err := d.MasterAddr() + if err != nil { + return nil, err + } + return net.DialTimeout("tcp", addr, d.opt.DialTimeout) +} + func (d *sentinelFailover) MasterAddr() (string, error) { d.mu.Lock() defer d.mu.Unlock() From 3ddda73a05f3ba7c3ea69ddce33b4667ff8f7aa2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 9 Jul 2017 13:10:07 +0300 Subject: [PATCH 0357/1746] Close connections to unused nodes --- cluster.go | 259 ++++++++++++++++++++++++++--------- cluster_test.go | 277 +++++++++++++++++++++----------------- export_test.go | 7 +- internal/internal.go | 13 +- internal/internal_test.go | 9 +- options.go | 13 +- redis.go | 24 ++-- 7 files changed, 395 insertions(+), 207 deletions(-) diff --git a/cluster.go b/cluster.go index 2641b16e43..30c5ce64f0 100644 --- a/cluster.go +++ b/cluster.go @@ -28,18 +28,19 @@ type ClusterOptions struct { // Default is 16. MaxRedirects int - // Enables read queries for a connection to a Redis Cluster slave node. + // Enables read-only commands on slave nodes. ReadOnly bool - - // Enables routing read-only queries to the closest master or slave node. + // Allows routing read-only commands to the closest master or slave node. RouteByLatency bool // Following options are copied from Options struct. OnConnect func(*Conn) error - MaxRetries int - Password string + MaxRetries int + MinRetryBackoff time.Duration + MaxRetryBackoff time.Duration + Password string DialTimeout time.Duration ReadTimeout time.Duration @@ -62,6 +63,19 @@ func (opt *ClusterOptions) init() { if opt.RouteByLatency { opt.ReadOnly = true } + + switch opt.MinRetryBackoff { + case -1: + opt.MinRetryBackoff = 0 + case 0: + opt.MinRetryBackoff = 8 * time.Millisecond + } + switch opt.MaxRetryBackoff { + case -1: + opt.MaxRetryBackoff = 0 + case 0: + opt.MaxRetryBackoff = 512 * time.Millisecond + } } func (opt *ClusterOptions) clientOptions() *Options { @@ -70,9 +84,11 @@ func (opt *ClusterOptions) clientOptions() *Options { return &Options{ OnConnect: opt.OnConnect, - MaxRetries: opt.MaxRetries, - Password: opt.Password, - ReadOnly: opt.ReadOnly, + MaxRetries: opt.MaxRetries, + MinRetryBackoff: opt.MinRetryBackoff, + MaxRetryBackoff: opt.MaxRetryBackoff, + Password: opt.Password, + ReadOnly: opt.ReadOnly, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, @@ -91,7 +107,9 @@ func (opt *ClusterOptions) clientOptions() *Options { type clusterNode struct { Client *Client Latency time.Duration - loading time.Time + + loading time.Time + generation uint32 } func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { @@ -122,6 +140,17 @@ func (n *clusterNode) Loading() bool { return !n.loading.IsZero() && time.Since(n.loading) < time.Minute } +func (n *clusterNode) Generation() uint32 { + return n.generation +} + +func (n *clusterNode) SetGeneration(gen uint32) { + if gen < n.generation { + panic("gen < n.generation") + } + n.generation = gen +} + //------------------------------------------------------------------------------ type clusterNodes struct { @@ -131,6 +160,8 @@ type clusterNodes struct { addrs []string nodes map[string]*clusterNode closed bool + + generation uint32 } func newClusterNodes(opt *ClusterOptions) *clusterNodes { @@ -161,6 +192,39 @@ func (c *clusterNodes) Close() error { return firstErr } +func (c *clusterNodes) NextGeneration() uint32 { + c.generation++ + return c.generation +} + +// GC removes unused nodes. +func (c *clusterNodes) GC(generation uint32) error { + var collected []*clusterNode + c.mu.Lock() + for i := 0; i < len(c.addrs); { + addr := c.addrs[i] + node := c.nodes[addr] + if node.Generation() >= generation { + i++ + continue + } + + c.addrs = append(c.addrs[:i], c.addrs[i+1:]...) + delete(c.nodes, addr) + collected = append(collected, node) + } + c.mu.Unlock() + + var firstErr error + for _, node := range collected { + if err := node.Client.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + + return firstErr +} + func (c *clusterNodes) All() ([]*clusterNode, error) { c.mu.RLock() defer c.mu.RUnlock() @@ -176,7 +240,7 @@ func (c *clusterNodes) All() ([]*clusterNode, error) { return nodes, nil } -func (c *clusterNodes) Get(addr string) (*clusterNode, error) { +func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { var node *clusterNode var ok bool @@ -223,7 +287,7 @@ func (c *clusterNodes) Random() (*clusterNode, error) { var nodeErr error for i := 0; i <= c.opt.MaxRedirects; i++ { n := rand.Intn(len(addrs)) - node, err := c.Get(addrs[n]) + node, err := c.GetOrCreate(addrs[n]) if err != nil { return nil, err } @@ -239,30 +303,45 @@ func (c *clusterNodes) Random() (*clusterNode, error) { //------------------------------------------------------------------------------ type clusterState struct { - nodes *clusterNodes + nodes *clusterNodes + masters []*clusterNode + slaves []*clusterNode + slots [][]*clusterNode + + generation uint32 } func newClusterState(nodes *clusterNodes, slots []ClusterSlot, origin string) (*clusterState, error) { c := clusterState{ - nodes: nodes, + nodes: nodes, + generation: nodes.NextGeneration(), + slots: make([][]*clusterNode, hashtag.SlotNumber), } isLoopbackOrigin := isLoopbackAddr(origin) for _, slot := range slots { var nodes []*clusterNode - for _, slotNode := range slot.Nodes { + for i, slotNode := range slot.Nodes { addr := slotNode.Addr if !isLoopbackOrigin && isLoopbackAddr(addr) { addr = origin } - node, err := c.nodes.Get(addr) + node, err := c.nodes.GetOrCreate(addr) if err != nil { return nil, err } + + node.SetGeneration(c.generation) nodes = append(nodes, node) + + if i == 0 { + c.masters = appendNode(c.masters, node) + } else { + c.slaves = appendNode(c.slaves, node) + } } for i := slot.Start; i <= slot.End; i++ { @@ -348,7 +427,7 @@ type ClusterClient struct { cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo - // Reports where slots reloading is in progress. + // Reports whether slots reloading is in progress. reloading uint32 } @@ -365,12 +444,12 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { // Add initial nodes. for _, addr := range opt.Addrs { - _, _ = c.nodes.Get(addr) + _, _ = c.nodes.GetOrCreate(addr) } // Preload cluster slots. for i := 0; i < 10; i++ { - state, err := c.reloadSlots() + state, err := c.reloadState() if err == nil { c._state.Store(state) break @@ -394,7 +473,7 @@ func (c *ClusterClient) state() *clusterState { if v != nil { return v.(*clusterState) } - c.lazyReloadSlots() + c.lazyReloadState() return nil } @@ -476,6 +555,10 @@ func (c *ClusterClient) Process(cmd Cmder) error { var ask bool for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { + if attempt > 0 { + time.Sleep(node.Client.retryBackoff(attempt)) + } + if ask { pipe := node.Client.Pipeline() pipe.Process(NewCmd("ASKING")) @@ -487,13 +570,14 @@ func (c *ClusterClient) Process(cmd Cmder) error { err = node.Client.Process(cmd) } - // If there is no (real) error - we are done. + // If there is no error - we are done. if err == nil { return nil } // If slave is loading - read from master. if c.opt.ReadOnly && internal.IsLoadingError(err) { + // TODO: race node.loading = time.Now() continue } @@ -516,11 +600,11 @@ func (c *ClusterClient) Process(cmd Cmder) error { if state != nil && slot >= 0 { master, _ := state.slotMasterNode(slot) if moved && (master == nil || master.Client.getAddr() != addr) { - c.lazyReloadSlots() + c.lazyReloadState() } } - node, err = c.nodes.Get(addr) + node, err = c.nodes.GetOrCreate(addr) if err != nil { cmd.setErr(err) return err @@ -535,17 +619,17 @@ func (c *ClusterClient) Process(cmd Cmder) error { return cmd.Err() } -// ForEachNode concurrently calls the fn on each ever known node in the cluster. +// ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. -func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { - nodes, err := c.nodes.All() - if err != nil { - return err +func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { + state := c.state() + if state == nil { + return errNilClusterState } var wg sync.WaitGroup errCh := make(chan error, 1) - for _, node := range nodes { + for _, master := range state.masters { wg.Add(1) go func(node *clusterNode) { defer wg.Done() @@ -556,7 +640,7 @@ func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { default: } } - }(node) + }(master) } wg.Wait() @@ -568,28 +652,17 @@ func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { } } -// ForEachMaster concurrently calls the fn on each master node in the cluster. +// ForEachSlave concurrently calls the fn on each slave node in the cluster. // It returns the first error if any. -func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { +func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { state := c.state() if state == nil { return errNilClusterState } var wg sync.WaitGroup - visited := make(map[*clusterNode]struct{}) errCh := make(chan error, 1) - for _, nodes := range state.slots { - if len(nodes) == 0 { - continue - } - - master := nodes[0] - if _, ok := visited[master]; ok { - continue - } - visited[master] = struct{}{} - + for _, slave := range state.slaves { wg.Add(1) go func(node *clusterNode) { defer wg.Done() @@ -600,7 +673,7 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { default: } } - }(master) + }(slave) } wg.Wait() @@ -612,16 +685,64 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { } } +// ForEachNode concurrently calls the fn on each known node in the cluster. +// It returns the first error if any. +func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { + state := c.state() + if state == nil { + return errNilClusterState + } + + var wg sync.WaitGroup + errCh := make(chan error, 1) + worker := func(node *clusterNode) { + defer wg.Done() + err := fn(node.Client) + if err != nil { + select { + case errCh <- err: + default: + } + } + } + + for _, node := range state.masters { + wg.Add(1) + go worker(node) + } + for _, node := range state.slaves { + wg.Add(1) + go worker(node) + } + + wg.Wait() + select { + case err := <-errCh: + return err + default: + return nil + } +} + // PoolStats returns accumulated connection pool stats. func (c *ClusterClient) PoolStats() *PoolStats { var acc PoolStats - nodes, err := c.nodes.All() - if err != nil { + state := c.state() + if state == nil { return &acc } - for _, node := range nodes { + for _, node := range state.masters { + s := node.Client.connPool.Stats() + acc.Requests += s.Requests + acc.Hits += s.Hits + acc.Timeouts += s.Timeouts + acc.TotalConns += s.TotalConns + acc.FreeConns += s.FreeConns + } + + for _, node := range state.slaves { s := node.Client.connPool.Stats() acc.Requests += s.Requests acc.Hits += s.Hits @@ -629,33 +750,42 @@ func (c *ClusterClient) PoolStats() *PoolStats { acc.TotalConns += s.TotalConns acc.FreeConns += s.FreeConns } + return &acc } -func (c *ClusterClient) lazyReloadSlots() { +func (c *ClusterClient) lazyReloadState() { if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { return } go func() { - for i := 0; i < 1000; i++ { - state, err := c.reloadSlots() + defer atomic.StoreUint32(&c.reloading, 0) + + var state *clusterState + for { + var err error + state, err = c.reloadState() if err == pool.ErrClosed { - break + return } - if err == nil { - c._state.Store(state) - break + + if err != nil { + time.Sleep(time.Millisecond) + continue } - time.Sleep(time.Millisecond) + + c._state.Store(state) + break } time.Sleep(3 * time.Second) - atomic.StoreUint32(&c.reloading, 0) + c.nodes.GC(state.generation) }() } -func (c *ClusterClient) reloadSlots() (*clusterState, error) { +// Not thread-safe. +func (c *ClusterClient) reloadState() (*clusterState, error) { node, err := c.nodes.Random() if err != nil { return nil, err @@ -799,9 +929,9 @@ func (c *ClusterClient) pipelineReadCmds( func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]Cmder) error { moved, ask, addr := internal.IsMovedError(cmd.Err()) if moved { - c.lazyReloadSlots() + c.lazyReloadState() - node, err := c.nodes.Get(addr) + node, err := c.nodes.GetOrCreate(addr) if err != nil { return err } @@ -809,7 +939,7 @@ func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]C failedCmds[node] = append(failedCmds[node], cmd) } if ask { - node, err := c.nodes.Get(addr) + node, err := c.nodes.GetOrCreate(addr) if err != nil { return err } @@ -1029,3 +1159,12 @@ func isLoopbackAddr(addr string) bool { return ip.IsLoopback() } + +func appendNode(nodes []*clusterNode, node *clusterNode) []*clusterNode { + for _, n := range nodes { + if n == node { + return nodes + } + } + return append(nodes, node) +} diff --git a/cluster_test.go b/cluster_test.go index 1dc7229501..0176c68538 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -75,7 +75,7 @@ func startCluster(scenario *clusterScenario) error { scenario.nodeIds[pos] = info[:40] } - // Meet cluster nodes + // Meet cluster nodes. for _, client := range scenario.clients { err := client.ClusterMeet("127.0.0.1", scenario.ports[0]).Err() if err != nil { @@ -83,7 +83,7 @@ func startCluster(scenario *clusterScenario) error { } } - // Bootstrap masters + // Bootstrap masters. slots := []int{0, 5000, 10000, 16384} for pos, master := range scenario.masters() { err := master.ClusterAddSlotsRange(slots[pos], slots[pos+1]-1).Err() @@ -92,7 +92,7 @@ func startCluster(scenario *clusterScenario) error { } } - // Bootstrap slaves + // Bootstrap slaves. for idx, slave := range scenario.slaves() { masterId := scenario.nodeIds[idx] @@ -115,7 +115,7 @@ func startCluster(scenario *clusterScenario) error { } } - // Wait until all nodes have consistent info + // Wait until all nodes have consistent info. for _, client := range scenario.clients { err := eventually(func() error { res, err := client.ClusterSlots().Result() @@ -189,62 +189,6 @@ var _ = Describe("ClusterClient", func() { var client *redis.ClusterClient assertClusterClient := func() { - It("should CLUSTER SLOTS", func() { - res, err := client.ClusterSlots().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(HaveLen(3)) - - wanted := []redis.ClusterSlot{ - {0, 4999, []redis.ClusterNode{{"", "127.0.0.1:8220"}, {"", "127.0.0.1:8223"}}}, - {5000, 9999, []redis.ClusterNode{{"", "127.0.0.1:8221"}, {"", "127.0.0.1:8224"}}}, - {10000, 16383, []redis.ClusterNode{{"", "127.0.0.1:8222"}, {"", "127.0.0.1:8225"}}}, - } - Expect(assertSlotsEqual(res, wanted)).NotTo(HaveOccurred()) - }) - - It("should CLUSTER NODES", func() { - res, err := client.ClusterNodes().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(len(res)).To(BeNumerically(">", 400)) - }) - - It("should CLUSTER INFO", func() { - res, err := client.ClusterInfo().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(ContainSubstring("cluster_known_nodes:6")) - }) - - It("should CLUSTER KEYSLOT", func() { - hashSlot, err := client.ClusterKeySlot("somekey").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(hashSlot).To(Equal(int64(hashtag.Slot("somekey")))) - }) - - It("should CLUSTER COUNT-FAILURE-REPORTS", func() { - n, err := client.ClusterCountFailureReports(cluster.nodeIds[0]).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(0))) - }) - - It("should CLUSTER COUNTKEYSINSLOT", func() { - n, err := client.ClusterCountKeysInSlot(10).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(0))) - }) - - It("should CLUSTER SAVECONFIG", func() { - res, err := client.ClusterSaveConfig().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal("OK")) - }) - - It("should CLUSTER SLAVES", func() { - nodesList, err := client.ClusterSlaves(cluster.nodeIds[0]).Result() - Expect(err).NotTo(HaveOccurred()) - Expect(nodesList).Should(ContainElement(ContainSubstring("slave"))) - Expect(nodesList).Should(HaveLen(1)) - }) - It("should GET/SET/DEL", func() { val, err := client.Get("A").Result() Expect(err).To(Equal(redis.Nil)) @@ -254,55 +198,24 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) Expect(val).To(Equal("OK")) - val, err = client.Get("A").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal("VALUE")) + Eventually(func() string { + return client.Get("A").Val() + }).Should(Equal("VALUE")) cnt, err := client.Del("A").Result() Expect(err).NotTo(HaveOccurred()) Expect(cnt).To(Equal(int64(1))) }) - It("returns pool stats", func() { - Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) - }) - - It("removes idle connections", func() { - stats := client.PoolStats() - Expect(stats.TotalConns).NotTo(BeZero()) - Expect(stats.FreeConns).NotTo(BeZero()) - - time.Sleep(2 * time.Second) - - stats = client.PoolStats() - Expect(stats.TotalConns).To(BeZero()) - Expect(stats.FreeConns).To(BeZero()) - }) - It("follows redirects", func() { Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) slot := hashtag.Slot("A") - Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) + client.SwapSlotNodes(slot) - val, err := client.Get("A").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal("VALUE")) - }) - - It("returns an error when there are no attempts left", func() { - opt := redisClusterOptions() - opt.MaxRedirects = -1 - client := cluster.clusterClient(opt) - - slot := hashtag.Slot("A") - Expect(client.SwapSlotNodes(slot)).To(Equal([]string{"127.0.0.1:8224", "127.0.0.1:8221"})) - - err := client.Get("A").Err() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("MOVED")) - - Expect(client.Close()).NotTo(HaveOccurred()) + Eventually(func() string { + return client.Get("A").Val() + }).Should(Equal("VALUE")) }) It("distributes keys", func() { @@ -311,9 +224,14 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) } - wanted := []string{"keys=31", "keys=29", "keys=40"} - for i, master := range cluster.masters() { - Expect(master.Info().Val()).To(ContainSubstring(wanted[i])) + for _, master := range cluster.masters() { + Eventually(func() string { + return master.Info("keyspace").Val() + }, 5*time.Second).Should(Or( + ContainSubstring("keys=31"), + ContainSubstring("keys=29"), + ContainSubstring("keys=40"), + )) } }) @@ -330,9 +248,14 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) } - wanted := []string{"keys=31", "keys=29", "keys=40"} - for i, master := range cluster.masters() { - Expect(master.Info().Val()).To(ContainSubstring(wanted[i])) + for _, master := range cluster.masters() { + Eventually(func() string { + return master.Info("keyspace").Val() + }, 5*time.Second).Should(Or( + ContainSubstring("keys=31"), + ContainSubstring("keys=29"), + ContainSubstring("keys=40"), + )) } }) @@ -447,7 +370,7 @@ var _ = Describe("ClusterClient", func() { }) } - Describe("Pipeline", func() { + Describe("with Pipeline", func() { BeforeEach(func() { pipe = client.Pipeline().(*redis.Pipeline) }) @@ -459,7 +382,7 @@ var _ = Describe("ClusterClient", func() { assertPipeline() }) - Describe("TxPipeline", func() { + Describe("with TxPipeline", func() { BeforeEach(func() { pipe = client.TxPipeline().(*redis.Pipeline) }) @@ -476,22 +399,70 @@ var _ = Describe("ClusterClient", func() { pubsub := client.Subscribe("mychannel") defer pubsub.Close() - msgi, err := pubsub.ReceiveTimeout(time.Second) - Expect(err).NotTo(HaveOccurred()) - subscr := msgi.(*redis.Subscription) - Expect(subscr.Kind).To(Equal("subscribe")) - Expect(subscr.Channel).To(Equal("mychannel")) - Expect(subscr.Count).To(Equal(1)) + Eventually(func() error { + _, err := client.Publish("mychannel", "hello").Result() + if err != nil { + return err + } - n, err := client.Publish("mychannel", "hello").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(1))) + msg, err := pubsub.ReceiveTimeout(time.Second) + if err != nil { + return err + } - msgi, err = pubsub.ReceiveTimeout(time.Second) - Expect(err).NotTo(HaveOccurred()) - msg := msgi.(*redis.Message) - Expect(msg.Channel).To(Equal("mychannel")) - Expect(msg.Payload).To(Equal("hello")) + _, ok := msg.(*redis.Message) + if !ok { + return fmt.Errorf("got %T, wanted *redis.Message", msg) + } + + return nil + }, 30*time.Second).ShouldNot(HaveOccurred()) + }) + } + + Describe("ClusterClient", func() { + BeforeEach(func() { + opt = redisClusterOptions() + client = cluster.clusterClient(opt) + + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) + }) + + AfterEach(func() { + Expect(client.Close()).NotTo(HaveOccurred()) + }) + + It("returns pool stats", func() { + Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) + }) + + It("removes idle connections", func() { + stats := client.PoolStats() + Expect(stats.TotalConns).NotTo(BeZero()) + Expect(stats.FreeConns).NotTo(BeZero()) + + time.Sleep(2 * time.Second) + + stats = client.PoolStats() + Expect(stats.TotalConns).To(BeZero()) + Expect(stats.FreeConns).To(BeZero()) + }) + + It("returns an error when there are no attempts left", func() { + opt := redisClusterOptions() + opt.MaxRedirects = -1 + client := cluster.clusterClient(opt) + + slot := hashtag.Slot("A") + client.SwapSlotNodes(slot) + + err := client.Get("A").Err() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("MOVED")) + + Expect(client.Close()).NotTo(HaveOccurred()) }) It("calls fn for every master node", func() { @@ -510,9 +481,67 @@ var _ = Describe("ClusterClient", func() { Expect(keys).To(HaveLen(0)) } }) - } - Describe("default ClusterClient", func() { + It("should CLUSTER SLOTS", func() { + res, err := client.ClusterSlots().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(3)) + + wanted := []redis.ClusterSlot{ + {0, 4999, []redis.ClusterNode{{"", "127.0.0.1:8220"}, {"", "127.0.0.1:8223"}}}, + {5000, 9999, []redis.ClusterNode{{"", "127.0.0.1:8221"}, {"", "127.0.0.1:8224"}}}, + {10000, 16383, []redis.ClusterNode{{"", "127.0.0.1:8222"}, {"", "127.0.0.1:8225"}}}, + } + Expect(assertSlotsEqual(res, wanted)).NotTo(HaveOccurred()) + }) + + It("should CLUSTER NODES", func() { + res, err := client.ClusterNodes().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(len(res)).To(BeNumerically(">", 400)) + }) + + It("should CLUSTER INFO", func() { + res, err := client.ClusterInfo().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ContainSubstring("cluster_known_nodes:6")) + }) + + It("should CLUSTER KEYSLOT", func() { + hashSlot, err := client.ClusterKeySlot("somekey").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(hashSlot).To(Equal(int64(hashtag.Slot("somekey")))) + }) + + It("should CLUSTER COUNT-FAILURE-REPORTS", func() { + n, err := client.ClusterCountFailureReports(cluster.nodeIds[0]).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) + }) + + It("should CLUSTER COUNTKEYSINSLOT", func() { + n, err := client.ClusterCountKeysInSlot(10).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(0))) + }) + + It("should CLUSTER SAVECONFIG", func() { + res, err := client.ClusterSaveConfig().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal("OK")) + }) + + It("should CLUSTER SLAVES", func() { + nodesList, err := client.ClusterSlaves(cluster.nodeIds[0]).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(nodesList).Should(ContainElement(ContainSubstring("slave"))) + Expect(nodesList).Should(HaveLen(1)) + }) + + assertClusterClient() + }) + + Describe("ClusterClient failover", func() { BeforeEach(func() { opt = redisClusterOptions() client = cluster.clusterClient(opt) @@ -520,6 +549,10 @@ var _ = Describe("ClusterClient", func() { _ = client.ForEachMaster(func(master *redis.Client) error { return master.FlushDB().Err() }) + + _ = client.ForEachSlave(func(slave *redis.Client) error { + return slave.ClusterFailover().Err() + }) }) AfterEach(func() { diff --git a/export_test.go b/export_test.go index b88e41be9e..3b7965d796 100644 --- a/export_test.go +++ b/export_test.go @@ -28,8 +28,9 @@ func (c *ClusterClient) SlotAddrs(slot int) []string { } // SwapSlot swaps a slot's master/slave address for testing MOVED redirects. -func (c *ClusterClient) SwapSlotNodes(slot int) []string { +func (c *ClusterClient) SwapSlotNodes(slot int) { nodes := c.state().slots[slot] - nodes[0], nodes[1] = nodes[1], nodes[0] - return c.SlotAddrs(slot) + if len(nodes) == 2 { + nodes[0], nodes[1] = nodes[1], nodes[0] + } } diff --git a/internal/internal.go b/internal/internal.go index fb4efa5f04..ad3fc3c9ff 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -5,19 +5,20 @@ import ( "time" ) -const retryBackoff = 8 * time.Millisecond - // Retry backoff with jitter sleep to prevent overloaded conditions during intervals // https://www.awsarchitectureblog.com/2015/03/backoff.html -func RetryBackoff(retry int, maxRetryBackoff time.Duration) time.Duration { +func RetryBackoff(retry int, minBackoff, maxBackoff time.Duration) time.Duration { if retry < 0 { retry = 0 } - backoff := retryBackoff << uint(retry) - if backoff > maxRetryBackoff { - backoff = maxRetryBackoff + backoff := minBackoff << uint(retry) + if backoff > maxBackoff || backoff < minBackoff { + backoff = maxBackoff } + if backoff == 0 { + return 0 + } return time.Duration(rand.Int63n(int64(backoff))) } diff --git a/internal/internal_test.go b/internal/internal_test.go index 5c7000e1ea..56ff611e10 100644 --- a/internal/internal_test.go +++ b/internal/internal_test.go @@ -2,15 +2,16 @@ package internal import ( "testing" - . "github.com/onsi/gomega" "time" + + . "github.com/onsi/gomega" ) func TestRetryBackoff(t *testing.T) { RegisterTestingT(t) - - for i := -1; i<= 8; i++ { - backoff := RetryBackoff(i, 512*time.Millisecond) + + for i := -1; i <= 16; i++ { + backoff := RetryBackoff(i, time.Millisecond, 512*time.Millisecond) Expect(backoff >= 0).To(BeTrue()) Expect(backoff <= 512*time.Millisecond).To(BeTrue()) } diff --git a/options.go b/options.go index cd6fa981a3..efa8f6aa6c 100644 --- a/options.go +++ b/options.go @@ -37,9 +37,11 @@ type Options struct { // Maximum number of retries before giving up. // Default is to not retry failed commands. MaxRetries int - + // Minimum backoff between each retry. + // Default is 8 milliseconds; -1 disables backoff. + MinRetryBackoff time.Duration // Maximum backoff between each retry. - // Default is 512 seconds; -1 disables backoff. + // Default is 512 milliseconds; -1 disables backoff. MaxRetryBackoff time.Duration // Dial timeout for establishing new connections. @@ -118,6 +120,13 @@ func (opt *Options) init() { if opt.IdleCheckFrequency == 0 { opt.IdleCheckFrequency = time.Minute } + + switch opt.MinRetryBackoff { + case -1: + opt.MinRetryBackoff = 0 + case 0: + opt.MinRetryBackoff = 8 * time.Millisecond + } switch opt.MaxRetryBackoff { case -1: opt.MaxRetryBackoff = 0 diff --git a/redis.go b/redis.go index 1a2ecb0b99..b3a215e403 100644 --- a/redis.go +++ b/redis.go @@ -107,13 +107,6 @@ func (c *baseClient) initConn(cn *pool.Conn) error { return nil } -func (c *baseClient) Process(cmd Cmder) error { - if c.process != nil { - return c.process(cmd) - } - return c.defaultProcess(cmd) -} - // WrapProcess replaces the process func. It takes a function createWrapper // which is supplied by the user. createWrapper takes the old process func as // an input and returns the new wrapper process func. createWrapper should @@ -122,10 +115,17 @@ func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func( c.process = fn(c.defaultProcess) } +func (c *baseClient) Process(cmd Cmder) error { + if c.process != nil { + return c.process(cmd) + } + return c.defaultProcess(cmd) +} + func (c *baseClient) defaultProcess(cmd Cmder) error { - for i := 0; i <= c.opt.MaxRetries; i++ { - if i > 0 { - time.Sleep(internal.RetryBackoff(i, c.opt.MaxRetryBackoff)) + for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) } cn, _, err := c.getConn() @@ -160,6 +160,10 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { return cmd.Err() } +func (c *baseClient) retryBackoff(attempt int) time.Duration { + return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) +} + func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration { if timeout := cmd.readTimeout(); timeout != nil { return *timeout From cf6c6dca84f776d737abe7e0d3c931138bee4c9f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 19 Jul 2017 15:32:50 +0300 Subject: [PATCH 0358/1746] Add Geo commands read-only variants --- command.go | 20 +++++++++++++++----- commands.go | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/command.go b/command.go index 361661adfb..0e5b2016e4 100644 --- a/command.go +++ b/command.go @@ -799,7 +799,9 @@ type GeoRadiusQuery struct { WithGeoHash bool Count int // Can be ASC or DESC. Default is no sort order. - Sort string + Sort string + Store string + StoreDist string } type GeoLocationCmd struct { @@ -817,20 +819,28 @@ func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { args = append(args, "km") } if q.WithCoord { - args = append(args, "WITHCOORD") + args = append(args, "withcoord") } if q.WithDist { - args = append(args, "WITHDIST") + args = append(args, "withdist") } if q.WithGeoHash { - args = append(args, "WITHHASH") + args = append(args, "withhash") } if q.Count > 0 { - args = append(args, "COUNT", q.Count) + args = append(args, "count", q.Count) } if q.Sort != "" { args = append(args, q.Sort) } + if q.Store != "" { + args = append(args, "store") + args = append(args, q.Store) + } + if q.StoreDist != "" { + args = append(args, "storedist") + args = append(args, q.StoreDist) + } return &GeoLocationCmd{ baseCmd: baseCmd{_args: args}, q: q, diff --git a/commands.go b/commands.go index 4ea78777c0..ac7c6cc12f 100644 --- a/commands.go +++ b/commands.go @@ -234,7 +234,9 @@ type Cmdable interface { GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd GeoPos(key string, members ...string) *GeoPosCmd GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd + GeoRadiusRO(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd + GeoRadiusByMemberRO(key, member string, query *GeoRadiusQuery) *GeoLocationCmd GeoDist(key string, member1, member2, unit string) *FloatCmd GeoHash(key string, members ...string) *StringSliceCmd Command() *CommandsInfoCmd @@ -2061,12 +2063,24 @@ func (c *cmdable) GeoRadius(key string, longitude, latitude float64, query *GeoR return cmd } +func (c *cmdable) GeoRadiusRO(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd { + cmd := NewGeoLocationCmd(query, "georadius_ro", key, longitude, latitude) + c.process(cmd) + return cmd +} + func (c *cmdable) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { cmd := NewGeoLocationCmd(query, "georadiusbymember", key, member) c.process(cmd) return cmd } +func (c *cmdable) GeoRadiusByMemberRO(key, member string, query *GeoRadiusQuery) *GeoLocationCmd { + cmd := NewGeoLocationCmd(query, "georadiusbymember_ro", key, member) + c.process(cmd) + return cmd +} + func (c *cmdable) GeoDist(key string, member1, member2, unit string) *FloatCmd { if unit == "" { unit = "km" From a005081ecd2d0d963a1a20efda049223205cf90a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 19 Jul 2017 15:41:16 +0300 Subject: [PATCH 0359/1746] Fix example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f3c61795e8..fd036496dc 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Some corner cases: vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result() EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" - vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, []string{"hello"}).Result() + vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result() ## Benchmark From 3cbacec8752afaa1195ae0c0f3509e47fde1d24d Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 20 Jul 2017 07:28:47 +0300 Subject: [PATCH 0360/1746] Use latest released Go versions --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8e0d652ee..f4666c5930 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,14 @@ services: - redis-server go: - - 1.4 - - 1.7 - - 1.8 + - 1.4.x + - 1.7.x + - 1.8.x - tip matrix: allow_failures: - - go: 1.4 + - go: 1.4.x - go: tip install: From 4e1d2a01db8ebcec6ddeae623050cbaa7df5ffcf Mon Sep 17 00:00:00 2001 From: "wenjun.yan" Date: Tue, 25 Jul 2017 10:35:41 +0900 Subject: [PATCH 0361/1746] Make readOnly a private field so that only cluster client can use it --- cluster.go | 2 +- options.go | 2 +- redis.go | 4 ++-- universal.go | 6 ++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cluster.go b/cluster.go index 30c5ce64f0..a8695c32e2 100644 --- a/cluster.go +++ b/cluster.go @@ -88,7 +88,7 @@ func (opt *ClusterOptions) clientOptions() *Options { MinRetryBackoff: opt.MinRetryBackoff, MaxRetryBackoff: opt.MaxRetryBackoff, Password: opt.Password, - ReadOnly: opt.ReadOnly, + readOnly: opt.ReadOnly, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, diff --git a/options.go b/options.go index efa8f6aa6c..dea0454531 100644 --- a/options.go +++ b/options.go @@ -73,7 +73,7 @@ type Options struct { IdleCheckFrequency time.Duration // Enables read only queries on slave nodes. - ReadOnly bool + readOnly bool // TLS Config to use. When set TLS will be negotiated. TLSConfig *tls.Config diff --git a/redis.go b/redis.go index b3a215e403..b18973cdb2 100644 --- a/redis.go +++ b/redis.go @@ -68,7 +68,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { if c.opt.Password == "" && c.opt.DB == 0 && - !c.opt.ReadOnly && + !c.opt.readOnly && c.opt.OnConnect == nil { return nil } @@ -91,7 +91,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { pipe.Select(c.opt.DB) } - if c.opt.ReadOnly { + if c.opt.readOnly { pipe.ReadOnly() } diff --git a/universal.go b/universal.go index 02ed51abd9..4aa579fa4f 100644 --- a/universal.go +++ b/universal.go @@ -17,12 +17,11 @@ type UniversalOptions struct { // Only single-node and failover clients. DB int + // Only cluster clients. + // Enables read only queries on slave nodes. - // Only cluster and single-node clients. ReadOnly bool - // Only cluster clients. - MaxRedirects int RouteByLatency bool @@ -93,7 +92,6 @@ func (o *UniversalOptions) simple() *Options { return &Options{ Addr: addr, DB: o.DB, - ReadOnly: o.ReadOnly, MaxRetries: o.MaxRetries, Password: o.Password, From dbcf95c85eb973acff76ba5d25c5e0b48126d899 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 1 Aug 2017 14:21:26 +0300 Subject: [PATCH 0362/1746] Fix PubSub.Subscribe deadlock --- pubsub.go | 26 +++++++++++++------------- pubsub_test.go | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pubsub.go b/pubsub.go index 74ac51c108..4a5c65f571 100644 --- a/pubsub.go +++ b/pubsub.go @@ -88,19 +88,19 @@ func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) return writeCmd(cn, cmd) } -func (c *PubSub) putConn(cn *pool.Conn, err error) { - if !internal.IsBadConn(err, true) { - return - } - +func (c *PubSub) releaseConn(cn *pool.Conn, err error) { c.mu.Lock() - if c.cn == cn { - _ = c.releaseConn() - } + c._releaseConn(cn, err) c.mu.Unlock() } -func (c *PubSub) releaseConn() error { +func (c *PubSub) _releaseConn(cn *pool.Conn, err error) { + if internal.IsBadConn(err, true) && c.cn == cn { + _ = c.closeTheCn() + } +} + +func (c *PubSub) closeTheCn() error { err := c.closeConn(c.cn) c.cn = nil return err @@ -116,7 +116,7 @@ func (c *PubSub) Close() error { c.closed = true if c.cn != nil { - return c.releaseConn() + return c.closeTheCn() } return nil } @@ -168,7 +168,7 @@ func (c *PubSub) subscribe(redisCmd string, channels ...string) error { } err = c._subscribe(cn, redisCmd, channels...) - c.putConn(cn, err) + c._releaseConn(cn, err) return err } @@ -186,7 +186,7 @@ func (c *PubSub) Ping(payload ...string) error { cn.SetWriteTimeout(c.opt.WriteTimeout) err = writeCmd(cn, cmd) - c.putConn(cn, err) + c.releaseConn(cn, err) return err } @@ -279,7 +279,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { cn.SetReadTimeout(timeout) err = c.cmd.readReply(cn) - c.putConn(cn, err) + c.releaseConn(cn, err) if err != nil { return nil, err } diff --git a/pubsub_test.go b/pubsub_test.go index 3cb9627b53..d44b1dd8ce 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -294,6 +294,22 @@ var _ = Describe("PubSub", func() { Expect(stats.Hits).To(Equal(uint32(1))) }) + It("returns an error when subscribe fails", func() { + pubsub := client.Subscribe() + defer pubsub.Close() + + pubsub.SetNetConn(&badConn{ + readErr: io.EOF, + writeErr: io.EOF, + }) + + err := pubsub.Subscribe("mychannel") + Expect(err).To(MatchError("EOF")) + + err = pubsub.Subscribe("mychannel") + Expect(err).NotTo(HaveOccurred()) + }) + expectReceiveMessageOnError := func(pubsub *redis.PubSub) { pubsub.SetNetConn(&badConn{ readErr: io.EOF, From 7ae26b74bcd4631279c50c88bb74d11723712e9c Mon Sep 17 00:00:00 2001 From: Nikolay Sivko Date: Fri, 4 Aug 2017 01:11:35 +0300 Subject: [PATCH 0363/1746] stop ConnPool.tryDial goroutine if pool was closed --- internal/pool/pool.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index a4e650847f..c769e01f7d 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -119,6 +119,9 @@ func (p *ConnPool) NewConn() (*Conn, error) { func (p *ConnPool) tryDial() { for { + if p.closed() { + return + } conn, err := p.opt.Dialer() if err != nil { p.setLastDialError(err) From 165f47fa41ad7d6895c5d95cef88e72f2c07ad06 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 4 Aug 2017 10:51:14 +0200 Subject: [PATCH 0364/1746] Using INCR as an atomic operation --- example_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example_test.go b/example_test.go index 319ea0ca2d..7e04cd4875 100644 --- a/example_test.go +++ b/example_test.go @@ -122,13 +122,13 @@ func ExampleClient_Set() { } func ExampleClient_Incr() { - if err := client.Incr("counter").Err(); err != nil { + result, err := client.Incr("counter").Result() + if err != nil { panic(err) } - n, err := client.Get("counter").Int64() - fmt.Println(n, err) - // Output: 1 + fmt.Println(result) + // Output: 1 } func ExampleClient_BLPop() { From a9364f117c80e51e9f7aea42903c903cda4d8842 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 15 Aug 2017 09:49:23 +0300 Subject: [PATCH 0365/1746] Add ZLexCount --- commands.go | 7 +++++++ commands_test.go | 28 ++++++++++++++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/commands.go b/commands.go index ac7c6cc12f..15f67a13fa 100644 --- a/commands.go +++ b/commands.go @@ -159,6 +159,7 @@ type Cmdable interface { ZIncrXX(key string, member Z) *FloatCmd ZCard(key string) *IntCmd ZCount(key, min, max string) *IntCmd + ZLexCount(key, min, max string) *IntCmd ZIncrBy(key string, increment float64, member string) *FloatCmd ZInterStore(destination string, store ZStore, keys ...string) *IntCmd ZRange(key string, start, stop int64) *StringSliceCmd @@ -1352,6 +1353,12 @@ func (c *cmdable) ZCount(key, min, max string) *IntCmd { return cmd } +func (c *cmdable) ZLexCount(key, min, max string) *IntCmd { + cmd := NewIntCmd("zlexcount", key, min, max) + c.process(cmd) + return cmd +} + func (c *cmdable) ZIncrBy(key string, increment float64, member string) *FloatCmd { cmd := NewFloatCmd("zincrby", key, increment, member) c.process(cmd) diff --git a/commands_test.go b/commands_test.go index e8cdb205e9..81e0e4a82f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2176,20 +2176,24 @@ var _ = Describe("Commands", func() { }) It("should ZCount", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{1, "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{2, "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{3, "three"}).Err() + Expect(err).NotTo(HaveOccurred()) + + count, err := client.ZCount("zset", "-inf", "+inf").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(int64(3))) - zCount := client.ZCount("zset", "-inf", "+inf") - Expect(zCount.Err()).NotTo(HaveOccurred()) - Expect(zCount.Val()).To(Equal(int64(3))) + count, err = client.ZCount("zset", "(1", "3").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(int64(2))) - zCount = client.ZCount("zset", "(1", "3") - Expect(zCount.Err()).NotTo(HaveOccurred()) - Expect(zCount.Val()).To(Equal(int64(2))) + count, err = client.ZLexCount("zset", "-", "+").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(int64(3))) }) It("should ZIncrBy", func() { From 63e3bc58c7d21783115fa0efc58ee6e09012b337 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 15 Aug 2017 10:12:43 +0300 Subject: [PATCH 0366/1746] Retry cluster down errors --- cluster.go | 2 +- cluster_test.go | 8 ++++---- internal/{errors.go => error.go} | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) rename internal/{errors.go => error.go} (88%) diff --git a/cluster.go b/cluster.go index a8695c32e2..647a25be39 100644 --- a/cluster.go +++ b/cluster.go @@ -583,7 +583,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { } // On network errors try random node. - if internal.IsRetryableError(err) { + if internal.IsRetryableError(err) || internal.IsClusterDownError(err) { node, err = c.nodes.Random() if err != nil { cmd.setErr(err) diff --git a/cluster_test.go b/cluster_test.go index 0176c68538..91b9e80e28 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -700,14 +700,14 @@ var _ = Describe("ClusterClient timeout", func() { testTimeout() }) - Context("network timeout", func() { + Context("ClientPause timeout", func() { const pause = time.Second BeforeEach(func() { opt := redisClusterOptions() - opt.ReadTimeout = 100 * time.Millisecond - opt.WriteTimeout = 100 * time.Millisecond - opt.MaxRedirects = 1 + opt.ReadTimeout = pause / 10 + opt.WriteTimeout = pause / 10 + opt.MaxRedirects = -1 client = cluster.clusterClient(opt) err := client.ForEachNode(func(client *redis.Client) error { diff --git a/internal/errors.go b/internal/error.go similarity index 88% rename from internal/errors.go rename to internal/error.go index c93e008182..90f6503a15 100644 --- a/internal/errors.go +++ b/internal/error.go @@ -67,9 +67,9 @@ func IsMovedError(err error) (moved bool, ask bool, addr string) { } func IsLoadingError(err error) bool { - return strings.HasPrefix(err.Error(), "LOADING") + return strings.HasPrefix(err.Error(), "LOADING ") } -func IsExecAbortError(err error) bool { - return strings.HasPrefix(err.Error(), "EXECABORT") +func IsClusterDownError(err error) bool { + return strings.HasPrefix(err.Error(), "CLUSTERDOWN ") } From 8ff417ca1810febd2f38abe9b3134996c6bd2f08 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 15 Aug 2017 10:34:05 +0300 Subject: [PATCH 0367/1746] Fix flaky tests --- cluster_test.go | 19 ++++++++++++------- commands.go | 11 +++++++---- commands_test.go | 8 ++++---- pubsub_test.go | 5 +++++ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 91b9e80e28..324bd1ce1e 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -342,7 +342,8 @@ var _ = Describe("ClusterClient", func() { Expect(get.Val()).To(Equal(key + "_value")) ttl := cmds[(i*2)+1].(*redis.DurationCmd) - Expect(ttl.Val()).To(BeNumerically("~", time.Duration(i+1)*time.Hour, time.Second)) + dur := time.Duration(i+1) * time.Hour + Expect(ttl.Val()).To(BeNumerically("~", dur, 5*time.Second)) } }) @@ -476,9 +477,9 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) for _, client := range cluster.masters() { - keys, err := client.Keys("*").Result() + size, err := client.DBSize().Result() Expect(err).NotTo(HaveOccurred()) - Expect(keys).To(HaveLen(0)) + Expect(size).To(Equal(int64(0))) } }) @@ -551,6 +552,9 @@ var _ = Describe("ClusterClient", func() { }) _ = client.ForEachSlave(func(slave *redis.Client) error { + Eventually(func() int64 { + return client.DBSize().Val() + }, 30*time.Second).Should(Equal(int64(0))) return slave.ClusterFailover().Err() }) }) @@ -717,11 +721,12 @@ var _ = Describe("ClusterClient timeout", func() { }) AfterEach(func() { - Eventually(func() error { - return client.ForEachNode(func(client *redis.Client) error { + client.ForEachNode(func(client *redis.Client) error { + Eventually(func() error { return client.Ping().Err() - }) - }, 2*pause).ShouldNot(HaveOccurred()) + }, 2*pause).ShouldNot(HaveOccurred()) + return nil + }) }) testTimeout() diff --git a/commands.go b/commands.go index 15f67a13fa..83b3824f81 100644 --- a/commands.go +++ b/commands.go @@ -191,7 +191,7 @@ type Cmdable interface { ConfigGet(parameter string) *SliceCmd ConfigResetStat() *StatusCmd ConfigSet(parameter, value string) *StatusCmd - DbSize() *IntCmd + DBSize() *IntCmd FlushAll() *StatusCmd FlushAllAsync() *StatusCmd FlushDB() *StatusCmd @@ -1684,7 +1684,12 @@ func (c *cmdable) ConfigSet(parameter, value string) *StatusCmd { return cmd } +// Deperecated. Use DBSize instead. func (c *cmdable) DbSize() *IntCmd { + return c.DBSize() +} + +func (c *cmdable) DBSize() *IntCmd { cmd := NewIntCmd("dbsize") c.process(cmd) return cmd @@ -1704,9 +1709,7 @@ func (c *cmdable) FlushAllAsync() *StatusCmd { // Deprecated. Use FlushDB instead. func (c *cmdable) FlushDb() *StatusCmd { - cmd := NewStatusCmd("flushdb") - c.process(cmd) - return cmd + return c.FlushDB() } func (c *cmdable) FlushDB() *StatusCmd { diff --git a/commands_test.go b/commands_test.go index 81e0e4a82f..4298cba68f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -139,10 +139,10 @@ var _ = Describe("Commands", func() { Expect(configSet.Val()).To(Equal("OK")) }) - It("should DbSize", func() { - dbSize := client.DbSize() - Expect(dbSize.Err()).NotTo(HaveOccurred()) - Expect(dbSize.Val()).To(Equal(int64(0))) + It("should DBSize", func() { + size, err := client.DBSize().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(0))) }) It("should Info", func() { diff --git a/pubsub_test.go b/pubsub_test.go index d44b1dd8ce..1d9dfcb997 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -400,8 +400,11 @@ var _ = Describe("PubSub", func() { pubsub := client.Subscribe() defer pubsub.Close() + var wg sync.WaitGroup + wg.Add(1) go func() { defer GinkgoRecover() + defer wg.Done() time.Sleep(2 * timeout) @@ -418,5 +421,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) + + wg.Wait() }) }) From 0daeac9c3e31a1b121318398249699f48bb8d2a4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 21 Aug 2017 10:19:13 +0300 Subject: [PATCH 0368/1746] readme: mention connection pooling --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fd036496dc..32877e6857 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Supports: - Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC. +- Automatic connection pooling with [circuit breaker](https://en.wikipedia.org/wiki/Circuit_breaker_design_pattern) support. - [Pub/Sub](https://godoc.org/github.com/go-redis/redis#PubSub). - [Transactions](https://godoc.org/github.com/go-redis/redis#Multi). - [Pipeline](https://godoc.org/github.com/go-redis/redis#example-Client-Pipeline) and [TxPipeline](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline). From dbd2c99ba93f77bf79885f12f7f0724e27be8b00 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 31 Aug 2017 15:22:47 +0300 Subject: [PATCH 0369/1746] Rework pipeline retrying --- cluster.go | 325 ++++++++++++++++++++++++--------------- cluster_test.go | 71 ++++----- command.go | 14 +- commands_test.go | 53 ++++--- export_test.go | 14 +- internal/error.go | 25 ++- internal/proto/reader.go | 2 +- internal/proto/scan.go | 2 +- main_test.go | 3 +- pubsub.go | 5 +- redis.go | 58 +++---- ring.go | 55 +++++-- tx.go | 13 +- 13 files changed, 386 insertions(+), 254 deletions(-) diff --git a/cluster.go b/cluster.go index 647a25be39..72bace76aa 100644 --- a/cluster.go +++ b/cluster.go @@ -14,8 +14,8 @@ import ( "github.com/go-redis/redis/internal/proto" ) -var errClusterNoNodes = internal.RedisError("redis: cluster has no nodes") -var errNilClusterState = internal.RedisError("redis: cannot load cluster slots") +var errClusterNoNodes = fmt.Errorf("redis: cluster has no nodes") +var errNilClusterState = fmt.Errorf("redis: cannot load cluster slots") // ClusterOptions are used to configure a cluster client and should be // passed to NewClusterClient. @@ -64,6 +64,19 @@ func (opt *ClusterOptions) init() { opt.ReadOnly = true } + switch opt.ReadTimeout { + case -1: + opt.ReadTimeout = 0 + case 0: + opt.ReadTimeout = 3 * time.Second + } + switch opt.WriteTimeout { + case -1: + opt.WriteTimeout = 0 + case 0: + opt.WriteTimeout = opt.ReadTimeout + } + switch opt.MinRetryBackoff { case -1: opt.MinRetryBackoff = 0 @@ -192,6 +205,19 @@ func (c *clusterNodes) Close() error { return firstErr } +func (c *clusterNodes) Err() error { + c.mu.RLock() + defer c.mu.RUnlock() + + if c.closed { + return pool.ErrClosed + } + if len(c.addrs) == 0 { + return errClusterNoNodes + } + return nil +} + func (c *clusterNodes) NextGeneration() uint32 { c.generation++ return c.generation @@ -468,13 +494,22 @@ func (c *ClusterClient) Options() *ClusterOptions { return c.opt } -func (c *ClusterClient) state() *clusterState { +func (c *ClusterClient) retryBackoff(attempt int) time.Duration { + return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) +} + +func (c *ClusterClient) state() (*clusterState, error) { v := c._state.Load() if v != nil { - return v.(*clusterState) + return v.(*clusterState), nil + } + + if err := c.nodes.Err(); err != nil { + return nil, err } + c.lazyReloadState() - return nil + return nil, errNilClusterState } func (c *ClusterClient) cmdInfo(name string) *CommandInfo { @@ -495,15 +530,20 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo { if err != nil { return nil } - return c.cmdsInfo[name] + info := c.cmdsInfo[name] + if info == nil { + internal.Logf("info for cmd=%s not found", name) + } + return info } -func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { - if state == nil { - node, err := c.nodes.Random() - return 0, node, err - } +func (c *ClusterClient) cmdSlot(cmd Cmder) int { + cmdInfo := c.cmdInfo(cmd.Name()) + firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + return hashtag.Slot(firstKey) +} +func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmdInfo(cmd.Name()) firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) slot := hashtag.Slot(firstKey) @@ -523,19 +563,51 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl } func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { - state := c.state() + if len(keys) == 0 { + return fmt.Errorf("redis: keys don't hash to the same slot") + } - var node *clusterNode - var err error - if state != nil && len(keys) > 0 { - node, err = state.slotMasterNode(hashtag.Slot(keys[0])) - } else { - node, err = c.nodes.Random() + state, err := c.state() + if err != nil { + return err + } + + slot := hashtag.Slot(keys[0]) + for _, key := range keys[1:] { + if hashtag.Slot(key) != slot { + return fmt.Errorf("redis: Watch requires all keys to be in the same slot") + } } + + node, err := state.slotMasterNode(slot) if err != nil { return err } - return node.Client.Watch(fn, keys...) + + for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } + + err = node.Client.Watch(fn, keys...) + if err == nil { + break + } + + moved, ask, addr := internal.IsMovedError(err) + if moved || ask { + c.lazyReloadState() + node, err = c.nodes.GetOrCreate(addr) + if err != nil { + return err + } + continue + } + + return err + } + + return err } // Close closes the cluster client, releasing any open resources. @@ -547,7 +619,13 @@ func (c *ClusterClient) Close() error { } func (c *ClusterClient) Process(cmd Cmder) error { - slot, node, err := c.cmdSlotAndNode(c.state(), cmd) + state, err := c.state() + if err != nil { + cmd.setErr(err) + return err + } + + _, node, err := c.cmdSlotAndNode(state, cmd) if err != nil { cmd.setErr(err) return err @@ -556,7 +634,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { var ask bool for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { if attempt > 0 { - time.Sleep(node.Client.retryBackoff(attempt)) + time.Sleep(c.retryBackoff(attempt)) } if ask { @@ -572,7 +650,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { // If there is no error - we are done. if err == nil { - return nil + break } // If slave is loading - read from master. @@ -582,12 +660,11 @@ func (c *ClusterClient) Process(cmd Cmder) error { continue } - // On network errors try random node. - if internal.IsRetryableError(err) || internal.IsClusterDownError(err) { - node, err = c.nodes.Random() - if err != nil { - cmd.setErr(err) - return err + if internal.IsRetryableError(err) { + var nodeErr error + node, nodeErr = c.nodes.Random() + if nodeErr != nil { + break } continue } @@ -596,20 +673,13 @@ func (c *ClusterClient) Process(cmd Cmder) error { var addr string moved, ask, addr = internal.IsMovedError(err) if moved || ask { - state := c.state() - if state != nil && slot >= 0 { - master, _ := state.slotMasterNode(slot) - if moved && (master == nil || master.Client.getAddr() != addr) { - c.lazyReloadState() - } - } + c.lazyReloadState() - node, err = c.nodes.GetOrCreate(addr) - if err != nil { - cmd.setErr(err) - return err + var nodeErr error + node, nodeErr = c.nodes.GetOrCreate(addr) + if nodeErr != nil { + break } - continue } @@ -622,9 +692,9 @@ func (c *ClusterClient) Process(cmd Cmder) error { // ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { - state := c.state() - if state == nil { - return errNilClusterState + state, err := c.state() + if err != nil { + return err } var wg sync.WaitGroup @@ -655,9 +725,9 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { // ForEachSlave concurrently calls the fn on each slave node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { - state := c.state() - if state == nil { - return errNilClusterState + state, err := c.state() + if err != nil { + return err } var wg sync.WaitGroup @@ -688,9 +758,9 @@ func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { // ForEachNode concurrently calls the fn on each known node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { - state := c.state() - if state == nil { - return errNilClusterState + state, err := c.state() + if err != nil { + return err } var wg sync.WaitGroup @@ -728,7 +798,7 @@ func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { func (c *ClusterClient) PoolStats() *PoolStats { var acc PoolStats - state := c.state() + state, _ := c.state() if state == nil { return &acc } @@ -762,10 +832,8 @@ func (c *ClusterClient) lazyReloadState() { go func() { defer atomic.StoreUint32(&c.reloading, 0) - var state *clusterState for { - var err error - state, err = c.reloadState() + state, err := c.reloadState() if err == pool.ErrClosed { return } @@ -776,11 +844,10 @@ func (c *ClusterClient) lazyReloadState() { } c._state.Store(state) + time.Sleep(5 * time.Second) + c.nodes.GC(state.generation) break } - - time.Sleep(3 * time.Second) - c.nodes.GC(state.generation) }() } @@ -843,10 +910,15 @@ func (c *ClusterClient) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { func (c *ClusterClient) pipelineExec(cmds []Cmder) error { cmdsMap, err := c.mapCmdsByNode(cmds) if err != nil { + setCmdsErr(cmds, err) return err } - for i := 0; i <= c.opt.MaxRedirects; i++ { + for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } + failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { @@ -856,8 +928,12 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { continue } - err = c.pipelineProcessCmds(cn, cmds, failedCmds) - node.Client.releaseConn(cn, err) + err = c.pipelineProcessCmds(node, cn, cmds, failedCmds) + if err == nil || internal.IsRedisError(err) { + _ = node.Client.connPool.Put(cn) + } else { + _ = node.Client.connPool.Remove(cn) + } } if len(failedCmds) == 0 { @@ -866,21 +942,20 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { cmdsMap = failedCmds } - var firstErr error - for _, cmd := range cmds { - if err := cmd.Err(); err != nil { - firstErr = err - break - } - } - return firstErr + return firstCmdsErr(cmds) } func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, error) { - state := c.state() + state, err := c.state() + if err != nil { + setCmdsErr(cmds, err) + return nil, err + } + cmdsMap := make(map[*clusterNode][]Cmder) for _, cmd := range cmds { - _, node, err := c.cmdSlotAndNode(state, cmd) + slot := c.cmdSlot(cmd) + node, err := state.slotMasterNode(slot) if err != nil { return nil, err } @@ -890,11 +965,12 @@ func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, e } func (c *ClusterClient) pipelineProcessCmds( - cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, + node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) error { cn.SetWriteTimeout(c.opt.WriteTimeout) if err := writeCmd(cn, cmds...); err != nil { setCmdsErr(cmds, err) + failedCmds[node] = cmds return err } @@ -907,46 +983,53 @@ func (c *ClusterClient) pipelineProcessCmds( func (c *ClusterClient) pipelineReadCmds( cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) error { - var firstErr error for _, cmd := range cmds { err := cmd.readReply(cn) if err == nil { continue } - if firstErr == nil { - firstErr = err + if c.checkMovedErr(cmd, err, failedCmds) { + continue } - err = c.checkMovedErr(cmd, failedCmds) - if err != nil && firstErr == nil { - firstErr = err + if internal.IsRedisError(err) { + continue } + + return err } - return firstErr + return nil } -func (c *ClusterClient) checkMovedErr(cmd Cmder, failedCmds map[*clusterNode][]Cmder) error { - moved, ask, addr := internal.IsMovedError(cmd.Err()) +func (c *ClusterClient) checkMovedErr( + cmd Cmder, err error, failedCmds map[*clusterNode][]Cmder, +) bool { + moved, ask, addr := internal.IsMovedError(err) + if moved { c.lazyReloadState() node, err := c.nodes.GetOrCreate(addr) if err != nil { - return err + return false } failedCmds[node] = append(failedCmds[node], cmd) + return true } + if ask { node, err := c.nodes.GetOrCreate(addr) if err != nil { - return err + return false } failedCmds[node] = append(failedCmds[node], NewCmd("ASKING"), cmd) + return true } - return nil + + return false } // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. @@ -963,25 +1046,25 @@ func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { } func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { - cmdsMap, err := c.mapCmdsBySlot(cmds) + state, err := c.state() if err != nil { return err } - state := c.state() - if state == nil { - return errNilClusterState - } - + cmdsMap := c.mapCmdsBySlot(cmds) for slot, cmds := range cmdsMap { node, err := state.slotMasterNode(slot) if err != nil { setCmdsErr(cmds, err) continue } - cmdsMap := map[*clusterNode][]Cmder{node: cmds} - for i := 0; i <= c.opt.MaxRedirects; i++ { + + for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } + failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { @@ -992,7 +1075,11 @@ func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { } err = c.txPipelineProcessCmds(node, cn, cmds, failedCmds) - node.Client.releaseConn(cn, err) + if err == nil || internal.IsRedisError(err) { + _ = node.Client.connPool.Put(cn) + } else { + _ = node.Client.connPool.Remove(cn) + } } if len(failedCmds) == 0 { @@ -1002,27 +1089,16 @@ func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { } } - var firstErr error - for _, cmd := range cmds { - if err := cmd.Err(); err != nil { - firstErr = err - break - } - } - return firstErr + return firstCmdsErr(cmds) } -func (c *ClusterClient) mapCmdsBySlot(cmds []Cmder) (map[int][]Cmder, error) { - state := c.state() +func (c *ClusterClient) mapCmdsBySlot(cmds []Cmder) map[int][]Cmder { cmdsMap := make(map[int][]Cmder) for _, cmd := range cmds { - slot, _, err := c.cmdSlotAndNode(state, cmd) - if err != nil { - return nil, err - } + slot := c.cmdSlot(cmd) cmdsMap[slot] = append(cmdsMap[slot], cmd) } - return cmdsMap, nil + return cmdsMap } func (c *ClusterClient) txPipelineProcessCmds( @@ -1039,22 +1115,20 @@ func (c *ClusterClient) txPipelineProcessCmds( cn.SetReadTimeout(c.opt.ReadTimeout) if err := c.txPipelineReadQueued(cn, cmds, failedCmds); err != nil { + setCmdsErr(cmds, err) return err } - _, err := pipelineReadCmds(cn, cmds) - return err + return pipelineReadCmds(cn, cmds) } func (c *ClusterClient) txPipelineReadQueued( cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) error { - var firstErr error - // Parse queued replies. var statusCmd StatusCmd - if err := statusCmd.readReply(cn); err != nil && firstErr == nil { - firstErr = err + if err := statusCmd.readReply(cn); err != nil { + return err } for _, cmd := range cmds { @@ -1063,15 +1137,11 @@ func (c *ClusterClient) txPipelineReadQueued( continue } - cmd.setErr(err) - if firstErr == nil { - firstErr = err + if c.checkMovedErr(cmd, err, failedCmds) || internal.IsRedisError(err) { + continue } - err = c.checkMovedErr(cmd, failedCmds) - if err != nil && firstErr == nil { - firstErr = err - } + return err } // Parse number of replies. @@ -1085,7 +1155,13 @@ func (c *ClusterClient) txPipelineReadQueued( switch line[0] { case proto.ErrorReply: - return proto.ParseErrorReply(line) + err := proto.ParseErrorReply(line) + for _, cmd := range cmds { + if !c.checkMovedErr(cmd, err, failedCmds) { + break + } + } + return err case proto.ArrayReply: // ok default: @@ -1093,7 +1169,7 @@ func (c *ClusterClient) txPipelineReadQueued( return err } - return firstErr + return nil } func (c *ClusterClient) pubSub(channels []string) *PubSub { @@ -1112,7 +1188,12 @@ func (c *ClusterClient) pubSub(channels []string) *PubSub { slot = -1 } - masterNode, err := c.state().slotMasterNode(slot) + state, err := c.state() + if err != nil { + return nil, err + } + + masterNode, err := state.slotMasterNode(slot) if err != nil { return nil, err } diff --git a/cluster_test.go b/cluster_test.go index 324bd1ce1e..b2106da9fc 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -200,7 +200,7 @@ var _ = Describe("ClusterClient", func() { Eventually(func() string { return client.Get("A").Val() - }).Should(Equal("VALUE")) + }, 30*time.Second).Should(Equal("VALUE")) cnt, err := client.Del("A").Result() Expect(err).NotTo(HaveOccurred()) @@ -215,7 +215,7 @@ var _ = Describe("ClusterClient", func() { Eventually(func() string { return client.Get("A").Val() - }).Should(Equal("VALUE")) + }, 30*time.Second).Should(Equal("VALUE")) }) It("distributes keys", func() { @@ -227,7 +227,7 @@ var _ = Describe("ClusterClient", func() { for _, master := range cluster.masters() { Eventually(func() string { return master.Info("keyspace").Val() - }, 5*time.Second).Should(Or( + }, 30*time.Second).Should(Or( ContainSubstring("keys=31"), ContainSubstring("keys=29"), ContainSubstring("keys=40"), @@ -251,7 +251,7 @@ var _ = Describe("ClusterClient", func() { for _, master := range cluster.masters() { Eventually(func() string { return master.Info("keyspace").Val() - }, 5*time.Second).Should(Or( + }, 30*time.Second).Should(Or( ContainSubstring("keys=31"), ContainSubstring("keys=29"), ContainSubstring("keys=40"), @@ -320,10 +320,6 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(14)) - if opt.RouteByLatency { - return - } - for _, key := range keys { slot := hashtag.Slot(key) client.SwapSlotNodes(slot) @@ -432,6 +428,9 @@ var _ = Describe("ClusterClient", func() { }) AfterEach(func() { + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -560,6 +559,9 @@ var _ = Describe("ClusterClient", func() { }) AfterEach(func() { + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -575,10 +577,19 @@ var _ = Describe("ClusterClient", func() { _ = client.ForEachMaster(func(master *redis.Client) error { return master.FlushDB().Err() }) + + _ = client.ForEachSlave(func(slave *redis.Client) error { + Eventually(func() int64 { + return client.DBSize().Val() + }, 30*time.Second).Should(Equal(int64(0))) + return nil + }) }) AfterEach(func() { - client.FlushDB() + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -597,7 +608,7 @@ var _ = Describe("ClusterClient without nodes", func() { Expect(client.Close()).NotTo(HaveOccurred()) }) - It("returns an error", func() { + It("Ping returns an error", func() { err := client.Ping().Err() Expect(err).To(MatchError("redis: cluster has no nodes")) }) @@ -626,7 +637,7 @@ var _ = Describe("ClusterClient without valid nodes", func() { It("returns an error", func() { err := client.Ping().Err() - Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + Expect(err).To(MatchError("redis: cannot load cluster slots")) }) It("pipeline returns an error", func() { @@ -634,7 +645,7 @@ var _ = Describe("ClusterClient without valid nodes", func() { pipe.Ping() return nil }) - Expect(err).To(MatchError("ERR This instance has cluster support disabled")) + Expect(err).To(MatchError("redis: cannot load cluster slots")) }) }) @@ -664,7 +675,7 @@ var _ = Describe("ClusterClient timeout", func() { It("Tx timeouts", func() { err := client.Watch(func(tx *redis.Tx) error { return tx.Ping().Err() - }) + }, "foo") Expect(err).To(HaveOccurred()) Expect(err.(net.Error).Timeout()).To(BeTrue()) }) @@ -676,42 +687,20 @@ var _ = Describe("ClusterClient timeout", func() { return nil }) return err - }) + }, "foo") Expect(err).To(HaveOccurred()) Expect(err.(net.Error).Timeout()).To(BeTrue()) }) } - Context("read timeout", func() { - BeforeEach(func() { - opt := redisClusterOptions() - opt.ReadTimeout = time.Nanosecond - opt.WriteTimeout = -1 - client = cluster.clusterClient(opt) - }) - - testTimeout() - }) + const pause = time.Second - Context("write timeout", func() { + Context("read/write timeout", func() { BeforeEach(func() { opt := redisClusterOptions() - opt.ReadTimeout = time.Nanosecond - opt.WriteTimeout = -1 - client = cluster.clusterClient(opt) - }) - - testTimeout() - }) - - Context("ClientPause timeout", func() { - const pause = time.Second - - BeforeEach(func() { - opt := redisClusterOptions() - opt.ReadTimeout = pause / 10 - opt.WriteTimeout = pause / 10 - opt.MaxRedirects = -1 + opt.ReadTimeout = 100 * time.Millisecond + opt.WriteTimeout = 100 * time.Millisecond + opt.MaxRedirects = 1 client = cluster.clusterClient(opt) err := client.ForEachNode(func(client *redis.Client) error { diff --git a/command.go b/command.go index 0e5b2016e4..a796a93fcc 100644 --- a/command.go +++ b/command.go @@ -46,8 +46,19 @@ type Cmder interface { func setCmdsErr(cmds []Cmder, e error) { for _, cmd := range cmds { - cmd.setErr(e) + if cmd.Err() == nil { + cmd.setErr(e) + } + } +} + +func firstCmdsErr(cmds []Cmder) error { + for _, cmd := range cmds { + if err := cmd.Err(); err != nil { + return err + } } + return nil } func writeCmd(cn *pool.Conn, cmds ...Cmder) error { @@ -95,7 +106,6 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { return 1 } if info == nil { - internal.Logf("info for cmd=%s not found", cmd.Name()) return -1 } return int(info.FirstKeyPos) diff --git a/commands_test.go b/commands_test.go index 4298cba68f..0811d12a02 100644 --- a/commands_test.go +++ b/commands_test.go @@ -27,11 +27,21 @@ var _ = Describe("Commands", func() { Describe("server", func() { It("should Auth", func() { - _, err := client.Pipelined(func(pipe redis.Pipeliner) error { + cmds, err := client.Pipelined(func(pipe redis.Pipeliner) error { pipe.Auth("password") + pipe.Auth("") return nil }) Expect(err).To(MatchError("ERR Client sent AUTH, but no password is set")) + Expect(cmds[0].Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) + Expect(cmds[1].Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) + + stats := client.Pool().Stats() + Expect(stats.Requests).To(Equal(uint32(2))) + Expect(stats.Hits).To(Equal(uint32(1))) + Expect(stats.Timeouts).To(Equal(uint32(0))) + Expect(stats.TotalConns).To(Equal(uint32(1))) + Expect(stats.FreeConns).To(Equal(uint32(1))) }) It("should Echo", func() { @@ -187,6 +197,29 @@ var _ = Describe("Commands", func() { Expect(tm).To(BeTemporally("~", time.Now(), 3*time.Second)) }) + It("Should Command", func() { + cmds, err := client.Command().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(len(cmds)).To(BeNumerically("~", 180, 10)) + + cmd := cmds["mget"] + Expect(cmd.Name).To(Equal("mget")) + Expect(cmd.Arity).To(Equal(int8(-2))) + Expect(cmd.Flags).To(ContainElement("readonly")) + Expect(cmd.FirstKeyPos).To(Equal(int8(1))) + Expect(cmd.LastKeyPos).To(Equal(int8(-1))) + Expect(cmd.StepCount).To(Equal(int8(1))) + + cmd = cmds["ping"] + Expect(cmd.Name).To(Equal("ping")) + Expect(cmd.Arity).To(Equal(int8(-1))) + Expect(cmd.Flags).To(ContainElement("stale")) + Expect(cmd.Flags).To(ContainElement("fast")) + Expect(cmd.FirstKeyPos).To(Equal(int8(0))) + Expect(cmd.LastKeyPos).To(Equal(int8(0))) + Expect(cmd.StepCount).To(Equal(int8(0))) + }) + }) Describe("debugging", func() { @@ -2887,24 +2920,6 @@ var _ = Describe("Commands", func() { }) - Describe("Command", func() { - - It("returns map of commands", func() { - cmds, err := client.Command().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(len(cmds)).To(BeNumerically("~", 180, 10)) - - cmd := cmds["mget"] - Expect(cmd.Name).To(Equal("mget")) - Expect(cmd.Arity).To(Equal(int8(-2))) - Expect(cmd.Flags).To(ContainElement("readonly")) - Expect(cmd.FirstKeyPos).To(Equal(int8(1))) - Expect(cmd.LastKeyPos).To(Equal(int8(-1))) - Expect(cmd.StepCount).To(Equal(int8(1))) - }) - - }) - Describe("Eval", func() { It("returns keys and values", func() { diff --git a/export_test.go b/export_test.go index 3b7965d796..bcc18c4576 100644 --- a/export_test.go +++ b/export_test.go @@ -20,8 +20,13 @@ func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) } func (c *ClusterClient) SlotAddrs(slot int) []string { + state, err := c.state() + if err != nil { + panic(err) + } + var addrs []string - for _, n := range c.state().slotNodes(slot) { + for _, n := range state.slotNodes(slot) { addrs = append(addrs, n.Client.getAddr()) } return addrs @@ -29,7 +34,12 @@ func (c *ClusterClient) SlotAddrs(slot int) []string { // SwapSlot swaps a slot's master/slave address for testing MOVED redirects. func (c *ClusterClient) SwapSlotNodes(slot int) { - nodes := c.state().slots[slot] + state, err := c.state() + if err != nil { + panic(err) + } + + nodes := state.slots[slot] if len(nodes) == 2 { nodes[0], nodes[1] = nodes[1], nodes[0] } diff --git a/internal/error.go b/internal/error.go index 90f6503a15..e1b8be6b60 100644 --- a/internal/error.go +++ b/internal/error.go @@ -13,10 +13,23 @@ type RedisError string func (e RedisError) Error() string { return string(e) } func IsRetryableError(err error) bool { - return IsNetworkError(err) || err.Error() == "ERR max number of clients reached" + if IsNetworkError(err) { + return true + } + s := err.Error() + if s == "ERR max number of clients reached" { + return true + } + if strings.HasPrefix(s, "LOADING ") { + return true + } + if strings.HasPrefix(s, "CLUSTERDOWN ") { + return true + } + return false } -func IsInternalError(err error) bool { +func IsRedisError(err error) bool { _, ok := err.(RedisError) return ok } @@ -33,7 +46,7 @@ func IsBadConn(err error, allowTimeout bool) bool { if err == nil { return false } - if IsInternalError(err) { + if IsRedisError(err) { return false } if allowTimeout { @@ -45,7 +58,7 @@ func IsBadConn(err error, allowTimeout bool) bool { } func IsMovedError(err error) (moved bool, ask bool, addr string) { - if !IsInternalError(err) { + if !IsRedisError(err) { return } @@ -69,7 +82,3 @@ func IsMovedError(err error) (moved bool, ask bool, addr string) { func IsLoadingError(err error) bool { return strings.HasPrefix(err.Error(), "LOADING ") } - -func IsClusterDownError(err error) bool { - return strings.HasPrefix(err.Error(), "CLUSTERDOWN ") -} diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 2159cf639a..cd94329d8d 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -63,7 +63,7 @@ func (p *Reader) ReadLine() ([]byte, error) { return nil, bufio.ErrBufferFull } if len(line) == 0 { - return nil, internal.RedisError("redis: reply is empty") + return nil, fmt.Errorf("redis: reply is empty") } if isNilReply(line) { return nil, internal.Nil diff --git a/internal/proto/scan.go b/internal/proto/scan.go index 3ab40b94f2..0431a877d0 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -11,7 +11,7 @@ import ( func Scan(b []byte, v interface{}) error { switch v := v.(type) { case nil: - return internal.RedisError("redis: Scan(nil)") + return fmt.Errorf("redis: Scan(nil)") case *string: *v = internal.BytesToString(b) return nil diff --git a/main_test.go b/main_test.go index 30f09c618b..64f25d9932 100644 --- a/main_test.go +++ b/main_test.go @@ -3,6 +3,7 @@ package redis_test import ( "errors" "fmt" + "log" "net" "os" "os/exec" @@ -51,7 +52,7 @@ var cluster = &clusterScenario{ } func init() { - //redis.SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) + redis.SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) } var _ = BeforeSuite(func() { diff --git a/pubsub.go b/pubsub.go index 4a5c65f571..e754a16f23 100644 --- a/pubsub.go +++ b/pubsub.go @@ -95,7 +95,10 @@ func (c *PubSub) releaseConn(cn *pool.Conn, err error) { } func (c *PubSub) _releaseConn(cn *pool.Conn, err error) { - if internal.IsBadConn(err, true) && c.cn == cn { + if c.cn != cn { + return + } + if internal.IsBadConn(err, true) { _ = c.closeTheCn() } } diff --git a/redis.go b/redis.go index b18973cdb2..db1f39c3a6 100644 --- a/redis.go +++ b/redis.go @@ -197,8 +197,11 @@ type pipelineProcessor func(*pool.Conn, []Cmder) (bool, error) func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { return func(cmds []Cmder) error { - var firstErr error - for i := 0; i <= c.opt.MaxRetries; i++ { + for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } + cn, _, err := c.getConn() if err != nil { setCmdsErr(cmds, err) @@ -206,18 +209,18 @@ func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { } canRetry, err := p(cn, cmds) - c.releaseConn(cn, err) - if err == nil { - return nil - } - if firstErr == nil { - firstErr = err + + if err == nil || internal.IsRedisError(err) { + _ = c.connPool.Put(cn) + break } + _ = c.connPool.Remove(cn) + if !canRetry || !internal.IsRetryableError(err) { break } } - return firstErr + return firstCmdsErr(cmds) } } @@ -230,23 +233,17 @@ func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, err // Set read timeout for all commands. cn.SetReadTimeout(c.opt.ReadTimeout) - return pipelineReadCmds(cn, cmds) + return true, pipelineReadCmds(cn, cmds) } -func pipelineReadCmds(cn *pool.Conn, cmds []Cmder) (retry bool, firstErr error) { - for i, cmd := range cmds { +func pipelineReadCmds(cn *pool.Conn, cmds []Cmder) error { + for _, cmd := range cmds { err := cmd.readReply(cn) - if err == nil { - continue - } - if i == 0 { - retry = true - } - if firstErr == nil { - firstErr = err + if err != nil && !internal.IsRedisError(err) { + return err } } - return + return nil } func (c *baseClient) txPipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) { @@ -260,11 +257,11 @@ func (c *baseClient) txPipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, e cn.SetReadTimeout(c.opt.ReadTimeout) if err := c.txPipelineReadQueued(cn, cmds); err != nil { + setCmdsErr(cmds, err) return false, err } - _, err := pipelineReadCmds(cn, cmds) - return false, err + return false, pipelineReadCmds(cn, cmds) } func txPipelineWriteMulti(cn *pool.Conn, cmds []Cmder) error { @@ -276,21 +273,16 @@ func txPipelineWriteMulti(cn *pool.Conn, cmds []Cmder) error { } func (c *baseClient) txPipelineReadQueued(cn *pool.Conn, cmds []Cmder) error { - var firstErr error - // Parse queued replies. var statusCmd StatusCmd - if err := statusCmd.readReply(cn); err != nil && firstErr == nil { - firstErr = err + if err := statusCmd.readReply(cn); err != nil { + return err } - for _, cmd := range cmds { + for _ = range cmds { err := statusCmd.readReply(cn) - if err != nil { - cmd.setErr(err) - if firstErr == nil { - firstErr = err - } + if err != nil && !internal.IsRedisError(err) { + return err } } diff --git a/ring.go b/ring.go index 72d52bf755..a9314fb5b8 100644 --- a/ring.go +++ b/ring.go @@ -34,7 +34,9 @@ type RingOptions struct { DB int Password string - MaxRetries int + MaxRetries int + MinRetryBackoff time.Duration + MaxRetryBackoff time.Duration DialTimeout time.Duration ReadTimeout time.Duration @@ -50,6 +52,19 @@ func (opt *RingOptions) init() { if opt.HeartbeatFrequency == 0 { opt.HeartbeatFrequency = 500 * time.Millisecond } + + switch opt.MinRetryBackoff { + case -1: + opt.MinRetryBackoff = 0 + case 0: + opt.MinRetryBackoff = 8 * time.Millisecond + } + switch opt.MaxRetryBackoff { + case -1: + opt.MaxRetryBackoff = 0 + case 0: + opt.MaxRetryBackoff = 512 * time.Millisecond + } } func (opt *RingOptions) clientOptions() *Options { @@ -165,6 +180,10 @@ func (c *Ring) Options() *RingOptions { return c.opt } +func (c *Ring) retryBackoff(attempt int) time.Duration { + return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) +} + // PoolStats returns accumulated connection pool stats. func (c *Ring) PoolStats() *PoolStats { var acc PoolStats @@ -241,6 +260,9 @@ func (c *Ring) ForEachShard(fn func(client *Client) error) error { func (c *Ring) cmdInfo(name string) *CommandInfo { err := c.cmdsInfoOnce.Do(func() error { + c.mu.RLock() + defer c.mu.RUnlock() + var firstErr error for _, shard := range c.shards { cmdsInfo, err := shard.Client.Command().Result() @@ -257,7 +279,11 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { if err != nil { return nil } - return c.cmdsInfo[name] + info := c.cmdsInfo[name] + if info == nil { + internal.Logf("info for cmd=%s not found", name) + } + return info } func (c *Ring) addClient(name string, cl *Client) { @@ -399,7 +425,7 @@ func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().pipelined(fn) } -func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { +func (c *Ring) pipelineExec(cmds []Cmder) error { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { cmdInfo := c.cmdInfo(cmd.Name()) @@ -410,36 +436,33 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { cmdsMap[name] = append(cmdsMap[name], cmd) } - for i := 0; i <= c.opt.MaxRetries; i++ { + for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } + var failedCmdsMap map[string][]Cmder for name, cmds := range cmdsMap { shard, err := c.shardByName(name) if err != nil { setCmdsErr(cmds, err) - if firstErr == nil { - firstErr = err - } continue } cn, _, err := shard.Client.getConn() if err != nil { setCmdsErr(cmds, err) - if firstErr == nil { - firstErr = err - } continue } canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds) - shard.Client.releaseConn(cn, err) - if err == nil { + if err == nil || internal.IsRedisError(err) { + _ = shard.Client.connPool.Put(cn) continue } - if firstErr == nil { - firstErr = err - } + _ = shard.Client.connPool.Remove(cn) + if canRetry && internal.IsRetryableError(err) { if failedCmdsMap == nil { failedCmdsMap = make(map[string][]Cmder) @@ -454,5 +477,5 @@ func (c *Ring) pipelineExec(cmds []Cmder) (firstErr error) { cmdsMap = failedCmdsMap } - return firstErr + return firstCmdsErr(cmds) } diff --git a/tx.go b/tx.go index 5ef89619ba..93c295d98f 100644 --- a/tx.go +++ b/tx.go @@ -36,11 +36,10 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return err } } - firstErr := fn(tx) - if err := tx.Close(); err != nil && firstErr == nil { - firstErr = err - } - return firstErr + + err := fn(tx) + _ = tx.Close() + return err } // close closes the transaction, releasing any open resources. @@ -53,7 +52,7 @@ func (c *Tx) Close() error { // of a transaction. func (c *Tx) Watch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "WATCH" + args[0] = "watch" for i, key := range keys { args[1+i] = key } @@ -65,7 +64,7 @@ func (c *Tx) Watch(keys ...string) *StatusCmd { // Unwatch flushes all the previously watched keys for a transaction. func (c *Tx) Unwatch(keys ...string) *StatusCmd { args := make([]interface{}, 1+len(keys)) - args[0] = "UNWATCH" + args[0] = "unwatch" for i, key := range keys { args[1+i] = key } From ddbd81ea6c66514fe1f857af890276fcac7921ce Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 3 Sep 2017 10:31:40 +0300 Subject: [PATCH 0370/1746] readme: fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32877e6857..0a2a67124c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Supports: - [Ring](https://godoc.org/github.com/go-redis/redis#NewRing). - [Instrumentation](https://godoc.org/github.com/go-redis/redis#ex-package--Instrumentation). - [Cache friendly](https://github.com/go-redis/cache). -- [Rate limiting](https://github.com/go-redis/rate). +- [Rate limiting](https://github.com/go-redis/redis_rate). - [Distributed Locks](https://github.com/bsm/redis-lock). API docs: https://godoc.org/github.com/go-redis/redis. From 1173a9589f1e25b15a43e155eb5443aa213453fb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 11 Sep 2017 08:58:56 +0300 Subject: [PATCH 0371/1746] Cleanup code --- cluster.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/cluster.go b/cluster.go index 72bace76aa..487faa9f99 100644 --- a/cluster.go +++ b/cluster.go @@ -205,17 +205,19 @@ func (c *clusterNodes) Close() error { return firstErr } -func (c *clusterNodes) Err() error { +func (c *clusterNodes) Addrs() ([]string, error) { c.mu.RLock() - defer c.mu.RUnlock() + closed := c.closed + addrs := c.addrs + c.mu.RUnlock() - if c.closed { - return pool.ErrClosed + if closed { + return nil, pool.ErrClosed } - if len(c.addrs) == 0 { - return errClusterNoNodes + if len(addrs) == 0 { + return nil, errClusterNoNodes } - return nil + return addrs, nil } func (c *clusterNodes) NextGeneration() uint32 { @@ -298,16 +300,9 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { } func (c *clusterNodes) Random() (*clusterNode, error) { - c.mu.RLock() - closed := c.closed - addrs := c.addrs - c.mu.RUnlock() - - if closed { - return nil, pool.ErrClosed - } - if len(addrs) == 0 { - return nil, errClusterNoNodes + addrs, err := c.Addrs() + if err != nil { + return nil, err } var nodeErr error @@ -504,7 +499,8 @@ func (c *ClusterClient) state() (*clusterState, error) { return v.(*clusterState), nil } - if err := c.nodes.Err(); err != nil { + _, err := c.nodes.Addrs() + if err != nil { return nil, err } From 5294b5dae18eff1e93fa83164d4f9dcf6f3aebe7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 11 Sep 2017 09:10:17 +0300 Subject: [PATCH 0372/1746] Add PoolStats.StaleConns and enable logging by default --- cluster.go | 15 +++++---------- internal/pool/pool.go | 19 +++++++++---------- main_test.go | 5 ----- options.go | 5 +++-- redis.go | 5 +++++ sentinel.go | 6 ++++-- 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/cluster.go b/cluster.go index 487faa9f99..6f435251a3 100644 --- a/cluster.go +++ b/cluster.go @@ -804,8 +804,10 @@ func (c *ClusterClient) PoolStats() *PoolStats { acc.Requests += s.Requests acc.Hits += s.Hits acc.Timeouts += s.Timeouts + acc.TotalConns += s.TotalConns acc.FreeConns += s.FreeConns + acc.StaleConns += s.StaleConns } for _, node := range state.slaves { @@ -813,8 +815,10 @@ func (c *ClusterClient) PoolStats() *PoolStats { acc.Requests += s.Requests acc.Hits += s.Hits acc.Timeouts += s.Timeouts + acc.TotalConns += s.TotalConns acc.FreeConns += s.FreeConns + acc.StaleConns += s.StaleConns } return &acc @@ -873,21 +877,12 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { break } - var n int for _, node := range nodes { - nn, err := node.Client.connPool.(*pool.ConnPool).ReapStaleConns() + _, err := node.Client.connPool.(*pool.ConnPool).ReapStaleConns() if err != nil { internal.Logf("ReapStaleConns failed: %s", err) - } else { - n += nn } } - - s := c.PoolStats() - internal.Logf( - "reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)", - n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts, - ) } } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 25e78aa3ce..26a891bf3b 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -27,8 +27,9 @@ type Stats struct { Hits uint32 // number of times free connection was found in the pool Timeouts uint32 // number of times a wait timeout occurred - TotalConns uint32 // the number of total connections in the pool - FreeConns uint32 // the number of free connections in the pool + TotalConns uint32 // number of total connections in the pool + FreeConns uint32 // number of free connections in the pool + StaleConns uint32 // number of stale connections removed from the pool } type Pooler interface { @@ -265,11 +266,13 @@ func (p *ConnPool) FreeLen() int { func (p *ConnPool) Stats() *Stats { return &Stats{ - Requests: atomic.LoadUint32(&p.stats.Requests), - Hits: atomic.LoadUint32(&p.stats.Hits), - Timeouts: atomic.LoadUint32(&p.stats.Timeouts), + Requests: atomic.LoadUint32(&p.stats.Requests), + Hits: atomic.LoadUint32(&p.stats.Hits), + Timeouts: atomic.LoadUint32(&p.stats.Timeouts), + TotalConns: uint32(p.Len()), FreeConns: uint32(p.FreeLen()), + StaleConns: atomic.LoadUint32(&p.stats.StaleConns), } } @@ -362,10 +365,6 @@ func (p *ConnPool) reaper(frequency time.Duration) { internal.Logf("ReapStaleConns failed: %s", err) continue } - s := p.Stats() - internal.Logf( - "reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)", - n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts, - ) + atomic.AddUint32(&p.stats.StaleConns, uint32(n)) } } diff --git a/main_test.go b/main_test.go index 64f25d9932..7c5a6a969f 100644 --- a/main_test.go +++ b/main_test.go @@ -3,7 +3,6 @@ package redis_test import ( "errors" "fmt" - "log" "net" "os" "os/exec" @@ -51,10 +50,6 @@ var cluster = &clusterScenario{ clients: make(map[string]*redis.Client, 6), } -func init() { - redis.SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) -} - var _ = BeforeSuite(func() { var err error diff --git a/options.go b/options.go index dea0454531..6c4e68d2e5 100644 --- a/options.go +++ b/options.go @@ -205,6 +205,7 @@ type PoolStats struct { Hits uint32 // number of times free connection was found in the pool Timeouts uint32 // number of times a wait timeout occurred - TotalConns uint32 // the number of total connections in the pool - FreeConns uint32 // the number of free connections in the pool + TotalConns uint32 // number of total connections in the pool + FreeConns uint32 // number of free connections in the pool + StaleConns uint32 // number of stale connections removed from the pool } diff --git a/redis.go b/redis.go index db1f39c3a6..d18a152e67 100644 --- a/redis.go +++ b/redis.go @@ -3,6 +3,7 @@ package redis import ( "fmt" "log" + "os" "time" "github.com/go-redis/redis/internal" @@ -13,6 +14,10 @@ import ( // Redis nil reply, .e.g. when key does not exist. const Nil = internal.Nil +func init() { + SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) +} + func SetLogger(logger *log.Logger) { internal.Logger = logger } diff --git a/sentinel.go b/sentinel.go index 3bfdb4a3f1..37d06b4821 100644 --- a/sentinel.go +++ b/sentinel.go @@ -301,8 +301,10 @@ func (d *sentinelFailover) listen(sentinel *sentinelClient) { msg, err := pubsub.ReceiveMessage() if err != nil { - internal.Logf("sentinel: ReceiveMessage failed: %s", err) - pubsub.Close() + if err != pool.ErrClosed { + internal.Logf("sentinel: ReceiveMessage failed: %s", err) + pubsub.Close() + } d.resetSentinel() return } From 0a7606651d026c521a6b51b2e947cbf58535a183 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 11 Sep 2017 09:19:30 +0300 Subject: [PATCH 0373/1746] Add Cluster.DBSize --- cluster_commands.go | 22 ++++++++++++++++++++++ cluster_test.go | 8 +++----- 2 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 cluster_commands.go diff --git a/cluster_commands.go b/cluster_commands.go new file mode 100644 index 0000000000..dff62c902d --- /dev/null +++ b/cluster_commands.go @@ -0,0 +1,22 @@ +package redis + +import "sync/atomic" + +func (c *ClusterClient) DBSize() *IntCmd { + cmd := NewIntCmd("dbsize") + var size int64 + err := c.ForEachMaster(func(master *Client) error { + n, err := master.DBSize().Result() + if err != nil { + return err + } + atomic.AddInt64(&size, n) + return nil + }) + if err != nil { + cmd.setErr(err) + return cmd + } + cmd.val = size + return cmd +} diff --git a/cluster_test.go b/cluster_test.go index b2106da9fc..6f3677b938 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -475,11 +475,9 @@ var _ = Describe("ClusterClient", func() { }) Expect(err).NotTo(HaveOccurred()) - for _, client := range cluster.masters() { - size, err := client.DBSize().Result() - Expect(err).NotTo(HaveOccurred()) - Expect(size).To(Equal(int64(0))) - } + size, err := client.DBSize().Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(0))) }) It("should CLUSTER SLOTS", func() { From 09176ef4fa3eae87ed310d02c51d9c6829347f93 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 11 Sep 2017 10:12:00 +0300 Subject: [PATCH 0374/1746] PoolStats is an alias for pool.Stats --- options.go | 11 ----------- pool_test.go | 2 ++ redis.go | 13 ++++--------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/options.go b/options.go index 6c4e68d2e5..75648053de 100644 --- a/options.go +++ b/options.go @@ -198,14 +198,3 @@ func newConnPool(opt *Options) *pool.ConnPool { IdleCheckFrequency: opt.IdleCheckFrequency, }) } - -// PoolStats contains pool state information and accumulated stats. -type PoolStats struct { - Requests uint32 // number of times a connection was requested by the pool - Hits uint32 // number of times free connection was found in the pool - Timeouts uint32 // number of times a wait timeout occurred - - TotalConns uint32 // number of total connections in the pool - FreeConns uint32 // number of free connections in the pool - StaleConns uint32 // number of stale connections removed from the pool -} diff --git a/pool_test.go b/pool_test.go index 34a548a639..2cc1cdfb7e 100644 --- a/pool_test.go +++ b/pool_test.go @@ -125,6 +125,7 @@ var _ = Describe("pool", func() { Timeouts: 0, TotalConns: 1, FreeConns: 1, + StaleConns: 0, })) time.Sleep(2 * time.Second) @@ -136,6 +137,7 @@ var _ = Describe("pool", func() { Timeouts: 0, TotalConns: 0, FreeConns: 0, + StaleConns: 1, })) }) }) diff --git a/redis.go b/redis.go index d18a152e67..70a17530ce 100644 --- a/redis.go +++ b/redis.go @@ -352,17 +352,12 @@ func (c *Client) Options() *Options { return c.opt } +type PoolStats pool.Stats + // PoolStats returns connection pool stats. func (c *Client) PoolStats() *PoolStats { - s := c.connPool.Stats() - return &PoolStats{ - Requests: s.Requests, - Hits: s.Hits, - Timeouts: s.Timeouts, - - TotalConns: s.TotalConns, - FreeConns: s.FreeConns, - } + stats := c.connPool.Stats() + return (*PoolStats)(stats) } func (c *Client) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { From 5c93788928f46e9881c1d0a41d377c1614022984 Mon Sep 17 00:00:00 2001 From: Davor Kapsa Date: Tue, 19 Sep 2017 16:23:54 +0200 Subject: [PATCH 0375/1746] travis: update go version --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f4666c5930..f49927ee84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ go: - 1.4.x - 1.7.x - 1.8.x + - 1.9.x - tip matrix: From bc5f9a68785e14425561c26a1383f7401ee31bf0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 22 Sep 2017 12:23:46 +0300 Subject: [PATCH 0376/1746] Replace PoolStats.Requests with PoolStats.Misses --- cluster.go | 4 ++-- commands_test.go | 6 ++--- internal/pool/pool.go | 8 +++---- pool_test.go | 8 +++---- pubsub_test.go | 6 ++--- ring.go | 52 +++++++++++++++++++++++++++---------------- 6 files changed, 49 insertions(+), 35 deletions(-) diff --git a/cluster.go b/cluster.go index 6f435251a3..019a2d445f 100644 --- a/cluster.go +++ b/cluster.go @@ -801,8 +801,8 @@ func (c *ClusterClient) PoolStats() *PoolStats { for _, node := range state.masters { s := node.Client.connPool.Stats() - acc.Requests += s.Requests acc.Hits += s.Hits + acc.Misses += s.Misses acc.Timeouts += s.Timeouts acc.TotalConns += s.TotalConns @@ -812,8 +812,8 @@ func (c *ClusterClient) PoolStats() *PoolStats { for _, node := range state.slaves { s := node.Client.connPool.Stats() - acc.Requests += s.Requests acc.Hits += s.Hits + acc.Misses += s.Misses acc.Timeouts += s.Timeouts acc.TotalConns += s.TotalConns diff --git a/commands_test.go b/commands_test.go index 0811d12a02..6b81f23cf7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -36,9 +36,9 @@ var _ = Describe("Commands", func() { Expect(cmds[0].Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) Expect(cmds[1].Err()).To(MatchError("ERR Client sent AUTH, but no password is set")) - stats := client.Pool().Stats() - Expect(stats.Requests).To(Equal(uint32(2))) + stats := client.PoolStats() Expect(stats.Hits).To(Equal(uint32(1))) + Expect(stats.Misses).To(Equal(uint32(1))) Expect(stats.Timeouts).To(Equal(uint32(0))) Expect(stats.TotalConns).To(Equal(uint32(1))) Expect(stats.FreeConns).To(Equal(uint32(1))) @@ -1391,8 +1391,8 @@ var _ = Describe("Commands", func() { Expect(client.Ping().Err()).NotTo(HaveOccurred()) stats := client.PoolStats() - Expect(stats.Requests).To(Equal(uint32(3))) Expect(stats.Hits).To(Equal(uint32(1))) + Expect(stats.Misses).To(Equal(uint32(2))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 26a891bf3b..836ec1045e 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -23,8 +23,8 @@ var timers = sync.Pool{ // Stats contains pool state information and accumulated stats. type Stats struct { - Requests uint32 // number of times a connection was requested by the pool Hits uint32 // number of times free connection was found in the pool + Misses uint32 // number of times free connection was NOT found in the pool Timeouts uint32 // number of times a wait timeout occurred TotalConns uint32 // number of total connections in the pool @@ -151,8 +151,6 @@ func (p *ConnPool) Get() (*Conn, bool, error) { return nil, false, ErrClosed } - atomic.AddUint32(&p.stats.Requests, 1) - select { case p.queue <- struct{}{}: default: @@ -190,6 +188,8 @@ func (p *ConnPool) Get() (*Conn, bool, error) { return cn, false, nil } + atomic.AddUint32(&p.stats.Misses, 1) + newcn, err := p.NewConn() if err != nil { <-p.queue @@ -266,8 +266,8 @@ func (p *ConnPool) FreeLen() int { func (p *ConnPool) Stats() *Stats { return &Stats{ - Requests: atomic.LoadUint32(&p.stats.Requests), Hits: atomic.LoadUint32(&p.stats.Hits), + Misses: atomic.LoadUint32(&p.stats.Misses), Timeouts: atomic.LoadUint32(&p.stats.Timeouts), TotalConns: uint32(p.Len()), diff --git a/pool_test.go b/pool_test.go index 2cc1cdfb7e..0ca09adc7c 100644 --- a/pool_test.go +++ b/pool_test.go @@ -95,8 +95,8 @@ var _ = Describe("pool", func() { Expect(pool.FreeLen()).To(Equal(1)) stats := pool.Stats() - Expect(stats.Requests).To(Equal(uint32(4))) Expect(stats.Hits).To(Equal(uint32(2))) + Expect(stats.Misses).To(Equal(uint32(2))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) @@ -112,16 +112,16 @@ var _ = Describe("pool", func() { Expect(pool.FreeLen()).To(Equal(1)) stats := pool.Stats() - Expect(stats.Requests).To(Equal(uint32(101))) Expect(stats.Hits).To(Equal(uint32(100))) + Expect(stats.Misses).To(Equal(uint32(1))) Expect(stats.Timeouts).To(Equal(uint32(0))) }) It("removes idle connections", func() { stats := client.PoolStats() Expect(stats).To(Equal(&redis.PoolStats{ - Requests: 1, Hits: 0, + Misses: 1, Timeouts: 0, TotalConns: 1, FreeConns: 1, @@ -132,8 +132,8 @@ var _ = Describe("pool", func() { stats = client.PoolStats() Expect(stats).To(Equal(&redis.PoolStats{ - Requests: 1, Hits: 0, + Misses: 1, Timeouts: 0, TotalConns: 0, FreeConns: 0, diff --git a/pubsub_test.go b/pubsub_test.go index 1d9dfcb997..6fc04a1989 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -68,7 +68,7 @@ var _ = Describe("PubSub", func() { } stats := client.PoolStats() - Expect(stats.Requests - stats.Hits).To(Equal(uint32(2))) + Expect(stats.Misses).To(Equal(uint32(2))) }) It("should pub/sub channels", func() { @@ -191,7 +191,7 @@ var _ = Describe("PubSub", func() { } stats := client.PoolStats() - Expect(stats.Requests - stats.Hits).To(Equal(uint32(2))) + Expect(stats.Misses).To(Equal(uint32(2))) }) It("should ping/pong", func() { @@ -290,8 +290,8 @@ var _ = Describe("PubSub", func() { Eventually(done).Should(Receive()) stats := client.PoolStats() - Expect(stats.Requests).To(Equal(uint32(2))) Expect(stats.Hits).To(Equal(uint32(1))) + Expect(stats.Misses).To(Equal(uint32(1))) }) It("returns an error when subscribe fails", func() { diff --git a/ring.go b/ring.go index a9314fb5b8..7b3d2d14de 100644 --- a/ring.go +++ b/ring.go @@ -145,9 +145,10 @@ type Ring struct { opt *RingOptions nreplicas int - mu sync.RWMutex - hash *consistenthash.Map - shards map[string]*ringShard + mu sync.RWMutex + hash *consistenthash.Map + shards map[string]*ringShard + shardsList []*ringShard cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo @@ -169,12 +170,21 @@ func NewRing(opt *RingOptions) *Ring { for name, addr := range opt.Addrs { clopt := opt.clientOptions() clopt.Addr = addr - ring.addClient(name, NewClient(clopt)) + ring.addShard(name, NewClient(clopt)) } go ring.heartbeat() return ring } +func (c *Ring) addShard(name string, cl *Client) { + shard := &ringShard{Client: cl} + c.mu.Lock() + c.hash.Add(name) + c.shards[name] = shard + c.shardsList = append(c.shardsList, shard) + c.mu.Unlock() +} + // Options returns read-only Options that were used to create the client. func (c *Ring) Options() *RingOptions { return c.opt @@ -186,11 +196,15 @@ func (c *Ring) retryBackoff(attempt int) time.Duration { // PoolStats returns accumulated connection pool stats. func (c *Ring) PoolStats() *PoolStats { + c.mu.RLock() + shards := c.shardsList + c.mu.RUnlock() + var acc PoolStats - for _, shard := range c.shards { + for _, shard := range shards { s := shard.Client.connPool.Stats() - acc.Requests += s.Requests acc.Hits += s.Hits + acc.Misses += s.Misses acc.Timeouts += s.Timeouts acc.TotalConns += s.TotalConns acc.FreeConns += s.FreeConns @@ -229,9 +243,13 @@ func (c *Ring) PSubscribe(channels ...string) *PubSub { // ForEachShard concurrently calls the fn on each live shard in the ring. // It returns the first error if any. func (c *Ring) ForEachShard(fn func(client *Client) error) error { + c.mu.RLock() + shards := c.shardsList + c.mu.RUnlock() + var wg sync.WaitGroup errCh := make(chan error, 1) - for _, shard := range c.shards { + for _, shard := range shards { if shard.IsDown() { continue } @@ -261,10 +279,11 @@ func (c *Ring) ForEachShard(fn func(client *Client) error) error { func (c *Ring) cmdInfo(name string) *CommandInfo { err := c.cmdsInfoOnce.Do(func() error { c.mu.RLock() - defer c.mu.RUnlock() + shards := c.shardsList + c.mu.RUnlock() var firstErr error - for _, shard := range c.shards { + for _, shard := range shards { cmdsInfo, err := shard.Client.Command().Result() if err == nil { c.cmdsInfo = cmdsInfo @@ -286,13 +305,6 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { return info } -func (c *Ring) addClient(name string, cl *Client) { - c.mu.Lock() - c.hash.Add(name) - c.shards[name] = &ringShard{Client: cl} - c.mu.Unlock() -} - func (c *Ring) shardByKey(key string) (*ringShard, error) { key = hashtag.Key(key) @@ -372,7 +384,10 @@ func (c *Ring) heartbeat() { break } - for _, shard := range c.shards { + shards := c.shardsList + c.mu.RUnlock() + + for _, shard := range shards { err := shard.Client.Ping().Err() if shard.Vote(err == nil || err == pool.ErrPoolTimeout) { internal.Logf("ring shard state changed: %s", shard) @@ -380,8 +395,6 @@ func (c *Ring) heartbeat() { } } - c.mu.RUnlock() - if rebalance { c.rebalance() } @@ -409,6 +422,7 @@ func (c *Ring) Close() error { } c.hash = nil c.shards = nil + c.shardsList = nil return firstErr } From f9307ab2fe5742bb714e98188cb4fd04c1651d7d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 25 Sep 2017 11:48:44 +0300 Subject: [PATCH 0377/1746] Add TxPipeline to Cmdable --- cluster.go | 4 ++-- commands.go | 3 +++ pipeline.go | 10 ++++++++-- redis.go | 8 ++++---- ring.go | 10 +++++++++- tx.go | 10 +++++++++- universal.go | 7 +++++-- 7 files changed, 40 insertions(+), 12 deletions(-) diff --git a/cluster.go b/cluster.go index 019a2d445f..c83da6a45a 100644 --- a/cluster.go +++ b/cluster.go @@ -895,7 +895,7 @@ func (c *ClusterClient) Pipeline() Pipeliner { } func (c *ClusterClient) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) + return c.Pipeline().Pipelined(fn) } func (c *ClusterClient) pipelineExec(cmds []Cmder) error { @@ -1033,7 +1033,7 @@ func (c *ClusterClient) TxPipeline() Pipeliner { } func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.TxPipeline().pipelined(fn) + return c.TxPipeline().Pipelined(fn) } func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { diff --git a/commands.go b/commands.go index 83b3824f81..7f0000f3ed 100644 --- a/commands.go +++ b/commands.go @@ -42,6 +42,9 @@ type Cmdable interface { Pipeline() Pipeliner Pipelined(fn func(Pipeliner) error) ([]Cmder, error) + TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) + TxPipeline() Pipeliner + ClientGetName() *StringCmd Echo(message interface{}) *StringCmd Ping() *StatusCmd diff --git a/pipeline.go b/pipeline.go index b66c0597f5..9349ef553e 100644 --- a/pipeline.go +++ b/pipeline.go @@ -13,9 +13,7 @@ type Pipeliner interface { Process(cmd Cmder) error Close() error Discard() error - discard() error Exec() ([]Cmder, error) - pipelined(fn func(Pipeliner) error) ([]Cmder, error) } var _ Pipeliner = (*Pipeline)(nil) @@ -104,3 +102,11 @@ func (c *Pipeline) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { func (c *Pipeline) Pipeline() Pipeliner { return c } + +func (c *Pipeline) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { + return c.pipelined(fn) +} + +func (c *Pipeline) TxPipeline() Pipeliner { + return c +} diff --git a/redis.go b/redis.go index 70a17530ce..615bf28d81 100644 --- a/redis.go +++ b/redis.go @@ -361,7 +361,7 @@ func (c *Client) PoolStats() *PoolStats { } func (c *Client) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) + return c.Pipeline().Pipelined(fn) } func (c *Client) Pipeline() Pipeliner { @@ -373,7 +373,7 @@ func (c *Client) Pipeline() Pipeliner { } func (c *Client) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.TxPipeline().pipelined(fn) + return c.TxPipeline().Pipelined(fn) } // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. @@ -425,7 +425,7 @@ type Conn struct { } func (c *Conn) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) + return c.Pipeline().Pipelined(fn) } func (c *Conn) Pipeline() Pipeliner { @@ -437,7 +437,7 @@ func (c *Conn) Pipeline() Pipeliner { } func (c *Conn) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.TxPipeline().pipelined(fn) + return c.TxPipeline().Pipelined(fn) } // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. diff --git a/ring.go b/ring.go index 7b3d2d14de..d558763b34 100644 --- a/ring.go +++ b/ring.go @@ -436,7 +436,7 @@ func (c *Ring) Pipeline() Pipeliner { } func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) + return c.Pipeline().Pipelined(fn) } func (c *Ring) pipelineExec(cmds []Cmder) error { @@ -493,3 +493,11 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { return firstCmdsErr(cmds) } + +func (c *Ring) TxPipeline() Pipeliner { + panic("not implemented") +} + +func (c *Ring) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { + panic("not implemented") +} diff --git a/tx.go b/tx.go index 93c295d98f..11d5d5cb00 100644 --- a/tx.go +++ b/tx.go @@ -91,5 +91,13 @@ func (c *Tx) Pipeline() Pipeliner { // TxFailedErr is returned. Otherwise Exec returns error of the first // failed command or nil. func (c *Tx) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.Pipeline().pipelined(fn) + return c.Pipeline().Pipelined(fn) +} + +func (c *Tx) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { + return c.Pipelined(fn) +} + +func (c *Tx) TxPipeline() Pipeliner { + return c.Pipeline() } diff --git a/universal.go b/universal.go index 4aa579fa4f..29eb12b188 100644 --- a/universal.go +++ b/universal.go @@ -90,8 +90,8 @@ func (o *UniversalOptions) simple() *Options { } return &Options{ - Addr: addr, - DB: o.DB, + Addr: addr, + DB: o.DB, MaxRetries: o.MaxRetries, Password: o.Password, @@ -117,6 +117,9 @@ type UniversalClient interface { Close() error } +var _ UniversalClient = (*Client)(nil) +var _ UniversalClient = (*ClusterClient)(nil) + // NewUniversalClient returns a new multi client. The type of client returned depends // on the following three conditions: // From 2a5293c99cf1d23f25c273c8d32c80b6db7d8702 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 26 Sep 2017 11:29:22 +0300 Subject: [PATCH 0378/1746] Export Cmder.Args --- cluster.go | 4 +-- command.go | 72 ++++++++++++++++++++++++++++++++++-------------------- ring.go | 4 +-- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/cluster.go b/cluster.go index c83da6a45a..43354d774e 100644 --- a/cluster.go +++ b/cluster.go @@ -535,13 +535,13 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo { func (c *ClusterClient) cmdSlot(cmd Cmder) int { cmdInfo := c.cmdInfo(cmd.Name()) - firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + firstKey := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) return hashtag.Slot(firstKey) } func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmdInfo(cmd.Name()) - firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + firstKey := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) slot := hashtag.Slot(firstKey) if cmdInfo != nil && cmdInfo.ReadOnly && c.opt.ReadOnly { diff --git a/command.go b/command.go index a796a93fcc..d2688082a2 100644 --- a/command.go +++ b/command.go @@ -12,28 +12,10 @@ import ( "github.com/go-redis/redis/internal/proto" ) -var ( - _ Cmder = (*Cmd)(nil) - _ Cmder = (*SliceCmd)(nil) - _ Cmder = (*StatusCmd)(nil) - _ Cmder = (*IntCmd)(nil) - _ Cmder = (*DurationCmd)(nil) - _ Cmder = (*BoolCmd)(nil) - _ Cmder = (*StringCmd)(nil) - _ Cmder = (*FloatCmd)(nil) - _ Cmder = (*StringSliceCmd)(nil) - _ Cmder = (*BoolSliceCmd)(nil) - _ Cmder = (*StringStringMapCmd)(nil) - _ Cmder = (*StringIntMapCmd)(nil) - _ Cmder = (*ZSliceCmd)(nil) - _ Cmder = (*ScanCmd)(nil) - _ Cmder = (*ClusterSlotsCmd)(nil) -) - type Cmder interface { - args() []interface{} - arg(int) string Name() string + Args() []interface{} + stringArg(int) string readReply(*pool.Conn) error setErr(error) @@ -64,7 +46,7 @@ func firstCmdsErr(cmds []Cmder) error { func writeCmd(cn *pool.Conn, cmds ...Cmder) error { cn.Wb.Reset() for _, cmd := range cmds { - if err := cn.Wb.Append(cmd.args()); err != nil { + if err := cn.Wb.Append(cmd.Args()); err != nil { return err } } @@ -75,7 +57,7 @@ func writeCmd(cn *pool.Conn, cmds ...Cmder) error { func cmdString(cmd Cmder, val interface{}) string { var ss []string - for _, arg := range cmd.args() { + for _, arg := range cmd.Args() { ss = append(ss, fmt.Sprint(arg)) } s := strings.Join(ss, " ") @@ -97,7 +79,7 @@ func cmdString(cmd Cmder, val interface{}) string { func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { switch cmd.Name() { case "eval", "evalsha": - if cmd.arg(2) != "0" { + if cmd.stringArg(2) != "0" { return 3 } else { return -1 @@ -120,15 +102,17 @@ type baseCmd struct { _readTimeout *time.Duration } +var _ Cmder = (*Cmd)(nil) + func (cmd *baseCmd) Err() error { return cmd.err } -func (cmd *baseCmd) args() []interface{} { +func (cmd *baseCmd) Args() []interface{} { return cmd._args } -func (cmd *baseCmd) arg(pos int) string { +func (cmd *baseCmd) stringArg(pos int) string { if pos < 0 || pos >= len(cmd._args) { return "" } @@ -139,7 +123,7 @@ func (cmd *baseCmd) arg(pos int) string { func (cmd *baseCmd) Name() string { if len(cmd._args) > 0 { // Cmd name must be lower cased. - s := internal.ToLower(cmd.arg(0)) + s := internal.ToLower(cmd.stringArg(0)) cmd._args[0] = s return s } @@ -204,6 +188,8 @@ type SliceCmd struct { val []interface{} } +var _ Cmder = (*SliceCmd)(nil) + func NewSliceCmd(args ...interface{}) *SliceCmd { return &SliceCmd{ baseCmd: baseCmd{_args: args}, @@ -240,6 +226,8 @@ type StatusCmd struct { val string } +var _ Cmder = (*StatusCmd)(nil) + func NewStatusCmd(args ...interface{}) *StatusCmd { return &StatusCmd{ baseCmd: baseCmd{_args: args}, @@ -271,6 +259,8 @@ type IntCmd struct { val int64 } +var _ Cmder = (*IntCmd)(nil) + func NewIntCmd(args ...interface{}) *IntCmd { return &IntCmd{ baseCmd: baseCmd{_args: args}, @@ -303,6 +293,8 @@ type DurationCmd struct { precision time.Duration } +var _ Cmder = (*DurationCmd)(nil) + func NewDurationCmd(precision time.Duration, args ...interface{}) *DurationCmd { return &DurationCmd{ baseCmd: baseCmd{_args: args}, @@ -340,6 +332,8 @@ type TimeCmd struct { val time.Time } +var _ Cmder = (*TimeCmd)(nil) + func NewTimeCmd(args ...interface{}) *TimeCmd { return &TimeCmd{ baseCmd: baseCmd{_args: args}, @@ -376,6 +370,8 @@ type BoolCmd struct { val bool } +var _ Cmder = (*BoolCmd)(nil) + func NewBoolCmd(args ...interface{}) *BoolCmd { return &BoolCmd{ baseCmd: baseCmd{_args: args}, @@ -431,6 +427,8 @@ type StringCmd struct { val []byte } +var _ Cmder = (*StringCmd)(nil) + func NewStringCmd(args ...interface{}) *StringCmd { return &StringCmd{ baseCmd: baseCmd{_args: args}, @@ -494,6 +492,8 @@ type FloatCmd struct { val float64 } +var _ Cmder = (*FloatCmd)(nil) + func NewFloatCmd(args ...interface{}) *FloatCmd { return &FloatCmd{ baseCmd: baseCmd{_args: args}, @@ -525,6 +525,8 @@ type StringSliceCmd struct { val []string } +var _ Cmder = (*StringSliceCmd)(nil) + func NewStringSliceCmd(args ...interface{}) *StringSliceCmd { return &StringSliceCmd{ baseCmd: baseCmd{_args: args}, @@ -565,6 +567,8 @@ type BoolSliceCmd struct { val []bool } +var _ Cmder = (*BoolSliceCmd)(nil) + func NewBoolSliceCmd(args ...interface{}) *BoolSliceCmd { return &BoolSliceCmd{ baseCmd: baseCmd{_args: args}, @@ -601,6 +605,8 @@ type StringStringMapCmd struct { val map[string]string } +var _ Cmder = (*StringStringMapCmd)(nil) + func NewStringStringMapCmd(args ...interface{}) *StringStringMapCmd { return &StringStringMapCmd{ baseCmd: baseCmd{_args: args}, @@ -637,6 +643,8 @@ type StringIntMapCmd struct { val map[string]int64 } +var _ Cmder = (*StringIntMapCmd)(nil) + func NewStringIntMapCmd(args ...interface{}) *StringIntMapCmd { return &StringIntMapCmd{ baseCmd: baseCmd{_args: args}, @@ -673,6 +681,8 @@ type ZSliceCmd struct { val []Z } +var _ Cmder = (*ZSliceCmd)(nil) + func NewZSliceCmd(args ...interface{}) *ZSliceCmd { return &ZSliceCmd{ baseCmd: baseCmd{_args: args}, @@ -712,6 +722,8 @@ type ScanCmd struct { process func(cmd Cmder) error } +var _ Cmder = (*ScanCmd)(nil) + func NewScanCmd(process func(cmd Cmder) error, args ...interface{}) *ScanCmd { return &ScanCmd{ baseCmd: baseCmd{_args: args}, @@ -762,6 +774,8 @@ type ClusterSlotsCmd struct { val []ClusterSlot } +var _ Cmder = (*ClusterSlotsCmd)(nil) + func NewClusterSlotsCmd(args ...interface{}) *ClusterSlotsCmd { return &ClusterSlotsCmd{ baseCmd: baseCmd{_args: args}, @@ -821,6 +835,8 @@ type GeoLocationCmd struct { locations []GeoLocation } +var _ Cmder = (*GeoLocationCmd)(nil) + func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { args = append(args, q.Radius) if q.Unit != "" { @@ -891,6 +907,8 @@ type GeoPosCmd struct { positions []*GeoPos } +var _ Cmder = (*GeoPosCmd)(nil) + func NewGeoPosCmd(args ...interface{}) *GeoPosCmd { return &GeoPosCmd{ baseCmd: baseCmd{_args: args}, @@ -937,6 +955,8 @@ type CommandsInfoCmd struct { val map[string]*CommandInfo } +var _ Cmder = (*CommandsInfoCmd)(nil) + func NewCommandsInfoCmd(args ...interface{}) *CommandsInfoCmd { return &CommandsInfoCmd{ baseCmd: baseCmd{_args: args}, diff --git a/ring.go b/ring.go index d558763b34..a058630fb4 100644 --- a/ring.go +++ b/ring.go @@ -343,7 +343,7 @@ func (c *Ring) shardByName(name string) (*ringShard, error) { func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { cmdInfo := c.cmdInfo(cmd.Name()) - firstKey := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + firstKey := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) return c.shardByKey(firstKey) } @@ -443,7 +443,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { cmdInfo := c.cmdInfo(cmd.Name()) - name := cmd.arg(cmdFirstKeyPos(cmd, cmdInfo)) + name := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) if name != "" { name = c.hash.Get(hashtag.Key(name)) } From 7cb146a31b2f5c6108d41063b2539a07f10ff034 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 27 Sep 2017 10:58:06 +0300 Subject: [PATCH 0379/1746] Increase read timeout for blocking commands and don't retry such commands --- cluster.go | 2 +- commands.go | 2 +- internal/error.go | 4 ++-- redis.go | 8 ++++---- ring.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cluster.go b/cluster.go index 43354d774e..c81fc1d57a 100644 --- a/cluster.go +++ b/cluster.go @@ -656,7 +656,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { continue } - if internal.IsRetryableError(err) { + if internal.IsRetryableError(err, true) { var nodeErr error node, nodeErr = c.nodes.Random() if nodeErr != nil { diff --git a/commands.go b/commands.go index 7f0000f3ed..a3b90f12d9 100644 --- a/commands.go +++ b/commands.go @@ -11,7 +11,7 @@ func readTimeout(timeout time.Duration) time.Duration { if timeout == 0 { return 0 } - return timeout + time.Second + return timeout + 10*time.Second } func usePrecise(dur time.Duration) bool { diff --git a/internal/error.go b/internal/error.go index e1b8be6b60..0898eeb622 100644 --- a/internal/error.go +++ b/internal/error.go @@ -12,9 +12,9 @@ type RedisError string func (e RedisError) Error() string { return string(e) } -func IsRetryableError(err error) bool { +func IsRetryableError(err error, retryNetError bool) bool { if IsNetworkError(err) { - return true + return retryNetError } s := err.Error() if s == "ERR max number of clients reached" { diff --git a/redis.go b/redis.go index 615bf28d81..230091b3e5 100644 --- a/redis.go +++ b/redis.go @@ -136,7 +136,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { cn, _, err := c.getConn() if err != nil { cmd.setErr(err) - if internal.IsRetryableError(err) { + if internal.IsRetryableError(err, true) { continue } return err @@ -146,7 +146,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { if err := writeCmd(cn, cmd); err != nil { c.releaseConn(cn, err) cmd.setErr(err) - if internal.IsRetryableError(err) { + if internal.IsRetryableError(err, true) { continue } return err @@ -155,7 +155,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { cn.SetReadTimeout(c.cmdTimeout(cmd)) err = cmd.readReply(cn) c.releaseConn(cn, err) - if err != nil && internal.IsRetryableError(err) { + if err != nil && internal.IsRetryableError(err, cmd.readTimeout() == nil) { continue } @@ -221,7 +221,7 @@ func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { } _ = c.connPool.Remove(cn) - if !canRetry || !internal.IsRetryableError(err) { + if !canRetry || !internal.IsRetryableError(err, true) { break } } diff --git a/ring.go b/ring.go index a058630fb4..a30c32102b 100644 --- a/ring.go +++ b/ring.go @@ -477,7 +477,7 @@ func (c *Ring) pipelineExec(cmds []Cmder) error { } _ = shard.Client.connPool.Remove(cn) - if canRetry && internal.IsRetryableError(err) { + if canRetry && internal.IsRetryableError(err, true) { if failedCmdsMap == nil { failedCmdsMap = make(map[string][]Cmder) } From 742a58164cb05c93b3244ab041f93d8cc53ce6cd Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sat, 30 Sep 2017 09:21:59 +0300 Subject: [PATCH 0380/1746] Add test for receive big message payload --- commands.go | 5 +++-- pubsub_test.go | 16 ++++++++++++++++ race_test.go | 9 ++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/commands.go b/commands.go index a3b90f12d9..c04b3c49b8 100644 --- a/commands.go +++ b/commands.go @@ -214,6 +214,7 @@ type Cmdable interface { ScriptKill() *StatusCmd ScriptLoad(script string) *StringCmd DebugObject(key string) *StringCmd + Publish(channel string, message interface{}) *IntCmd PubSubChannels(pattern string) *StringSliceCmd PubSubNumSub(channels ...string) *StringIntMapCmd PubSubNumPat() *IntCmd @@ -1880,8 +1881,8 @@ func (c *cmdable) DebugObject(key string) *StringCmd { //------------------------------------------------------------------------------ // Publish posts the message to the channel. -func (c *cmdable) Publish(channel, message string) *IntCmd { - cmd := NewIntCmd("PUBLISH", channel, message) +func (c *cmdable) Publish(channel string, message interface{}) *IntCmd { + cmd := NewIntCmd("publish", channel, message) c.process(cmd) return cmd } diff --git a/pubsub_test.go b/pubsub_test.go index 6fc04a1989..6a85bd0389 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -424,4 +424,20 @@ var _ = Describe("PubSub", func() { wg.Wait() }) + + It("handles big message payload", func() { + pubsub := client.Subscribe("mychannel") + defer pubsub.Close() + + ch := pubsub.Channel() + + bigVal := bigVal() + err := client.Publish("mychannel", bigVal).Err() + Expect(err).NotTo(HaveOccurred()) + + var msg *redis.Message + Eventually(ch).Should(Receive(&msg)) + Expect(msg.Channel).To(Equal("mychannel")) + Expect(msg.Payload).To(Equal(string(bigVal))) + }) }) diff --git a/race_test.go b/race_test.go index 5bcb0768e2..14264086cd 100644 --- a/race_test.go +++ b/race_test.go @@ -105,7 +105,7 @@ var _ = Describe("races", func() { It("should handle big vals in Get", func() { C, N = 4, 100 - bigVal := bytes.Repeat([]byte{'*'}, 1<<17) // 128kb + bigVal := bigVal() err := client.Set("key", bigVal, 0).Err() Expect(err).NotTo(HaveOccurred()) @@ -126,8 +126,7 @@ var _ = Describe("races", func() { It("should handle big vals in Set", func() { C, N = 4, 100 - bigVal := bytes.Repeat([]byte{'*'}, 1<<17) // 128kb - + bigVal := bigVal() perform(C, func(id int) { for i := 0; i < N; i++ { err := client.Set("key", bigVal, 0).Err() @@ -245,3 +244,7 @@ var _ = Describe("races", func() { Expect(n).To(Equal(int64(N))) }) }) + +func bigVal() []byte { + return bytes.Repeat([]byte{'*'}, 1<<17) // 128kb +} From dac1820e476ae285d7b7fe069268a7f5d1f5a549 Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 11 Oct 2017 16:03:55 +0100 Subject: [PATCH 0381/1746] Fix pool panics --- internal/pool/pool.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index 836ec1045e..ae81905ea8 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -60,8 +60,10 @@ type Options struct { type ConnPool struct { opt *Options - dialErrorsNum uint32 // atomic - _lastDialError atomic.Value + dialErrorsNum uint32 // atomic + + lastDialError error + lastDialErrorMu sync.RWMutex queue chan struct{} @@ -98,7 +100,7 @@ func (p *ConnPool) NewConn() (*Conn, error) { } if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.opt.PoolSize) { - return nil, p.lastDialError() + return nil, p.getLastDialError() } netConn, err := p.opt.Dialer() @@ -138,11 +140,16 @@ func (p *ConnPool) tryDial() { } func (p *ConnPool) setLastDialError(err error) { - p._lastDialError.Store(err) + p.lastDialErrorMu.Lock() + p.lastDialError = err + p.lastDialErrorMu.Unlock() } -func (p *ConnPool) lastDialError() error { - return p._lastDialError.Load().(error) +func (p *ConnPool) getLastDialError() error { + p.lastDialErrorMu.RLock() + err := p.lastDialError + p.lastDialErrorMu.RUnlock() + return err } // Get returns existed connection from the pool or creates a new one. From 15f14b83059e8b093b47376e2cd435035bf65970 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 30 Oct 2017 12:09:57 +0200 Subject: [PATCH 0382/1746] Create PubSub channel once --- pubsub.go | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/pubsub.go b/pubsub.go index e754a16f23..01f8a61aad 100644 --- a/pubsub.go +++ b/pubsub.go @@ -29,6 +29,9 @@ type PubSub struct { closed bool cmd *Cmd + + chOnce sync.Once + ch chan *Message } func (c *PubSub) conn() (*pool.Conn, error) { @@ -346,24 +349,27 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { } } -// Channel returns a channel for concurrently receiving messages. -// The channel is closed with PubSub. +// Channel returns a Go channel for concurrently receiving messages. +// The channel is closed with PubSub. Receive or ReceiveMessage APIs +// can not be used after channel is created. func (c *PubSub) Channel() <-chan *Message { - ch := make(chan *Message, 100) - go func() { - for { - msg, err := c.ReceiveMessage() - if err != nil { - if err == pool.ErrClosed { - break + c.chOnce.Do(func() { + c.ch = make(chan *Message, 100) + go func() { + for { + msg, err := c.ReceiveMessage() + if err != nil { + if err == pool.ErrClosed { + break + } + continue } - continue + c.ch <- msg } - ch <- msg - } - close(ch) - }() - return ch + close(c.ch) + }() + }) + return c.ch } func appendIfNotExists(ss []string, es ...string) []string { From 48b70050d496a538ee06d5886a5981c79f7a90a0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 1 Nov 2017 15:33:53 +0200 Subject: [PATCH 0383/1746] Fix slice next elem func --- internal/proto/scan.go | 3 ++- internal/util.go | 37 ++++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/internal/proto/scan.go b/internal/proto/scan.go index 0431a877d0..03c8b59aa0 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -120,8 +120,9 @@ func ScanSlice(data []string, slice interface{}) error { return fmt.Errorf("redis: ScanSlice(non-slice %T)", slice) } + next := internal.MakeSliceNextElemFunc(v) for i, s := range data { - elem := internal.SliceNextElem(v) + elem := next() if err := Scan(internal.StringToBytes(s), elem.Addr().Interface()); err != nil { return fmt.Errorf("redis: ScanSlice(index=%d value=%q) failed: %s", i, s, err) } diff --git a/internal/util.go b/internal/util.go index 520596fd97..1ba9805fe3 100644 --- a/internal/util.go +++ b/internal/util.go @@ -28,20 +28,35 @@ func isLower(s string) bool { return true } -func SliceNextElem(v reflect.Value) reflect.Value { - if v.Len() < v.Cap() { - v.Set(v.Slice(0, v.Len()+1)) - return v.Index(v.Len() - 1) - } - +func MakeSliceNextElemFunc(v reflect.Value) func() reflect.Value { elemType := v.Type().Elem() if elemType.Kind() == reflect.Ptr { - elem := reflect.New(elemType.Elem()) - v.Set(reflect.Append(v, elem)) - return elem.Elem() + elemType = elemType.Elem() + return func() reflect.Value { + if v.Len() < v.Cap() { + v.Set(v.Slice(0, v.Len()+1)) + elem := v.Index(v.Len() - 1) + if elem.IsNil() { + elem.Set(reflect.New(elemType)) + } + return elem.Elem() + } + + elem := reflect.New(elemType) + v.Set(reflect.Append(v, elem)) + return elem.Elem() + } } - v.Set(reflect.Append(v, reflect.Zero(elemType))) - return v.Index(v.Len() - 1) + zero := reflect.Zero(elemType) + return func() reflect.Value { + if v.Len() < v.Cap() { + v.Set(v.Slice(0, v.Len()+1)) + return v.Index(v.Len() - 1) + } + + v.Set(reflect.Append(v, zero)) + return v.Index(v.Len() - 1) + } } From 7aa0130b2e568a6c0e540e3cce65f3a23e9b8f7b Mon Sep 17 00:00:00 2001 From: Crimson Date: Sun, 19 Nov 2017 17:36:23 +0100 Subject: [PATCH 0384/1746] Add SMembersMap function --- command.go | 38 ++++++++++++++++++++++++++++++++++++++ commands.go | 7 +++++++ commands_test.go | 11 +++++++++++ parser.go | 14 ++++++++++++++ 4 files changed, 70 insertions(+) diff --git a/command.go b/command.go index d2688082a2..601a2882da 100644 --- a/command.go +++ b/command.go @@ -675,6 +675,44 @@ func (cmd *StringIntMapCmd) readReply(cn *pool.Conn) error { //------------------------------------------------------------------------------ +type StringStructMapCmd struct { + baseCmd + + val map[string]struct{} +} + +var _ Cmder = (*StringStructMapCmd)(nil) + +func NewStringStructMapCmd(args ...interface{}) *StringStructMapCmd { + return &StringStructMapCmd{ + baseCmd: baseCmd{_args: args}, + } +} + +func (cmd *StringStructMapCmd) Val() map[string]struct{} { + return cmd.val +} + +func (cmd *StringStructMapCmd) Result() (map[string]struct{}, error) { + return cmd.val, cmd.err +} + +func (cmd *StringStructMapCmd) String() string { + return cmdString(cmd, cmd.val) +} + +func (cmd *StringStructMapCmd) readReply(cn *pool.Conn) error { + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(stringStructMapParser) + if cmd.err != nil { + return cmd.err + } + cmd.val = v.(map[string]struct{}) + return nil +} + +//------------------------------------------------------------------------------ + type ZSliceCmd struct { baseCmd diff --git a/commands.go b/commands.go index c04b3c49b8..7a9637547d 100644 --- a/commands.go +++ b/commands.go @@ -143,6 +143,7 @@ type Cmdable interface { SInterStore(destination string, keys ...string) *IntCmd SIsMember(key string, member interface{}) *BoolCmd SMembers(key string) *StringSliceCmd + SMembersMap(key string) *StringStructMapCmd SMove(source, destination string, member interface{}) *BoolCmd SPop(key string) *StringCmd SPopN(key string, count int64) *StringSliceCmd @@ -1169,6 +1170,12 @@ func (c *cmdable) SMembers(key string) *StringSliceCmd { return cmd } +func (c *cmdable) SMembersMap(key string) *StringStructMapCmd { + cmd := NewStringStructMapCmd("smembers", key) + c.process(cmd) + return cmd +} + func (c *cmdable) SMove(source, destination string, member interface{}) *BoolCmd { cmd := NewBoolCmd("smove", source, destination, member) c.process(cmd) diff --git a/commands_test.go b/commands_test.go index 6b81f23cf7..7153795569 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1848,6 +1848,17 @@ var _ = Describe("Commands", func() { Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) }) + It("should SMembersMap", func() { + sAdd := client.SAdd("set", "Hello") + Expect(sAdd.Err()).NotTo(HaveOccurred()) + sAdd = client.SAdd("set", "World") + Expect(sAdd.Err()).NotTo(HaveOccurred()) + + sMembersMap := client.SMembersMap("set") + Expect(sMembersMap.Err()).NotTo(HaveOccurred()) + Expect(sMembersMap.Val()).To(Equal(map[string]struct{}{"Hello": struct{}{}, "World": struct{}{}})) + }) + It("should SMove", func() { sAdd := client.SAdd("set1", "one") Expect(sAdd.Err()).NotTo(HaveOccurred()) diff --git a/parser.go b/parser.go index 1d7ec630e4..b378abc4ee 100644 --- a/parser.go +++ b/parser.go @@ -97,6 +97,20 @@ func stringIntMapParser(rd *proto.Reader, n int64) (interface{}, error) { return m, nil } +// Implements proto.MultiBulkParse +func stringStructMapParser(rd *proto.Reader, n int64) (interface{}, error) { + m := make(map[string]struct{}, n) + for i := int64(0); i < n; i++ { + key, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + m[key] = struct{}{} + } + return m, nil +} + // Implements proto.MultiBulkParse func zSliceParser(rd *proto.Reader, n int64) (interface{}, error) { zz := make([]Z, n/2) From b2e8f5832a8679ff5c2d3b602433ca62b0a281ac Mon Sep 17 00:00:00 2001 From: Crimson Date: Sun, 19 Nov 2017 17:56:54 +0100 Subject: [PATCH 0385/1746] Add godoc commentary to SMembers and SMembersMap --- commands.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands.go b/commands.go index 7a9637547d..aa98aa736c 100644 --- a/commands.go +++ b/commands.go @@ -1164,12 +1164,14 @@ func (c *cmdable) SIsMember(key string, member interface{}) *BoolCmd { return cmd } +// Redis `SMEMBERS key` command output as a slice func (c *cmdable) SMembers(key string) *StringSliceCmd { cmd := NewStringSliceCmd("smembers", key) c.process(cmd) return cmd } +// Redis `SMEMBERS key` command output as a map func (c *cmdable) SMembersMap(key string) *StringStructMapCmd { cmd := NewStringStructMapCmd("smembers", key) c.process(cmd) From 2cf19e8b9a537c0fc622832e03a23384ab60cd77 Mon Sep 17 00:00:00 2001 From: Matt Kelly Date: Fri, 24 Nov 2017 12:33:21 -0500 Subject: [PATCH 0386/1746] Fix grammar in quickstart --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a2a67124c..ca4b41d2ae 100644 --- a/README.md +++ b/README.md @@ -66,14 +66,14 @@ func ExampleClient() { val2, err := client.Get("key2").Result() if err == redis.Nil { - fmt.Println("key2 does not exists") + fmt.Println("key2 does not exist") } else if err != nil { panic(err) } else { fmt.Println("key2", val2) } // Output: key value - // key2 does not exists + // key2 does not exist } ``` From bdbad50f7e1cd706511fbb39575d0e18e1d7bb75 Mon Sep 17 00:00:00 2001 From: Matt Kelly Date: Sat, 25 Nov 2017 12:37:26 -0500 Subject: [PATCH 0387/1746] Fix grammar in ExampleClient --- example_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example_test.go b/example_test.go index 7e04cd4875..4d18ddb945 100644 --- a/example_test.go +++ b/example_test.go @@ -96,14 +96,14 @@ func ExampleClient() { val2, err := client.Get("key2").Result() if err == redis.Nil { - fmt.Println("key2 does not exists") + fmt.Println("key2 does not exist") } else if err != nil { panic(err) } else { fmt.Println("key2", val2) } // Output: key value - // key2 does not exists + // key2 does not exist } func ExampleClient_Set() { From 08ae2d0555fe40e124639f423c9ae7f5e0458932 Mon Sep 17 00:00:00 2001 From: Sudhir Jonathan Date: Thu, 14 Dec 2017 22:06:57 +0530 Subject: [PATCH 0388/1746] Added subscription interfaces to universal client --- commands.go | 5 +++++ universal.go | 1 + 2 files changed, 6 insertions(+) diff --git a/commands.go b/commands.go index aa98aa736c..d6e6f2a596 100644 --- a/commands.go +++ b/commands.go @@ -257,6 +257,11 @@ type StatefulCmdable interface { ReadWrite() *StatusCmd } +type Subscribable interface { + Subscribe(channels ...string) *PubSub + PSubscribe(channels ...string) *PubSub +} + var _ Cmdable = (*Client)(nil) var _ Cmdable = (*Tx)(nil) var _ Cmdable = (*Ring)(nil) diff --git a/universal.go b/universal.go index 29eb12b188..339bb4e1d6 100644 --- a/universal.go +++ b/universal.go @@ -113,6 +113,7 @@ func (o *UniversalOptions) simple() *Options { // applications locally. type UniversalClient interface { Cmdable + Subscribable Process(cmd Cmder) error Close() error } From 58bf6f4abe6ffd4d5d77e8eb7306fa74474c1787 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 27 Dec 2017 13:10:36 +0200 Subject: [PATCH 0389/1746] Add Airbrake badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ca4b41d2ae..9f349764af 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis) [![GoDoc](https://godoc.org/github.com/go-redis/redis?status.svg)](https://godoc.org/github.com/go-redis/redis) +[![Airbrake](https://img.shields.io/badge/kudos-airbrake.io-orange.svg)](https://airbrake.io) Supports: From 624096af5ea5332b498557edb12aa59554a012e6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 27 Dec 2017 13:29:46 +0200 Subject: [PATCH 0390/1746] Mention redis.Nil --- commands.go | 1 + redis.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index aa98aa736c..569342cfa1 100644 --- a/commands.go +++ b/commands.go @@ -677,6 +677,7 @@ func (c *cmdable) DecrBy(key string, decrement int64) *IntCmd { return cmd } +// Redis `GET key` command. It returns redis.Nil error when key does not exist. func (c *cmdable) Get(key string) *StringCmd { cmd := NewStringCmd("get", key) c.process(cmd) diff --git a/redis.go b/redis.go index 230091b3e5..37ffafd97b 100644 --- a/redis.go +++ b/redis.go @@ -11,7 +11,7 @@ import ( "github.com/go-redis/redis/internal/proto" ) -// Redis nil reply, .e.g. when key does not exist. +// Redis nil reply returned when key does not exist. const Nil = internal.Nil func init() { From 3de5605ab2401c963647b63935049b1631121709 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 28 Dec 2017 16:41:05 +0200 Subject: [PATCH 0391/1746] Pick random node when command does not have keys --- cluster.go | 14 ++++++++++---- cluster_test.go | 26 ++++++++++++++++++++++++++ command.go | 4 ++-- internal/hashtag/hashtag.go | 8 ++++++-- ring.go | 6 +++++- 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/cluster.go b/cluster.go index c81fc1d57a..afaf3f3510 100644 --- a/cluster.go +++ b/cluster.go @@ -533,16 +533,22 @@ func (c *ClusterClient) cmdInfo(name string) *CommandInfo { return info } +func cmdSlot(cmd Cmder, pos int) int { + if pos == 0 { + return hashtag.RandomSlot() + } + firstKey := cmd.stringArg(pos) + return hashtag.Slot(firstKey) +} + func (c *ClusterClient) cmdSlot(cmd Cmder) int { cmdInfo := c.cmdInfo(cmd.Name()) - firstKey := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) - return hashtag.Slot(firstKey) + return cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) } func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { cmdInfo := c.cmdInfo(cmd.Name()) - firstKey := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) - slot := hashtag.Slot(firstKey) + slot := cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) if cmdInfo != nil && cmdInfo.ReadOnly && c.opt.ReadOnly { if c.opt.RouteByLatency { diff --git a/cluster_test.go b/cluster_test.go index 6f3677b938..43f3261bcd 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -536,6 +536,32 @@ var _ = Describe("ClusterClient", func() { Expect(nodesList).Should(HaveLen(1)) }) + It("should RANDOMKEY", func() { + const nkeys = 100 + + for i := 0; i < nkeys; i++ { + err := client.Set(fmt.Sprintf("key%d", i), "value", 0).Err() + Expect(err).NotTo(HaveOccurred()) + } + + var keys []string + addKey := func(key string) { + for _, k := range keys { + if k == key { + return + } + } + keys = append(keys, key) + } + + for i := 0; i < nkeys*10; i++ { + key := client.RandomKey().Val() + addKey(key) + } + + Expect(len(keys)).To(BeNumerically("~", nkeys, nkeys/10)) + }) + assertClusterClient() }) diff --git a/command.go b/command.go index 601a2882da..598ed98002 100644 --- a/command.go +++ b/command.go @@ -82,13 +82,13 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { if cmd.stringArg(2) != "0" { return 3 } else { - return -1 + return 0 } case "publish": return 1 } if info == nil { - return -1 + return 0 } return int(info.FirstKeyPos) } diff --git a/internal/hashtag/hashtag.go b/internal/hashtag/hashtag.go index 2866488e59..8c7ebbfa64 100644 --- a/internal/hashtag/hashtag.go +++ b/internal/hashtag/hashtag.go @@ -55,13 +55,17 @@ func Key(key string) string { return key } +func RandomSlot() int { + return rand.Intn(SlotNumber) +} + // hashSlot returns a consistent slot number between 0 and 16383 // for any given string key. func Slot(key string) int { - key = Key(key) if key == "" { - return rand.Intn(SlotNumber) + return RandomSlot() } + key = Key(key) return int(crc16sum(key)) % SlotNumber } diff --git a/ring.go b/ring.go index a30c32102b..0697b3e131 100644 --- a/ring.go +++ b/ring.go @@ -343,7 +343,11 @@ func (c *Ring) shardByName(name string) (*ringShard, error) { func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { cmdInfo := c.cmdInfo(cmd.Name()) - firstKey := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) + pos := cmdFirstKeyPos(cmd, cmdInfo) + if pos == 0 { + return c.randomShard() + } + firstKey := cmd.stringArg(pos) return c.shardByKey(firstKey) } From e5040d9ce782bd7e89c9eb686ed27395ddf94aeb Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 28 Dec 2017 17:00:22 +0200 Subject: [PATCH 0392/1746] Don't test in Go 1.4 --- .travis.yml | 2 -- ring.go | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f49927ee84..c95b3e6c6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ services: - redis-server go: - - 1.4.x - 1.7.x - 1.8.x - 1.9.x @@ -13,7 +12,6 @@ go: matrix: allow_failures: - - go: 1.4.x - go: tip install: diff --git a/ring.go b/ring.go index 0697b3e131..c11ef6bc2f 100644 --- a/ring.go +++ b/ring.go @@ -298,6 +298,9 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { if err != nil { return nil } + if c.cmdsInfo == nil { + return nil + } info := c.cmdsInfo[name] if info == nil { internal.Logf("info for cmd=%s not found", name) From ea762c666ed8ad9c930caaaa4cf516025d10c5f9 Mon Sep 17 00:00:00 2001 From: Eldar Rakhimberdin Date: Thu, 4 Jan 2018 15:20:49 +0300 Subject: [PATCH 0393/1746] removed quotes surrounding %q %q is a single-quoted character literal safely escaped with Go syntax. It doesn't need additional quotes. --- options_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options_test.go b/options_test.go index 6a4af7169a..211f6b1950 100644 --- a/options_test.go +++ b/options_test.go @@ -71,7 +71,7 @@ func TestParseURL(t *testing.T) { t.Run(c.u, func(t *testing.T) { o, err := ParseURL(c.u) if c.err == nil && err != nil { - t.Fatalf("unexpected error: '%q'", err) + t.Fatalf("unexpected error: %q", err) return } if c.err != nil && err != nil { From a8340c6b74299c6521176f26583c020aec5f2b93 Mon Sep 17 00:00:00 2001 From: FJSDS <821431199@qq.com> Date: Thu, 11 Jan 2018 16:07:24 +0800 Subject: [PATCH 0394/1746] fix unsafe.go --- internal/unsafe.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/internal/unsafe.go b/internal/unsafe.go index c18b25c17a..a54a34a794 100644 --- a/internal/unsafe.go +++ b/internal/unsafe.go @@ -3,25 +3,20 @@ package internal import ( - "reflect" "unsafe" ) +// BytesToString converts byte slice to string. func BytesToString(b []byte) string { - bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - strHeader := reflect.StringHeader{ - Data: bytesHeader.Data, - Len: bytesHeader.Len, - } - return *(*string)(unsafe.Pointer(&strHeader)) + return *(*string)(unsafe.Pointer(&b)) } +// StringToBytes converts string to byte slice. func StringToBytes(s string) []byte { - sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) - bh := reflect.SliceHeader{ - Data: sh.Data, - Len: sh.Len, - Cap: sh.Len, - } - return *(*[]byte)(unsafe.Pointer(&bh)) + return *(*[]byte)(unsafe.Pointer( + &struct { + string + Cap int + }{s,len(s)}, + )) } From abb85b0fb8a7f0ef7551677888d8d674d153c991 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 15 Jan 2018 16:15:20 +0200 Subject: [PATCH 0395/1746] Remove StringToBytes --- internal/proto/scan.go | 5 +++-- internal/safe.go | 4 ---- internal/unsafe.go | 10 ---------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/internal/proto/scan.go b/internal/proto/scan.go index 03c8b59aa0..0329ffd991 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -123,8 +123,9 @@ func ScanSlice(data []string, slice interface{}) error { next := internal.MakeSliceNextElemFunc(v) for i, s := range data { elem := next() - if err := Scan(internal.StringToBytes(s), elem.Addr().Interface()); err != nil { - return fmt.Errorf("redis: ScanSlice(index=%d value=%q) failed: %s", i, s, err) + if err := Scan([]byte(s), elem.Addr().Interface()); err != nil { + err = fmt.Errorf("redis: ScanSlice index=%d value=%q failed: %s", i, s, err) + return err } } diff --git a/internal/safe.go b/internal/safe.go index 870fe541f0..dc5f4cc8a4 100644 --- a/internal/safe.go +++ b/internal/safe.go @@ -5,7 +5,3 @@ package internal func BytesToString(b []byte) string { return string(b) } - -func StringToBytes(s string) []byte { - return []byte(s) -} diff --git a/internal/unsafe.go b/internal/unsafe.go index a54a34a794..3ae48c14b9 100644 --- a/internal/unsafe.go +++ b/internal/unsafe.go @@ -10,13 +10,3 @@ import ( func BytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } - -// StringToBytes converts string to byte slice. -func StringToBytes(s string) []byte { - return *(*[]byte)(unsafe.Pointer( - &struct { - string - Cap int - }{s,len(s)}, - )) -} From a84876237b646d12e176987b37cd2c3a5523ff24 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 17 Jan 2018 12:55:20 +0200 Subject: [PATCH 0396/1746] Retry if node is closed; close nodes with delay --- cluster.go | 62 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/cluster.go b/cluster.go index afaf3f3510..accdb3d272 100644 --- a/cluster.go +++ b/cluster.go @@ -226,7 +226,7 @@ func (c *clusterNodes) NextGeneration() uint32 { } // GC removes unused nodes. -func (c *clusterNodes) GC(generation uint32) error { +func (c *clusterNodes) GC(generation uint32) { var collected []*clusterNode c.mu.Lock() for i := 0; i < len(c.addrs); { @@ -243,14 +243,11 @@ func (c *clusterNodes) GC(generation uint32) error { } c.mu.Unlock() - var firstErr error - for _, node := range collected { - if err := node.Client.Close(); err != nil && firstErr == nil { - firstErr = err + time.AfterFunc(time.Minute, func() { + for _, node := range collected { + _ = node.Client.Close() } - } - - return firstErr + }) } func (c *clusterNodes) All() ([]*clusterNode, error) { @@ -596,6 +593,10 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { break } + if internal.IsRetryableError(err, true) { + continue + } + moved, ask, addr := internal.IsMovedError(err) if moved || ask { c.lazyReloadState() @@ -606,6 +607,13 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { continue } + if err == pool.ErrClosed { + node, err = state.slotMasterNode(slot) + if err != nil { + return err + } + } + return err } @@ -641,10 +649,10 @@ func (c *ClusterClient) Process(cmd Cmder) error { if ask { pipe := node.Client.Pipeline() - pipe.Process(NewCmd("ASKING")) - pipe.Process(cmd) + _ = pipe.Process(NewCmd("ASKING")) + _ = pipe.Process(cmd) _, err = pipe.Exec() - pipe.Close() + _ = pipe.Close() ask = false } else { err = node.Client.Process(cmd) @@ -685,6 +693,14 @@ func (c *ClusterClient) Process(cmd Cmder) error { continue } + if err == pool.ErrClosed { + _, node, err = c.cmdSlotAndNode(state, cmd) + if err != nil { + cmd.setErr(err) + return err + } + } + break } @@ -921,7 +937,11 @@ func (c *ClusterClient) pipelineExec(cmds []Cmder) error { for node, cmds := range cmdsMap { cn, _, err := node.Client.getConn() if err != nil { - setCmdsErr(cmds, err) + if err == pool.ErrClosed { + c.remapCmds(cmds, failedCmds) + } else { + setCmdsErr(cmds, err) + } continue } @@ -961,6 +981,18 @@ func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, e return cmdsMap, nil } +func (c *ClusterClient) remapCmds(cmds []Cmder, failedCmds map[*clusterNode][]Cmder) { + remappedCmds, err := c.mapCmdsByNode(cmds) + if err != nil { + setCmdsErr(cmds, err) + return + } + + for node, cmds := range remappedCmds { + failedCmds[node] = cmds + } +} + func (c *ClusterClient) pipelineProcessCmds( node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) error { @@ -1067,7 +1099,11 @@ func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { for node, cmds := range cmdsMap { cn, _, err := node.Client.getConn() if err != nil { - setCmdsErr(cmds, err) + if err == pool.ErrClosed { + c.remapCmds(cmds, failedCmds) + } else { + setCmdsErr(cmds, err) + } continue } From ea4d6c3490285e115c475c81dfb9a4e7a2c4bcae Mon Sep 17 00:00:00 2001 From: Veselkov Konstantin Date: Wed, 24 Jan 2018 22:38:47 +0400 Subject: [PATCH 0397/1746] golint warnings are removed --- command.go | 4 +-- internal/proto/reader.go | 64 ++++++++++++++++++++-------------------- pubsub.go | 10 +++---- redis.go | 6 ++-- tx.go | 4 +-- 5 files changed, 44 insertions(+), 44 deletions(-) diff --git a/command.go b/command.go index 598ed98002..480a5ce19a 100644 --- a/command.go +++ b/command.go @@ -81,9 +81,9 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { case "eval", "evalsha": if cmd.stringArg(2) != "0" { return 3 - } else { - return 0 } + + return 0 case "publish": return 1 } diff --git a/internal/proto/reader.go b/internal/proto/reader.go index cd94329d8d..964519d353 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -37,25 +37,25 @@ func (r *Reader) Reset(rd io.Reader) { r.src.Reset(rd) } -func (p *Reader) PeekBuffered() []byte { - if n := p.src.Buffered(); n != 0 { - b, _ := p.src.Peek(n) +func (r *Reader) PeekBuffered() []byte { + if n := r.src.Buffered(); n != 0 { + b, _ := r.src.Peek(n) return b } return nil } -func (p *Reader) ReadN(n int) ([]byte, error) { - b, err := readN(p.src, p.buf, n) +func (r *Reader) ReadN(n int) ([]byte, error) { + b, err := readN(r.src, r.buf, n) if err != nil { return nil, err } - p.buf = b + r.buf = b return b, nil } -func (p *Reader) ReadLine() ([]byte, error) { - line, isPrefix, err := p.src.ReadLine() +func (r *Reader) ReadLine() ([]byte, error) { + line, isPrefix, err := r.src.ReadLine() if err != nil { return nil, err } @@ -71,8 +71,8 @@ func (p *Reader) ReadLine() ([]byte, error) { return line, nil } -func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { - line, err := p.ReadLine() +func (r *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { + line, err := r.ReadLine() if err != nil { return nil, err } @@ -85,19 +85,19 @@ func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { case IntReply: return parseInt(line[1:], 10, 64) case StringReply: - return p.readTmpBytesValue(line) + return r.readTmpBytesValue(line) case ArrayReply: n, err := parseArrayLen(line) if err != nil { return nil, err } - return m(p, n) + return m(r, n) } return nil, fmt.Errorf("redis: can't parse %.100q", line) } -func (p *Reader) ReadIntReply() (int64, error) { - line, err := p.ReadLine() +func (r *Reader) ReadIntReply() (int64, error) { + line, err := r.ReadLine() if err != nil { return 0, err } @@ -111,8 +111,8 @@ func (p *Reader) ReadIntReply() (int64, error) { } } -func (p *Reader) ReadTmpBytesReply() ([]byte, error) { - line, err := p.ReadLine() +func (r *Reader) ReadTmpBytesReply() ([]byte, error) { + line, err := r.ReadLine() if err != nil { return nil, err } @@ -120,7 +120,7 @@ func (p *Reader) ReadTmpBytesReply() ([]byte, error) { case ErrorReply: return nil, ParseErrorReply(line) case StringReply: - return p.readTmpBytesValue(line) + return r.readTmpBytesValue(line) case StatusReply: return parseStatusValue(line), nil default: @@ -138,24 +138,24 @@ func (r *Reader) ReadBytesReply() ([]byte, error) { return cp, nil } -func (p *Reader) ReadStringReply() (string, error) { - b, err := p.ReadTmpBytesReply() +func (r *Reader) ReadStringReply() (string, error) { + b, err := r.ReadTmpBytesReply() if err != nil { return "", err } return string(b), nil } -func (p *Reader) ReadFloatReply() (float64, error) { - b, err := p.ReadTmpBytesReply() +func (r *Reader) ReadFloatReply() (float64, error) { + b, err := r.ReadTmpBytesReply() if err != nil { return 0, err } return parseFloat(b, 64) } -func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { - line, err := p.ReadLine() +func (r *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { + line, err := r.ReadLine() if err != nil { return nil, err } @@ -167,14 +167,14 @@ func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { if err != nil { return nil, err } - return m(p, n) + return m(r, n) default: return nil, fmt.Errorf("redis: can't parse array reply: %.100q", line) } } -func (p *Reader) ReadArrayLen() (int64, error) { - line, err := p.ReadLine() +func (r *Reader) ReadArrayLen() (int64, error) { + line, err := r.ReadLine() if err != nil { return 0, err } @@ -188,8 +188,8 @@ func (p *Reader) ReadArrayLen() (int64, error) { } } -func (p *Reader) ReadScanReply() ([]string, uint64, error) { - n, err := p.ReadArrayLen() +func (r *Reader) ReadScanReply() ([]string, uint64, error) { + n, err := r.ReadArrayLen() if err != nil { return nil, 0, err } @@ -202,14 +202,14 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) { return nil, 0, err } - n, err = p.ReadArrayLen() + n, err = r.ReadArrayLen() if err != nil { return nil, 0, err } keys := make([]string, n) for i := int64(0); i < n; i++ { - key, err := p.ReadStringReply() + key, err := r.ReadStringReply() if err != nil { return nil, 0, err } @@ -219,7 +219,7 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) { return keys, cursor, err } -func (p *Reader) readTmpBytesValue(line []byte) ([]byte, error) { +func (r *Reader) readTmpBytesValue(line []byte) ([]byte, error) { if isNilReply(line) { return nil, internal.Nil } @@ -229,7 +229,7 @@ func (p *Reader) readTmpBytesValue(line []byte) ([]byte, error) { return nil, err } - b, err := p.ReadN(replyLen + 2) + b, err := r.ReadN(replyLen + 2) if err != nil { return nil, err } diff --git a/pubsub.go b/pubsub.go index 01f8a61aad..3ee4ea9d01 100644 --- a/pubsub.go +++ b/pubsub.go @@ -127,7 +127,7 @@ func (c *PubSub) Close() error { return nil } -// Subscribes the client to the specified channels. It returns +// Subscribe the client to the specified channels. It returns // empty subscription if there are no channels. func (c *PubSub) Subscribe(channels ...string) error { c.mu.Lock() @@ -137,7 +137,7 @@ func (c *PubSub) Subscribe(channels ...string) error { return err } -// Subscribes the client to the given patterns. It returns +// PSubscribe the client to the given patterns. It returns // empty subscription if there are no patterns. func (c *PubSub) PSubscribe(patterns ...string) error { c.mu.Lock() @@ -147,7 +147,7 @@ func (c *PubSub) PSubscribe(patterns ...string) error { return err } -// Unsubscribes the client from the given channels, or from all of +// Unsubscribe the client from the given channels, or from all of // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { c.mu.Lock() @@ -157,7 +157,7 @@ func (c *PubSub) Unsubscribe(channels ...string) error { return err } -// Unsubscribes the client from the given patterns, or from all of +// PUnsubscribe the client from the given patterns, or from all of // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { c.mu.Lock() @@ -196,7 +196,7 @@ func (c *PubSub) Ping(payload ...string) error { return err } -// Message received after a successful subscription to channel. +// Subscription received after a successful subscription to channel. type Subscription struct { // Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe". Kind string diff --git a/redis.go b/redis.go index 37ffafd97b..ab7beb7cf0 100644 --- a/redis.go +++ b/redis.go @@ -11,7 +11,7 @@ import ( "github.com/go-redis/redis/internal/proto" ) -// Redis nil reply returned when key does not exist. +// Nil reply redis returned when key does not exist. const Nil = internal.Nil func init() { @@ -172,9 +172,9 @@ func (c *baseClient) retryBackoff(attempt int) time.Duration { func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration { if timeout := cmd.readTimeout(); timeout != nil { return *timeout - } else { - return c.opt.ReadTimeout } + + return c.opt.ReadTimeout } // Close closes the client, releasing any open resources. diff --git a/tx.go b/tx.go index 11d5d5cb00..eaea66d332 100644 --- a/tx.go +++ b/tx.go @@ -5,7 +5,7 @@ import ( "github.com/go-redis/redis/internal/pool" ) -// Redis transaction failed. +// TxFailedErr transaction redis failed. const TxFailedErr = internal.RedisError("redis: transaction failed") // Tx implements Redis transactions as described in @@ -42,7 +42,7 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { return err } -// close closes the transaction, releasing any open resources. +// Close closes the transaction, releasing any open resources. func (c *Tx) Close() error { _ = c.Unwatch().Err() return c.baseClient.Close() From fbec95a796d4200f6f88bd2bf18dc16d4acf7db4 Mon Sep 17 00:00:00 2001 From: Veselkov Konstantin Date: Thu, 25 Jan 2018 11:15:44 +0400 Subject: [PATCH 0398/1746] fix rename --- internal/proto/reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/proto/reader.go b/internal/proto/reader.go index 964519d353..e5ae8a03e3 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -197,7 +197,7 @@ func (r *Reader) ReadScanReply() ([]string, uint64, error) { return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n) } - cursor, err := p.ReadUint() + cursor, err := r.ReadUint() if err != nil { return nil, 0, err } From 8b4fa6d443e35ca8b1c37be877285252430b06a3 Mon Sep 17 00:00:00 2001 From: "yifei.huang" Date: Sat, 20 Jan 2018 18:26:33 +0800 Subject: [PATCH 0399/1746] Add WrapProcessPipeline --- cluster.go | 42 ++++++++++++--- example_instrumentation_test.go | 69 ++++++++++-------------- redis.go | 96 +++++++++++++++++++-------------- redis_context.go | 5 +- redis_no_context.go | 5 +- ring.go | 29 ++++++++-- sentinel.go | 14 ++--- tx.go | 7 +-- 8 files changed, 166 insertions(+), 101 deletions(-) diff --git a/cluster.go b/cluster.go index accdb3d272..a2c18b3877 100644 --- a/cluster.go +++ b/cluster.go @@ -445,6 +445,10 @@ type ClusterClient struct { cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo + process func(Cmder) error + processPipeline func([]Cmder) error + processTxPipeline func([]Cmder) error + // Reports whether slots reloading is in progress. reloading uint32 } @@ -458,7 +462,12 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt: opt, nodes: newClusterNodes(opt), } - c.setProcessor(c.Process) + + c.process = c.defaultProcess + c.processPipeline = c.defaultProcessPipeline + c.processTxPipeline = c.defaultProcessTxPipeline + + c.cmdable.setProcessor(c.Process) // Add initial nodes. for _, addr := range opt.Addrs { @@ -628,7 +637,20 @@ func (c *ClusterClient) Close() error { return c.nodes.Close() } +func (c *ClusterClient) WrapProcess( + fn func(oldProcess func(Cmder) error) func(Cmder) error, +) { + c.process = fn(c.process) +} + func (c *ClusterClient) Process(cmd Cmder) error { + if c.process != nil { + return c.process(cmd) + } + return c.defaultProcess(cmd) +} + +func (c *ClusterClient) defaultProcess(cmd Cmder) error { state, err := c.state() if err != nil { cmd.setErr(err) @@ -910,9 +932,9 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { func (c *ClusterClient) Pipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExec, + exec: c.processPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } @@ -920,7 +942,13 @@ func (c *ClusterClient) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().Pipelined(fn) } -func (c *ClusterClient) pipelineExec(cmds []Cmder) error { +func (c *ClusterClient) WrapProcessPipeline( + fn func(oldProcess func([]Cmder) error) func([]Cmder) error, +) { + c.processPipeline = fn(c.processPipeline) +} + +func (c *ClusterClient) defaultProcessPipeline(cmds []Cmder) error { cmdsMap, err := c.mapCmdsByNode(cmds) if err != nil { setCmdsErr(cmds, err) @@ -1064,9 +1092,9 @@ func (c *ClusterClient) checkMovedErr( // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. func (c *ClusterClient) TxPipeline() Pipeliner { pipe := Pipeline{ - exec: c.txPipelineExec, + exec: c.processTxPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } @@ -1074,7 +1102,7 @@ func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.TxPipeline().Pipelined(fn) } -func (c *ClusterClient) txPipelineExec(cmds []Cmder) error { +func (c *ClusterClient) defaultProcessTxPipeline(cmds []Cmder) error { state, err := c.state() if err != nil { return err diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go index 02051f9c9c..85abbd7440 100644 --- a/example_instrumentation_test.go +++ b/example_instrumentation_test.go @@ -2,58 +2,47 @@ package redis_test import ( "fmt" - "sync/atomic" - "time" "github.com/go-redis/redis" ) func Example_instrumentation() { - ring := redis.NewRing(&redis.RingOptions{ - Addrs: map[string]string{ - "shard1": ":6379", - }, + cl := redis.NewClient(&redis.Options{ + Addr: ":6379", }) - ring.ForEachShard(func(client *redis.Client) error { - wrapRedisProcess(client) - return nil + cl.WrapProcess(func(old func(cmd redis.Cmder) error) func(cmd redis.Cmder) error { + return func(cmd redis.Cmder) error { + fmt.Printf("starting processing: <%s>\n", cmd) + err := old(cmd) + fmt.Printf("finished processing: <%s>\n", cmd) + return err + } }) - for { - ring.Ping() - } + cl.Ping() + // Output: starting processing: + // finished processing: } -func wrapRedisProcess(client *redis.Client) { - const precision = time.Microsecond - var count, avgDur uint32 - - go func() { - for range time.Tick(3 * time.Second) { - n := atomic.LoadUint32(&count) - dur := time.Duration(atomic.LoadUint32(&avgDur)) * precision - fmt.Printf("%s: processed=%d avg_dur=%s\n", client, n, dur) - } - }() - - client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error { - return func(cmd redis.Cmder) error { - start := time.Now() - err := oldProcess(cmd) - dur := time.Since(start) - - const decay = float64(1) / 100 - ms := float64(dur / precision) - for { - avg := atomic.LoadUint32(&avgDur) - newAvg := uint32((1-decay)*float64(avg) + decay*ms) - if atomic.CompareAndSwapUint32(&avgDur, avg, newAvg) { - break - } - } - atomic.AddUint32(&count, 1) +func Example_Pipeline_instrumentation() { + client := redis.NewClient(&redis.Options{ + Addr: ":6379", + }) + client.WrapProcessPipeline(func(old func([]redis.Cmder) error) func([]redis.Cmder) error { + return func(cmds []redis.Cmder) error { + fmt.Printf("pipeline starting processing: %v\n", cmds) + err := old(cmds) + fmt.Printf("pipeline finished processing: %v\n", cmds) return err } }) + + client.Pipelined(func(pipe redis.Pipeliner) error { + pipe.Ping() + pipe.Ping() + return nil + }) + // Output: pipeline starting processing: [ping: ping: ] + // pipeline finished processing: [ping: PONG ping: PONG] } diff --git a/redis.go b/redis.go index ab7beb7cf0..cf402986de 100644 --- a/redis.go +++ b/redis.go @@ -22,6 +22,12 @@ func SetLogger(logger *log.Logger) { internal.Logger = logger } +func (c *baseClient) init() { + c.process = c.defaultProcess + c.processPipeline = c.defaultProcessPipeline + c.processTxPipeline = c.defaultProcessTxPipeline +} + func (c *baseClient) String() string { return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB) } @@ -85,7 +91,8 @@ func (c *baseClient) initConn(cn *pool.Conn) error { connPool: pool.NewSingleConnPool(cn), }, } - conn.setProcessor(conn.Process) + conn.baseClient.init() + conn.statefulCmdable.setProcessor(conn.Process) _, err := conn.Pipelined(func(pipe Pipeliner) error { if c.opt.Password != "" { @@ -117,14 +124,11 @@ func (c *baseClient) initConn(cn *pool.Conn) error { // an input and returns the new wrapper process func. createWrapper should // use call the old process func within the new process func. func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { - c.process = fn(c.defaultProcess) + c.process = fn(c.process) } func (c *baseClient) Process(cmd Cmder) error { - if c.process != nil { - return c.process(cmd) - } - return c.defaultProcess(cmd) + return c.process(cmd) } func (c *baseClient) defaultProcess(cmd Cmder) error { @@ -198,35 +202,48 @@ func (c *baseClient) getAddr() string { return c.opt.Addr } +func (c *baseClient) WrapProcessPipeline( + fn func(oldProcess func([]Cmder) error) func([]Cmder) error, +) { + c.processPipeline = fn(c.processPipeline) + c.processTxPipeline = fn(c.processTxPipeline) +} + +func (c *baseClient) defaultProcessPipeline(cmds []Cmder) error { + return c.generalProcessPipeline(cmds, c.pipelineProcessCmds) +} + +func (c *baseClient) defaultProcessTxPipeline(cmds []Cmder) error { + return c.generalProcessPipeline(cmds, c.txPipelineProcessCmds) +} + type pipelineProcessor func(*pool.Conn, []Cmder) (bool, error) -func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer { - return func(cmds []Cmder) error { - for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { - if attempt > 0 { - time.Sleep(c.retryBackoff(attempt)) - } +func (c *baseClient) generalProcessPipeline(cmds []Cmder, p pipelineProcessor) error { + for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } - cn, _, err := c.getConn() - if err != nil { - setCmdsErr(cmds, err) - return err - } + cn, _, err := c.getConn() + if err != nil { + setCmdsErr(cmds, err) + return err + } - canRetry, err := p(cn, cmds) + canRetry, err := p(cn, cmds) - if err == nil || internal.IsRedisError(err) { - _ = c.connPool.Put(cn) - break - } - _ = c.connPool.Remove(cn) + if err == nil || internal.IsRedisError(err) { + _ = c.connPool.Put(cn) + break + } + _ = c.connPool.Remove(cn) - if !canRetry || !internal.IsRetryableError(err, true) { - break - } + if !canRetry || !internal.IsRetryableError(err, true) { + break } - return firstCmdsErr(cmds) } + return firstCmdsErr(cmds) } func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) { @@ -324,14 +341,15 @@ type Client struct { } func newClient(opt *Options, pool pool.Pooler) *Client { - client := Client{ + c := Client{ baseClient: baseClient{ opt: opt, connPool: pool, }, } - client.setProcessor(client.Process) - return &client + c.baseClient.init() + c.cmdable.setProcessor(c.Process) + return &c } // NewClient returns a client to the Redis Server specified by Options. @@ -343,7 +361,7 @@ func NewClient(opt *Options) *Client { func (c *Client) copy() *Client { c2 := new(Client) *c2 = *c - c2.setProcessor(c2.Process) + c2.cmdable.setProcessor(c2.Process) return c2 } @@ -366,9 +384,9 @@ func (c *Client) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { func (c *Client) Pipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExecer(c.pipelineProcessCmds), + exec: c.processPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } @@ -379,9 +397,9 @@ func (c *Client) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. func (c *Client) TxPipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExecer(c.txPipelineProcessCmds), + exec: c.processTxPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } @@ -430,9 +448,9 @@ func (c *Conn) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { func (c *Conn) Pipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExecer(c.pipelineProcessCmds), + exec: c.processPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } @@ -443,8 +461,8 @@ func (c *Conn) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { // TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. func (c *Conn) TxPipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExecer(c.txPipelineProcessCmds), + exec: c.processTxPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } diff --git a/redis_context.go b/redis_context.go index 6ec811ca5c..c00e505f6c 100644 --- a/redis_context.go +++ b/redis_context.go @@ -12,7 +12,10 @@ type baseClient struct { connPool pool.Pooler opt *Options - process func(Cmder) error + process func(Cmder) error + processPipeline func([]Cmder) error + processTxPipeline func([]Cmder) error + onClose func() error // hook called when client is closed ctx context.Context diff --git a/redis_no_context.go b/redis_no_context.go index 0752192f15..8555c5c093 100644 --- a/redis_no_context.go +++ b/redis_no_context.go @@ -10,6 +10,9 @@ type baseClient struct { connPool pool.Pooler opt *Options - process func(Cmder) error + process func(Cmder) error + processPipeline func([]Cmder) error + processTxPipeline func([]Cmder) error + onClose func() error // hook called when client is closed } diff --git a/ring.go b/ring.go index c11ef6bc2f..10f33ed006 100644 --- a/ring.go +++ b/ring.go @@ -150,6 +150,8 @@ type Ring struct { shards map[string]*ringShard shardsList []*ringShard + processPipeline func([]Cmder) error + cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo @@ -158,7 +160,9 @@ type Ring struct { func NewRing(opt *RingOptions) *Ring { const nreplicas = 100 + opt.init() + ring := &Ring{ opt: opt, nreplicas: nreplicas, @@ -166,13 +170,17 @@ func NewRing(opt *RingOptions) *Ring { hash: consistenthash.New(nreplicas, nil), shards: make(map[string]*ringShard), } - ring.setProcessor(ring.Process) + ring.processPipeline = ring.defaultProcessPipeline + ring.cmdable.setProcessor(ring.Process) + for name, addr := range opt.Addrs { clopt := opt.clientOptions() clopt.Addr = addr ring.addShard(name, NewClient(clopt)) } + go ring.heartbeat() + return ring } @@ -354,6 +362,13 @@ func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { return c.shardByKey(firstKey) } +func (c *Ring) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { + c.ForEachShard(func(c *Client) error { + c.WrapProcess(fn) + return nil + }) +} + func (c *Ring) Process(cmd Cmder) error { shard, err := c.cmdShard(cmd) if err != nil { @@ -436,9 +451,9 @@ func (c *Ring) Close() error { func (c *Ring) Pipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExec, + exec: c.processPipeline, } - pipe.setProcessor(pipe.Process) + pipe.cmdable.setProcessor(pipe.Process) return &pipe } @@ -446,7 +461,13 @@ func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().Pipelined(fn) } -func (c *Ring) pipelineExec(cmds []Cmder) error { +func (c *Ring) WrapProcessPipeline( + fn func(oldProcess func([]Cmder) error) func([]Cmder) error, +) { + c.processPipeline = fn(c.processPipeline) +} + +func (c *Ring) defaultProcessPipeline(cmds []Cmder) error { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { cmdInfo := c.cmdInfo(cmd.Name()) diff --git a/sentinel.go b/sentinel.go index 37d06b4821..3f56f08b33 100644 --- a/sentinel.go +++ b/sentinel.go @@ -76,7 +76,7 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { opt: opt, } - client := Client{ + c := Client{ baseClient: baseClient{ opt: opt, connPool: failover.Pool(), @@ -86,9 +86,10 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { }, }, } - client.setProcessor(client.Process) + c.baseClient.init() + c.setProcessor(c.Process) - return &client + return &c } //------------------------------------------------------------------------------ @@ -100,14 +101,15 @@ type sentinelClient struct { func newSentinel(opt *Options) *sentinelClient { opt.init() - client := sentinelClient{ + c := sentinelClient{ baseClient: baseClient{ opt: opt, connPool: newConnPool(opt), }, } - client.cmdable = cmdable{client.Process} - return &client + c.baseClient.init() + c.cmdable.setProcessor(c.Process) + return &c } func (c *sentinelClient) PubSub() *PubSub { diff --git a/tx.go b/tx.go index eaea66d332..26c29bef55 100644 --- a/tx.go +++ b/tx.go @@ -24,7 +24,8 @@ func (c *Client) newTx() *Tx { connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true), }, } - tx.setProcessor(tx.Process) + tx.baseClient.init() + tx.statefulCmdable.setProcessor(tx.Process) return &tx } @@ -75,9 +76,9 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd { func (c *Tx) Pipeline() Pipeliner { pipe := Pipeline{ - exec: c.pipelineExecer(c.txPipelineProcessCmds), + exec: c.processTxPipeline, } - pipe.setProcessor(pipe.Process) + pipe.statefulCmdable.setProcessor(pipe.Process) return &pipe } From 1a4d34448d71983c99178509f18ef683762d4584 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 12 Feb 2018 16:15:40 +0200 Subject: [PATCH 0400/1746] Update cluster node latency asynchronously --- cluster.go | 53 ++++++++++++++++++++++++++++++++++++++---------- cluster_test.go | 12 +++++++++-- commands_test.go | 6 +++--- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/cluster.go b/cluster.go index a2c18b3877..628f513edc 100644 --- a/cluster.go +++ b/cluster.go @@ -2,6 +2,7 @@ package redis import ( "fmt" + "math" "math/rand" "net" "sync" @@ -118,11 +119,11 @@ func (opt *ClusterOptions) clientOptions() *Options { //------------------------------------------------------------------------------ type clusterNode struct { - Client *Client - Latency time.Duration + Client *Client - loading time.Time + latency uint32 // atomic generation uint32 + loading int64 // atomic } func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { @@ -132,8 +133,9 @@ func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { Client: NewClient(opt), } + node.latency = math.MaxUint32 if clOpt.RouteByLatency { - node.updateLatency() + go node.updateLatency() } return &node @@ -141,16 +143,46 @@ func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { func (n *clusterNode) updateLatency() { const probes = 10 + + var latency uint32 for i := 0; i < probes; i++ { start := time.Now() n.Client.Ping() - n.Latency += time.Since(start) + probe := uint32(time.Since(start) / time.Microsecond) + latency = (latency + probe) / 2 } - n.Latency = n.Latency / probes + atomic.StoreUint32(&n.latency, latency) +} + +func (n *clusterNode) Close() error { + return n.Client.Close() +} + +func (n *clusterNode) Test() error { + return n.Client.ClusterInfo().Err() +} + +func (n *clusterNode) Latency() time.Duration { + latency := atomic.LoadUint32(&n.latency) + return time.Duration(latency) * time.Microsecond +} + +func (n *clusterNode) MarkAsLoading() { + atomic.StoreInt64(&n.loading, time.Now().Unix()) } func (n *clusterNode) Loading() bool { - return !n.loading.IsZero() && time.Since(n.loading) < time.Minute + const minute = int64(time.Minute / time.Second) + + loading := atomic.LoadInt64(&n.loading) + if loading == 0 { + return false + } + if time.Now().Unix()-loading < minute { + return true + } + atomic.StoreInt64(&n.loading, 0) + return false } func (n *clusterNode) Generation() uint32 { @@ -310,7 +342,7 @@ func (c *clusterNodes) Random() (*clusterNode, error) { return nil, err } - nodeErr = node.Client.ClusterInfo().Err() + nodeErr = node.Test() if nodeErr == nil { return node, nil } @@ -416,7 +448,7 @@ func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { if n.Loading() { continue } - if node == nil || node.Latency-n.Latency > threshold { + if node == nil || node.Latency()-n.Latency() > threshold { node = n } } @@ -687,8 +719,7 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { // If slave is loading - read from master. if c.opt.ReadOnly && internal.IsLoadingError(err) { - // TODO: race - node.loading = time.Now() + node.MarkAsLoading() continue } diff --git a/cluster_test.go b/cluster_test.go index 43f3261bcd..a142f8c06f 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -320,6 +320,14 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmds).To(HaveLen(14)) + _ = client.ForEachNode(func(node *redis.Client) error { + defer GinkgoRecover() + Eventually(func() int64 { + return node.DBSize().Val() + }, 30*time.Second).ShouldNot(BeZero()) + return nil + }) + for _, key := range keys { slot := hashtag.Slot(key) client.SwapSlotNodes(slot) @@ -576,7 +584,7 @@ var _ = Describe("ClusterClient", func() { _ = client.ForEachSlave(func(slave *redis.Client) error { Eventually(func() int64 { - return client.DBSize().Val() + return slave.DBSize().Val() }, 30*time.Second).Should(Equal(int64(0))) return slave.ClusterFailover().Err() }) @@ -717,7 +725,7 @@ var _ = Describe("ClusterClient timeout", func() { }) } - const pause = time.Second + const pause = 2 * time.Second Context("read/write timeout", func() { BeforeEach(func() { diff --git a/commands_test.go b/commands_test.go index 7153795569..067ffccdfb 100644 --- a/commands_test.go +++ b/commands_test.go @@ -447,7 +447,7 @@ var _ = Describe("Commands", func() { pttl := client.PTTL("key") Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val()).To(BeNumerically("~", expiration, 10*time.Millisecond)) + Expect(pttl.Val()).To(BeNumerically("~", expiration, 100*time.Millisecond)) }) It("should PExpireAt", func() { @@ -466,7 +466,7 @@ var _ = Describe("Commands", func() { pttl := client.PTTL("key") Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val()).To(BeNumerically("~", expiration, 10*time.Millisecond)) + Expect(pttl.Val()).To(BeNumerically("~", expiration, 100*time.Millisecond)) }) It("should PTTL", func() { @@ -481,7 +481,7 @@ var _ = Describe("Commands", func() { pttl := client.PTTL("key") Expect(pttl.Err()).NotTo(HaveOccurred()) - Expect(pttl.Val()).To(BeNumerically("~", expiration, 10*time.Millisecond)) + Expect(pttl.Val()).To(BeNumerically("~", expiration, 100*time.Millisecond)) }) It("should RandomKey", func() { From e456ee71487b9fa3e21ca85e5a5c4ac40602aa53 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 13 Feb 2018 16:08:11 +0200 Subject: [PATCH 0401/1746] Add SortStore and fix Sort signature --- commands.go | 29 ++++++++++++++++++----------- commands_test.go | 32 +++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/commands.go b/commands.go index 569342cfa1..545fcc03aa 100644 --- a/commands.go +++ b/commands.go @@ -70,8 +70,9 @@ type Cmdable interface { RenameNX(key, newkey string) *BoolCmd Restore(key string, ttl time.Duration, value string) *StatusCmd RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd - Sort(key string, sort Sort) *StringSliceCmd - SortInterfaces(key string, sort Sort) *SliceCmd + Sort(key string, sort *Sort) *StringSliceCmd + SortStore(key, store string, sort *Sort) *IntCmd + SortInterfaces(key string, sort *Sort) *SliceCmd TTL(key string) *DurationCmd Type(key string) *StatusCmd Scan(cursor uint64, match string, count int64) *ScanCmd @@ -484,11 +485,10 @@ func (c *cmdable) RestoreReplace(key string, ttl time.Duration, value string) *S type Sort struct { By string - Offset, Count float64 + Offset, Count int64 Get []string Order string - IsAlpha bool - Store string + Alpha bool } func (sort *Sort) args(key string) []interface{} { @@ -505,22 +505,29 @@ func (sort *Sort) args(key string) []interface{} { if sort.Order != "" { args = append(args, sort.Order) } - if sort.IsAlpha { + if sort.Alpha { args = append(args, "alpha") } - if sort.Store != "" { - args = append(args, "store", sort.Store) - } return args } -func (c *cmdable) Sort(key string, sort Sort) *StringSliceCmd { +func (c *cmdable) Sort(key string, sort *Sort) *StringSliceCmd { cmd := NewStringSliceCmd(sort.args(key)...) c.process(cmd) return cmd } -func (c *cmdable) SortInterfaces(key string, sort Sort) *SliceCmd { +func (c *cmdable) SortStore(key, store string, sort *Sort) *IntCmd { + args := sort.args(key) + if store != "" { + args = append(args, "store", store) + } + cmd := NewIntCmd(args...) + c.process(cmd) + return cmd +} + +func (c *cmdable) SortInterfaces(key string, sort *Sort) *SliceCmd { cmd := NewSliceCmd(sort.args(key)...) c.process(cmd) return cmd diff --git a/commands_test.go b/commands_test.go index 067ffccdfb..eb6c7a7759 100644 --- a/commands_test.go +++ b/commands_test.go @@ -582,7 +582,7 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(size).To(Equal(int64(3))) - els, err := client.Sort("list", redis.Sort{ + els, err := client.Sort("list", &redis.Sort{ Offset: 0, Count: 2, Order: "ASC", @@ -608,7 +608,7 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) { - els, err := client.Sort("list", redis.Sort{ + els, err := client.Sort("list", &redis.Sort{ Get: []string{"object_*"}, }).Result() Expect(err).NotTo(HaveOccurred()) @@ -616,7 +616,7 @@ var _ = Describe("Commands", func() { } { - els, err := client.SortInterfaces("list", redis.Sort{ + els, err := client.SortInterfaces("list", &redis.Sort{ Get: []string{"object_*"}, }).Result() Expect(err).NotTo(HaveOccurred()) @@ -624,6 +624,32 @@ var _ = Describe("Commands", func() { } }) + It("should Sort and Store", func() { + size, err := client.LPush("list", "1").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(1))) + + size, err = client.LPush("list", "3").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(2))) + + size, err = client.LPush("list", "2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(size).To(Equal(int64(3))) + + n, err := client.SortStore("list", "list2", &redis.Sort{ + Offset: 0, + Count: 2, + Order: "ASC", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(2))) + + els, err := client.LRange("list2", 0, -1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(els).To(Equal([]string{"1", "2"})) + }) + It("should TTL", func() { ttl := client.TTL("key") Expect(ttl.Err()).NotTo(HaveOccurred()) From 6b10d46cfb7c2ce98385ade576eea5507693f463 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 13 Feb 2018 16:23:49 +0200 Subject: [PATCH 0402/1746] Fix tests --- cluster_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster_test.go b/cluster_test.go index a142f8c06f..8ca922c4fa 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -583,6 +583,7 @@ var _ = Describe("ClusterClient", func() { }) _ = client.ForEachSlave(func(slave *redis.Client) error { + defer GinkgoRecover() Eventually(func() int64 { return slave.DBSize().Val() }, 30*time.Second).Should(Equal(int64(0))) From daab7c60d0a516845ee36c9807dc54f43e8c31e2 Mon Sep 17 00:00:00 2001 From: Huan Du Date: Wed, 14 Feb 2018 12:42:19 +0800 Subject: [PATCH 0403/1746] add new command Touch and SwapDB --- commands.go | 19 +++++++++++++++++++ commands_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/commands.go b/commands.go index 545fcc03aa..92a8b7a5e3 100644 --- a/commands.go +++ b/commands.go @@ -73,6 +73,7 @@ type Cmdable interface { Sort(key string, sort *Sort) *StringSliceCmd SortStore(key, store string, sort *Sort) *IntCmd SortInterfaces(key string, sort *Sort) *SliceCmd + Touch(keys ...string) *IntCmd TTL(key string) *DurationCmd Type(key string) *StatusCmd Scan(cursor uint64, match string, count int64) *ScanCmd @@ -253,6 +254,7 @@ type StatefulCmdable interface { Cmdable Auth(password string) *StatusCmd Select(index int) *StatusCmd + SwapDB(index1, index2 int) *StatusCmd ClientSetName(name string) *BoolCmd ReadOnly() *StatusCmd ReadWrite() *StatusCmd @@ -317,6 +319,12 @@ func (c *statefulCmdable) Select(index int) *StatusCmd { return cmd } +func (c *statefulCmdable) SwapDB(index1, index2 int) *StatusCmd { + cmd := NewStatusCmd("swapdb", index1, index2) + c.process(cmd) + return cmd +} + //------------------------------------------------------------------------------ func (c *cmdable) Del(keys ...string) *IntCmd { @@ -533,6 +541,17 @@ func (c *cmdable) SortInterfaces(key string, sort *Sort) *SliceCmd { return cmd } +func (c *cmdable) Touch(keys ...string) *IntCmd { + args := make([]interface{}, len(keys)+1) + args[0] = "touch" + for i, key := range keys { + args[i+1] = key + } + cmd := NewIntCmd(args...) + c.process(cmd) + return cmd +} + func (c *cmdable) TTL(key string) *DurationCmd { cmd := NewDurationCmd(time.Second, "ttl", key) c.process(cmd) diff --git a/commands_test.go b/commands_test.go index eb6c7a7759..5b3ee97ec5 100644 --- a/commands_test.go +++ b/commands_test.go @@ -79,6 +79,16 @@ var _ = Describe("Commands", func() { Expect(sel.Val()).To(Equal("OK")) }) + It("should SwapDB", func() { + pipe := client.Pipeline() + sel := pipe.SwapDB(1, 2) + _, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + + Expect(sel.Err()).NotTo(HaveOccurred()) + Expect(sel.Val()).To(Equal("OK")) + }) + It("should BgRewriteAOF", func() { Skip("flaky test") @@ -650,6 +660,20 @@ var _ = Describe("Commands", func() { Expect(els).To(Equal([]string{"1", "2"})) }) + It("should Touch", func() { + set1 := client.Set("touch1", "hello", 0) + Expect(set1.Err()).NotTo(HaveOccurred()) + Expect(set1.Val()).To(Equal("OK")) + + set2 := client.Set("touch2", "hello", 0) + Expect(set2.Err()).NotTo(HaveOccurred()) + Expect(set2.Val()).To(Equal("OK")) + + touch := client.Touch("touch1", "touch2", "touch3") + Expect(touch.Err()).NotTo(HaveOccurred()) + Expect(touch.Val()).To(Equal(int64(2))) + }) + It("should TTL", func() { ttl := client.TTL("key") Expect(ttl.Err()).NotTo(HaveOccurred()) From 4598ed0eac6a41e5df6e9657c8488db766fe0566 Mon Sep 17 00:00:00 2001 From: Huan Du Date: Fri, 16 Feb 2018 19:39:56 +0800 Subject: [PATCH 0404/1746] Correctly parse EVAL response containing customized error (#710) * always parse all content in an array including error --- commands_test.go | 10 ++++++++++ parser.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/commands_test.go b/commands_test.go index 5b3ee97ec5..524c3ac71c 100644 --- a/commands_test.go +++ b/commands_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/gomega" "github.com/go-redis/redis" + "github.com/go-redis/redis/internal" ) var _ = Describe("Commands", func() { @@ -2993,6 +2994,15 @@ var _ = Describe("Commands", func() { Expect(vals).To(Equal([]interface{}{"key", "hello"})) }) + It("returns all values after an error", func() { + vals, err := client.Eval( + `return {12, {err="error"}, "abc"}`, + nil, + ).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]interface{}{int64(12), internal.RedisError("error"), "abc"})) + }) + }) }) diff --git a/parser.go b/parser.go index b378abc4ee..e77efeeb97 100644 --- a/parser.go +++ b/parser.go @@ -17,7 +17,7 @@ func sliceParser(rd *proto.Reader, n int64) (interface{}, error) { if err == Nil { vals = append(vals, nil) } else if err != nil { - return nil, err + vals = append(vals, err) } else { switch vv := v.(type) { case []byte: From fa7f64f7f27348658ecfa71dd71dfb2d112e2f86 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 15 Feb 2018 13:00:54 +0200 Subject: [PATCH 0405/1746] Rework retrying --- cluster.go | 246 ++++++++++++++------------ cluster_test.go | 28 +-- internal/error.go | 4 + internal/singleflight/singleflight.go | 64 +++++++ 4 files changed, 216 insertions(+), 126 deletions(-) create mode 100644 internal/singleflight/singleflight.go diff --git a/cluster.go b/cluster.go index 628f513edc..ea677ae213 100644 --- a/cluster.go +++ b/cluster.go @@ -13,10 +13,10 @@ import ( "github.com/go-redis/redis/internal/hashtag" "github.com/go-redis/redis/internal/pool" "github.com/go-redis/redis/internal/proto" + "github.com/go-redis/redis/internal/singleflight" ) var errClusterNoNodes = fmt.Errorf("redis: cluster has no nodes") -var errNilClusterState = fmt.Errorf("redis: cannot load cluster slots") // ClusterOptions are used to configure a cluster client and should be // passed to NewClusterClient. @@ -122,8 +122,8 @@ type clusterNode struct { Client *Client latency uint32 // atomic - generation uint32 - loading int64 // atomic + generation uint32 // atomic + loading int64 // atomic } func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { @@ -141,6 +141,14 @@ func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { return &node } +func (n *clusterNode) Close() error { + return n.Client.Close() +} + +func (n *clusterNode) Test() error { + return n.Client.ClusterInfo().Err() +} + func (n *clusterNode) updateLatency() { const probes = 10 @@ -154,14 +162,6 @@ func (n *clusterNode) updateLatency() { atomic.StoreUint32(&n.latency, latency) } -func (n *clusterNode) Close() error { - return n.Client.Close() -} - -func (n *clusterNode) Test() error { - return n.Client.ClusterInfo().Err() -} - func (n *clusterNode) Latency() time.Duration { latency := atomic.LoadUint32(&n.latency) return time.Duration(latency) * time.Microsecond @@ -186,14 +186,16 @@ func (n *clusterNode) Loading() bool { } func (n *clusterNode) Generation() uint32 { - return n.generation + return atomic.LoadUint32(&n.generation) } func (n *clusterNode) SetGeneration(gen uint32) { - if gen < n.generation { - panic("gen < n.generation") + for { + v := atomic.LoadUint32(&n.generation) + if gen < v || atomic.CompareAndSwapUint32(&n.generation, v, gen) { + break + } } - n.generation = gen } //------------------------------------------------------------------------------ @@ -201,18 +203,23 @@ func (n *clusterNode) SetGeneration(gen uint32) { type clusterNodes struct { opt *ClusterOptions - mu sync.RWMutex - addrs []string - nodes map[string]*clusterNode - closed bool + mu sync.RWMutex + allAddrs []string + addrs []string + nodes map[string]*clusterNode + closed bool + + nodeCreateGroup singleflight.Group generation uint32 } func newClusterNodes(opt *ClusterOptions) *clusterNodes { return &clusterNodes{ - opt: opt, - nodes: make(map[string]*clusterNode), + opt: opt, + + allAddrs: opt.Addrs, + nodes: make(map[string]*clusterNode), } } @@ -231,6 +238,7 @@ func (c *clusterNodes) Close() error { firstErr = err } } + c.addrs = nil c.nodes = nil @@ -238,9 +246,16 @@ func (c *clusterNodes) Close() error { } func (c *clusterNodes) Addrs() ([]string, error) { + var addrs []string c.mu.RLock() closed := c.closed - addrs := c.addrs + if !closed { + if len(c.addrs) > 0 { + addrs = c.addrs + } else { + addrs = c.allAddrs + } + } c.mu.RUnlock() if closed { @@ -310,6 +325,14 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { return node, nil } + v, err := c.nodeCreateGroup.Do(addr, func() (interface{}, error) { + node := newClusterNode(c.opt, addr) + return node, node.Test() + }) + if err != nil { + return nil, err + } + c.mu.Lock() defer c.mu.Unlock() @@ -319,12 +342,15 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { node, ok = c.nodes[addr] if ok { + _ = v.(*clusterNode).Close() return node, nil } + node = v.(*clusterNode) + c.allAddrs = appendIfNotExists(c.allAddrs, addr) c.addrs = append(c.addrs, addr) - node = newClusterNode(c.opt, addr) c.nodes[addr] = node + return node, nil } @@ -334,20 +360,8 @@ func (c *clusterNodes) Random() (*clusterNode, error) { return nil, err } - var nodeErr error - for i := 0; i <= c.opt.MaxRedirects; i++ { - n := rand.Intn(len(addrs)) - node, err := c.GetOrCreate(addrs[n]) - if err != nil { - return nil, err - } - - nodeErr = node.Test() - if nodeErr == nil { - return node, nil - } - } - return nil, nodeErr + n := rand.Intn(len(addrs)) + return c.GetOrCreate(addrs[n]) } //------------------------------------------------------------------------------ @@ -470,9 +484,12 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode { type ClusterClient struct { cmdable - opt *ClusterOptions - nodes *clusterNodes - _state atomic.Value + opt *ClusterOptions + nodes *clusterNodes + + _state atomic.Value + stateErrMu sync.RWMutex + stateErr error cmdsInfoOnce internal.Once cmdsInfo map[string]*CommandInfo @@ -501,20 +518,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { c.cmdable.setProcessor(c.Process) - // Add initial nodes. - for _, addr := range opt.Addrs { - _, _ = c.nodes.GetOrCreate(addr) - } - - // Preload cluster slots. - for i := 0; i < 10; i++ { - state, err := c.reloadState() - if err == nil { - c._state.Store(state) - break - } - } - + c.reloadState() if opt.IdleCheckFrequency > 0 { go c.reaper(opt.IdleCheckFrequency) } @@ -531,21 +535,6 @@ func (c *ClusterClient) retryBackoff(attempt int) time.Duration { return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) } -func (c *ClusterClient) state() (*clusterState, error) { - v := c._state.Load() - if v != nil { - return v.(*clusterState), nil - } - - _, err := c.nodes.Addrs() - if err != nil { - return nil, err - } - - c.lazyReloadState() - return nil, errNilClusterState -} - func (c *ClusterClient) cmdInfo(name string) *CommandInfo { err := c.cmdsInfoOnce.Do(func() error { node, err := c.nodes.Random() @@ -584,7 +573,12 @@ func (c *ClusterClient) cmdSlot(cmd Cmder) int { return cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) } -func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *clusterNode, error) { +func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { + state, err := c.state() + if err != nil { + return 0, nil, err + } + cmdInfo := c.cmdInfo(cmd.Name()) slot := cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) @@ -602,16 +596,24 @@ func (c *ClusterClient) cmdSlotAndNode(state *clusterState, cmd Cmder) (int, *cl return slot, node, err } +func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { + state, err := c.state() + if err != nil { + return nil, err + } + + nodes := state.slotNodes(slot) + if len(nodes) > 0 { + return nodes[0], nil + } + return c.nodes.Random() +} + func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { if len(keys) == 0 { return fmt.Errorf("redis: keys don't hash to the same slot") } - state, err := c.state() - if err != nil { - return err - } - slot := hashtag.Slot(keys[0]) for _, key := range keys[1:] { if hashtag.Slot(key) != slot { @@ -619,7 +621,7 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { } } - node, err := state.slotMasterNode(slot) + node, err := c.slotMasterNode(slot) if err != nil { return err } @@ -649,10 +651,11 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { } if err == pool.ErrClosed { - node, err = state.slotMasterNode(slot) + node, err = c.slotMasterNode(slot) if err != nil { return err } + continue } return err @@ -683,13 +686,7 @@ func (c *ClusterClient) Process(cmd Cmder) error { } func (c *ClusterClient) defaultProcess(cmd Cmder) error { - state, err := c.state() - if err != nil { - cmd.setErr(err) - return err - } - - _, node, err := c.cmdSlotAndNode(state, cmd) + _, node, err := c.cmdSlotAndNode(cmd) if err != nil { cmd.setErr(err) return err @@ -747,11 +744,12 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { } if err == pool.ErrClosed { - _, node, err = c.cmdSlotAndNode(state, cmd) + _, node, err = c.cmdSlotAndNode(cmd) if err != nil { cmd.setErr(err) - return err + break } + continue } break @@ -903,31 +901,37 @@ func (c *ClusterClient) lazyReloadState() { if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { return } - go func() { - defer atomic.StoreUint32(&c.reloading, 0) - - for { - state, err := c.reloadState() - if err == pool.ErrClosed { - return - } + if c.reloadState() { + time.Sleep(time.Second) + } + atomic.StoreUint32(&c.reloading, 0) + }() +} - if err != nil { - time.Sleep(time.Millisecond) - continue - } +func (c *ClusterClient) reloadState() bool { + for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { + if attempt > 0 { + time.Sleep(c.retryBackoff(attempt)) + } + state, err := c.loadState() + if err == nil { c._state.Store(state) - time.Sleep(5 * time.Second) c.nodes.GC(state.generation) - break + return true } - }() + + c.setStateErr(err) + switch err { + case pool.ErrClosed, errClusterNoNodes: + return false + } + } + return false } -// Not thread-safe. -func (c *ClusterClient) reloadState() (*clusterState, error) { +func (c *ClusterClient) loadState() (*clusterState, error) { node, err := c.nodes.Random() if err != nil { return nil, err @@ -941,6 +945,27 @@ func (c *ClusterClient) reloadState() (*clusterState, error) { return newClusterState(c.nodes, slots, node.Client.opt.Addr) } +func (c *ClusterClient) state() (*clusterState, error) { + v := c._state.Load() + if v != nil { + return v.(*clusterState), nil + } + return nil, c.getStateErr() +} + +func (c *ClusterClient) setStateErr(err error) { + c.stateErrMu.Lock() + c.stateErr = err + c.stateErrMu.Unlock() +} + +func (c *ClusterClient) getStateErr() error { + c.stateErrMu.RLock() + err := c.stateErr + c.stateErrMu.RUnlock() + return err +} + // reaper closes idle connections to the cluster. func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { ticker := time.NewTicker(idleCheckFrequency) @@ -1055,15 +1080,17 @@ func (c *ClusterClient) remapCmds(cmds []Cmder, failedCmds map[*clusterNode][]Cm func (c *ClusterClient) pipelineProcessCmds( node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) error { - cn.SetWriteTimeout(c.opt.WriteTimeout) - if err := writeCmd(cn, cmds...); err != nil { + _ = cn.SetWriteTimeout(c.opt.WriteTimeout) + + err := writeCmd(cn, cmds...) + if err != nil { setCmdsErr(cmds, err) failedCmds[node] = cmds return err } // Set read timeout for all commands. - cn.SetReadTimeout(c.opt.ReadTimeout) + _ = cn.SetReadTimeout(c.opt.ReadTimeout) return c.pipelineReadCmds(cn, cmds, failedCmds) } @@ -1280,12 +1307,7 @@ func (c *ClusterClient) pubSub(channels []string) *PubSub { slot = -1 } - state, err := c.state() - if err != nil { - return nil, err - } - - masterNode, err := state.slotMasterNode(slot) + masterNode, err := c.slotMasterNode(slot) if err != nil { return nil, err } diff --git a/cluster_test.go b/cluster_test.go index 8ca922c4fa..61cac99850 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -190,13 +190,11 @@ var _ = Describe("ClusterClient", func() { assertClusterClient := func() { It("should GET/SET/DEL", func() { - val, err := client.Get("A").Result() + err := client.Get("A").Err() Expect(err).To(Equal(redis.Nil)) - Expect(val).To(Equal("")) - val, err = client.Set("A", "VALUE", 0).Result() + err = client.Set("A", "VALUE", 0).Err() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal("OK")) Eventually(func() string { return client.Get("A").Val() @@ -295,9 +293,9 @@ var _ = Describe("ClusterClient", func() { } wg.Wait() - n, err := client.Get("key").Int64() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(100))) + Eventually(func() string { + return client.Get("key").Val() + }, 30*time.Second).Should(Equal("100")) }) Describe("pipelining", func() { @@ -578,12 +576,13 @@ var _ = Describe("ClusterClient", func() { opt = redisClusterOptions() client = cluster.clusterClient(opt) - _ = client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDB().Err() - }) - _ = client.ForEachSlave(func(slave *redis.Client) error { defer GinkgoRecover() + + _ = client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) + Eventually(func() int64 { return slave.DBSize().Val() }, 30*time.Second).Should(Equal(int64(0))) @@ -670,7 +669,7 @@ var _ = Describe("ClusterClient without valid nodes", func() { It("returns an error", func() { err := client.Ping().Err() - Expect(err).To(MatchError("redis: cannot load cluster slots")) + Expect(err).To(MatchError("ERR This instance has cluster support disabled")) }) It("pipeline returns an error", func() { @@ -678,7 +677,7 @@ var _ = Describe("ClusterClient without valid nodes", func() { pipe.Ping() return nil }) - Expect(err).To(MatchError("redis: cannot load cluster slots")) + Expect(err).To(MatchError("ERR This instance has cluster support disabled")) }) }) @@ -743,7 +742,8 @@ var _ = Describe("ClusterClient timeout", func() { }) AfterEach(func() { - client.ForEachNode(func(client *redis.Client) error { + _ = client.ForEachNode(func(client *redis.Client) error { + defer GinkgoRecover() Eventually(func() error { return client.Ping().Err() }, 2*pause).ShouldNot(HaveOccurred()) diff --git a/internal/error.go b/internal/error.go index 0898eeb622..c190f46989 100644 --- a/internal/error.go +++ b/internal/error.go @@ -42,6 +42,10 @@ func IsNetworkError(err error) bool { return ok } +func IsReadOnlyError(err error) bool { + return strings.HasPrefix(err.Error(), "READONLY ") +} + func IsBadConn(err error, allowTimeout bool) bool { if err == nil { return false diff --git a/internal/singleflight/singleflight.go b/internal/singleflight/singleflight.go new file mode 100644 index 0000000000..3b17417245 --- /dev/null +++ b/internal/singleflight/singleflight.go @@ -0,0 +1,64 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight + +import "sync" + +// call is an in-flight or completed Do call +type call struct { + wg sync.WaitGroup + val interface{} + err error +} + +// Group represents a class of work and forms a namespace in which +// units of work can be executed with duplicate suppression. +type Group struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + g.mu.Unlock() + c.wg.Wait() + return c.val, c.err + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + c.val, c.err = fn() + c.wg.Done() + + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() + + return c.val, c.err +} From fd2200b0515bd30a38d5019afcec09f18665883d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 20 Feb 2018 10:05:11 +0200 Subject: [PATCH 0406/1746] Add TxPipeline race test --- race_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/race_test.go b/race_test.go index 14264086cd..05f338c78e 100644 --- a/race_test.go +++ b/race_test.go @@ -243,6 +243,21 @@ var _ = Describe("races", func() { Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(N))) }) + + It("should TxPipeline", func() { + pipe := client.TxPipeline() + perform(N, func(id int) { + pipe.Incr("key") + }) + + cmds, err := pipe.Exec() + Expect(err).NotTo(HaveOccurred()) + Expect(cmds).To(HaveLen(N)) + + n, err := client.Get("key").Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(N))) + }) }) func bigVal() []byte { From 036680734d7ad94dad96bda7a537174e47f79a89 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 20 Feb 2018 10:14:24 +0200 Subject: [PATCH 0407/1746] Increase backoff in failover tests --- cluster_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster_test.go b/cluster_test.go index 61cac99850..6e0d7c089e 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -574,6 +574,8 @@ var _ = Describe("ClusterClient", func() { Describe("ClusterClient failover", func() { BeforeEach(func() { opt = redisClusterOptions() + opt.MinRetryBackoff = 100 * time.Millisecond + opt.MaxRetryBackoff = 3 * time.Second client = cluster.clusterClient(opt) _ = client.ForEachSlave(func(slave *redis.Client) error { From 4e89aeeae456e29942003f3debb6e71b0f189c86 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 20 Feb 2018 10:28:34 +0200 Subject: [PATCH 0408/1746] Increase timeout --- cluster_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 6e0d7c089e..fd21bbe0e7 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -727,13 +727,13 @@ var _ = Describe("ClusterClient timeout", func() { }) } - const pause = 2 * time.Second + const pause = 3 * time.Second Context("read/write timeout", func() { BeforeEach(func() { opt := redisClusterOptions() - opt.ReadTimeout = 100 * time.Millisecond - opt.WriteTimeout = 100 * time.Millisecond + opt.ReadTimeout = 200 * time.Millisecond + opt.WriteTimeout = 200 * time.Millisecond opt.MaxRedirects = 1 client = cluster.clusterClient(opt) From 71ed499c465106db0d1457f19d2cb4bea8b1106e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 20 Feb 2018 11:00:19 +0200 Subject: [PATCH 0409/1746] Decrease accurracy --- cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_test.go b/cluster_test.go index fd21bbe0e7..076f4b674c 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -345,7 +345,7 @@ var _ = Describe("ClusterClient", func() { ttl := cmds[(i*2)+1].(*redis.DurationCmd) dur := time.Duration(i+1) * time.Hour - Expect(ttl.Val()).To(BeNumerically("~", dur, 5*time.Second)) + Expect(ttl.Val()).To(BeNumerically("~", dur, 10*time.Second)) } }) From 56dea1f39a5f4a64fe8c639464c966c75f226e3e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 22 Feb 2018 14:14:30 +0200 Subject: [PATCH 0410/1746] Fix proto.RedisError in slices --- cluster_test.go | 4 +-- command.go | 3 +- commands.go | 3 +- commands_test.go | 4 +-- internal/error.go | 10 ++---- internal/proto/reader.go | 48 ++++++++++++-------------- internal/proto/scan.go | 63 ++++++++++++++++++++++++++--------- internal/util.go | 37 ++------------------ internal/{ => util}/safe.go | 2 +- internal/util/strconv.go | 19 +++++++++++ internal/{ => util}/unsafe.go | 2 +- parser.go | 26 +++++++++------ redis.go | 2 +- tx.go | 4 +-- 14 files changed, 122 insertions(+), 105 deletions(-) rename internal/{ => util}/safe.go (82%) create mode 100644 internal/util/strconv.go rename internal/{ => util}/unsafe.go (91%) diff --git a/cluster_test.go b/cluster_test.go index 076f4b674c..2bf5d91617 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -574,8 +574,8 @@ var _ = Describe("ClusterClient", func() { Describe("ClusterClient failover", func() { BeforeEach(func() { opt = redisClusterOptions() - opt.MinRetryBackoff = 100 * time.Millisecond - opt.MaxRetryBackoff = 3 * time.Second + opt.MinRetryBackoff = 250 * time.Millisecond + opt.MaxRetryBackoff = time.Second client = cluster.clusterClient(opt) _ = client.ForEachSlave(func(slave *redis.Client) error { diff --git a/command.go b/command.go index 480a5ce19a..7d45c3fada 100644 --- a/command.go +++ b/command.go @@ -10,6 +10,7 @@ import ( "github.com/go-redis/redis/internal" "github.com/go-redis/redis/internal/pool" "github.com/go-redis/redis/internal/proto" + "github.com/go-redis/redis/internal/util" ) type Cmder interface { @@ -436,7 +437,7 @@ func NewStringCmd(args ...interface{}) *StringCmd { } func (cmd *StringCmd) Val() string { - return internal.BytesToString(cmd.val) + return util.BytesToString(cmd.val) } func (cmd *StringCmd) Result() (string, error) { diff --git a/commands.go b/commands.go index 92a8b7a5e3..45bb0aed14 100644 --- a/commands.go +++ b/commands.go @@ -1,6 +1,7 @@ package redis import ( + "errors" "io" "time" @@ -1802,7 +1803,7 @@ func (c *cmdable) shutdown(modifier string) *StatusCmd { } } else { // Server did not quit. String reply contains the reason. - cmd.err = internal.RedisError(cmd.val) + cmd.err = errors.New(cmd.val) cmd.val = "" } return cmd diff --git a/commands_test.go b/commands_test.go index 524c3ac71c..b173e0d4e9 100644 --- a/commands_test.go +++ b/commands_test.go @@ -10,7 +10,7 @@ import ( . "github.com/onsi/gomega" "github.com/go-redis/redis" - "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/proto" ) var _ = Describe("Commands", func() { @@ -3000,7 +3000,7 @@ var _ = Describe("Commands", func() { nil, ).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]interface{}{int64(12), internal.RedisError("error"), "abc"})) + Expect(vals).To(Equal([]interface{}{int64(12), proto.RedisError("error"), "abc"})) }) }) diff --git a/internal/error.go b/internal/error.go index c190f46989..7b419577ed 100644 --- a/internal/error.go +++ b/internal/error.go @@ -4,13 +4,9 @@ import ( "io" "net" "strings" -) - -const Nil = RedisError("redis: nil") -type RedisError string - -func (e RedisError) Error() string { return string(e) } + "github.com/go-redis/redis/internal/proto" +) func IsRetryableError(err error, retryNetError bool) bool { if IsNetworkError(err) { @@ -30,7 +26,7 @@ func IsRetryableError(err error, retryNetError bool) bool { } func IsRedisError(err error) bool { - _, ok := err.(RedisError) + _, ok := err.(proto.RedisError) return ok } diff --git a/internal/proto/reader.go b/internal/proto/reader.go index e5ae8a03e3..d5d6953585 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -6,7 +6,7 @@ import ( "io" "strconv" - "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/util" ) const bytesAllocLimit = 1024 * 1024 // 1mb @@ -19,6 +19,16 @@ const ( ArrayReply = '*' ) +//------------------------------------------------------------------------------ + +const Nil = RedisError("redis: nil") + +type RedisError string + +func (e RedisError) Error() string { return string(e) } + +//------------------------------------------------------------------------------ + type MultiBulkParse func(*Reader, int64) (interface{}, error) type Reader struct { @@ -66,7 +76,7 @@ func (r *Reader) ReadLine() ([]byte, error) { return nil, fmt.Errorf("redis: reply is empty") } if isNilReply(line) { - return nil, internal.Nil + return nil, Nil } return line, nil } @@ -83,7 +93,7 @@ func (r *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { case StatusReply: return parseStatusValue(line), nil case IntReply: - return parseInt(line[1:], 10, 64) + return util.ParseInt(line[1:], 10, 64) case StringReply: return r.readTmpBytesValue(line) case ArrayReply: @@ -105,7 +115,7 @@ func (r *Reader) ReadIntReply() (int64, error) { case ErrorReply: return 0, ParseErrorReply(line) case IntReply: - return parseInt(line[1:], 10, 64) + return util.ParseInt(line[1:], 10, 64) default: return 0, fmt.Errorf("redis: can't parse int reply: %.100q", line) } @@ -151,7 +161,7 @@ func (r *Reader) ReadFloatReply() (float64, error) { if err != nil { return 0, err } - return parseFloat(b, 64) + return util.ParseFloat(b, 64) } func (r *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) { @@ -221,7 +231,7 @@ func (r *Reader) ReadScanReply() ([]string, uint64, error) { func (r *Reader) readTmpBytesValue(line []byte) ([]byte, error) { if isNilReply(line) { - return nil, internal.Nil + return nil, Nil } replyLen, err := strconv.Atoi(string(line[1:])) @@ -241,7 +251,7 @@ func (r *Reader) ReadInt() (int64, error) { if err != nil { return 0, err } - return parseInt(b, 10, 64) + return util.ParseInt(b, 10, 64) } func (r *Reader) ReadUint() (uint64, error) { @@ -249,7 +259,7 @@ func (r *Reader) ReadUint() (uint64, error) { if err != nil { return 0, err } - return parseUint(b, 10, 64) + return util.ParseUint(b, 10, 64) } // -------------------------------------------------------------------- @@ -303,7 +313,7 @@ func isNilReply(b []byte) bool { } func ParseErrorReply(line []byte) error { - return internal.RedisError(string(line[1:])) + return RedisError(string(line[1:])) } func parseStatusValue(line []byte) []byte { @@ -312,23 +322,7 @@ func parseStatusValue(line []byte) []byte { func parseArrayLen(line []byte) (int64, error) { if isNilReply(line) { - return 0, internal.Nil + return 0, Nil } - return parseInt(line[1:], 10, 64) -} - -func atoi(b []byte) (int, error) { - return strconv.Atoi(internal.BytesToString(b)) -} - -func parseInt(b []byte, base int, bitSize int) (int64, error) { - return strconv.ParseInt(internal.BytesToString(b), base, bitSize) -} - -func parseUint(b []byte, base int, bitSize int) (uint64, error) { - return strconv.ParseUint(internal.BytesToString(b), base, bitSize) -} - -func parseFloat(b []byte, bitSize int) (float64, error) { - return strconv.ParseFloat(internal.BytesToString(b), bitSize) + return util.ParseInt(line[1:], 10, 64) } diff --git a/internal/proto/scan.go b/internal/proto/scan.go index 0329ffd991..3bdb33f9db 100644 --- a/internal/proto/scan.go +++ b/internal/proto/scan.go @@ -5,7 +5,7 @@ import ( "fmt" "reflect" - "github.com/go-redis/redis/internal" + "github.com/go-redis/redis/internal/util" ) func Scan(b []byte, v interface{}) error { @@ -13,80 +13,80 @@ func Scan(b []byte, v interface{}) error { case nil: return fmt.Errorf("redis: Scan(nil)") case *string: - *v = internal.BytesToString(b) + *v = util.BytesToString(b) return nil case *[]byte: *v = b return nil case *int: var err error - *v, err = atoi(b) + *v, err = util.Atoi(b) return err case *int8: - n, err := parseInt(b, 10, 8) + n, err := util.ParseInt(b, 10, 8) if err != nil { return err } *v = int8(n) return nil case *int16: - n, err := parseInt(b, 10, 16) + n, err := util.ParseInt(b, 10, 16) if err != nil { return err } *v = int16(n) return nil case *int32: - n, err := parseInt(b, 10, 32) + n, err := util.ParseInt(b, 10, 32) if err != nil { return err } *v = int32(n) return nil case *int64: - n, err := parseInt(b, 10, 64) + n, err := util.ParseInt(b, 10, 64) if err != nil { return err } *v = n return nil case *uint: - n, err := parseUint(b, 10, 64) + n, err := util.ParseUint(b, 10, 64) if err != nil { return err } *v = uint(n) return nil case *uint8: - n, err := parseUint(b, 10, 8) + n, err := util.ParseUint(b, 10, 8) if err != nil { return err } *v = uint8(n) return nil case *uint16: - n, err := parseUint(b, 10, 16) + n, err := util.ParseUint(b, 10, 16) if err != nil { return err } *v = uint16(n) return nil case *uint32: - n, err := parseUint(b, 10, 32) + n, err := util.ParseUint(b, 10, 32) if err != nil { return err } *v = uint32(n) return nil case *uint64: - n, err := parseUint(b, 10, 64) + n, err := util.ParseUint(b, 10, 64) if err != nil { return err } *v = n return nil case *float32: - n, err := parseFloat(b, 32) + n, err := util.ParseFloat(b, 32) if err != nil { return err } @@ -94,7 +94,7 @@ func Scan(b []byte, v interface{}) error { return err case *float64: var err error - *v, err = parseFloat(b, 64) + *v, err = util.ParseFloat(b, 64) return err case *bool: *v = len(b) == 1 && b[0] == '1' @@ -120,7 +120,7 @@ func ScanSlice(data []string, slice interface{}) error { return fmt.Errorf("redis: ScanSlice(non-slice %T)", slice) } - next := internal.MakeSliceNextElemFunc(v) + next := makeSliceNextElemFunc(v) for i, s := range data { elem := next() if err := Scan([]byte(s), elem.Addr().Interface()); err != nil { @@ -131,3 +131,36 @@ func ScanSlice(data []string, slice interface{}) error { return nil } + +func makeSliceNextElemFunc(v reflect.Value) func() reflect.Value { + elemType := v.Type().Elem() + + if elemType.Kind() == reflect.Ptr { + elemType = elemType.Elem() + return func() reflect.Value { + if v.Len() < v.Cap() { + v.Set(v.Slice(0, v.Len()+1)) + elem := v.Index(v.Len() - 1) + if elem.IsNil() { + elem.Set(reflect.New(elemType)) + } + return elem.Elem() + } + + elem := reflect.New(elemType) + v.Set(reflect.Append(v, elem)) + return elem.Elem() + } + } + + zero := reflect.Zero(elemType) + return func() reflect.Value { + if v.Len() < v.Cap() { + v.Set(v.Slice(0, v.Len()+1)) + return v.Index(v.Len() - 1) + } + + v.Set(reflect.Append(v, zero)) + return v.Index(v.Len() - 1) + } +} diff --git a/internal/util.go b/internal/util.go index 1ba9805fe3..ffd2353e0e 100644 --- a/internal/util.go +++ b/internal/util.go @@ -1,6 +1,6 @@ package internal -import "reflect" +import "github.com/go-redis/redis/internal/util" func ToLower(s string) string { if isLower(s) { @@ -15,7 +15,7 @@ func ToLower(s string) string { } b[i] = c } - return BytesToString(b) + return util.BytesToString(b) } func isLower(s string) bool { @@ -27,36 +27,3 @@ func isLower(s string) bool { } return true } - -func MakeSliceNextElemFunc(v reflect.Value) func() reflect.Value { - elemType := v.Type().Elem() - - if elemType.Kind() == reflect.Ptr { - elemType = elemType.Elem() - return func() reflect.Value { - if v.Len() < v.Cap() { - v.Set(v.Slice(0, v.Len()+1)) - elem := v.Index(v.Len() - 1) - if elem.IsNil() { - elem.Set(reflect.New(elemType)) - } - return elem.Elem() - } - - elem := reflect.New(elemType) - v.Set(reflect.Append(v, elem)) - return elem.Elem() - } - } - - zero := reflect.Zero(elemType) - return func() reflect.Value { - if v.Len() < v.Cap() { - v.Set(v.Slice(0, v.Len()+1)) - return v.Index(v.Len() - 1) - } - - v.Set(reflect.Append(v, zero)) - return v.Index(v.Len() - 1) - } -} diff --git a/internal/safe.go b/internal/util/safe.go similarity index 82% rename from internal/safe.go rename to internal/util/safe.go index dc5f4cc8a4..cd89183306 100644 --- a/internal/safe.go +++ b/internal/util/safe.go @@ -1,6 +1,6 @@ // +build appengine -package internal +package util func BytesToString(b []byte) string { return string(b) diff --git a/internal/util/strconv.go b/internal/util/strconv.go new file mode 100644 index 0000000000..db5033802a --- /dev/null +++ b/internal/util/strconv.go @@ -0,0 +1,19 @@ +package util + +import "strconv" + +func Atoi(b []byte) (int, error) { + return strconv.Atoi(BytesToString(b)) +} + +func ParseInt(b []byte, base int, bitSize int) (int64, error) { + return strconv.ParseInt(BytesToString(b), base, bitSize) +} + +func ParseUint(b []byte, base int, bitSize int) (uint64, error) { + return strconv.ParseUint(BytesToString(b), base, bitSize) +} + +func ParseFloat(b []byte, bitSize int) (float64, error) { + return strconv.ParseFloat(BytesToString(b), bitSize) +} diff --git a/internal/unsafe.go b/internal/util/unsafe.go similarity index 91% rename from internal/unsafe.go rename to internal/util/unsafe.go index 3ae48c14b9..93a89c55c7 100644 --- a/internal/unsafe.go +++ b/internal/util/unsafe.go @@ -1,6 +1,6 @@ // +build !appengine -package internal +package util import ( "unsafe" diff --git a/parser.go b/parser.go index e77efeeb97..f0dc67f0e1 100644 --- a/parser.go +++ b/parser.go @@ -14,17 +14,23 @@ func sliceParser(rd *proto.Reader, n int64) (interface{}, error) { vals := make([]interface{}, 0, n) for i := int64(0); i < n; i++ { v, err := rd.ReadReply(sliceParser) - if err == Nil { - vals = append(vals, nil) - } else if err != nil { - vals = append(vals, err) - } else { - switch vv := v.(type) { - case []byte: - vals = append(vals, string(vv)) - default: - vals = append(vals, v) + if err != nil { + if err == Nil { + vals = append(vals, nil) + continue + } + if err, ok := err.(proto.RedisError); ok { + vals = append(vals, err) + continue } + return nil, err + } + + switch v := v.(type) { + case []byte: + vals = append(vals, string(v)) + default: + vals = append(vals, v) } } return vals, nil diff --git a/redis.go b/redis.go index cf402986de..20f1ed1c7d 100644 --- a/redis.go +++ b/redis.go @@ -12,7 +12,7 @@ import ( ) // Nil reply redis returned when key does not exist. -const Nil = internal.Nil +const Nil = proto.Nil func init() { SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile)) diff --git a/tx.go b/tx.go index 26c29bef55..6a753b6a09 100644 --- a/tx.go +++ b/tx.go @@ -1,12 +1,12 @@ package redis import ( - "github.com/go-redis/redis/internal" "github.com/go-redis/redis/internal/pool" + "github.com/go-redis/redis/internal/proto" ) // TxFailedErr transaction redis failed. -const TxFailedErr = internal.RedisError("redis: transaction failed") +const TxFailedErr = proto.RedisError("redis: transaction failed") // Tx implements Redis transactions as described in // http://redis.io/topics/transactions. It's NOT safe for concurrent use From e5648e8c7c8ef178ad34188aec4492d11796bf51 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 22 Feb 2018 15:01:48 +0200 Subject: [PATCH 0411/1746] Fix flaky test --- commands_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/commands_test.go b/commands_test.go index b173e0d4e9..5750bdea11 100644 --- a/commands_test.go +++ b/commands_test.go @@ -62,12 +62,14 @@ var _ = Describe("Commands", func() { }) It("should Wait", func() { + const wait = 3 * time.Second + // assume testing on single redis instance start := time.Now() - wait := client.Wait(1, time.Second) - Expect(wait.Err()).NotTo(HaveOccurred()) - Expect(wait.Val()).To(Equal(int64(0))) - Expect(time.Now()).To(BeTemporally("~", start.Add(time.Second), 800*time.Millisecond)) + val, err := client.Wait(1, wait).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal(int64(0))) + Expect(time.Now()).To(BeTemporally("~", start.Add(wait), time.Second)) }) It("should Select", func() { From 9df09066e2882c4b835e830241782726161318bd Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 27 Feb 2018 14:35:03 +0200 Subject: [PATCH 0412/1746] Store all created cluster nodes in allNodes --- cluster.go | 84 ++++++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/cluster.go b/cluster.go index ea677ae213..783e27de3e 100644 --- a/cluster.go +++ b/cluster.go @@ -203,11 +203,11 @@ func (n *clusterNode) SetGeneration(gen uint32) { type clusterNodes struct { opt *ClusterOptions - mu sync.RWMutex - allAddrs []string - addrs []string - nodes map[string]*clusterNode - closed bool + mu sync.RWMutex + allAddrs []string + allNodes map[string]*clusterNode + clusterAddrs []string + closed bool nodeCreateGroup singleflight.Group @@ -219,7 +219,7 @@ func newClusterNodes(opt *ClusterOptions) *clusterNodes { opt: opt, allAddrs: opt.Addrs, - nodes: make(map[string]*clusterNode), + allNodes: make(map[string]*clusterNode), } } @@ -233,14 +233,14 @@ func (c *clusterNodes) Close() error { c.closed = true var firstErr error - for _, node := range c.nodes { + for _, node := range c.allNodes { if err := node.Client.Close(); err != nil && firstErr == nil { firstErr = err } } - c.addrs = nil - c.nodes = nil + c.allNodes = nil + c.clusterAddrs = nil return firstErr } @@ -250,8 +250,8 @@ func (c *clusterNodes) Addrs() ([]string, error) { c.mu.RLock() closed := c.closed if !closed { - if len(c.addrs) > 0 { - addrs = c.addrs + if len(c.clusterAddrs) > 0 { + addrs = c.clusterAddrs } else { addrs = c.allAddrs } @@ -276,25 +276,20 @@ func (c *clusterNodes) NextGeneration() uint32 { func (c *clusterNodes) GC(generation uint32) { var collected []*clusterNode c.mu.Lock() - for i := 0; i < len(c.addrs); { - addr := c.addrs[i] - node := c.nodes[addr] + for addr, node := range c.allNodes { if node.Generation() >= generation { - i++ continue } - c.addrs = append(c.addrs[:i], c.addrs[i+1:]...) - delete(c.nodes, addr) + c.clusterAddrs = remove(c.clusterAddrs, addr) + delete(c.allNodes, addr) collected = append(collected, node) } c.mu.Unlock() - time.AfterFunc(time.Minute, func() { - for _, node := range collected { - _ = node.Client.Close() - } - }) + for _, node := range collected { + _ = node.Client.Close() + } } func (c *clusterNodes) All() ([]*clusterNode, error) { @@ -305,23 +300,28 @@ func (c *clusterNodes) All() ([]*clusterNode, error) { return nil, pool.ErrClosed } - nodes := make([]*clusterNode, 0, len(c.nodes)) - for _, node := range c.nodes { - nodes = append(nodes, node) + cp := make([]*clusterNode, 0, len(c.allNodes)) + for _, node := range c.allNodes { + cp = append(cp, node) } - return nodes, nil + return cp, nil } func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { var node *clusterNode - var ok bool + var err error c.mu.RLock() - if !c.closed { - node, ok = c.nodes[addr] + if c.closed { + err = pool.ErrClosed + } else { + node = c.allNodes[addr] } c.mu.RUnlock() - if ok { + if err != nil { + return nil, err + } + if node != nil { return node, nil } @@ -329,9 +329,6 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { node := newClusterNode(c.opt, addr) return node, node.Test() }) - if err != nil { - return nil, err - } c.mu.Lock() defer c.mu.Unlock() @@ -340,18 +337,20 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { return nil, pool.ErrClosed } - node, ok = c.nodes[addr] + node, ok := c.allNodes[addr] if ok { _ = v.(*clusterNode).Close() - return node, nil + return node, err } node = v.(*clusterNode) c.allAddrs = appendIfNotExists(c.allAddrs, addr) - c.addrs = append(c.addrs, addr) - c.nodes[addr] = node + if err == nil { + c.clusterAddrs = append(c.clusterAddrs, addr) + } + c.allNodes[addr] = node - return node, nil + return node, err } func (c *clusterNodes) Random() (*clusterNode, error) { @@ -679,10 +678,7 @@ func (c *ClusterClient) WrapProcess( } func (c *ClusterClient) Process(cmd Cmder) error { - if c.process != nil { - return c.process(cmd) - } - return c.defaultProcess(cmd) + return c.process(cmd) } func (c *ClusterClient) defaultProcess(cmd Cmder) error { @@ -918,7 +914,9 @@ func (c *ClusterClient) reloadState() bool { state, err := c.loadState() if err == nil { c._state.Store(state) - c.nodes.GC(state.generation) + time.AfterFunc(time.Minute, func() { + c.nodes.GC(state.generation) + }) return true } From cc47cf2439b7c4cd883f1814fd4b4b73237dc297 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 27 Feb 2018 14:50:08 +0200 Subject: [PATCH 0413/1746] Use fresh cluster state after sleeping --- cluster.go | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/cluster.go b/cluster.go index 783e27de3e..8b01a5182c 100644 --- a/cluster.go +++ b/cluster.go @@ -682,18 +682,23 @@ func (c *ClusterClient) Process(cmd Cmder) error { } func (c *ClusterClient) defaultProcess(cmd Cmder) error { - _, node, err := c.cmdSlotAndNode(cmd) - if err != nil { - cmd.setErr(err) - return err - } - + var node *clusterNode var ask bool for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { if attempt > 0 { time.Sleep(c.retryBackoff(attempt)) } + if node == nil { + var err error + _, node, err = c.cmdSlotAndNode(cmd) + if err != nil { + cmd.setErr(err) + break + } + } + + var err error if ask { pipe := node.Client.Pipeline() _ = pipe.Process(NewCmd("ASKING")) @@ -717,9 +722,8 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { } if internal.IsRetryableError(err, true) { - var nodeErr error - node, nodeErr = c.nodes.Random() - if nodeErr != nil { + node, err = c.nodes.Random() + if err != nil { break } continue @@ -731,20 +735,15 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { if moved || ask { c.lazyReloadState() - var nodeErr error - node, nodeErr = c.nodes.GetOrCreate(addr) - if nodeErr != nil { + node, err = c.nodes.GetOrCreate(addr) + if err != nil { break } continue } if err == pool.ErrClosed { - _, node, err = c.cmdSlotAndNode(cmd) - if err != nil { - cmd.setErr(err) - break - } + node = nil continue } From cac7aa8c367f11064663d3980472a76db1486ad0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 27 Feb 2018 15:01:10 +0200 Subject: [PATCH 0414/1746] Reduce ClusterOptions.MaxRedirects from 16 to 8 --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 8b01a5182c..dc4db244dd 100644 --- a/cluster.go +++ b/cluster.go @@ -58,7 +58,7 @@ func (opt *ClusterOptions) init() { if opt.MaxRedirects == -1 { opt.MaxRedirects = 0 } else if opt.MaxRedirects == 0 { - opt.MaxRedirects = 16 + opt.MaxRedirects = 8 } if opt.RouteByLatency { From e075ad33d33d48923b5e964f9be7abd2e67795fa Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 27 Feb 2018 16:09:55 +0200 Subject: [PATCH 0415/1746] Simplify test --- redis_test.go | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/redis_test.go b/redis_test.go index 49d3fb3295..ad8aa11a06 100644 --- a/redis_test.go +++ b/redis_test.go @@ -157,30 +157,15 @@ var _ = Describe("Client", func() { }) It("should retry with backoff", func() { - Expect(client.Close()).NotTo(HaveOccurred()) - - // use up all the available connections to force a fail - connectionHogClient := redis.NewClient(&redis.Options{ - Addr: redisAddr, - MaxRetries: 1, - }) - defer connectionHogClient.Close() - - for i := 0; i <= 1002; i++ { - connectionHogClient.Pool().NewConn() - } - clientNoRetry := redis.NewClient(&redis.Options{ - Addr: redisAddr, - PoolSize: 1, - MaxRetryBackoff: -1, + Addr: ":1234", + MaxRetries: 0, }) defer clientNoRetry.Close() clientRetry := redis.NewClient(&redis.Options{ - Addr: redisAddr, + Addr: ":1234", MaxRetries: 5, - PoolSize: 1, MaxRetryBackoff: 128 * time.Millisecond, }) defer clientRetry.Close() @@ -195,7 +180,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) elapseRetry := time.Since(startRetry) - Expect(elapseRetry > elapseNoRetry).To(BeTrue()) + Expect(elapseRetry).To(BeNumerically(">", elapseNoRetry, 10*time.Millisecond)) }) It("should update conn.UsedAt on read/write", func() { From 68362cfda1eeb3a69316e7bc00169a9a8de4823a Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 27 Feb 2018 16:11:25 +0200 Subject: [PATCH 0416/1746] Fix doc --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index dc4db244dd..781dedb850 100644 --- a/cluster.go +++ b/cluster.go @@ -26,7 +26,7 @@ type ClusterOptions struct { // The maximum number of retries before giving up. Command is retried // on network errors and MOVED/ASK redirects. - // Default is 16. + // Default is 8. MaxRedirects int // Enables read-only commands on slave nodes. From 55c9929dbaa1362cf9af13fab0f46bf8fde8fe84 Mon Sep 17 00:00:00 2001 From: Tess Thyer Date: Tue, 27 Feb 2018 10:05:21 -0800 Subject: [PATCH 0417/1746] Change Makefile to use sed syntax that will work for both GNU and BSD sed (#719) * Use sed syntax that will work with both GNU and BSD sed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 50fdc55a1a..d6c184e98d 100644 --- a/Makefile +++ b/Makefile @@ -15,5 +15,5 @@ testdata/redis: wget -qO- https://github.com/antirez/redis/archive/unstable.tar.gz | tar xvz --strip-components=1 -C $@ testdata/redis/src/redis-server: testdata/redis - sed -i 's/libjemalloc.a/libjemalloc.a -lrt/g' $ Date: Thu, 1 Mar 2018 09:37:51 +0100 Subject: [PATCH 0418/1746] Impement Config Rewrite command (#722) This commit adds support for config rewrite as documented in https://redis.io/commands/config-rewrite . --- commands.go | 7 +++++++ commands_test.go | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/commands.go b/commands.go index 45bb0aed14..175cd851f4 100644 --- a/commands.go +++ b/commands.go @@ -198,6 +198,7 @@ type Cmdable interface { ConfigGet(parameter string) *SliceCmd ConfigResetStat() *StatusCmd ConfigSet(parameter, value string) *StatusCmd + ConfigRewrite() *StatusCmd DBSize() *IntCmd FlushAll() *StatusCmd FlushAllAsync() *StatusCmd @@ -1725,6 +1726,12 @@ func (c *cmdable) ConfigSet(parameter, value string) *StatusCmd { return cmd } +func (c *cmdable) ConfigRewrite() *StatusCmd { + cmd := NewStatusCmd("config", "rewrite") + c.process(cmd) + return cmd +} + // Deperecated. Use DBSize instead. func (c *cmdable) DbSize() *IntCmd { return c.DBSize() diff --git a/commands_test.go b/commands_test.go index 5750bdea11..a4e5250dea 100644 --- a/commands_test.go +++ b/commands_test.go @@ -162,6 +162,12 @@ var _ = Describe("Commands", func() { Expect(configSet.Val()).To(Equal("OK")) }) + It("should ConfigRewrite", func() { + configRewrite := client.ConfigRewrite() + Expect(configRewrite.Err()).NotTo(HaveOccurred()) + Expect(configRewrite.Val()).To(Equal("OK")) + }) + It("should DBSize", func() { size, err := client.DBSize().Result() Expect(err).NotTo(HaveOccurred()) From 7acec74c599074e0a8e21e1b6aeb6ee80f442ab4 Mon Sep 17 00:00:00 2001 From: Evan Goldschmidt Date: Mon, 5 Mar 2018 00:54:11 -0800 Subject: [PATCH 0419/1746] Script: Fix `Exists` to use hash instead of source (#726) `SCRIPT EXISTS` accepts a hash, not the raw source: https://redis.io/commands/script-exists --- commands.go | 10 +++++----- script.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/commands.go b/commands.go index 175cd851f4..6609a74f06 100644 --- a/commands.go +++ b/commands.go @@ -214,7 +214,7 @@ type Cmdable interface { Time() *TimeCmd Eval(script string, keys []string, args ...interface{}) *Cmd EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd - ScriptExists(scripts ...string) *BoolSliceCmd + ScriptExists(hashes ...string) *BoolSliceCmd ScriptFlush() *StatusCmd ScriptKill() *StatusCmd ScriptLoad(script string) *StringCmd @@ -1884,12 +1884,12 @@ func (c *cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd return cmd } -func (c *cmdable) ScriptExists(scripts ...string) *BoolSliceCmd { - args := make([]interface{}, 2+len(scripts)) +func (c *cmdable) ScriptExists(hashes ...string) *BoolSliceCmd { + args := make([]interface{}, 2+len(hashes)) args[0] = "script" args[1] = "exists" - for i, script := range scripts { - args[2+i] = script + for i, hash := range hashes { + args[2+i] = hash } cmd := NewBoolSliceCmd(args...) c.process(cmd) diff --git a/script.go b/script.go index 74135f5a5c..09f36d9320 100644 --- a/script.go +++ b/script.go @@ -10,7 +10,7 @@ import ( type scripter interface { Eval(script string, keys []string, args ...interface{}) *Cmd EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd - ScriptExists(scripts ...string) *BoolSliceCmd + ScriptExists(hashes ...string) *BoolSliceCmd ScriptLoad(script string) *StringCmd } @@ -40,7 +40,7 @@ func (s *Script) Load(c scripter) *StringCmd { } func (s *Script) Exists(c scripter) *BoolSliceCmd { - return c.ScriptExists(s.src) + return c.ScriptExists(s.hash) } func (s *Script) Eval(c scripter, keys []string, args ...interface{}) *Cmd { From 0082bdcd4b39dd5678de09d088a0403cdc35452b Mon Sep 17 00:00:00 2001 From: Pavel Prokopenko Date: Tue, 6 Mar 2018 09:15:14 +0200 Subject: [PATCH 0420/1746] add WrapProcess to UniversalClient interface (#728) --- universal.go | 1 + 1 file changed, 1 insertion(+) diff --git a/universal.go b/universal.go index ea42f69847..fde3c41506 100644 --- a/universal.go +++ b/universal.go @@ -114,6 +114,7 @@ func (o *UniversalOptions) simple() *Options { type UniversalClient interface { Cmdable Process(cmd Cmder) error + WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) Subscribe(channels ...string) *PubSub PSubscribe(channels ...string) *PubSub Close() error From f13fc5381c615f0798a5719338f3d7aa8c5d8847 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 6 Mar 2018 14:50:48 +0200 Subject: [PATCH 0421/1746] Extract commands info cache --- cluster.go | 30 +++++++++++------------------- command.go | 23 +++++++++++++++++++++++ redis.go | 7 ++----- ring.go | 37 ++++++++++++++----------------------- 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/cluster.go b/cluster.go index 781dedb850..7a4c2c8445 100644 --- a/cluster.go +++ b/cluster.go @@ -483,22 +483,20 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode { type ClusterClient struct { cmdable - opt *ClusterOptions - nodes *clusterNodes + opt *ClusterOptions + nodes *clusterNodes + cmdsInfoCache *cmdsInfoCache _state atomic.Value stateErrMu sync.RWMutex stateErr error - cmdsInfoOnce internal.Once - cmdsInfo map[string]*CommandInfo - process func(Cmder) error processPipeline func([]Cmder) error processTxPipeline func([]Cmder) error // Reports whether slots reloading is in progress. - reloading uint32 + reloading uint32 // atomic } // NewClusterClient returns a Redis Cluster client as described in @@ -507,8 +505,9 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt.init() c := &ClusterClient{ - opt: opt, - nodes: newClusterNodes(opt), + opt: opt, + nodes: newClusterNodes(opt), + cmdsInfoCache: newCmdsInfoCache(), } c.process = c.defaultProcess @@ -535,24 +534,17 @@ func (c *ClusterClient) retryBackoff(attempt int) time.Duration { } func (c *ClusterClient) cmdInfo(name string) *CommandInfo { - err := c.cmdsInfoOnce.Do(func() error { + cmdsInfo, err := c.cmdsInfoCache.Do(func() (map[string]*CommandInfo, error) { node, err := c.nodes.Random() if err != nil { - return err - } - - cmdsInfo, err := node.Client.Command().Result() - if err != nil { - return err + return nil, err } - - c.cmdsInfo = cmdsInfo - return nil + return node.Client.Command().Result() }) if err != nil { return nil } - info := c.cmdsInfo[name] + info := cmdsInfo[name] if info == nil { internal.Logf("info for cmd=%s not found", name) } diff --git a/command.go b/command.go index 7d45c3fada..1588ca2519 100644 --- a/command.go +++ b/command.go @@ -1023,3 +1023,26 @@ func (cmd *CommandsInfoCmd) readReply(cn *pool.Conn) error { cmd.val = v.(map[string]*CommandInfo) return nil } + +//------------------------------------------------------------------------------ + +type cmdsInfoCache struct { + once internal.Once + cmds map[string]*CommandInfo +} + +func newCmdsInfoCache() *cmdsInfoCache { + return &cmdsInfoCache{} +} + +func (c *cmdsInfoCache) Do(fn func() (map[string]*CommandInfo, error)) (map[string]*CommandInfo, error) { + err := c.once.Do(func() error { + cmds, err := fn() + if err != nil { + return err + } + c.cmds = cmds + return nil + }) + return c.cmds, err +} diff --git a/redis.go b/redis.go index 20f1ed1c7d..1f2167aa71 100644 --- a/redis.go +++ b/redis.go @@ -11,7 +11,7 @@ import ( "github.com/go-redis/redis/internal/proto" ) -// Nil reply redis returned when key does not exist. +// Nil reply Redis returns when key does not exist. const Nil = proto.Nil func init() { @@ -119,10 +119,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { return nil } -// WrapProcess replaces the process func. It takes a function createWrapper -// which is supplied by the user. createWrapper takes the old process func as -// an input and returns the new wrapper process func. createWrapper should -// use call the old process func within the new process func. +// WrapProcess wraps function that processes Redis commands. func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { c.process = fn(c.process) } diff --git a/ring.go b/ring.go index 10f33ed006..d07e806e38 100644 --- a/ring.go +++ b/ring.go @@ -15,6 +15,8 @@ import ( "github.com/go-redis/redis/internal/pool" ) +const nreplicas = 100 + var errRingShardsDown = errors.New("redis: all ring shards are down") // RingOptions are used to configure a ring client and should be @@ -142,30 +144,25 @@ func (shard *ringShard) Vote(up bool) bool { type Ring struct { cmdable - opt *RingOptions - nreplicas int + opt *RingOptions + cmdsInfoCache *cmdsInfoCache mu sync.RWMutex hash *consistenthash.Map - shards map[string]*ringShard - shardsList []*ringShard + shards map[string]*ringShard // read only + shardsList []*ringShard // read only processPipeline func([]Cmder) error - cmdsInfoOnce internal.Once - cmdsInfo map[string]*CommandInfo - closed bool } func NewRing(opt *RingOptions) *Ring { - const nreplicas = 100 - opt.init() ring := &Ring{ - opt: opt, - nreplicas: nreplicas, + opt: opt, + cmdsInfoCache: newCmdsInfoCache(), hash: consistenthash.New(nreplicas, nil), shards: make(map[string]*ringShard), @@ -186,11 +183,9 @@ func NewRing(opt *RingOptions) *Ring { func (c *Ring) addShard(name string, cl *Client) { shard := &ringShard{Client: cl} - c.mu.Lock() c.hash.Add(name) c.shards[name] = shard c.shardsList = append(c.shardsList, shard) - c.mu.Unlock() } // Options returns read-only Options that were used to create the client. @@ -285,31 +280,27 @@ func (c *Ring) ForEachShard(fn func(client *Client) error) error { } func (c *Ring) cmdInfo(name string) *CommandInfo { - err := c.cmdsInfoOnce.Do(func() error { + cmdsInfo, err := c.cmdsInfoCache.Do(func() (map[string]*CommandInfo, error) { c.mu.RLock() shards := c.shardsList c.mu.RUnlock() - var firstErr error + firstErr := errRingShardsDown for _, shard := range shards { cmdsInfo, err := shard.Client.Command().Result() if err == nil { - c.cmdsInfo = cmdsInfo - return nil + return cmdsInfo, nil } if firstErr == nil { firstErr = err } } - return firstErr + return nil, firstErr }) if err != nil { return nil } - if c.cmdsInfo == nil { - return nil - } - info := c.cmdsInfo[name] + info := cmdsInfo[name] if info == nil { internal.Logf("info for cmd=%s not found", name) } @@ -380,7 +371,7 @@ func (c *Ring) Process(cmd Cmder) error { // rebalance removes dead shards from the Ring. func (c *Ring) rebalance() { - hash := consistenthash.New(c.nreplicas, nil) + hash := consistenthash.New(nreplicas, nil) for name, shard := range c.shards { if shard.IsUp() { hash.Add(name) From 5e72be1cbf9726a528eef3ea6aeccd54f5f00aa3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 6 Mar 2018 17:10:01 +0200 Subject: [PATCH 0422/1746] Extract clusterStateHolder --- cluster.go | 207 +++++++++++++++++++++++++++---------------------- export_test.go | 4 +- 2 files changed, 115 insertions(+), 96 deletions(-) diff --git a/cluster.go b/cluster.go index 7a4c2c8445..ba562cf144 100644 --- a/cluster.go +++ b/cluster.go @@ -1,6 +1,7 @@ package redis import ( + "errors" "fmt" "math" "math/rand" @@ -292,21 +293,6 @@ func (c *clusterNodes) GC(generation uint32) { } } -func (c *clusterNodes) All() ([]*clusterNode, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.closed { - return nil, pool.ErrClosed - } - - cp := make([]*clusterNode, 0, len(c.allNodes)) - for _, node := range c.allNodes { - cp = append(cp, node) - } - return cp, nil -} - func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { var node *clusterNode var err error @@ -353,6 +339,21 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { return node, err } +func (c *clusterNodes) All() ([]*clusterNode, error) { + c.mu.RLock() + defer c.mu.RUnlock() + + if c.closed { + return nil, pool.ErrClosed + } + + cp := make([]*clusterNode, 0, len(c.allNodes)) + for _, node := range c.allNodes { + cp = append(cp, node) + } + return cp, nil +} + func (c *clusterNodes) Random() (*clusterNode, error) { addrs, err := c.Addrs() if err != nil { @@ -412,6 +413,10 @@ func newClusterState(nodes *clusterNodes, slots []ClusterSlot, origin string) (* } } + time.AfterFunc(time.Minute, func() { + nodes.GC(c.generation) + }) + return &c, nil } @@ -477,6 +482,66 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode { //------------------------------------------------------------------------------ +type clusterStateHolder struct { + load func() (*clusterState, error) + reloading uint32 // atomic + + state atomic.Value + + lastErrMu sync.RWMutex + lastErr error +} + +func newClusterStateHolder(fn func() (*clusterState, error)) *clusterStateHolder { + return &clusterStateHolder{ + load: fn, + } +} + +func (c *clusterStateHolder) Load() (*clusterState, error) { + state, err := c.load() + if err != nil { + c.lastErrMu.Lock() + c.lastErr = err + c.lastErrMu.Unlock() + return nil, err + } + c.state.Store(state) + return state, nil +} + +func (c *clusterStateHolder) LazyReload() { + if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { + return + } + go func() { + defer atomic.StoreUint32(&c.reloading, 0) + + _, err := c.Load() + if err == nil { + time.Sleep(time.Second) + } + }() +} + +func (c *clusterStateHolder) Get() (*clusterState, error) { + v := c.state.Load() + if v != nil { + return v.(*clusterState), nil + } + + c.lastErrMu.RLock() + err := c.lastErr + c.lastErrMu.RUnlock() + if err != nil { + return nil, err + } + + return nil, errors.New("redis: cluster has no state") +} + +//------------------------------------------------------------------------------ + // ClusterClient is a Redis Cluster client representing a pool of zero // or more underlying connections. It's safe for concurrent use by // multiple goroutines. @@ -485,18 +550,12 @@ type ClusterClient struct { opt *ClusterOptions nodes *clusterNodes + state *clusterStateHolder cmdsInfoCache *cmdsInfoCache - _state atomic.Value - stateErrMu sync.RWMutex - stateErr error - process func(Cmder) error processPipeline func([]Cmder) error processTxPipeline func([]Cmder) error - - // Reports whether slots reloading is in progress. - reloading uint32 // atomic } // NewClusterClient returns a Redis Cluster client as described in @@ -509,6 +568,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { nodes: newClusterNodes(opt), cmdsInfoCache: newCmdsInfoCache(), } + c.state = newClusterStateHolder(c.loadState) c.process = c.defaultProcess c.processPipeline = c.defaultProcessPipeline @@ -516,7 +576,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { c.cmdable.setProcessor(c.Process) - c.reloadState() + _, _ = c.state.Load() if opt.IdleCheckFrequency > 0 { go c.reaper(opt.IdleCheckFrequency) } @@ -565,7 +625,7 @@ func (c *ClusterClient) cmdSlot(cmd Cmder) int { } func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { - state, err := c.state() + state, err := c.state.Get() if err != nil { return 0, nil, err } @@ -588,7 +648,7 @@ func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { } func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { - state, err := c.state() + state, err := c.state.Get() if err != nil { return nil, err } @@ -633,7 +693,7 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { moved, ask, addr := internal.IsMovedError(err) if moved || ask { - c.lazyReloadState() + c.state.LazyReload() node, err = c.nodes.GetOrCreate(addr) if err != nil { return err @@ -725,7 +785,7 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { var addr string moved, ask, addr = internal.IsMovedError(err) if moved || ask { - c.lazyReloadState() + c.state.LazyReload() node, err = c.nodes.GetOrCreate(addr) if err != nil { @@ -748,7 +808,7 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { // ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { - state, err := c.state() + state, err := c.state.Get() if err != nil { return err } @@ -781,7 +841,7 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { // ForEachSlave concurrently calls the fn on each slave node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { - state, err := c.state() + state, err := c.state.Get() if err != nil { return err } @@ -814,7 +874,7 @@ func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { // ForEachNode concurrently calls the fn on each known node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { - state, err := c.state() + state, err := c.state.Get() if err != nil { return err } @@ -854,7 +914,7 @@ func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { func (c *ClusterClient) PoolStats() *PoolStats { var acc PoolStats - state, _ := c.state() + state, _ := c.state.Get() if state == nil { return &acc } @@ -884,75 +944,34 @@ func (c *ClusterClient) PoolStats() *PoolStats { return &acc } -func (c *ClusterClient) lazyReloadState() { - if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { - return - } - go func() { - if c.reloadState() { - time.Sleep(time.Second) - } - atomic.StoreUint32(&c.reloading, 0) - }() -} - -func (c *ClusterClient) reloadState() bool { - for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { - if attempt > 0 { - time.Sleep(c.retryBackoff(attempt)) - } - - state, err := c.loadState() - if err == nil { - c._state.Store(state) - time.AfterFunc(time.Minute, func() { - c.nodes.GC(state.generation) - }) - return true - } - - c.setStateErr(err) - switch err { - case pool.ErrClosed, errClusterNoNodes: - return false - } - } - return false -} - func (c *ClusterClient) loadState() (*clusterState, error) { - node, err := c.nodes.Random() + addrs, err := c.nodes.Addrs() if err != nil { return nil, err } - slots, err := node.Client.ClusterSlots().Result() - if err != nil { - return nil, err - } + var firstErr error + for _, addr := range addrs { + node, err := c.nodes.GetOrCreate(addr) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } - return newClusterState(c.nodes, slots, node.Client.opt.Addr) -} + slots, err := node.Client.ClusterSlots().Result() + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } -func (c *ClusterClient) state() (*clusterState, error) { - v := c._state.Load() - if v != nil { - return v.(*clusterState), nil + return newClusterState(c.nodes, slots, node.Client.opt.Addr) } - return nil, c.getStateErr() -} -func (c *ClusterClient) setStateErr(err error) { - c.stateErrMu.Lock() - c.stateErr = err - c.stateErrMu.Unlock() -} - -func (c *ClusterClient) getStateErr() error { - c.stateErrMu.RLock() - err := c.stateErr - c.stateErrMu.RUnlock() - return err + return nil, firstErr } // reaper closes idle connections to the cluster. @@ -1036,7 +1055,7 @@ func (c *ClusterClient) defaultProcessPipeline(cmds []Cmder) error { } func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, error) { - state, err := c.state() + state, err := c.state.Get() if err != nil { setCmdsErr(cmds, err) return nil, err @@ -1112,7 +1131,7 @@ func (c *ClusterClient) checkMovedErr( moved, ask, addr := internal.IsMovedError(err) if moved { - c.lazyReloadState() + c.state.LazyReload() node, err := c.nodes.GetOrCreate(addr) if err != nil { @@ -1150,7 +1169,7 @@ func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { } func (c *ClusterClient) defaultProcessTxPipeline(cmds []Cmder) error { - state, err := c.state() + state, err := c.state.Get() if err != nil { return err } diff --git a/export_test.go b/export_test.go index bcc18c4576..288e86f047 100644 --- a/export_test.go +++ b/export_test.go @@ -20,7 +20,7 @@ func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) } func (c *ClusterClient) SlotAddrs(slot int) []string { - state, err := c.state() + state, err := c.state.Get() if err != nil { panic(err) } @@ -34,7 +34,7 @@ func (c *ClusterClient) SlotAddrs(slot int) []string { // SwapSlot swaps a slot's master/slave address for testing MOVED redirects. func (c *ClusterClient) SwapSlotNodes(slot int) { - state, err := c.state() + state, err := c.state.Get() if err != nil { panic(err) } From e9e73a92c0759a95478c8018cd8210a7f713683c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 6 Mar 2018 17:18:15 +0200 Subject: [PATCH 0423/1746] travis: test on Go 1.10 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c95b3e6c6c..39ffc2becb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ go: - 1.7.x - 1.8.x - 1.9.x + - 1.10.x - tip matrix: From 852a60d5208bc8faab70eb5ea589e944d4d64db4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 11:56:24 +0200 Subject: [PATCH 0424/1746] Fix go vet --- bench_test.go | 6 +- cluster_test.go | 68 +++- commands_test.go | 693 +++++++++++++++++++++----------- example_instrumentation_test.go | 2 +- 4 files changed, 536 insertions(+), 233 deletions(-) diff --git a/bench_test.go b/bench_test.go index f6b75c72a2..b7c4081e1b 100644 --- a/bench_test.go +++ b/bench_test.go @@ -208,7 +208,11 @@ func BenchmarkZAdd(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.ZAdd("key", redis.Z{float64(1), "hello"}).Err(); err != nil { + err := client.ZAdd("key", redis.Z{ + Score: float64(1), + Member: "hello", + }).Err() + if err != nil { b.Fatal(err) } } diff --git a/cluster_test.go b/cluster_test.go index 2bf5d91617..6788599a9f 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -123,9 +123,37 @@ func startCluster(scenario *clusterScenario) error { return err } wanted := []redis.ClusterSlot{ - {0, 4999, []redis.ClusterNode{{"", "127.0.0.1:8220"}, {"", "127.0.0.1:8223"}}}, - {5000, 9999, []redis.ClusterNode{{"", "127.0.0.1:8221"}, {"", "127.0.0.1:8224"}}}, - {10000, 16383, []redis.ClusterNode{{"", "127.0.0.1:8222"}, {"", "127.0.0.1:8225"}}}, + { + Start: 0, + End: 4999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8220", + }, { + Id: "", + Addr: "127.0.0.1:8223", + }}, + }, { + Start: 5000, + End: 9999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8221", + }, { + Id: "", + Addr: "127.0.0.1:8224", + }}, + }, { + Start: 10000, + End: 16383, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8222", + }, { + Id: "", + Addr: "127.0.0.1:8225", + }}, + }, } return assertSlotsEqual(res, wanted) }, 30*time.Second) @@ -492,9 +520,37 @@ var _ = Describe("ClusterClient", func() { Expect(res).To(HaveLen(3)) wanted := []redis.ClusterSlot{ - {0, 4999, []redis.ClusterNode{{"", "127.0.0.1:8220"}, {"", "127.0.0.1:8223"}}}, - {5000, 9999, []redis.ClusterNode{{"", "127.0.0.1:8221"}, {"", "127.0.0.1:8224"}}}, - {10000, 16383, []redis.ClusterNode{{"", "127.0.0.1:8222"}, {"", "127.0.0.1:8225"}}}, + { + Start: 0, + End: 4999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8220", + }, { + Id: "", + Addr: "127.0.0.1:8223", + }}, + }, { + Start: 5000, + End: 9999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8221", + }, { + Id: "", + Addr: "127.0.0.1:8224", + }}, + }, { + Start: 10000, + End: 16383, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8222", + }, { + Id: "", + Addr: "127.0.0.1:8225", + }}, + }, } Expect(assertSlotsEqual(res, wanted)).NotTo(HaveOccurred()) }) diff --git a/commands_test.go b/commands_test.go index a4e5250dea..5e4d260f95 100644 --- a/commands_test.go +++ b/commands_test.go @@ -753,8 +753,11 @@ var _ = Describe("Commands", func() { It("should ZScan", func() { for i := 0; i < 1000; i++ { - sadd := client.ZAdd("myset", redis.Z{float64(i), fmt.Sprintf("member%d", i)}) - Expect(sadd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("myset", redis.Z{ + Score: float64(i), + Member: fmt.Sprintf("member%d", i), + }).Err() + Expect(err).NotTo(HaveOccurred()) } keys, cursor, err := client.ZScan("myset", 0, "", 0).Result() @@ -794,11 +797,17 @@ var _ = Describe("Commands", func() { Expect(bitCount.Err()).NotTo(HaveOccurred()) Expect(bitCount.Val()).To(Equal(int64(26))) - bitCount = client.BitCount("key", &redis.BitCount{0, 0}) + bitCount = client.BitCount("key", &redis.BitCount{ + Start: 0, + End: 0, + }) Expect(bitCount.Err()).NotTo(HaveOccurred()) Expect(bitCount.Val()).To(Equal(int64(4))) - bitCount = client.BitCount("key", &redis.BitCount{1, 1}) + bitCount = client.BitCount("key", &redis.BitCount{ + Start: 1, + End: 1, + }) Expect(bitCount.Err()).NotTo(HaveOccurred()) Expect(bitCount.Val()).To(Equal(int64(6))) }) @@ -2076,69 +2085,120 @@ var _ = Describe("Commands", func() { Describe("sorted sets", func() { It("should ZAdd", func() { - added, err := client.ZAdd("zset", redis.Z{1, "one"}).Result() + added, err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAdd("zset", redis.Z{1, "uno"}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "uno", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAdd("zset", redis.Z{2, "two"}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 2, + Member: "two", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAdd("zset", redis.Z{3, "two"}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 3, + Member: "two", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(0))) vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}, {1, "uno"}, {3, "two"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }, { + Score: 1, + Member: "uno", + }, { + Score: 3, + Member: "two", + }})) }) It("should ZAdd bytes", func() { - added, err := client.ZAdd("zset", redis.Z{1, []byte("one")}).Result() + added, err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: []byte("one"), + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAdd("zset", redis.Z{1, []byte("uno")}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 1, + Member: []byte("uno"), + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAdd("zset", redis.Z{2, []byte("two")}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 2, + Member: []byte("two"), + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAdd("zset", redis.Z{3, []byte("two")}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 3, + Member: []byte("two"), + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(0))) - val, err := client.ZRangeWithScores("zset", 0, -1).Result() + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}, {1, "uno"}, {3, "two"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }, { + Score: 1, + Member: "uno", + }, { + Score: 3, + Member: "two", + }})) }) It("should ZAddNX", func() { - added, err := client.ZAddNX("zset", redis.Z{1, "one"}).Result() + added, err := client.ZAddNX("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) - added, err = client.ZAddNX("zset", redis.Z{2, "one"}).Result() + added, err = client.ZAddNX("zset", redis.Z{ + Score: 2, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(0))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) }) It("should ZAddXX", func() { - added, err := client.ZAddXX("zset", redis.Z{1, "one"}).Result() + added, err := client.ZAddXX("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(0))) @@ -2146,49 +2206,73 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(vals).To(BeEmpty()) - added, err = client.ZAdd("zset", redis.Z{1, "one"}).Result() + added, err = client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - added, err = client.ZAddXX("zset", redis.Z{2, "one"}).Result() + added, err = client.ZAddXX("zset", redis.Z{ + Score: 2, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(0))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 2, Member: "one"}})) }) It("should ZAddCh", func() { - changed, err := client.ZAddCh("zset", redis.Z{1, "one"}).Result() + changed, err := client.ZAddCh("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(changed).To(Equal(int64(1))) - changed, err = client.ZAddCh("zset", redis.Z{1, "one"}).Result() + changed, err = client.ZAddCh("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(changed).To(Equal(int64(0))) }) It("should ZAddNXCh", func() { - changed, err := client.ZAddNXCh("zset", redis.Z{1, "one"}).Result() + changed, err := client.ZAddNXCh("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(changed).To(Equal(int64(1))) vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) - changed, err = client.ZAddNXCh("zset", redis.Z{2, "one"}).Result() + changed, err = client.ZAddNXCh("zset", redis.Z{ + Score: 2, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(changed).To(Equal(int64(0))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }})) }) It("should ZAddXXCh", func() { - changed, err := client.ZAddXXCh("zset", redis.Z{1, "one"}).Result() + changed, err := client.ZAddXXCh("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(changed).To(Equal(int64(0))) @@ -2196,57 +2280,75 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(vals).To(BeEmpty()) - added, err := client.ZAdd("zset", redis.Z{1, "one"}).Result() + added, err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - changed, err = client.ZAddXXCh("zset", redis.Z{2, "one"}).Result() + changed, err = client.ZAddXXCh("zset", redis.Z{ + Score: 2, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(changed).To(Equal(int64(1))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 2, Member: "one"}})) }) It("should ZIncr", func() { - score, err := client.ZIncr("zset", redis.Z{1, "one"}).Result() + score, err := client.ZIncr("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(score).To(Equal(float64(1))) vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) - score, err = client.ZIncr("zset", redis.Z{1, "one"}).Result() + score, err = client.ZIncr("zset", redis.Z{Score: 1, Member: "one"}).Result() Expect(err).NotTo(HaveOccurred()) Expect(score).To(Equal(float64(2))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 2, Member: "one"}})) }) It("should ZIncrNX", func() { - score, err := client.ZIncrNX("zset", redis.Z{1, "one"}).Result() + score, err := client.ZIncrNX("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(score).To(Equal(float64(1))) vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) - score, err = client.ZIncrNX("zset", redis.Z{1, "one"}).Result() + score, err = client.ZIncrNX("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).To(Equal(redis.Nil)) Expect(score).To(Equal(float64(0))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) }) It("should ZIncrXX", func() { - score, err := client.ZIncrXX("zset", redis.Z{1, "one"}).Result() + score, err := client.ZIncrXX("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).To(Equal(redis.Nil)) Expect(score).To(Equal(float64(0))) @@ -2254,36 +2356,57 @@ var _ = Describe("Commands", func() { Expect(err).NotTo(HaveOccurred()) Expect(vals).To(BeEmpty()) - added, err := client.ZAdd("zset", redis.Z{1, "one"}).Result() + added, err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(added).To(Equal(int64(1))) - score, err = client.ZIncrXX("zset", redis.Z{1, "one"}).Result() + score, err = client.ZIncrXX("zset", redis.Z{ + Score: 1, + Member: "one", + }).Result() Expect(err).NotTo(HaveOccurred()) Expect(score).To(Equal(float64(2))) vals, err = client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{2, "one"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 2, Member: "one"}})) }) It("should ZCard", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{ + Score: 2, + Member: "two", + }).Err() + Expect(err).NotTo(HaveOccurred()) - zCard := client.ZCard("zset") - Expect(zCard.Err()).NotTo(HaveOccurred()) - Expect(zCard.Val()).To(Equal(int64(2))) + card, err := client.ZCard("zset").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(card).To(Equal(int64(2))) }) It("should ZCount", func() { - err := client.ZAdd("zset", redis.Z{1, "one"}).Err() + err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Err() Expect(err).NotTo(HaveOccurred()) - err = client.ZAdd("zset", redis.Z{2, "two"}).Err() + err = client.ZAdd("zset", redis.Z{ + Score: 2, + Member: "two", + }).Err() Expect(err).NotTo(HaveOccurred()) - err = client.ZAdd("zset", redis.Z{3, "three"}).Err() + err = client.ZAdd("zset", redis.Z{ + Score: 3, + Member: "three", + }).Err() Expect(err).NotTo(HaveOccurred()) count, err := client.ZCount("zset", "-inf", "+inf").Result() @@ -2300,50 +2423,74 @@ var _ = Describe("Commands", func() { }) It("should ZIncrBy", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{ + Score: 1, + Member: "one", + }).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{ + Score: 2, + Member: "two", + }).Err() + Expect(err).NotTo(HaveOccurred()) - zIncrBy := client.ZIncrBy("zset", 2, "one") - Expect(zIncrBy.Err()).NotTo(HaveOccurred()) - Expect(zIncrBy.Val()).To(Equal(float64(3))) + n, err := client.ZIncrBy("zset", 2, "one").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(float64(3))) val, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{2, "two"}, {3, "one"}})) + Expect(val).To(Equal([]redis.Z{{ + Score: 2, + Member: "two", + }, { + Score: 3, + Member: "one", + }})) }) It("should ZInterStore", func() { - zAdd := client.ZAdd("zset1", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset1", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset1", redis.Z{ + Score: 1, + Member: "one", + }).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset1", redis.Z{ + Score: 2, + Member: "two", + }).Err() + Expect(err).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset2", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset2", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset3", redis.Z{3, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err = client.ZAdd("zset2", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset2", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset3", redis.Z{Score: 3, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) zInterStore := client.ZInterStore( "out", redis.ZStore{Weights: []float64{2, 3}}, "zset1", "zset2") Expect(zInterStore.Err()).NotTo(HaveOccurred()) Expect(zInterStore.Val()).To(Equal(int64(2))) - val, err := client.ZRangeWithScores("out", 0, -1).Result() + vals, err := client.ZRangeWithScores("out", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{5, "one"}, {10, "two"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 5, + Member: "one", + }, { + Score: 10, + Member: "two", + }})) }) It("should ZRange", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRange := client.ZRange("zset", 0, -1) Expect(zRange.Err()).NotTo(HaveOccurred()) @@ -2359,33 +2506,48 @@ var _ = Describe("Commands", func() { }) It("should ZRangeWithScores", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) - val, err := client.ZRangeWithScores("zset", 0, -1).Result() + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}, {3, "three"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }, { + Score: 2, + Member: "two", + }, { + Score: 3, + Member: "three", + }})) - val, err = client.ZRangeWithScores("zset", 2, 3).Result() + vals, err = client.ZRangeWithScores("zset", 2, 3).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{3, "three"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 3, Member: "three"}})) - val, err = client.ZRangeWithScores("zset", -2, -1).Result() + vals, err = client.ZRangeWithScores("zset", -2, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{2, "two"}, {3, "three"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 2, + Member: "two", + }, { + Score: 3, + Member: "three", + }})) }) It("should ZRangeByScore", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRangeByScore := client.ZRangeByScore("zset", redis.ZRangeBy{ Min: "-inf", @@ -2417,12 +2579,21 @@ var _ = Describe("Commands", func() { }) It("should ZRangeByLex", func() { - zAdd := client.ZAdd("zset", redis.Z{0, "a"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{0, "b"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{0, "c"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{ + Score: 0, + Member: "a", + }).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{ + Score: 0, + Member: "b", + }).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{ + Score: 0, + Member: "c", + }).Err() + Expect(err).NotTo(HaveOccurred()) zRangeByLex := client.ZRangeByLex("zset", redis.ZRangeBy{ Min: "-", @@ -2454,49 +2625,64 @@ var _ = Describe("Commands", func() { }) It("should ZRangeByScoreWithScoresMap", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) - val, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "-inf", Max: "+inf", }).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}, {3, "three"}})) - - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }, { + Score: 2, + Member: "two", + }, { + Score: 3, + Member: "three", + }})) + + vals, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}, {2, "two"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }, { + Score: 2, + Member: "two", + }})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + vals, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "(1", Max: "2", }).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{2, "two"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 2, Member: "two"}})) - val, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + vals, err = client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ Min: "(1", Max: "(2", }).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{})) + Expect(vals).To(Equal([]redis.Z{})) }) It("should ZRank", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRank := client.ZRank("zset", "three") Expect(zRank.Err()).NotTo(HaveOccurred()) @@ -2508,68 +2694,83 @@ var _ = Describe("Commands", func() { }) It("should ZRem", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRem := client.ZRem("zset", "two") Expect(zRem.Err()).NotTo(HaveOccurred()) Expect(zRem.Val()).To(Equal(int64(1))) - val, err := client.ZRangeWithScores("zset", 0, -1).Result() + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}, {3, "three"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 1, + Member: "one", + }, { + Score: 3, + Member: "three", + }})) }) It("should ZRemRangeByRank", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRemRangeByRank := client.ZRemRangeByRank("zset", 0, 1) Expect(zRemRangeByRank.Err()).NotTo(HaveOccurred()) Expect(zRemRangeByRank.Val()).To(Equal(int64(2))) - val, err := client.ZRangeWithScores("zset", 0, -1).Result() + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{3, "three"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 3, + Member: "three", + }})) }) It("should ZRemRangeByScore", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRemRangeByScore := client.ZRemRangeByScore("zset", "-inf", "(2") Expect(zRemRangeByScore.Err()).NotTo(HaveOccurred()) Expect(zRemRangeByScore.Val()).To(Equal(int64(1))) - val, err := client.ZRangeWithScores("zset", 0, -1).Result() + vals, err := client.ZRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{2, "two"}, {3, "three"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 2, + Member: "two", + }, { + Score: 3, + Member: "three", + }})) }) It("should ZRemRangeByLex", func() { zz := []redis.Z{ - {0, "aaaa"}, - {0, "b"}, - {0, "c"}, - {0, "d"}, - {0, "e"}, - {0, "foo"}, - {0, "zap"}, - {0, "zip"}, - {0, "ALPHA"}, - {0, "alpha"}, + {Score: 0, Member: "aaaa"}, + {Score: 0, Member: "b"}, + {Score: 0, Member: "c"}, + {Score: 0, Member: "d"}, + {Score: 0, Member: "e"}, + {Score: 0, Member: "foo"}, + {Score: 0, Member: "zap"}, + {Score: 0, Member: "zip"}, + {Score: 0, Member: "ALPHA"}, + {Score: 0, Member: "alpha"}, } for _, z := range zz { err := client.ZAdd("zset", z).Err() @@ -2586,12 +2787,12 @@ var _ = Describe("Commands", func() { }) It("should ZRevRange", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRevRange := client.ZRevRange("zset", 0, -1) Expect(zRevRange.Err()).NotTo(HaveOccurred()) @@ -2607,33 +2808,48 @@ var _ = Describe("Commands", func() { }) It("should ZRevRangeWithScoresMap", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) val, err := client.ZRevRangeWithScores("zset", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) + Expect(val).To(Equal([]redis.Z{{ + Score: 3, + Member: "three", + }, { + Score: 2, + Member: "two", + }, { + Score: 1, + Member: "one", + }})) val, err = client.ZRevRangeWithScores("zset", 2, 3).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{1, "one"}})) + Expect(val).To(Equal([]redis.Z{{Score: 1, Member: "one"}})) val, err = client.ZRevRangeWithScores("zset", -2, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{2, "two"}, {1, "one"}})) + Expect(val).To(Equal([]redis.Z{{ + Score: 2, + Member: "two", + }, { + Score: 1, + Member: "one", + }})) }) It("should ZRevRangeByScore", func() { - zadd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) - zadd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) - zadd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScore( "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() @@ -2652,12 +2868,12 @@ var _ = Describe("Commands", func() { }) It("should ZRevRangeByLex", func() { - zadd := client.ZAdd("zset", redis.Z{0, "a"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) - zadd = client.ZAdd("zset", redis.Z{0, "b"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) - zadd = client.ZAdd("zset", redis.Z{0, "c"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 0, Member: "a"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 0, Member: "b"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 0, Member: "c"}).Err() + Expect(err).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByLex( "zset", redis.ZRangeBy{Max: "+", Min: "-"}).Result() @@ -2676,50 +2892,68 @@ var _ = Describe("Commands", func() { }) It("should ZRevRangeByScoreWithScores", func() { - zadd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) - zadd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) - zadd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zadd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) vals, err := client.ZRevRangeByScoreWithScores( "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) - Expect(vals).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) + Expect(vals).To(Equal([]redis.Z{{ + Score: 3, + Member: "three", + }, { + Score: 2, + Member: "two", + }, { + Score: 1, + Member: "one", + }})) }) It("should ZRevRangeByScoreWithScoresMap", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) - val, err := client.ZRevRangeByScoreWithScores( + vals, err := client.ZRevRangeByScoreWithScores( "zset", redis.ZRangeBy{Max: "+inf", Min: "-inf"}).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{3, "three"}, {2, "two"}, {1, "one"}})) - - val, err = client.ZRevRangeByScoreWithScores( + Expect(vals).To(Equal([]redis.Z{{ + Score: 3, + Member: "three", + }, { + Score: 2, + Member: "two", + }, { + Score: 1, + Member: "one", + }})) + + vals, err = client.ZRevRangeByScoreWithScores( "zset", redis.ZRangeBy{Max: "2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{2, "two"}})) + Expect(vals).To(Equal([]redis.Z{{Score: 2, Member: "two"}})) - val, err = client.ZRevRangeByScoreWithScores( + vals, err = client.ZRevRangeByScoreWithScores( "zset", redis.ZRangeBy{Max: "(2", Min: "(1"}).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{})) + Expect(vals).To(Equal([]redis.Z{})) }) It("should ZRevRank", func() { - zAdd := client.ZAdd("zset", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zRevRank := client.ZRevRank("zset", "one") Expect(zRevRank.Err()).NotTo(HaveOccurred()) @@ -2731,7 +2965,7 @@ var _ = Describe("Commands", func() { }) It("should ZScore", func() { - zAdd := client.ZAdd("zset", redis.Z{1.001, "one"}) + zAdd := client.ZAdd("zset", redis.Z{Score: 1.001, Member: "one"}) Expect(zAdd.Err()).NotTo(HaveOccurred()) zScore := client.ZScore("zset", "one") @@ -2740,17 +2974,17 @@ var _ = Describe("Commands", func() { }) It("should ZUnionStore", func() { - zAdd := client.ZAdd("zset1", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset1", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err := client.ZAdd("zset1", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset1", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset2", redis.Z{1, "one"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset2", redis.Z{2, "two"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) - zAdd = client.ZAdd("zset2", redis.Z{3, "three"}) - Expect(zAdd.Err()).NotTo(HaveOccurred()) + err = client.ZAdd("zset2", redis.Z{Score: 1, Member: "one"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset2", redis.Z{Score: 2, Member: "two"}).Err() + Expect(err).NotTo(HaveOccurred()) + err = client.ZAdd("zset2", redis.Z{Score: 3, Member: "three"}).Err() + Expect(err).NotTo(HaveOccurred()) zUnionStore := client.ZUnionStore( "out", redis.ZStore{Weights: []float64{2, 3}}, "zset1", "zset2") @@ -2759,7 +2993,16 @@ var _ = Describe("Commands", func() { val, err := client.ZRangeWithScores("out", 0, -1).Result() Expect(err).NotTo(HaveOccurred()) - Expect(val).To(Equal([]redis.Z{{5, "one"}, {9, "three"}, {10, "two"}})) + Expect(val).To(Equal([]redis.Z{{ + Score: 5, + Member: "one", + }, { + Score: 9, + Member: "three", + }, { + Score: 10, + Member: "two", + }})) }) }) diff --git a/example_instrumentation_test.go b/example_instrumentation_test.go index 85abbd7440..82f655f2e1 100644 --- a/example_instrumentation_test.go +++ b/example_instrumentation_test.go @@ -24,7 +24,7 @@ func Example_instrumentation() { // finished processing: } -func Example_Pipeline_instrumentation() { +func ExamplePipeline_instrumentation() { client := redis.NewClient(&redis.Options{ Addr: ":6379", }) From 063393987ad5010fac0dc88b1237d3de9694ed44 Mon Sep 17 00:00:00 2001 From: "T. Thyer" Date: Wed, 7 Mar 2018 02:23:38 -0800 Subject: [PATCH 0425/1746] Add option to balance load between master node and replica nodes (#729) * Add option to balance load between master node and replica nodes --- cluster.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cluster.go b/cluster.go index ba562cf144..319f1e2cdf 100644 --- a/cluster.go +++ b/cluster.go @@ -34,6 +34,8 @@ type ClusterOptions struct { ReadOnly bool // Allows routing read-only commands to the closest master or slave node. RouteByLatency bool + // Allows routing read-only commands to the random master or slave node. + RouteRandomly bool // Following options are copied from Options struct. @@ -473,6 +475,12 @@ func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { return node, nil } +func (c *clusterState) slotRandomNode(slot int) *clusterNode { + nodes := c.slotNodes(slot) + n := rand.Intn(len(nodes)) + return nodes[n] +} + func (c *clusterState) slotNodes(slot int) []*clusterNode { if slot >= 0 && slot < len(c.slots) { return c.slots[slot] @@ -639,6 +647,11 @@ func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { return slot, node, err } + if c.opt.RouteRandomly { + node := state.slotRandomNode(slot) + return slot, node, nil + } + node, err := state.slotSlaveNode(slot) return slot, node, err } From a64d3e1ef1fbd678d54138e3a0cf2c742a1056f8 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 12:20:26 +0200 Subject: [PATCH 0426/1746] Fix build on 32bit arch --- Makefile | 1 + cluster.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d6c184e98d..1fbdac91c5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ all: testdeps go test ./... go test ./... -short -race + env GOOS=linux GOARCH=386 go test ./... go vet testdeps: testdata/redis/src/redis-server diff --git a/cluster.go b/cluster.go index 319f1e2cdf..4175b64fae 100644 --- a/cluster.go +++ b/cluster.go @@ -126,7 +126,7 @@ type clusterNode struct { latency uint32 // atomic generation uint32 // atomic - loading int64 // atomic + loading uint32 // atomic } func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { @@ -171,20 +171,20 @@ func (n *clusterNode) Latency() time.Duration { } func (n *clusterNode) MarkAsLoading() { - atomic.StoreInt64(&n.loading, time.Now().Unix()) + atomic.StoreUint32(&n.loading, uint32(time.Now().Unix())) } func (n *clusterNode) Loading() bool { const minute = int64(time.Minute / time.Second) - loading := atomic.LoadInt64(&n.loading) + loading := atomic.LoadUint32(&n.loading) if loading == 0 { return false } - if time.Now().Unix()-loading < minute { + if time.Now().Unix()-int64(loading) < minute { return true } - atomic.StoreInt64(&n.loading, 0) + atomic.StoreUint32(&n.loading, 0) return false } From 3a38e90858f3af4a2d7f426207c4d205bd06eb6d Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 13:42:20 +0200 Subject: [PATCH 0427/1746] Require Go 1.7+ and use context.Context --- redis.go | 30 ++++++++++++++++++++++++++++++ redis_context.go | 38 -------------------------------------- redis_no_context.go | 18 ------------------ 3 files changed, 30 insertions(+), 56 deletions(-) delete mode 100644 redis_context.go delete mode 100644 redis_no_context.go diff --git a/redis.go b/redis.go index 1f2167aa71..7f5ac5ff8d 100644 --- a/redis.go +++ b/redis.go @@ -1,6 +1,7 @@ package redis import ( + "context" "fmt" "log" "os" @@ -22,6 +23,19 @@ func SetLogger(logger *log.Logger) { internal.Logger = logger } +type baseClient struct { + connPool pool.Pooler + opt *Options + + process func(Cmder) error + processPipeline func([]Cmder) error + processTxPipeline func([]Cmder) error + + onClose func() error // hook called when client is closed + + ctx context.Context +} + func (c *baseClient) init() { c.process = c.defaultProcess c.processPipeline = c.defaultProcessPipeline @@ -355,6 +369,22 @@ func NewClient(opt *Options) *Client { return newClient(opt, newConnPool(opt)) } +func (c *Client) Context() context.Context { + if c.ctx != nil { + return c.ctx + } + return context.Background() +} + +func (c *Client) WithContext(ctx context.Context) *Client { + if ctx == nil { + panic("nil context") + } + c2 := c.copy() + c2.ctx = ctx + return c2 +} + func (c *Client) copy() *Client { c2 := new(Client) *c2 = *c diff --git a/redis_context.go b/redis_context.go deleted file mode 100644 index c00e505f6c..0000000000 --- a/redis_context.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build go1.7 - -package redis - -import ( - "context" - - "github.com/go-redis/redis/internal/pool" -) - -type baseClient struct { - connPool pool.Pooler - opt *Options - - process func(Cmder) error - processPipeline func([]Cmder) error - processTxPipeline func([]Cmder) error - - onClose func() error // hook called when client is closed - - ctx context.Context -} - -func (c *Client) Context() context.Context { - if c.ctx != nil { - return c.ctx - } - return context.Background() -} - -func (c *Client) WithContext(ctx context.Context) *Client { - if ctx == nil { - panic("nil context") - } - c2 := c.copy() - c2.ctx = ctx - return c2 -} diff --git a/redis_no_context.go b/redis_no_context.go deleted file mode 100644 index 8555c5c093..0000000000 --- a/redis_no_context.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !go1.7 - -package redis - -import ( - "github.com/go-redis/redis/internal/pool" -) - -type baseClient struct { - connPool pool.Pooler - opt *Options - - process func(Cmder) error - processPipeline func([]Cmder) error - processTxPipeline func([]Cmder) error - - onClose func() error // hook called when client is closed -} From d6cb688ea756d865f588932a322fc92e9368e338 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 13:50:14 +0200 Subject: [PATCH 0428/1746] Cleanup context implementation --- cluster.go | 5 +++++ redis.go | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cluster.go b/cluster.go index 4175b64fae..89f1e9a434 100644 --- a/cluster.go +++ b/cluster.go @@ -592,6 +592,11 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } +func (c *ClusterClient) copy() *ClusterClient { + cp := *c + return &cp +} + // Options returns read-only Options that were used to create the client. func (c *ClusterClient) Options() *ClusterOptions { return c.opt diff --git a/redis.go b/redis.go index 7f5ac5ff8d..e0b6464481 100644 --- a/redis.go +++ b/redis.go @@ -24,16 +24,14 @@ func SetLogger(logger *log.Logger) { } type baseClient struct { - connPool pool.Pooler opt *Options + connPool pool.Pooler process func(Cmder) error processPipeline func([]Cmder) error processTxPipeline func([]Cmder) error onClose func() error // hook called when client is closed - - ctx context.Context } func (c *baseClient) init() { @@ -349,6 +347,8 @@ func (c *baseClient) txPipelineReadQueued(cn *pool.Conn, cmds []Cmder) error { type Client struct { baseClient cmdable + + ctx context.Context } func newClient(opt *Options, pool pool.Pooler) *Client { @@ -386,10 +386,8 @@ func (c *Client) WithContext(ctx context.Context) *Client { } func (c *Client) copy() *Client { - c2 := new(Client) - *c2 = *c - c2.cmdable.setProcessor(c2.Process) - return c2 + cp := *c + return &cp } // Options returns read-only Options that were used to create the client. From 0e33c36bd1dccbe6371cca7ac11f14cdd9d8eda0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 13:42:20 +0200 Subject: [PATCH 0429/1746] Require Go 1.7+ and use context.Context --- redis.go | 30 ++++++++++++++++++++++++++++++ redis_context.go | 38 -------------------------------------- redis_no_context.go | 18 ------------------ 3 files changed, 30 insertions(+), 56 deletions(-) delete mode 100644 redis_context.go delete mode 100644 redis_no_context.go diff --git a/redis.go b/redis.go index 1f2167aa71..7f5ac5ff8d 100644 --- a/redis.go +++ b/redis.go @@ -1,6 +1,7 @@ package redis import ( + "context" "fmt" "log" "os" @@ -22,6 +23,19 @@ func SetLogger(logger *log.Logger) { internal.Logger = logger } +type baseClient struct { + connPool pool.Pooler + opt *Options + + process func(Cmder) error + processPipeline func([]Cmder) error + processTxPipeline func([]Cmder) error + + onClose func() error // hook called when client is closed + + ctx context.Context +} + func (c *baseClient) init() { c.process = c.defaultProcess c.processPipeline = c.defaultProcessPipeline @@ -355,6 +369,22 @@ func NewClient(opt *Options) *Client { return newClient(opt, newConnPool(opt)) } +func (c *Client) Context() context.Context { + if c.ctx != nil { + return c.ctx + } + return context.Background() +} + +func (c *Client) WithContext(ctx context.Context) *Client { + if ctx == nil { + panic("nil context") + } + c2 := c.copy() + c2.ctx = ctx + return c2 +} + func (c *Client) copy() *Client { c2 := new(Client) *c2 = *c diff --git a/redis_context.go b/redis_context.go deleted file mode 100644 index c00e505f6c..0000000000 --- a/redis_context.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build go1.7 - -package redis - -import ( - "context" - - "github.com/go-redis/redis/internal/pool" -) - -type baseClient struct { - connPool pool.Pooler - opt *Options - - process func(Cmder) error - processPipeline func([]Cmder) error - processTxPipeline func([]Cmder) error - - onClose func() error // hook called when client is closed - - ctx context.Context -} - -func (c *Client) Context() context.Context { - if c.ctx != nil { - return c.ctx - } - return context.Background() -} - -func (c *Client) WithContext(ctx context.Context) *Client { - if ctx == nil { - panic("nil context") - } - c2 := c.copy() - c2.ctx = ctx - return c2 -} diff --git a/redis_no_context.go b/redis_no_context.go deleted file mode 100644 index 8555c5c093..0000000000 --- a/redis_no_context.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !go1.7 - -package redis - -import ( - "github.com/go-redis/redis/internal/pool" -) - -type baseClient struct { - connPool pool.Pooler - opt *Options - - process func(Cmder) error - processPipeline func([]Cmder) error - processTxPipeline func([]Cmder) error - - onClose func() error // hook called when client is closed -} From c2fb5132c003876e045ef1809112e6b6774e7e6f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 13:50:14 +0200 Subject: [PATCH 0430/1746] Cleanup context implementation --- cluster.go | 5 +++++ redis.go | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cluster.go b/cluster.go index 4175b64fae..89f1e9a434 100644 --- a/cluster.go +++ b/cluster.go @@ -592,6 +592,11 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } +func (c *ClusterClient) copy() *ClusterClient { + cp := *c + return &cp +} + // Options returns read-only Options that were used to create the client. func (c *ClusterClient) Options() *ClusterOptions { return c.opt diff --git a/redis.go b/redis.go index 7f5ac5ff8d..e0b6464481 100644 --- a/redis.go +++ b/redis.go @@ -24,16 +24,14 @@ func SetLogger(logger *log.Logger) { } type baseClient struct { - connPool pool.Pooler opt *Options + connPool pool.Pooler process func(Cmder) error processPipeline func([]Cmder) error processTxPipeline func([]Cmder) error onClose func() error // hook called when client is closed - - ctx context.Context } func (c *baseClient) init() { @@ -349,6 +347,8 @@ func (c *baseClient) txPipelineReadQueued(cn *pool.Conn, cmds []Cmder) error { type Client struct { baseClient cmdable + + ctx context.Context } func newClient(opt *Options, pool pool.Pooler) *Client { @@ -386,10 +386,8 @@ func (c *Client) WithContext(ctx context.Context) *Client { } func (c *Client) copy() *Client { - c2 := new(Client) - *c2 = *c - c2.cmdable.setProcessor(c2.Process) - return c2 + cp := *c + return &cp } // Options returns read-only Options that were used to create the client. From db04210af4fef2acbb259e0e6faca844408a10fe Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 14:08:40 +0200 Subject: [PATCH 0431/1746] Extract ringShards --- ring.go | 323 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 175 insertions(+), 148 deletions(-) diff --git a/ring.go b/ring.go index d07e806e38..c1c16d770b 100644 --- a/ring.go +++ b/ring.go @@ -87,6 +87,8 @@ func (opt *RingOptions) clientOptions() *Options { } } +//------------------------------------------------------------------------------ + type ringShard struct { Client *Client down int32 @@ -127,6 +129,150 @@ func (shard *ringShard) Vote(up bool) bool { return shard.IsDown() } +//------------------------------------------------------------------------------ + +type ringShards struct { + mu sync.RWMutex + hash *consistenthash.Map + shards map[string]*ringShard // read only + list []*ringShard // read only + closed bool +} + +func newRingShards() *ringShards { + return &ringShards{ + hash: consistenthash.New(nreplicas, nil), + shards: make(map[string]*ringShard), + } +} + +func (c *ringShards) Add(name string, cl *Client) { + shard := &ringShard{Client: cl} + c.hash.Add(name) + c.shards[name] = shard + c.list = append(c.list, shard) +} + +func (c *ringShards) List() []*ringShard { + c.mu.RLock() + list := c.list + c.mu.RUnlock() + return list +} + +func (c *ringShards) Hash(key string) string { + c.mu.RLock() + hash := c.hash.Get(key) + c.mu.RUnlock() + return hash +} + +func (c *ringShards) GetByKey(key string) (*ringShard, error) { + key = hashtag.Key(key) + + c.mu.RLock() + + if c.closed { + c.mu.RUnlock() + return nil, pool.ErrClosed + } + + hash := c.hash.Get(key) + if hash == "" { + c.mu.RUnlock() + return nil, errRingShardsDown + } + + shard := c.shards[hash] + c.mu.RUnlock() + + return shard, nil +} + +func (c *ringShards) GetByHash(name string) (*ringShard, error) { + if name == "" { + return c.Random() + } + + c.mu.RLock() + shard := c.shards[name] + c.mu.RUnlock() + return shard, nil +} + +func (c *ringShards) Random() (*ringShard, error) { + return c.GetByKey(strconv.Itoa(rand.Int())) +} + +// heartbeat monitors state of each shard in the ring. +func (c *ringShards) Heartbeat(frequency time.Duration) { + ticker := time.NewTicker(frequency) + defer ticker.Stop() + for range ticker.C { + var rebalance bool + + c.mu.RLock() + + if c.closed { + c.mu.RUnlock() + break + } + + shards := c.list + c.mu.RUnlock() + + for _, shard := range shards { + err := shard.Client.Ping().Err() + if shard.Vote(err == nil || err == pool.ErrPoolTimeout) { + internal.Logf("ring shard state changed: %s", shard) + rebalance = true + } + } + + if rebalance { + c.rebalance() + } + } +} + +// rebalance removes dead shards from the Ring. +func (c *ringShards) rebalance() { + hash := consistenthash.New(nreplicas, nil) + for name, shard := range c.shards { + if shard.IsUp() { + hash.Add(name) + } + } + + c.mu.Lock() + c.hash = hash + c.mu.Unlock() +} + +func (c *ringShards) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return nil + } + c.closed = true + + var firstErr error + for _, shard := range c.shards { + if err := shard.Client.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + c.hash = nil + c.shards = nil + c.list = nil + + return firstErr +} + +//------------------------------------------------------------------------------ + // Ring is a Redis client that uses constistent hashing to distribute // keys across multiple Redis servers (shards). It's safe for // concurrent use by multiple goroutines. @@ -143,18 +289,11 @@ func (shard *ringShard) Vote(up bool) bool { // Otherwise you should use Redis Cluster. type Ring struct { cmdable - opt *RingOptions + shards *ringShards cmdsInfoCache *cmdsInfoCache - mu sync.RWMutex - hash *consistenthash.Map - shards map[string]*ringShard // read only - shardsList []*ringShard // read only - processPipeline func([]Cmder) error - - closed bool } func NewRing(opt *RingOptions) *Ring { @@ -162,10 +301,8 @@ func NewRing(opt *RingOptions) *Ring { ring := &Ring{ opt: opt, + shards: newRingShards(), cmdsInfoCache: newCmdsInfoCache(), - - hash: consistenthash.New(nreplicas, nil), - shards: make(map[string]*ringShard), } ring.processPipeline = ring.defaultProcessPipeline ring.cmdable.setProcessor(ring.Process) @@ -173,19 +310,17 @@ func NewRing(opt *RingOptions) *Ring { for name, addr := range opt.Addrs { clopt := opt.clientOptions() clopt.Addr = addr - ring.addShard(name, NewClient(clopt)) + ring.shards.Add(name, NewClient(clopt)) } - go ring.heartbeat() + go ring.shards.Heartbeat(opt.HeartbeatFrequency) return ring } -func (c *Ring) addShard(name string, cl *Client) { - shard := &ringShard{Client: cl} - c.hash.Add(name) - c.shards[name] = shard - c.shardsList = append(c.shardsList, shard) +func (c *Ring) copy() *Ring { + cp := *c + return &cp } // Options returns read-only Options that were used to create the client. @@ -199,10 +334,7 @@ func (c *Ring) retryBackoff(attempt int) time.Duration { // PoolStats returns accumulated connection pool stats. func (c *Ring) PoolStats() *PoolStats { - c.mu.RLock() - shards := c.shardsList - c.mu.RUnlock() - + shards := c.shards.List() var acc PoolStats for _, shard := range shards { s := shard.Client.connPool.Stats() @@ -221,7 +353,7 @@ func (c *Ring) Subscribe(channels ...string) *PubSub { panic("at least one channel is required") } - shard, err := c.shardByKey(channels[0]) + shard, err := c.shards.GetByKey(channels[0]) if err != nil { // TODO: return PubSub with sticky error panic(err) @@ -235,7 +367,7 @@ func (c *Ring) PSubscribe(channels ...string) *PubSub { panic("at least one channel is required") } - shard, err := c.shardByKey(channels[0]) + shard, err := c.shards.GetByKey(channels[0]) if err != nil { // TODO: return PubSub with sticky error panic(err) @@ -246,10 +378,7 @@ func (c *Ring) PSubscribe(channels ...string) *PubSub { // ForEachShard concurrently calls the fn on each live shard in the ring. // It returns the first error if any. func (c *Ring) ForEachShard(fn func(client *Client) error) error { - c.mu.RLock() - shards := c.shardsList - c.mu.RUnlock() - + shards := c.shards.List() var wg sync.WaitGroup errCh := make(chan error, 1) for _, shard := range shards { @@ -281,10 +410,7 @@ func (c *Ring) ForEachShard(fn func(client *Client) error) error { func (c *Ring) cmdInfo(name string) *CommandInfo { cmdsInfo, err := c.cmdsInfoCache.Do(func() (map[string]*CommandInfo, error) { - c.mu.RLock() - shards := c.shardsList - c.mu.RUnlock() - + shards := c.shards.List() firstErr := errRingShardsDown for _, shard := range shards { cmdsInfo, err := shard.Client.Command().Result() @@ -307,50 +433,14 @@ func (c *Ring) cmdInfo(name string) *CommandInfo { return info } -func (c *Ring) shardByKey(key string) (*ringShard, error) { - key = hashtag.Key(key) - - c.mu.RLock() - - if c.closed { - c.mu.RUnlock() - return nil, pool.ErrClosed - } - - name := c.hash.Get(key) - if name == "" { - c.mu.RUnlock() - return nil, errRingShardsDown - } - - shard := c.shards[name] - c.mu.RUnlock() - return shard, nil -} - -func (c *Ring) randomShard() (*ringShard, error) { - return c.shardByKey(strconv.Itoa(rand.Int())) -} - -func (c *Ring) shardByName(name string) (*ringShard, error) { - if name == "" { - return c.randomShard() - } - - c.mu.RLock() - shard := c.shards[name] - c.mu.RUnlock() - return shard, nil -} - func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) { cmdInfo := c.cmdInfo(cmd.Name()) pos := cmdFirstKeyPos(cmd, cmdInfo) if pos == 0 { - return c.randomShard() + return c.shards.Random() } firstKey := cmd.stringArg(pos) - return c.shardByKey(firstKey) + return c.shards.GetByKey(firstKey) } func (c *Ring) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) { @@ -369,77 +459,6 @@ func (c *Ring) Process(cmd Cmder) error { return shard.Client.Process(cmd) } -// rebalance removes dead shards from the Ring. -func (c *Ring) rebalance() { - hash := consistenthash.New(nreplicas, nil) - for name, shard := range c.shards { - if shard.IsUp() { - hash.Add(name) - } - } - - c.mu.Lock() - c.hash = hash - c.mu.Unlock() -} - -// heartbeat monitors state of each shard in the ring. -func (c *Ring) heartbeat() { - ticker := time.NewTicker(c.opt.HeartbeatFrequency) - defer ticker.Stop() - for range ticker.C { - var rebalance bool - - c.mu.RLock() - - if c.closed { - c.mu.RUnlock() - break - } - - shards := c.shardsList - c.mu.RUnlock() - - for _, shard := range shards { - err := shard.Client.Ping().Err() - if shard.Vote(err == nil || err == pool.ErrPoolTimeout) { - internal.Logf("ring shard state changed: %s", shard) - rebalance = true - } - } - - if rebalance { - c.rebalance() - } - } -} - -// Close closes the ring client, releasing any open resources. -// -// It is rare to Close a Ring, as the Ring is meant to be long-lived -// and shared between many goroutines. -func (c *Ring) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.closed { - return nil - } - c.closed = true - - var firstErr error - for _, shard := range c.shards { - if err := shard.Client.Close(); err != nil && firstErr == nil { - firstErr = err - } - } - c.hash = nil - c.shards = nil - c.shardsList = nil - - return firstErr -} - func (c *Ring) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.processPipeline, @@ -462,11 +481,11 @@ func (c *Ring) defaultProcessPipeline(cmds []Cmder) error { cmdsMap := make(map[string][]Cmder) for _, cmd := range cmds { cmdInfo := c.cmdInfo(cmd.Name()) - name := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) - if name != "" { - name = c.hash.Get(hashtag.Key(name)) + hash := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo)) + if hash != "" { + hash = c.shards.Hash(hashtag.Key(hash)) } - cmdsMap[name] = append(cmdsMap[name], cmd) + cmdsMap[hash] = append(cmdsMap[hash], cmd) } for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ { @@ -476,8 +495,8 @@ func (c *Ring) defaultProcessPipeline(cmds []Cmder) error { var failedCmdsMap map[string][]Cmder - for name, cmds := range cmdsMap { - shard, err := c.shardByName(name) + for hash, cmds := range cmdsMap { + shard, err := c.shards.GetByHash(hash) if err != nil { setCmdsErr(cmds, err) continue @@ -500,7 +519,7 @@ func (c *Ring) defaultProcessPipeline(cmds []Cmder) error { if failedCmdsMap == nil { failedCmdsMap = make(map[string][]Cmder) } - failedCmdsMap[name] = cmds + failedCmdsMap[hash] = cmds } } @@ -520,3 +539,11 @@ func (c *Ring) TxPipeline() Pipeliner { func (c *Ring) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { panic("not implemented") } + +// Close closes the ring client, releasing any open resources. +// +// It is rare to Close a Ring, as the Ring is meant to be long-lived +// and shared between many goroutines. +func (c *Ring) Close() error { + return c.shards.Close() +} From 4fe9f93940dd8a3e1d7dab2c0eb46d1cb9bace1c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 7 Mar 2018 14:38:18 +0200 Subject: [PATCH 0432/1746] Re-order atomic field to please race detector --- cluster.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 89f1e9a434..767df5f139 100644 --- a/cluster.go +++ b/cluster.go @@ -491,13 +491,14 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode { //------------------------------------------------------------------------------ type clusterStateHolder struct { - load func() (*clusterState, error) - reloading uint32 // atomic + load func() (*clusterState, error) state atomic.Value lastErrMu sync.RWMutex lastErr error + + reloading uint32 // atomic } func newClusterStateHolder(fn func() (*clusterState, error)) *clusterStateHolder { From 731dd72b84618db9d6dd300e43e5b096e837c37a Mon Sep 17 00:00:00 2001 From: Gabriel Aszalos Date: Wed, 7 Mar 2018 14:39:56 +0100 Subject: [PATCH 0433/1746] {cluster,ring}: add support for context to ClusterClient and Ring --- cluster.go | 19 +++++++++++++++++++ ring.go | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/cluster.go b/cluster.go index 767df5f139..1e787adda1 100644 --- a/cluster.go +++ b/cluster.go @@ -1,6 +1,7 @@ package redis import ( + "context" "errors" "fmt" "math" @@ -557,6 +558,8 @@ func (c *clusterStateHolder) Get() (*clusterState, error) { type ClusterClient struct { cmdable + ctx context.Context + opt *ClusterOptions nodes *clusterNodes state *clusterStateHolder @@ -593,6 +596,22 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } +func (c *ClusterClient) Context() context.Context { + if c.ctx != nil { + return c.ctx + } + return context.Background() +} + +func (c *ClusterClient) WithContext(ctx context.Context) *ClusterClient { + if ctx == nil { + panic("nil context") + } + c2 := c.copy() + c2.ctx = ctx + return c2 +} + func (c *ClusterClient) copy() *ClusterClient { cp := *c return &cp diff --git a/ring.go b/ring.go index c1c16d770b..6d28774136 100644 --- a/ring.go +++ b/ring.go @@ -1,6 +1,7 @@ package redis import ( + "context" "errors" "fmt" "math/rand" @@ -289,6 +290,9 @@ func (c *ringShards) Close() error { // Otherwise you should use Redis Cluster. type Ring struct { cmdable + + ctx context.Context + opt *RingOptions shards *ringShards cmdsInfoCache *cmdsInfoCache @@ -318,6 +322,22 @@ func NewRing(opt *RingOptions) *Ring { return ring } +func (c *Ring) Context() context.Context { + if c.ctx != nil { + return c.ctx + } + return context.Background() +} + +func (c *Ring) WithContext(ctx context.Context) *Ring { + if ctx == nil { + panic("nil context") + } + c2 := c.copy() + c2.ctx = ctx + return c2 +} + func (c *Ring) copy() *Ring { cp := *c return &cp From 11ca0e65c640dd1b1dd9f9a9b64c500a0af51ad4 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 8 Mar 2018 10:16:53 +0200 Subject: [PATCH 0434/1746] Add race test for BLPop --- cluster_test.go | 4 ++-- main_test.go | 7 ++++++- race_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index 6788599a9f..58bf738980 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -788,8 +788,8 @@ var _ = Describe("ClusterClient timeout", func() { Context("read/write timeout", func() { BeforeEach(func() { opt := redisClusterOptions() - opt.ReadTimeout = 200 * time.Millisecond - opt.WriteTimeout = 200 * time.Millisecond + opt.ReadTimeout = 300 * time.Millisecond + opt.WriteTimeout = 300 * time.Millisecond opt.MaxRedirects = 1 client = cluster.clusterClient(opt) diff --git a/main_test.go b/main_test.go index 7c5a6a969f..4eddc1ee3d 100644 --- a/main_test.go +++ b/main_test.go @@ -142,7 +142,7 @@ func redisRingOptions() *redis.RingOptions { } } -func perform(n int, cbs ...func(int)) { +func performAsync(n int, cbs ...func(int)) *sync.WaitGroup { var wg sync.WaitGroup for _, cb := range cbs { for i := 0; i < n; i++ { @@ -155,6 +155,11 @@ func perform(n int, cbs ...func(int)) { }(cb, i) } } + return &wg +} + +func perform(n int, cbs ...func(int)) { + wg := performAsync(n, cbs...) wg.Wait() } diff --git a/race_test.go b/race_test.go index 05f338c78e..789c8ba8a1 100644 --- a/race_test.go +++ b/race_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "strconv" + "sync/atomic" "testing" "time" @@ -258,6 +259,30 @@ var _ = Describe("races", func() { Expect(err).NotTo(HaveOccurred()) Expect(n).To(Equal(int64(N))) }) + + It("should BLPop", func() { + var received uint32 + wg := performAsync(C, func(id int) { + for { + v, err := client.BLPop(3*time.Second, "list").Result() + if err != nil { + break + } + Expect(v).To(Equal([]string{"list", "hello"})) + atomic.AddUint32(&received, 1) + } + }) + + perform(C, func(id int) { + for i := 0; i < N; i++ { + err := client.LPush("list", "hello").Err() + Expect(err).NotTo(HaveOccurred()) + } + }) + + wg.Wait() + Expect(received).To(Equal(uint32(C * N))) + }) }) func bigVal() []byte { From 46dd7afbbf95205808592008813f24affa09aca9 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 8 Mar 2018 13:50:16 +0200 Subject: [PATCH 0435/1746] Add cluster race test --- race_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/race_test.go b/race_test.go index 789c8ba8a1..acecf85690 100644 --- a/race_test.go +++ b/race_test.go @@ -285,6 +285,53 @@ var _ = Describe("races", func() { }) }) +var _ = Describe("cluster races", func() { + var client *redis.ClusterClient + var C, N int + + BeforeEach(func() { + opt := redisClusterOptions() + client = cluster.clusterClient(opt) + + C, N = 10, 1000 + if testing.Short() { + C = 4 + N = 100 + } + }) + + AfterEach(func() { + err := client.Close() + Expect(err).NotTo(HaveOccurred()) + }) + + It("should echo", func() { + perform(C, func(id int) { + for i := 0; i < N; i++ { + msg := fmt.Sprintf("echo %d %d", id, i) + echo, err := client.Echo(msg).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(echo).To(Equal(msg)) + } + }) + }) + + It("should incr", func() { + key := "TestIncrFromGoroutines" + + perform(C, func(id int) { + for i := 0; i < N; i++ { + err := client.Incr(key).Err() + Expect(err).NotTo(HaveOccurred()) + } + }) + + val, err := client.Get(key).Int64() + Expect(err).NotTo(HaveOccurred()) + Expect(val).To(Equal(int64(C * N))) + }) +}) + func bigVal() []byte { return bytes.Repeat([]byte{'*'}, 1<<17) // 128kb } From 3aa80da65408aab8562a3447f9eb56d07174ab6b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 8 Mar 2018 14:00:21 +0200 Subject: [PATCH 0436/1746] internal/proto: cleanup code --- internal/proto/write_buffer.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/proto/write_buffer.go b/internal/proto/write_buffer.go index 096b6d76af..cc4014fb4f 100644 --- a/internal/proto/write_buffer.go +++ b/internal/proto/write_buffer.go @@ -71,17 +71,15 @@ func (w *WriteBuffer) append(val interface{}) error { } else { w.AppendString("0") } - default: - if bm, ok := val.(encoding.BinaryMarshaler); ok { - bb, err := bm.MarshalBinary() - if err != nil { - return err - } - w.AppendBytes(bb) - } else { - return fmt.Errorf( - "redis: can't marshal %T (consider implementing encoding.BinaryMarshaler)", val) + case encoding.BinaryMarshaler: + b, err := v.MarshalBinary() + if err != nil { + return err } + w.AppendBytes(b) + default: + return fmt.Errorf( + "redis: can't marshal %T (consider implementing encoding.BinaryMarshaler)", val) } return nil } From 0c76bc80b7a0ff15bfb384ab5211d0b52b0ceafc Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 8 Mar 2018 14:30:27 +0200 Subject: [PATCH 0437/1746] Support []string slices in commands that accept multiple values --- commands.go | 78 ++++++++++++++++++++++-------------------------- commands_test.go | 11 +++++++ 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/commands.go b/commands.go index 6609a74f06..a3dacacd25 100644 --- a/commands.go +++ b/commands.go @@ -39,6 +39,22 @@ func formatSec(dur time.Duration) int64 { return int64(dur / time.Second) } +func appendArgs(dst, src []interface{}) []interface{} { + if len(src) == 1 { + if ss, ok := src[0].([]string); ok { + for _, s := range ss { + dst = append(dst, s) + } + return dst + } + } + + for _, v := range src { + dst = append(dst, v) + } + return dst +} + type Cmdable interface { Pipeline() Pipeliner Pipelined(fn func(Pipeliner) error) ([]Cmder, error) @@ -760,22 +776,18 @@ func (c *cmdable) MGet(keys ...string) *SliceCmd { } func (c *cmdable) MSet(pairs ...interface{}) *StatusCmd { - args := make([]interface{}, 1+len(pairs)) + args := make([]interface{}, 1, 1+len(pairs)) args[0] = "mset" - for i, pair := range pairs { - args[1+i] = pair - } + args = appendArgs(args, pairs) cmd := NewStatusCmd(args...) c.process(cmd) return cmd } func (c *cmdable) MSetNX(pairs ...interface{}) *BoolCmd { - args := make([]interface{}, 1+len(pairs)) + args := make([]interface{}, 1, 1+len(pairs)) args[0] = "msetnx" - for i, pair := range pairs { - args[1+i] = pair - } + args = appendArgs(args, pairs) cmd := NewBoolCmd(args...) c.process(cmd) return cmd @@ -1040,12 +1052,10 @@ func (c *cmdable) LPop(key string) *StringCmd { } func (c *cmdable) LPush(key string, values ...interface{}) *IntCmd { - args := make([]interface{}, 2+len(values)) + args := make([]interface{}, 2, 2+len(values)) args[0] = "lpush" args[1] = key - for i, value := range values { - args[2+i] = value - } + args = appendArgs(args, values) cmd := NewIntCmd(args...) c.process(cmd) return cmd @@ -1104,12 +1114,10 @@ func (c *cmdable) RPopLPush(source, destination string) *StringCmd { } func (c *cmdable) RPush(key string, values ...interface{}) *IntCmd { - args := make([]interface{}, 2+len(values)) + args := make([]interface{}, 2, 2+len(values)) args[0] = "rpush" args[1] = key - for i, value := range values { - args[2+i] = value - } + args = appendArgs(args, values) cmd := NewIntCmd(args...) c.process(cmd) return cmd @@ -1124,12 +1132,10 @@ func (c *cmdable) RPushX(key string, value interface{}) *IntCmd { //------------------------------------------------------------------------------ func (c *cmdable) SAdd(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2+len(members)) + args := make([]interface{}, 2, 2+len(members)) args[0] = "sadd" args[1] = key - for i, member := range members { - args[2+i] = member - } + args = appendArgs(args, members) cmd := NewIntCmd(args...) c.process(cmd) return cmd @@ -1242,12 +1248,10 @@ func (c *cmdable) SRandMemberN(key string, count int64) *StringSliceCmd { } func (c *cmdable) SRem(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2+len(members)) + args := make([]interface{}, 2, 2+len(members)) args[0] = "srem" args[1] = key - for i, member := range members { - args[2+i] = member - } + args = appendArgs(args, members) cmd := NewIntCmd(args...) c.process(cmd) return cmd @@ -1507,12 +1511,10 @@ func (c *cmdable) ZRank(key, member string) *IntCmd { } func (c *cmdable) ZRem(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2+len(members)) + args := make([]interface{}, 2, 2+len(members)) args[0] = "zrem" args[1] = key - for i, member := range members { - args[2+i] = member - } + args = appendArgs(args, members) cmd := NewIntCmd(args...) c.process(cmd) return cmd @@ -1628,12 +1630,10 @@ func (c *cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd //------------------------------------------------------------------------------ func (c *cmdable) PFAdd(key string, els ...interface{}) *IntCmd { - args := make([]interface{}, 2+len(els)) + args := make([]interface{}, 2, 2+len(els)) args[0] = "pfadd" args[1] = key - for i, el := range els { - args[2+i] = el - } + args = appendArgs(args, els) cmd := NewIntCmd(args...) c.process(cmd) return cmd @@ -1851,34 +1851,28 @@ func (c *cmdable) Time() *TimeCmd { //------------------------------------------------------------------------------ func (c *cmdable) Eval(script string, keys []string, args ...interface{}) *Cmd { - cmdArgs := make([]interface{}, 3+len(keys)+len(args)) + cmdArgs := make([]interface{}, 3+len(keys), 3+len(keys)+len(args)) cmdArgs[0] = "eval" cmdArgs[1] = script cmdArgs[2] = len(keys) for i, key := range keys { cmdArgs[3+i] = key } - pos := 3 + len(keys) - for i, arg := range args { - cmdArgs[pos+i] = arg - } + cmdArgs = appendArgs(cmdArgs, args) cmd := NewCmd(cmdArgs...) c.process(cmd) return cmd } func (c *cmdable) EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd { - cmdArgs := make([]interface{}, 3+len(keys)+len(args)) + cmdArgs := make([]interface{}, 3+len(keys), 3+len(keys)+len(args)) cmdArgs[0] = "evalsha" cmdArgs[1] = sha1 cmdArgs[2] = len(keys) for i, key := range keys { cmdArgs[3+i] = key } - pos := 3 + len(keys) - for i, arg := range args { - cmdArgs[pos+i] = arg - } + cmdArgs = appendArgs(cmdArgs, args) cmd := NewCmd(cmdArgs...) c.process(cmd) return cmd diff --git a/commands_test.go b/commands_test.go index 5e4d260f95..156b122184 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1790,6 +1790,17 @@ var _ = Describe("Commands", func() { Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) }) + It("should SAdd strings", func() { + set := []string{"Hello", "World", "World"} + sAdd := client.SAdd("set", set) + Expect(sAdd.Err()).NotTo(HaveOccurred()) + Expect(sAdd.Val()).To(Equal(int64(2))) + + sMembers := client.SMembers("set") + Expect(sMembers.Err()).NotTo(HaveOccurred()) + Expect(sMembers.Val()).To(ConsistOf([]string{"Hello", "World"})) + }) + It("should SCard", func() { sAdd := client.SAdd("set", "Hello") Expect(sAdd.Err()).NotTo(HaveOccurred()) From 20363d149b416d974462deff808f0499e0ea0d13 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Tue, 13 Mar 2018 15:51:38 +0200 Subject: [PATCH 0438/1746] Fix WithContext followed by WrapProcess --- redis.go | 40 +++++++++++++++++++++++----------------- redis_test.go | 32 +++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/redis.go b/redis.go index e0b6464481..7a606b70ed 100644 --- a/redis.go +++ b/redis.go @@ -96,16 +96,7 @@ func (c *baseClient) initConn(cn *pool.Conn) error { return nil } - // Temp client to initialize connection. - conn := &Conn{ - baseClient: baseClient{ - opt: c.opt, - connPool: pool.NewSingleConnPool(cn), - }, - } - conn.baseClient.init() - conn.statefulCmdable.setProcessor(conn.Process) - + conn := newConn(c.opt, cn) _, err := conn.Pipelined(func(pipe Pipeliner) error { if c.opt.Password != "" { pipe.Auth(c.opt.Password) @@ -351,22 +342,24 @@ type Client struct { ctx context.Context } -func newClient(opt *Options, pool pool.Pooler) *Client { +// NewClient returns a client to the Redis Server specified by Options. +func NewClient(opt *Options) *Client { + opt.init() + c := Client{ baseClient: baseClient{ opt: opt, - connPool: pool, + connPool: newConnPool(opt), }, } c.baseClient.init() - c.cmdable.setProcessor(c.Process) + c.init() + return &c } -// NewClient returns a client to the Redis Server specified by Options. -func NewClient(opt *Options) *Client { - opt.init() - return newClient(opt, newConnPool(opt)) +func (c *Client) init() { + c.cmdable.setProcessor(c.Process) } func (c *Client) Context() context.Context { @@ -387,6 +380,7 @@ func (c *Client) WithContext(ctx context.Context) *Client { func (c *Client) copy() *Client { cp := *c + cp.init() return &cp } @@ -467,6 +461,18 @@ type Conn struct { statefulCmdable } +func newConn(opt *Options, cn *pool.Conn) *Conn { + c := Conn{ + baseClient: baseClient{ + opt: opt, + connPool: pool.NewSingleConnPool(cn), + }, + } + c.baseClient.init() + c.statefulCmdable.setProcessor(c.Process) + return &c +} + func (c *Conn) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().Pipelined(fn) } diff --git a/redis_test.go b/redis_test.go index ad8aa11a06..df2485d394 100644 --- a/redis_test.go +++ b/redis_test.go @@ -228,18 +228,40 @@ var _ = Describe("Client", func() { }) It("should call WrapProcess", func() { - var wrapperFnCalled bool + var fnCalled bool - client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error { + client.WrapProcess(func(old func(redis.Cmder) error) func(redis.Cmder) error { return func(cmd redis.Cmder) error { - wrapperFnCalled = true - return oldProcess(cmd) + fnCalled = true + return old(cmd) } }) Expect(client.Ping().Err()).NotTo(HaveOccurred()) + Expect(fnCalled).To(BeTrue()) + }) + + It("should call WrapProcess after WithContext", func() { + var fn1Called, fn2Called bool + + client.WrapProcess(func(old func(cmd redis.Cmder) error) func(cmd redis.Cmder) error { + return func(cmd redis.Cmder) error { + fn1Called = true + return old(cmd) + } + }) + + client2 := client.WithContext(client.Context()) + client2.WrapProcess(func(old func(cmd redis.Cmder) error) func(cmd redis.Cmder) error { + return func(cmd redis.Cmder) error { + fn2Called = true + return old(cmd) + } + }) - Expect(wrapperFnCalled).To(BeTrue()) + Expect(client2.Ping().Err()).NotTo(HaveOccurred()) + Expect(fn2Called).To(BeTrue()) + Expect(fn1Called).To(BeTrue()) }) }) From 877867d2845fbaf86798befe410b6ceb6f5c29a3 Mon Sep 17 00:00:00 2001 From: superkinglabs Date: Wed, 14 Mar 2018 16:12:51 +0530 Subject: [PATCH 0439/1746] Remove costly 'appendIfNotExists' and 'remove' call from PubSub (#743) * remove costly 'appendIfNotExists' and 'remove' call from pubsub --- cluster.go | 28 ++++++++++++++++++++++ pubsub.go | 70 ++++++++++++++++++++++++++---------------------------- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/cluster.go b/cluster.go index 1e787adda1..4a2951157c 100644 --- a/cluster.go +++ b/cluster.go @@ -1409,3 +1409,31 @@ func appendNode(nodes []*clusterNode, node *clusterNode) []*clusterNode { } return append(nodes, node) } + +func appendIfNotExists(ss []string, es ...string) []string { +loop: + for _, e := range es { + for _, s := range ss { + if s == e { + continue loop + } + } + ss = append(ss, e) + } + return ss +} + +func remove(ss []string, es ...string) []string { + if len(es) == 0 { + return ss[:0] + } + for _, e := range es { + for i, s := range ss { + if s == e { + ss = append(ss[:i], ss[i+1:]...) + break + } + } + } + return ss +} diff --git a/pubsub.go b/pubsub.go index 3ee4ea9d01..b56728f3e6 100644 --- a/pubsub.go +++ b/pubsub.go @@ -24,8 +24,8 @@ type PubSub struct { mu sync.Mutex cn *pool.Conn - channels []string - patterns []string + channels map[string]struct{} + patterns map[string]struct{} closed bool cmd *Cmd @@ -67,12 +67,24 @@ func (c *PubSub) _conn(channels []string) (*pool.Conn, error) { func (c *PubSub) resubscribe(cn *pool.Conn) error { var firstErr error if len(c.channels) > 0 { - if err := c._subscribe(cn, "subscribe", c.channels...); err != nil && firstErr == nil { + channels := make([]string, len(c.channels)) + i := 0 + for channel := range c.channels { + channels[i] = channel + i++ + } + if err := c._subscribe(cn, "subscribe", channels...); err != nil && firstErr == nil { firstErr = err } } if len(c.patterns) > 0 { - if err := c._subscribe(cn, "psubscribe", c.patterns...); err != nil && firstErr == nil { + patterns := make([]string, len(c.patterns)) + i := 0 + for pattern := range c.patterns { + patterns[i] = pattern + i++ + } + if err := c._subscribe(cn, "psubscribe", patterns...); err != nil && firstErr == nil { firstErr = err } } @@ -132,7 +144,12 @@ func (c *PubSub) Close() error { func (c *PubSub) Subscribe(channels ...string) error { c.mu.Lock() err := c.subscribe("subscribe", channels...) - c.channels = appendIfNotExists(c.channels, channels...) + if c.channels == nil { + c.channels = make(map[string]struct{}) + } + for _, channel := range channels { + c.channels[channel] = struct{}{} + } c.mu.Unlock() return err } @@ -142,7 +159,12 @@ func (c *PubSub) Subscribe(channels ...string) error { func (c *PubSub) PSubscribe(patterns ...string) error { c.mu.Lock() err := c.subscribe("psubscribe", patterns...) - c.patterns = appendIfNotExists(c.patterns, patterns...) + if c.patterns == nil { + c.patterns = make(map[string]struct{}) + } + for _, pattern := range patterns { + c.patterns[pattern] = struct{}{} + } c.mu.Unlock() return err } @@ -152,7 +174,9 @@ func (c *PubSub) PSubscribe(patterns ...string) error { func (c *PubSub) Unsubscribe(channels ...string) error { c.mu.Lock() err := c.subscribe("unsubscribe", channels...) - c.channels = remove(c.channels, channels...) + for _, channel := range channels { + delete(c.channels, channel) + } c.mu.Unlock() return err } @@ -162,7 +186,9 @@ func (c *PubSub) Unsubscribe(channels ...string) error { func (c *PubSub) PUnsubscribe(patterns ...string) error { c.mu.Lock() err := c.subscribe("punsubscribe", patterns...) - c.patterns = remove(c.patterns, patterns...) + for _, pattern := range patterns { + delete(c.patterns, pattern) + } c.mu.Unlock() return err } @@ -371,31 +397,3 @@ func (c *PubSub) Channel() <-chan *Message { }) return c.ch } - -func appendIfNotExists(ss []string, es ...string) []string { -loop: - for _, e := range es { - for _, s := range ss { - if s == e { - continue loop - } - } - ss = append(ss, e) - } - return ss -} - -func remove(ss []string, es ...string) []string { - if len(es) == 0 { - return ss[:0] - } - for _, e := range es { - for i, s := range ss { - if s == e { - ss = append(ss[:i], ss[i+1:]...) - break - } - } - } - return ss -} From 1b1fc80e204776fb2b8931884146ebd0979766cb Mon Sep 17 00:00:00 2001 From: "Olve S. Hansen" Date: Mon, 16 Apr 2018 09:15:52 +0200 Subject: [PATCH 0440/1746] Added OnConnect callback to UniversalOptions (#747) * Added OnConnect callback to UniversalOptions --- commands_test.go | 2 +- universal.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/commands_test.go b/commands_test.go index 156b122184..595581fcd3 100644 --- a/commands_test.go +++ b/commands_test.go @@ -219,7 +219,7 @@ var _ = Describe("Commands", func() { It("Should Command", func() { cmds, err := client.Command().Result() Expect(err).NotTo(HaveOccurred()) - Expect(len(cmds)).To(BeNumerically("~", 180, 10)) + Expect(len(cmds)).To(BeNumerically("~", 185, 10)) cmd := cmds["mget"] Expect(cmd.Name).To(Equal("mget")) diff --git a/universal.go b/universal.go index fde3c41506..28df2c799d 100644 --- a/universal.go +++ b/universal.go @@ -27,6 +27,7 @@ type UniversalOptions struct { // Common options + OnConnect func(*Conn) error MaxRetries int Password string DialTimeout time.Duration @@ -49,6 +50,7 @@ func (o *UniversalOptions) cluster() *ClusterOptions { RouteByLatency: o.RouteByLatency, ReadOnly: o.ReadOnly, + OnConnect: o.OnConnect, MaxRetries: o.MaxRetries, Password: o.Password, DialTimeout: o.DialTimeout, @@ -71,6 +73,7 @@ func (o *UniversalOptions) failover() *FailoverOptions { MasterName: o.MasterName, DB: o.DB, + OnConnect: o.OnConnect, MaxRetries: o.MaxRetries, Password: o.Password, DialTimeout: o.DialTimeout, @@ -93,6 +96,7 @@ func (o *UniversalOptions) simple() *Options { Addr: addr, DB: o.DB, + OnConnect: o.OnConnect, MaxRetries: o.MaxRetries, Password: o.Password, DialTimeout: o.DialTimeout, From 9ccc23344a52164531ed90362e2516b798e3296c Mon Sep 17 00:00:00 2001 From: Cyrille Hemidy Date: Tue, 17 Apr 2018 08:18:16 +0200 Subject: [PATCH 0441/1746] Update result.go (#754) fix misspelling --- result.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/result.go b/result.go index 28cea5ca83..e086e8e34c 100644 --- a/result.go +++ b/result.go @@ -2,7 +2,7 @@ package redis import "time" -// NewCmdResult returns a Cmd initalised with val and err for testing +// NewCmdResult returns a Cmd initialised with val and err for testing func NewCmdResult(val interface{}, err error) *Cmd { var cmd Cmd cmd.val = val @@ -10,7 +10,7 @@ func NewCmdResult(val interface{}, err error) *Cmd { return &cmd } -// NewSliceResult returns a SliceCmd initalised with val and err for testing +// NewSliceResult returns a SliceCmd initialised with val and err for testing func NewSliceResult(val []interface{}, err error) *SliceCmd { var cmd SliceCmd cmd.val = val @@ -18,7 +18,7 @@ func NewSliceResult(val []interface{}, err error) *SliceCmd { return &cmd } -// NewStatusResult returns a StatusCmd initalised with val and err for testing +// NewStatusResult returns a StatusCmd initialised with val and err for testing func NewStatusResult(val string, err error) *StatusCmd { var cmd StatusCmd cmd.val = val @@ -26,7 +26,7 @@ func NewStatusResult(val string, err error) *StatusCmd { return &cmd } -// NewIntResult returns an IntCmd initalised with val and err for testing +// NewIntResult returns an IntCmd initialised with val and err for testing func NewIntResult(val int64, err error) *IntCmd { var cmd IntCmd cmd.val = val @@ -34,7 +34,7 @@ func NewIntResult(val int64, err error) *IntCmd { return &cmd } -// NewDurationResult returns a DurationCmd initalised with val and err for testing +// NewDurationResult returns a DurationCmd initialised with val and err for testing func NewDurationResult(val time.Duration, err error) *DurationCmd { var cmd DurationCmd cmd.val = val @@ -42,7 +42,7 @@ func NewDurationResult(val time.Duration, err error) *DurationCmd { return &cmd } -// NewBoolResult returns a BoolCmd initalised with val and err for testing +// NewBoolResult returns a BoolCmd initialised with val and err for testing func NewBoolResult(val bool, err error) *BoolCmd { var cmd BoolCmd cmd.val = val @@ -50,7 +50,7 @@ func NewBoolResult(val bool, err error) *BoolCmd { return &cmd } -// NewStringResult returns a StringCmd initalised with val and err for testing +// NewStringResult returns a StringCmd initialised with val and err for testing func NewStringResult(val string, err error) *StringCmd { var cmd StringCmd cmd.val = []byte(val) @@ -58,7 +58,7 @@ func NewStringResult(val string, err error) *StringCmd { return &cmd } -// NewFloatResult returns a FloatCmd initalised with val and err for testing +// NewFloatResult returns a FloatCmd initialised with val and err for testing func NewFloatResult(val float64, err error) *FloatCmd { var cmd FloatCmd cmd.val = val @@ -66,7 +66,7 @@ func NewFloatResult(val float64, err error) *FloatCmd { return &cmd } -// NewStringSliceResult returns a StringSliceCmd initalised with val and err for testing +// NewStringSliceResult returns a StringSliceCmd initialised with val and err for testing func NewStringSliceResult(val []string, err error) *StringSliceCmd { var cmd StringSliceCmd cmd.val = val @@ -74,7 +74,7 @@ func NewStringSliceResult(val []string, err error) *StringSliceCmd { return &cmd } -// NewBoolSliceResult returns a BoolSliceCmd initalised with val and err for testing +// NewBoolSliceResult returns a BoolSliceCmd initialised with val and err for testing func NewBoolSliceResult(val []bool, err error) *BoolSliceCmd { var cmd BoolSliceCmd cmd.val = val @@ -82,7 +82,7 @@ func NewBoolSliceResult(val []bool, err error) *BoolSliceCmd { return &cmd } -// NewStringStringMapResult returns a StringStringMapCmd initalised with val and err for testing +// NewStringStringMapResult returns a StringStringMapCmd initialised with val and err for testing func NewStringStringMapResult(val map[string]string, err error) *StringStringMapCmd { var cmd StringStringMapCmd cmd.val = val @@ -90,7 +90,7 @@ func NewStringStringMapResult(val map[string]string, err error) *StringStringMap return &cmd } -// NewStringIntMapCmdResult returns a StringIntMapCmd initalised with val and err for testing +// NewStringIntMapCmdResult returns a StringIntMapCmd initialised with val and err for testing func NewStringIntMapCmdResult(val map[string]int64, err error) *StringIntMapCmd { var cmd StringIntMapCmd cmd.val = val @@ -98,7 +98,7 @@ func NewStringIntMapCmdResult(val map[string]int64, err error) *StringIntMapCmd return &cmd } -// NewZSliceCmdResult returns a ZSliceCmd initalised with val and err for testing +// NewZSliceCmdResult returns a ZSliceCmd initialised with val and err for testing func NewZSliceCmdResult(val []Z, err error) *ZSliceCmd { var cmd ZSliceCmd cmd.val = val @@ -106,7 +106,7 @@ func NewZSliceCmdResult(val []Z, err error) *ZSliceCmd { return &cmd } -// NewScanCmdResult returns a ScanCmd initalised with val and err for testing +// NewScanCmdResult returns a ScanCmd initialised with val and err for testing func NewScanCmdResult(keys []string, cursor uint64, err error) *ScanCmd { var cmd ScanCmd cmd.page = keys @@ -115,7 +115,7 @@ func NewScanCmdResult(keys []string, cursor uint64, err error) *ScanCmd { return &cmd } -// NewClusterSlotsCmdResult returns a ClusterSlotsCmd initalised with val and err for testing +// NewClusterSlotsCmdResult returns a ClusterSlotsCmd initialised with val and err for testing func NewClusterSlotsCmdResult(val []ClusterSlot, err error) *ClusterSlotsCmd { var cmd ClusterSlotsCmd cmd.val = val @@ -123,7 +123,7 @@ func NewClusterSlotsCmdResult(val []ClusterSlot, err error) *ClusterSlotsCmd { return &cmd } -// NewGeoLocationCmdResult returns a GeoLocationCmd initalised with val and err for testing +// NewGeoLocationCmdResult returns a GeoLocationCmd initialised with val and err for testing func NewGeoLocationCmdResult(val []GeoLocation, err error) *GeoLocationCmd { var cmd GeoLocationCmd cmd.locations = val @@ -131,7 +131,7 @@ func NewGeoLocationCmdResult(val []GeoLocation, err error) *GeoLocationCmd { return &cmd } -// NewCommandsInfoCmdResult returns a CommandsInfoCmd initalised with val and err for testing +// NewCommandsInfoCmdResult returns a CommandsInfoCmd initialised with val and err for testing func NewCommandsInfoCmdResult(val map[string]*CommandInfo, err error) *CommandsInfoCmd { var cmd CommandsInfoCmd cmd.val = val From ea254bbfda76377a603e7c272154f8adf667201d Mon Sep 17 00:00:00 2001 From: Ilya Sinelnikov Date: Mon, 7 May 2018 20:28:11 +0300 Subject: [PATCH 0442/1746] Add Watch func to UniversalClient interface --- universal.go | 1 + 1 file changed, 1 insertion(+) diff --git a/universal.go b/universal.go index 28df2c799d..5838d7a55e 100644 --- a/universal.go +++ b/universal.go @@ -117,6 +117,7 @@ func (o *UniversalOptions) simple() *Options { // applications locally. type UniversalClient interface { Cmdable + Watch(fn func(*Tx) error, keys ...string) error Process(cmd Cmder) error WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) Subscribe(channels ...string) *PubSub From 09b9a99666bccd3bd1c736a03298567823254498 Mon Sep 17 00:00:00 2001 From: Vadim Liman Date: Fri, 11 May 2018 09:46:09 +0300 Subject: [PATCH 0443/1746] Add TLS configuration support for Universal Client --- cluster.go | 5 +++++ sentinel.go | 5 +++++ universal.go | 9 ++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 4a2951157c..4f5f5b55a5 100644 --- a/cluster.go +++ b/cluster.go @@ -2,6 +2,7 @@ package redis import ( "context" + "crypto/tls" "errors" "fmt" "math" @@ -56,6 +57,8 @@ type ClusterOptions struct { PoolTimeout time.Duration IdleTimeout time.Duration IdleCheckFrequency time.Duration + + TLSConfig *tls.Config } func (opt *ClusterOptions) init() { @@ -117,6 +120,8 @@ func (opt *ClusterOptions) clientOptions() *Options { IdleTimeout: opt.IdleTimeout, IdleCheckFrequency: disableIdleCheck, + + TLSConfig: opt.TLSConfig, } } diff --git a/sentinel.go b/sentinel.go index 3f56f08b33..64248a52d1 100644 --- a/sentinel.go +++ b/sentinel.go @@ -1,6 +1,7 @@ package redis import ( + "crypto/tls" "errors" "net" "strings" @@ -38,6 +39,8 @@ type FailoverOptions struct { PoolTimeout time.Duration IdleTimeout time.Duration IdleCheckFrequency time.Duration + + TLSConfig *tls.Config } func (opt *FailoverOptions) options() *Options { @@ -59,6 +62,8 @@ func (opt *FailoverOptions) options() *Options { PoolTimeout: opt.PoolTimeout, IdleTimeout: opt.IdleTimeout, IdleCheckFrequency: opt.IdleCheckFrequency, + + TLSConfig: opt.TLSConfig, } } diff --git a/universal.go b/universal.go index 5838d7a55e..9e30c81d93 100644 --- a/universal.go +++ b/universal.go @@ -1,6 +1,9 @@ package redis -import "time" +import ( + "crypto/tls" + "time" +) // UniversalOptions information is required by UniversalClient to establish // connections. @@ -37,6 +40,7 @@ type UniversalOptions struct { PoolTimeout time.Duration IdleTimeout time.Duration IdleCheckFrequency time.Duration + TLSConfig *tls.Config } func (o *UniversalOptions) cluster() *ClusterOptions { @@ -60,6 +64,7 @@ func (o *UniversalOptions) cluster() *ClusterOptions { PoolTimeout: o.PoolTimeout, IdleTimeout: o.IdleTimeout, IdleCheckFrequency: o.IdleCheckFrequency, + TLSConfig: o.TLSConfig, } } @@ -83,6 +88,7 @@ func (o *UniversalOptions) failover() *FailoverOptions { PoolTimeout: o.PoolTimeout, IdleTimeout: o.IdleTimeout, IdleCheckFrequency: o.IdleCheckFrequency, + TLSConfig: o.TLSConfig, } } @@ -106,6 +112,7 @@ func (o *UniversalOptions) simple() *Options { PoolTimeout: o.PoolTimeout, IdleTimeout: o.IdleTimeout, IdleCheckFrequency: o.IdleCheckFrequency, + TLSConfig: o.TLSConfig, } } From 34facee367d1efc471832627e9d16dc7f732334e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 17 May 2018 14:36:51 +0300 Subject: [PATCH 0444/1746] Add more race tests --- cluster.go | 5 ++-- cluster_test.go | 70 ++++++++++++++++++++++++------------------------ commands_test.go | 2 +- race_test.go | 10 +++++++ 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/cluster.go b/cluster.go index 4f5f5b55a5..5991afddb8 100644 --- a/cluster.go +++ b/cluster.go @@ -220,7 +220,7 @@ type clusterNodes struct { nodeCreateGroup singleflight.Group - generation uint32 + _generation uint32 // atomic } func newClusterNodes(opt *ClusterOptions) *clusterNodes { @@ -277,8 +277,7 @@ func (c *clusterNodes) Addrs() ([]string, error) { } func (c *clusterNodes) NextGeneration() uint32 { - c.generation++ - return c.generation + return atomic.AddUint32(&c._generation, 1) } // GC removes unused nodes. diff --git a/cluster_test.go b/cluster_test.go index 58bf738980..24ea4e13eb 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -519,39 +519,37 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) Expect(res).To(HaveLen(3)) - wanted := []redis.ClusterSlot{ - { - Start: 0, - End: 4999, - Nodes: []redis.ClusterNode{{ - Id: "", - Addr: "127.0.0.1:8220", - }, { - Id: "", - Addr: "127.0.0.1:8223", - }}, + wanted := []redis.ClusterSlot{{ + Start: 0, + End: 4999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8220", }, { - Start: 5000, - End: 9999, - Nodes: []redis.ClusterNode{{ - Id: "", - Addr: "127.0.0.1:8221", - }, { - Id: "", - Addr: "127.0.0.1:8224", - }}, + Id: "", + Addr: "127.0.0.1:8223", + }}, + }, { + Start: 5000, + End: 9999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8221", }, { - Start: 10000, - End: 16383, - Nodes: []redis.ClusterNode{{ - Id: "", - Addr: "127.0.0.1:8222", - }, { - Id: "", - Addr: "127.0.0.1:8225", - }}, - }, - } + Id: "", + Addr: "127.0.0.1:8224", + }}, + }, { + Start: 10000, + End: 16383, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8222", + }, { + Id: "", + Addr: "127.0.0.1:8225", + }}, + }} Expect(assertSlotsEqual(res, wanted)).NotTo(HaveOccurred()) }) @@ -634,16 +632,18 @@ var _ = Describe("ClusterClient", func() { opt.MaxRetryBackoff = time.Second client = cluster.clusterClient(opt) + err := client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) + Expect(err).NotTo(HaveOccurred()) + _ = client.ForEachSlave(func(slave *redis.Client) error { defer GinkgoRecover() - _ = client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDB().Err() - }) - Eventually(func() int64 { return slave.DBSize().Val() }, 30*time.Second).Should(Equal(int64(0))) + return slave.ClusterFailover().Err() }) }) diff --git a/commands_test.go b/commands_test.go index 595581fcd3..f4f794f554 100644 --- a/commands_test.go +++ b/commands_test.go @@ -219,7 +219,7 @@ var _ = Describe("Commands", func() { It("Should Command", func() { cmds, err := client.Command().Result() Expect(err).NotTo(HaveOccurred()) - Expect(len(cmds)).To(BeNumerically("~", 185, 10)) + Expect(len(cmds)).To(BeNumerically("~", 200, 20)) cmd := cmds["mget"] Expect(cmd.Name).To(Equal("mget")) diff --git a/race_test.go b/race_test.go index acecf85690..04effc46bf 100644 --- a/race_test.go +++ b/race_test.go @@ -316,6 +316,16 @@ var _ = Describe("cluster races", func() { }) }) + It("should get", func() { + perform(C, func(id int) { + for i := 0; i < N; i++ { + key := fmt.Sprintf("key_%d_%d", id, i) + _, err := client.Get(key).Result() + Expect(err).To(Equal(redis.Nil)) + } + }) + }) + It("should incr", func() { key := "TestIncrFromGoroutines" From 092971361bcfb3fd8c0cb21d55f959ddd5cecbca Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 17 May 2018 15:02:35 +0300 Subject: [PATCH 0445/1746] cluster: retry same node on retryable error --- cluster.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster.go b/cluster.go index 5991afddb8..5e6b52c67f 100644 --- a/cluster.go +++ b/cluster.go @@ -816,6 +816,12 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { } if internal.IsRetryableError(err, true) { + // Firstly retry the same node. + if attempt == 0 { + continue + } + + // Secondly try random node. node, err = c.nodes.Random() if err != nil { break From 18b2e30835f248bffe475cb69c7267204de1bc1e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 17 May 2018 15:21:51 +0300 Subject: [PATCH 0446/1746] Cleanup cmds info --- cluster.go | 51 +++++++++++++++++++++++++++++++++++++++------------ command.go | 12 ++++++++---- ring.go | 37 ++++++++++++++++++++----------------- 3 files changed, 67 insertions(+), 33 deletions(-) diff --git a/cluster.go b/cluster.go index 5e6b52c67f..67e07e35b0 100644 --- a/cluster.go +++ b/cluster.go @@ -300,10 +300,9 @@ func (c *clusterNodes) GC(generation uint32) { } } -func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { +func (c *clusterNodes) Get(addr string) (*clusterNode, error) { var node *clusterNode var err error - c.mu.RLock() if c.closed { err = pool.ErrClosed @@ -311,6 +310,11 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { node = c.allNodes[addr] } c.mu.RUnlock() + return node, err +} + +func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { + node, err := c.Get(addr) if err != nil { return nil, err } @@ -580,11 +584,11 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { opt.init() c := &ClusterClient{ - opt: opt, - nodes: newClusterNodes(opt), - cmdsInfoCache: newCmdsInfoCache(), + opt: opt, + nodes: newClusterNodes(opt), } c.state = newClusterStateHolder(c.loadState) + c.cmdsInfoCache = newCmdsInfoCache(c.cmdsInfo) c.process = c.defaultProcess c.processPipeline = c.defaultProcessPipeline @@ -630,17 +634,39 @@ func (c *ClusterClient) retryBackoff(attempt int) time.Duration { return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) } -func (c *ClusterClient) cmdInfo(name string) *CommandInfo { - cmdsInfo, err := c.cmdsInfoCache.Do(func() (map[string]*CommandInfo, error) { - node, err := c.nodes.Random() +func (c *ClusterClient) cmdsInfo() (map[string]*CommandInfo, error) { + addrs, err := c.nodes.Addrs() + if err != nil { + return nil, err + } + + var firstErr error + for _, addr := range addrs { + node, err := c.nodes.Get(addr) if err != nil { return nil, err } - return node.Client.Command().Result() - }) + if node == nil { + continue + } + + info, err := node.Client.Command().Result() + if err == nil { + return info, nil + } + if firstErr == nil { + firstErr = err + } + } + return nil, firstErr +} + +func (c *ClusterClient) cmdInfo(name string) *CommandInfo { + cmdsInfo, err := c.cmdsInfoCache.Get() if err != nil { return nil } + info := cmdsInfo[name] if info == nil { internal.Logf("info for cmd=%s not found", name) @@ -704,13 +730,14 @@ func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { if len(keys) == 0 { - return fmt.Errorf("redis: keys don't hash to the same slot") + return fmt.Errorf("redis: Watch requires at least one key") } slot := hashtag.Slot(keys[0]) for _, key := range keys[1:] { if hashtag.Slot(key) != slot { - return fmt.Errorf("redis: Watch requires all keys to be in the same slot") + err := fmt.Errorf("redis: Watch requires all keys to be in the same slot") + return err } } diff --git a/command.go b/command.go index 1588ca2519..552c897bb8 100644 --- a/command.go +++ b/command.go @@ -1027,17 +1027,21 @@ func (cmd *CommandsInfoCmd) readReply(cn *pool.Conn) error { //------------------------------------------------------------------------------ type cmdsInfoCache struct { + fn func() (map[string]*CommandInfo, error) + once internal.Once cmds map[string]*CommandInfo } -func newCmdsInfoCache() *cmdsInfoCache { - return &cmdsInfoCache{} +func newCmdsInfoCache(fn func() (map[string]*CommandInfo, error)) *cmdsInfoCache { + return &cmdsInfoCache{ + fn: fn, + } } -func (c *cmdsInfoCache) Do(fn func() (map[string]*CommandInfo, error)) (map[string]*CommandInfo, error) { +func (c *cmdsInfoCache) Get() (map[string]*CommandInfo, error) { err := c.once.Do(func() error { - cmds, err := fn() + cmds, err := c.fn() if err != nil { return err } diff --git a/ring.go b/ring.go index 6d28774136..362bd0319a 100644 --- a/ring.go +++ b/ring.go @@ -304,10 +304,11 @@ func NewRing(opt *RingOptions) *Ring { opt.init() ring := &Ring{ - opt: opt, - shards: newRingShards(), - cmdsInfoCache: newCmdsInfoCache(), + opt: opt, + shards: newRingShards(), } + ring.cmdsInfoCache = newCmdsInfoCache(ring.cmdsInfo) + ring.processPipeline = ring.defaultProcessPipeline ring.cmdable.setProcessor(ring.Process) @@ -428,21 +429,23 @@ func (c *Ring) ForEachShard(fn func(client *Client) error) error { } } -func (c *Ring) cmdInfo(name string) *CommandInfo { - cmdsInfo, err := c.cmdsInfoCache.Do(func() (map[string]*CommandInfo, error) { - shards := c.shards.List() - firstErr := errRingShardsDown - for _, shard := range shards { - cmdsInfo, err := shard.Client.Command().Result() - if err == nil { - return cmdsInfo, nil - } - if firstErr == nil { - firstErr = err - } +func (c *Ring) cmdsInfo() (map[string]*CommandInfo, error) { + shards := c.shards.List() + firstErr := errRingShardsDown + for _, shard := range shards { + cmdsInfo, err := shard.Client.Command().Result() + if err == nil { + return cmdsInfo, nil } - return nil, firstErr - }) + if firstErr == nil { + firstErr = err + } + } + return nil, firstErr +} + +func (c *Ring) cmdInfo(name string) *CommandInfo { + cmdsInfo, err := c.cmdsInfoCache.Get() if err != nil { return nil } From 5c742fff786a79fb2e1ad959e4096630ec5781f3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 17 May 2018 16:09:56 +0300 Subject: [PATCH 0447/1746] cluster: cleanup tests --- cluster.go | 81 +++++++++++++++----- cluster_test.go | 191 ++++++++++++++++++++++++++++++++---------------- commands.go | 8 +- export_test.go | 32 ++++++-- 4 files changed, 222 insertions(+), 90 deletions(-) diff --git a/cluster.go b/cluster.go index 67e07e35b0..d24438fc3a 100644 --- a/cluster.go +++ b/cluster.go @@ -8,6 +8,7 @@ import ( "math" "math/rand" "net" + "strings" "sync" "sync/atomic" "time" @@ -35,6 +36,7 @@ type ClusterOptions struct { // Enables read-only commands on slave nodes. ReadOnly bool // Allows routing read-only commands to the closest master or slave node. + // It automatically enables ReadOnly. RouteByLatency bool // Allows routing read-only commands to the random master or slave node. RouteRandomly bool @@ -150,6 +152,10 @@ func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { return &node } +func (n *clusterNode) String() string { + return n.Client.String() +} + func (n *clusterNode) Close() error { return n.Client.Close() } @@ -379,15 +385,17 @@ func (c *clusterNodes) Random() (*clusterNode, error) { type clusterState struct { nodes *clusterNodes - masters []*clusterNode - slaves []*clusterNode + Masters []*clusterNode + Slaves []*clusterNode slots [][]*clusterNode generation uint32 } -func newClusterState(nodes *clusterNodes, slots []ClusterSlot, origin string) (*clusterState, error) { +func newClusterState( + nodes *clusterNodes, slots []ClusterSlot, origin string, +) (*clusterState, error) { c := clusterState{ nodes: nodes, generation: nodes.NextGeneration(), @@ -413,9 +421,9 @@ func newClusterState(nodes *clusterNodes, slots []ClusterSlot, origin string) (* nodes = append(nodes, node) if i == 0 { - c.masters = appendNode(c.masters, node) + c.Masters = appendUniqueNode(c.Masters, node) } else { - c.slaves = appendNode(c.slaves, node) + c.Slaves = appendUniqueNode(c.Slaves, node) } } @@ -497,6 +505,28 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode { return nil } +func (c *clusterState) IsConsistent() bool { + if len(c.Masters) > len(c.Slaves) { + return false + } + + for _, master := range c.Masters { + s := master.Client.Info("replication").Val() + if !strings.Contains(s, "role:master") { + return false + } + } + + for _, slave := range c.Slaves { + s := slave.Client.Info("replication").Val() + if !strings.Contains(s, "role:slave") { + return false + } + } + + return true +} + //------------------------------------------------------------------------------ type clusterStateHolder struct { @@ -516,7 +546,18 @@ func newClusterStateHolder(fn func() (*clusterState, error)) *clusterStateHolder } } -func (c *clusterStateHolder) Load() (*clusterState, error) { +func (c *clusterStateHolder) Reload() (*clusterState, error) { + state, err := c.reload() + if err != nil { + return nil, err + } + if !state.IsConsistent() { + c.LazyReload() + } + return state, nil +} + +func (c *clusterStateHolder) reload() (*clusterState, error) { state, err := c.load() if err != nil { c.lastErrMu.Lock() @@ -535,9 +576,15 @@ func (c *clusterStateHolder) LazyReload() { go func() { defer atomic.StoreUint32(&c.reloading, 0) - _, err := c.Load() - if err == nil { - time.Sleep(time.Second) + for { + state, err := c.reload() + if err != nil { + return + } + time.Sleep(100 * time.Millisecond) + if state.IsConsistent() { + return + } } }() } @@ -596,7 +643,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { c.cmdable.setProcessor(c.Process) - _, _ = c.state.Load() + _, _ = c.state.Reload() if opt.IdleCheckFrequency > 0 { go c.reaper(opt.IdleCheckFrequency) } @@ -890,7 +937,7 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { var wg sync.WaitGroup errCh := make(chan error, 1) - for _, master := range state.masters { + for _, master := range state.Masters { wg.Add(1) go func(node *clusterNode) { defer wg.Done() @@ -923,7 +970,7 @@ func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { var wg sync.WaitGroup errCh := make(chan error, 1) - for _, slave := range state.slaves { + for _, slave := range state.Slaves { wg.Add(1) go func(node *clusterNode) { defer wg.Done() @@ -967,11 +1014,11 @@ func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { } } - for _, node := range state.masters { + for _, node := range state.Masters { wg.Add(1) go worker(node) } - for _, node := range state.slaves { + for _, node := range state.Slaves { wg.Add(1) go worker(node) } @@ -994,7 +1041,7 @@ func (c *ClusterClient) PoolStats() *PoolStats { return &acc } - for _, node := range state.masters { + for _, node := range state.Masters { s := node.Client.connPool.Stats() acc.Hits += s.Hits acc.Misses += s.Misses @@ -1005,7 +1052,7 @@ func (c *ClusterClient) PoolStats() *PoolStats { acc.StaleConns += s.StaleConns } - for _, node := range state.slaves { + for _, node := range state.Slaves { s := node.Client.connPool.Stats() acc.Hits += s.Hits acc.Misses += s.Misses @@ -1438,7 +1485,7 @@ func isLoopbackAddr(addr string) bool { return ip.IsLoopback() } -func appendNode(nodes []*clusterNode, node *clusterNode) []*clusterNode { +func appendUniqueNode(nodes []*clusterNode, node *clusterNode) []*clusterNode { for _, n := range nodes { if n == node { return nodes diff --git a/cluster_test.go b/cluster_test.go index 24ea4e13eb..db0728d726 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -50,7 +50,15 @@ func (s *clusterScenario) addrs() []string { func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.ClusterClient { opt.Addrs = s.addrs() - return redis.NewClusterClient(opt) + client := redis.NewClusterClient(opt) + Eventually(func() bool { + state, err := client.GetState() + if err != nil { + return false + } + return state.IsConsistent() + }, 30*time.Second).Should(BeTrue()) + return client } func startCluster(scenario *clusterScenario) error { @@ -116,45 +124,43 @@ func startCluster(scenario *clusterScenario) error { } // Wait until all nodes have consistent info. + wanted := []redis.ClusterSlot{{ + Start: 0, + End: 4999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8220", + }, { + Id: "", + Addr: "127.0.0.1:8223", + }}, + }, { + Start: 5000, + End: 9999, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8221", + }, { + Id: "", + Addr: "127.0.0.1:8224", + }}, + }, { + Start: 10000, + End: 16383, + Nodes: []redis.ClusterNode{{ + Id: "", + Addr: "127.0.0.1:8222", + }, { + Id: "", + Addr: "127.0.0.1:8225", + }}, + }} for _, client := range scenario.clients { err := eventually(func() error { res, err := client.ClusterSlots().Result() if err != nil { return err } - wanted := []redis.ClusterSlot{ - { - Start: 0, - End: 4999, - Nodes: []redis.ClusterNode{{ - Id: "", - Addr: "127.0.0.1:8220", - }, { - Id: "", - Addr: "127.0.0.1:8223", - }}, - }, { - Start: 5000, - End: 9999, - Nodes: []redis.ClusterNode{{ - Id: "", - Addr: "127.0.0.1:8221", - }, { - Id: "", - Addr: "127.0.0.1:8224", - }}, - }, { - Start: 10000, - End: 16383, - Nodes: []redis.ClusterNode{{ - Id: "", - Addr: "127.0.0.1:8222", - }, { - Id: "", - Addr: "127.0.0.1:8225", - }}, - }, - } return assertSlotsEqual(res, wanted) }, 30*time.Second) if err != nil { @@ -213,6 +219,7 @@ func stopCluster(scenario *clusterScenario) error { //------------------------------------------------------------------------------ var _ = Describe("ClusterClient", func() { + var failover bool var opt *redis.ClusterOptions var client *redis.ClusterClient @@ -233,15 +240,42 @@ var _ = Describe("ClusterClient", func() { Expect(cnt).To(Equal(int64(1))) }) - It("follows redirects", func() { - Expect(client.Set("A", "VALUE", 0).Err()).NotTo(HaveOccurred()) + It("GET follows redirects", func() { + err := client.Set("A", "VALUE", 0).Err() + Expect(err).NotTo(HaveOccurred()) - slot := hashtag.Slot("A") - client.SwapSlotNodes(slot) + if !failover { + Eventually(func() int64 { + nodes, err := client.Nodes("A") + if err != nil { + return 0 + } + return nodes[1].Client.DBSize().Val() + }, 30*time.Second).Should(Equal(int64(1))) - Eventually(func() string { - return client.Get("A").Val() - }, 30*time.Second).Should(Equal("VALUE")) + Eventually(func() error { + return client.SwapNodes("A") + }, 30*time.Second).ShouldNot(HaveOccurred()) + } + + v, err := client.Get("A").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(v).To(Equal("VALUE")) + }) + + It("SET follows redirects", func() { + if !failover { + Eventually(func() error { + return client.SwapNodes("A") + }, 30*time.Second).ShouldNot(HaveOccurred()) + } + + err := client.Set("A", "VALUE", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + v, err := client.Get("A").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(v).To(Equal("VALUE")) }) It("distributes keys", func() { @@ -250,7 +284,8 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) } - for _, master := range cluster.masters() { + client.ForEachMaster(func(master *redis.Client) error { + defer GinkgoRecover() Eventually(func() string { return master.Info("keyspace").Val() }, 30*time.Second).Should(Or( @@ -258,7 +293,8 @@ var _ = Describe("ClusterClient", func() { ContainSubstring("keys=29"), ContainSubstring("keys=40"), )) - } + return nil + }) }) It("distributes keys when using EVAL", func() { @@ -333,9 +369,12 @@ var _ = Describe("ClusterClient", func() { keys := []string{"A", "B", "C", "D", "E", "F", "G"} It("follows redirects", func() { - for _, key := range keys { - slot := hashtag.Slot(key) - client.SwapSlotNodes(slot) + if !failover { + for _, key := range keys { + Eventually(func() error { + return client.SwapNodes(key) + }, 30*time.Second).ShouldNot(HaveOccurred()) + } } for i, key := range keys { @@ -354,9 +393,12 @@ var _ = Describe("ClusterClient", func() { return nil }) - for _, key := range keys { - slot := hashtag.Slot(key) - client.SwapSlotNodes(slot) + if !failover { + for _, key := range keys { + Eventually(func() error { + return client.SwapNodes(key) + }, 30*time.Second).ShouldNot(HaveOccurred()) + } } for _, key := range keys { @@ -456,9 +498,10 @@ var _ = Describe("ClusterClient", func() { opt = redisClusterOptions() client = cluster.clusterClient(opt) - _ = client.ForEachMaster(func(master *redis.Client) error { + err := client.ForEachMaster(func(master *redis.Client) error { return master.FlushDB().Err() }) + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -469,7 +512,8 @@ var _ = Describe("ClusterClient", func() { }) It("returns pool stats", func() { - Expect(client.PoolStats()).To(BeAssignableToTypeOf(&redis.PoolStats{})) + stats := client.PoolStats() + Expect(stats).To(BeAssignableToTypeOf(&redis.PoolStats{})) }) It("removes idle connections", func() { @@ -489,8 +533,9 @@ var _ = Describe("ClusterClient", func() { opt.MaxRedirects = -1 client := cluster.clusterClient(opt) - slot := hashtag.Slot("A") - client.SwapSlotNodes(slot) + Eventually(func() error { + return client.SwapNodes("A") + }, 30*time.Second).ShouldNot(HaveOccurred()) err := client.Get("A").Err() Expect(err).To(HaveOccurred()) @@ -627,6 +672,8 @@ var _ = Describe("ClusterClient", func() { Describe("ClusterClient failover", func() { BeforeEach(func() { + failover = true + opt = redisClusterOptions() opt.MinRetryBackoff = 250 * time.Millisecond opt.MaxRetryBackoff = time.Second @@ -637,21 +684,34 @@ var _ = Describe("ClusterClient", func() { }) Expect(err).NotTo(HaveOccurred()) - _ = client.ForEachSlave(func(slave *redis.Client) error { + err = client.ForEachSlave(func(slave *redis.Client) error { defer GinkgoRecover() Eventually(func() int64 { return slave.DBSize().Val() }, 30*time.Second).Should(Equal(int64(0))) - return slave.ClusterFailover().Err() + return nil }) + Expect(err).NotTo(HaveOccurred()) + + state, err := client.GetState() + Expect(err).NotTo(HaveOccurred()) + Expect(state.IsConsistent()).To(BeTrue()) + + for _, slave := range state.Slaves { + err = slave.Client.ClusterFailover().Err() + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() bool { + state, _ := client.LoadState() + return state.IsConsistent() + }, 30*time.Second).Should(BeTrue()) + } }) AfterEach(func() { - _ = client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDB().Err() - }) + failover = false Expect(client.Close()).NotTo(HaveOccurred()) }) @@ -664,23 +724,28 @@ var _ = Describe("ClusterClient", func() { opt.RouteByLatency = true client = cluster.clusterClient(opt) - _ = client.ForEachMaster(func(master *redis.Client) error { + err := client.ForEachMaster(func(master *redis.Client) error { return master.FlushDB().Err() }) + Expect(err).NotTo(HaveOccurred()) - _ = client.ForEachSlave(func(slave *redis.Client) error { + err = client.ForEachSlave(func(slave *redis.Client) error { Eventually(func() int64 { return client.DBSize().Val() }, 30*time.Second).Should(Equal(int64(0))) return nil }) + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { - _ = client.ForEachMaster(func(master *redis.Client) error { - return master.FlushDB().Err() + err := client.ForEachSlave(func(slave *redis.Client) error { + return slave.ReadWrite().Err() }) - Expect(client.Close()).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) + + err = client.Close() + Expect(err).NotTo(HaveOccurred()) }) assertClusterClient() diff --git a/commands.go b/commands.go index a3dacacd25..c6a88154e5 100644 --- a/commands.go +++ b/commands.go @@ -266,6 +266,8 @@ type Cmdable interface { GeoDist(key string, member1, member2, unit string) *FloatCmd GeoHash(key string, members ...string) *StringSliceCmd Command() *CommandsInfoCmd + ReadOnly() *StatusCmd + ReadWrite() *StatusCmd } type StatefulCmdable interface { @@ -274,8 +276,6 @@ type StatefulCmdable interface { Select(index int) *StatusCmd SwapDB(index1, index2 int) *StatusCmd ClientSetName(name string) *BoolCmd - ReadOnly() *StatusCmd - ReadWrite() *StatusCmd } var _ Cmdable = (*Client)(nil) @@ -2054,13 +2054,13 @@ func (c *cmdable) ClusterSlaves(nodeID string) *StringSliceCmd { return cmd } -func (c *statefulCmdable) ReadOnly() *StatusCmd { +func (c *cmdable) ReadOnly() *StatusCmd { cmd := NewStatusCmd("readonly") c.process(cmd) return cmd } -func (c *statefulCmdable) ReadWrite() *StatusCmd { +func (c *cmdable) ReadWrite() *StatusCmd { cmd := NewStatusCmd("readwrite") c.process(cmd) return cmd diff --git a/export_test.go b/export_test.go index 288e86f047..fcb7fa0d7e 100644 --- a/export_test.go +++ b/export_test.go @@ -1,9 +1,11 @@ package redis import ( + "fmt" "net" "time" + "github.com/go-redis/redis/internal/hashtag" "github.com/go-redis/redis/internal/pool" ) @@ -19,6 +21,14 @@ func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) return c.receiveMessage(timeout) } +func (c *ClusterClient) GetState() (*clusterState, error) { + return c.state.Get() +} + +func (c *ClusterClient) LoadState() (*clusterState, error) { + return c.loadState() +} + func (c *ClusterClient) SlotAddrs(slot int) []string { state, err := c.state.Get() if err != nil { @@ -32,15 +42,25 @@ func (c *ClusterClient) SlotAddrs(slot int) []string { return addrs } -// SwapSlot swaps a slot's master/slave address for testing MOVED redirects. -func (c *ClusterClient) SwapSlotNodes(slot int) { - state, err := c.state.Get() +func (c *ClusterClient) Nodes(key string) ([]*clusterNode, error) { + state, err := c.state.Reload() if err != nil { - panic(err) + return nil, err } + slot := hashtag.Slot(key) nodes := state.slots[slot] - if len(nodes) == 2 { - nodes[0], nodes[1] = nodes[1], nodes[0] + if len(nodes) != 2 { + return nil, fmt.Errorf("slot=%d does not have enough nodes: %v", slot, nodes) + } + return nodes, nil +} + +func (c *ClusterClient) SwapNodes(key string) error { + nodes, err := c.Nodes(key) + if err != nil { + return err } + nodes[0], nodes[1] = nodes[1], nodes[0] + return nil } From 471caa3d91116ec0608a8d72cfd0281cee9d5250 Mon Sep 17 00:00:00 2001 From: Jeffrey Hutchins Date: Mon, 21 May 2018 16:33:41 +0300 Subject: [PATCH 0448/1746] cluster: reload state for ForEach functions and every 1 minute --- cluster.go | 52 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/cluster.go b/cluster.go index d24438fc3a..c08c00db8e 100644 --- a/cluster.go +++ b/cluster.go @@ -391,16 +391,19 @@ type clusterState struct { slots [][]*clusterNode generation uint32 + createdAt time.Time } func newClusterState( nodes *clusterNodes, slots []ClusterSlot, origin string, ) (*clusterState, error) { c := clusterState{ - nodes: nodes, - generation: nodes.NextGeneration(), + nodes: nodes, slots: make([][]*clusterNode, hashtag.SlotNumber), + + generation: nodes.NextGeneration(), + createdAt: time.Now(), } isLoopbackOrigin := isLoopbackAddr(origin) @@ -534,8 +537,8 @@ type clusterStateHolder struct { state atomic.Value - lastErrMu sync.RWMutex - lastErr error + firstErrMu sync.RWMutex + firstErr error reloading uint32 // atomic } @@ -560,9 +563,11 @@ func (c *clusterStateHolder) Reload() (*clusterState, error) { func (c *clusterStateHolder) reload() (*clusterState, error) { state, err := c.load() if err != nil { - c.lastErrMu.Lock() - c.lastErr = err - c.lastErrMu.Unlock() + c.firstErrMu.Lock() + if c.firstErr == nil { + c.firstErr = err + } + c.firstErrMu.Unlock() return nil, err } c.state.Store(state) @@ -592,12 +597,16 @@ func (c *clusterStateHolder) LazyReload() { func (c *clusterStateHolder) Get() (*clusterState, error) { v := c.state.Load() if v != nil { - return v.(*clusterState), nil + state := v.(*clusterState) + if time.Since(state.createdAt) > time.Minute { + c.LazyReload() + } + return state, nil } - c.lastErrMu.RLock() - err := c.lastErr - c.lastErrMu.RUnlock() + c.firstErrMu.RLock() + err := c.firstErr + c.firstErrMu.RUnlock() if err != nil { return nil, err } @@ -930,9 +939,12 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { // ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { - state, err := c.state.Get() + state, err := c.state.Reload() if err != nil { - return err + state, err = c.state.Get() + if err != nil { + return err + } } var wg sync.WaitGroup @@ -963,9 +975,12 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { // ForEachSlave concurrently calls the fn on each slave node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { - state, err := c.state.Get() + state, err := c.state.Reload() if err != nil { - return err + state, err = c.state.Get() + if err != nil { + return err + } } var wg sync.WaitGroup @@ -996,9 +1011,12 @@ func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { // ForEachNode concurrently calls the fn on each known node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { - state, err := c.state.Get() + state, err := c.state.Reload() if err != nil { - return err + state, err = c.state.Get() + if err != nil { + return err + } } var wg sync.WaitGroup From d790448589242fab1e0c7434d1f39b1f5b557f95 Mon Sep 17 00:00:00 2001 From: Nathan Ziebart Date: Wed, 23 May 2018 17:30:50 -0700 Subject: [PATCH 0449/1746] reset cmdable.process when copying cluster client --- cluster.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index c08c00db8e..6f0855eb78 100644 --- a/cluster.go +++ b/cluster.go @@ -650,7 +650,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { c.processPipeline = c.defaultProcessPipeline c.processTxPipeline = c.defaultProcessTxPipeline - c.cmdable.setProcessor(c.Process) + c.init() _, _ = c.state.Reload() if opt.IdleCheckFrequency > 0 { @@ -660,6 +660,10 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } +func (c *ClusterClient) init() { + c.cmdable.setProcessor(c.Process) +} + func (c *ClusterClient) Context() context.Context { if c.ctx != nil { return c.ctx @@ -678,6 +682,7 @@ func (c *ClusterClient) WithContext(ctx context.Context) *ClusterClient { func (c *ClusterClient) copy() *ClusterClient { cp := *c + cp.init() return &cp } From faf5666fbd4d53631c20a3cdabc4719e3a9c55c2 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 28 May 2018 17:27:24 +0300 Subject: [PATCH 0450/1746] Cleanup pool --- cluster.go | 12 +- internal/pool/bench_test.go | 12 +- internal/pool/pool.go | 205 +++++++++++++++++++++-------------- internal/pool/pool_single.go | 12 +- internal/pool/pool_sticky.go | 54 ++++----- internal/pool/pool_test.go | 57 +++++----- options.go | 3 +- pool_test.go | 21 ++-- redis.go | 27 ++--- redis_test.go | 12 +- ring.go | 6 +- tx_test.go | 5 +- 12 files changed, 219 insertions(+), 207 deletions(-) diff --git a/cluster.go b/cluster.go index 6f0855eb78..5fad373d7b 100644 --- a/cluster.go +++ b/cluster.go @@ -1172,7 +1172,7 @@ func (c *ClusterClient) defaultProcessPipeline(cmds []Cmder) error { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { - cn, _, err := node.Client.getConn() + cn, err := node.Client.getConn() if err != nil { if err == pool.ErrClosed { c.remapCmds(cmds, failedCmds) @@ -1184,9 +1184,9 @@ func (c *ClusterClient) defaultProcessPipeline(cmds []Cmder) error { err = c.pipelineProcessCmds(node, cn, cmds, failedCmds) if err == nil || internal.IsRedisError(err) { - _ = node.Client.connPool.Put(cn) + node.Client.connPool.Put(cn) } else { - _ = node.Client.connPool.Remove(cn) + node.Client.connPool.Remove(cn) } } @@ -1336,7 +1336,7 @@ func (c *ClusterClient) defaultProcessTxPipeline(cmds []Cmder) error { failedCmds := make(map[*clusterNode][]Cmder) for node, cmds := range cmdsMap { - cn, _, err := node.Client.getConn() + cn, err := node.Client.getConn() if err != nil { if err == pool.ErrClosed { c.remapCmds(cmds, failedCmds) @@ -1348,9 +1348,9 @@ func (c *ClusterClient) defaultProcessTxPipeline(cmds []Cmder) error { err = c.txPipelineProcessCmds(node, cn, cmds, failedCmds) if err == nil || internal.IsRedisError(err) { - _ = node.Client.connPool.Put(cn) + node.Client.connPool.Put(cn) } else { - _ = node.Client.connPool.Remove(cn) + node.Client.connPool.Remove(cn) } } diff --git a/internal/pool/bench_test.go b/internal/pool/bench_test.go index e0bb524466..e80c9c00ca 100644 --- a/internal/pool/bench_test.go +++ b/internal/pool/bench_test.go @@ -20,13 +20,11 @@ func benchmarkPoolGetPut(b *testing.B, poolSize int) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - cn, _, err := connPool.Get() + cn, err := connPool.Get() if err != nil { b.Fatal(err) } - if err = connPool.Put(cn); err != nil { - b.Fatal(err) - } + connPool.Put(cn) } }) } @@ -56,13 +54,11 @@ func benchmarkPoolGetRemove(b *testing.B, poolSize int) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - cn, _, err := connPool.Get() + cn, err := connPool.Get() if err != nil { b.Fatal(err) } - if err := connPool.Remove(cn); err != nil { - b.Fatal(err) - } + connPool.Remove(cn) } }) } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index ae81905ea8..cab66904a3 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -28,7 +28,8 @@ type Stats struct { Timeouts uint32 // number of times a wait timeout occurred TotalConns uint32 // number of total connections in the pool - FreeConns uint32 // number of free connections in the pool + FreeConns uint32 // deprecated - use IdleConns + IdleConns uint32 // number of idle connections in the pool StaleConns uint32 // number of stale connections removed from the pool } @@ -36,12 +37,12 @@ type Pooler interface { NewConn() (*Conn, error) CloseConn(*Conn) error - Get() (*Conn, bool, error) - Put(*Conn) error - Remove(*Conn) error + Get() (*Conn, error) + Put(*Conn) + Remove(*Conn) Len() int - FreeLen() int + IdleLen() int Stats() *Stats Close() error @@ -70,8 +71,8 @@ type ConnPool struct { connsMu sync.Mutex conns []*Conn - freeConnsMu sync.Mutex - freeConns []*Conn + idleConnsMu sync.RWMutex + idleConns []*Conn stats Stats @@ -86,15 +87,29 @@ func NewConnPool(opt *Options) *ConnPool { queue: make(chan struct{}, opt.PoolSize), conns: make([]*Conn, 0, opt.PoolSize), - freeConns: make([]*Conn, 0, opt.PoolSize), + idleConns: make([]*Conn, 0, opt.PoolSize), } + if opt.IdleTimeout > 0 && opt.IdleCheckFrequency > 0 { go p.reaper(opt.IdleCheckFrequency) } + return p } func (p *ConnPool) NewConn() (*Conn, error) { + cn, err := p.newConn() + if err != nil { + return nil, err + } + + p.connsMu.Lock() + p.conns = append(p.conns, cn) + p.connsMu.Unlock() + return cn, nil +} + +func (p *ConnPool) newConn() (*Conn, error) { if p.closed() { return nil, ErrClosed } @@ -112,12 +127,7 @@ func (p *ConnPool) NewConn() (*Conn, error) { return nil, err } - cn := NewConn(netConn) - p.connsMu.Lock() - p.conns = append(p.conns, cn) - p.connsMu.Unlock() - - return cn, nil + return NewConn(netConn), nil } func (p *ConnPool) tryDial() { @@ -153,34 +163,20 @@ func (p *ConnPool) getLastDialError() error { } // Get returns existed connection from the pool or creates a new one. -func (p *ConnPool) Get() (*Conn, bool, error) { +func (p *ConnPool) Get() (*Conn, error) { if p.closed() { - return nil, false, ErrClosed + return nil, ErrClosed } - select { - case p.queue <- struct{}{}: - default: - timer := timers.Get().(*time.Timer) - timer.Reset(p.opt.PoolTimeout) - - select { - case p.queue <- struct{}{}: - if !timer.Stop() { - <-timer.C - } - timers.Put(timer) - case <-timer.C: - timers.Put(timer) - atomic.AddUint32(&p.stats.Timeouts, 1) - return nil, false, ErrPoolTimeout - } + err := p.waitTurn() + if err != nil { + return nil, err } for { - p.freeConnsMu.Lock() - cn := p.popFree() - p.freeConnsMu.Unlock() + p.idleConnsMu.Lock() + cn := p.popIdle() + p.idleConnsMu.Unlock() if cn == nil { break @@ -192,50 +188,89 @@ func (p *ConnPool) Get() (*Conn, bool, error) { } atomic.AddUint32(&p.stats.Hits, 1) - return cn, false, nil + return cn, nil } atomic.AddUint32(&p.stats.Misses, 1) newcn, err := p.NewConn() if err != nil { - <-p.queue - return nil, false, err + p.freeTurn() + return nil, err } - return newcn, true, nil + return newcn, nil +} + +func (p *ConnPool) getTurn() { + p.queue <- struct{}{} } -func (p *ConnPool) popFree() *Conn { - if len(p.freeConns) == 0 { +func (p *ConnPool) waitTurn() error { + select { + case p.queue <- struct{}{}: + return nil + default: + timer := timers.Get().(*time.Timer) + timer.Reset(p.opt.PoolTimeout) + + select { + case p.queue <- struct{}{}: + if !timer.Stop() { + <-timer.C + } + timers.Put(timer) + return nil + case <-timer.C: + timers.Put(timer) + atomic.AddUint32(&p.stats.Timeouts, 1) + return ErrPoolTimeout + } + } +} + +func (p *ConnPool) freeTurn() { + <-p.queue +} + +func (p *ConnPool) popIdle() *Conn { + if len(p.idleConns) == 0 { return nil } - idx := len(p.freeConns) - 1 - cn := p.freeConns[idx] - p.freeConns = p.freeConns[:idx] + idx := len(p.idleConns) - 1 + cn := p.idleConns[idx] + p.idleConns = p.idleConns[:idx] + return cn } -func (p *ConnPool) Put(cn *Conn) error { - if data := cn.Rd.PeekBuffered(); data != nil { - internal.Logf("connection has unread data: %q", data) - return p.Remove(cn) +func (p *ConnPool) Put(cn *Conn) { + buf := cn.Rd.PeekBuffered() + if buf != nil { + internal.Logf("connection has unread data: %.100q", buf) + p.Remove(cn) + return } - p.freeConnsMu.Lock() - p.freeConns = append(p.freeConns, cn) - p.freeConnsMu.Unlock() - <-p.queue - return nil + + p.idleConnsMu.Lock() + p.idleConns = append(p.idleConns, cn) + p.idleConnsMu.Unlock() + p.freeTurn() } -func (p *ConnPool) Remove(cn *Conn) error { - _ = p.CloseConn(cn) - <-p.queue - return nil +func (p *ConnPool) Remove(cn *Conn) { + p.removeConn(cn) + p.freeTurn() + _ = p.closeConn(cn) } func (p *ConnPool) CloseConn(cn *Conn) error { + p.removeConn(cn) + return p.closeConn(cn) +} + +func (p *ConnPool) removeConn(cn *Conn) { p.connsMu.Lock() for i, c := range p.conns { if c == cn { @@ -244,8 +279,6 @@ func (p *ConnPool) CloseConn(cn *Conn) error { } } p.connsMu.Unlock() - - return p.closeConn(cn) } func (p *ConnPool) closeConn(cn *Conn) error { @@ -263,22 +296,24 @@ func (p *ConnPool) Len() int { return l } -// FreeLen returns number of free connections. -func (p *ConnPool) FreeLen() int { - p.freeConnsMu.Lock() - l := len(p.freeConns) - p.freeConnsMu.Unlock() +// FreeLen returns number of idle connections. +func (p *ConnPool) IdleLen() int { + p.idleConnsMu.RLock() + l := len(p.idleConns) + p.idleConnsMu.RUnlock() return l } func (p *ConnPool) Stats() *Stats { + idleLen := p.IdleLen() return &Stats{ Hits: atomic.LoadUint32(&p.stats.Hits), Misses: atomic.LoadUint32(&p.stats.Misses), Timeouts: atomic.LoadUint32(&p.stats.Timeouts), TotalConns: uint32(p.Len()), - FreeConns: uint32(p.FreeLen()), + FreeConns: uint32(idleLen), + IdleConns: uint32(idleLen), StaleConns: atomic.LoadUint32(&p.stats.StaleConns), } } @@ -316,41 +351,45 @@ func (p *ConnPool) Close() error { p.conns = nil p.connsMu.Unlock() - p.freeConnsMu.Lock() - p.freeConns = nil - p.freeConnsMu.Unlock() + p.idleConnsMu.Lock() + p.idleConns = nil + p.idleConnsMu.Unlock() return firstErr } -func (p *ConnPool) reapStaleConn() bool { - if len(p.freeConns) == 0 { - return false +func (p *ConnPool) reapStaleConn() *Conn { + if len(p.idleConns) == 0 { + return nil } - cn := p.freeConns[0] + cn := p.idleConns[0] if !cn.IsStale(p.opt.IdleTimeout) { - return false + return nil } - p.CloseConn(cn) - p.freeConns = append(p.freeConns[:0], p.freeConns[1:]...) + p.idleConns = append(p.idleConns[:0], p.idleConns[1:]...) - return true + return cn } func (p *ConnPool) ReapStaleConns() (int, error) { var n int for { - p.queue <- struct{}{} - p.freeConnsMu.Lock() + p.getTurn() - reaped := p.reapStaleConn() + p.idleConnsMu.Lock() + cn := p.reapStaleConn() + p.idleConnsMu.Unlock() + + if cn != nil { + p.removeConn(cn) + } - p.freeConnsMu.Unlock() - <-p.queue + p.freeTurn() - if reaped { + if cn != nil { + p.closeConn(cn) n++ } else { break diff --git a/internal/pool/pool_single.go b/internal/pool/pool_single.go index ff91279b36..b35b78afbd 100644 --- a/internal/pool/pool_single.go +++ b/internal/pool/pool_single.go @@ -20,29 +20,27 @@ func (p *SingleConnPool) CloseConn(*Conn) error { panic("not implemented") } -func (p *SingleConnPool) Get() (*Conn, bool, error) { - return p.cn, false, nil +func (p *SingleConnPool) Get() (*Conn, error) { + return p.cn, nil } -func (p *SingleConnPool) Put(cn *Conn) error { +func (p *SingleConnPool) Put(cn *Conn) { if p.cn != cn { panic("p.cn != cn") } - return nil } -func (p *SingleConnPool) Remove(cn *Conn) error { +func (p *SingleConnPool) Remove(cn *Conn) { if p.cn != cn { panic("p.cn != cn") } - return nil } func (p *SingleConnPool) Len() int { return 1 } -func (p *SingleConnPool) FreeLen() int { +func (p *SingleConnPool) IdleLen() int { return 0 } diff --git a/internal/pool/pool_sticky.go b/internal/pool/pool_sticky.go index 17f163858b..91bd913330 100644 --- a/internal/pool/pool_sticky.go +++ b/internal/pool/pool_sticky.go @@ -28,55 +28,40 @@ func (p *StickyConnPool) CloseConn(*Conn) error { panic("not implemented") } -func (p *StickyConnPool) Get() (*Conn, bool, error) { +func (p *StickyConnPool) Get() (*Conn, error) { p.mu.Lock() defer p.mu.Unlock() if p.closed { - return nil, false, ErrClosed + return nil, ErrClosed } if p.cn != nil { - return p.cn, false, nil + return p.cn, nil } - cn, _, err := p.pool.Get() + cn, err := p.pool.Get() if err != nil { - return nil, false, err + return nil, err } + p.cn = cn - return cn, true, nil + return cn, nil } -func (p *StickyConnPool) putUpstream() (err error) { - err = p.pool.Put(p.cn) +func (p *StickyConnPool) putUpstream() { + p.pool.Put(p.cn) p.cn = nil - return err } -func (p *StickyConnPool) Put(cn *Conn) error { - p.mu.Lock() - defer p.mu.Unlock() - - if p.closed { - return ErrClosed - } - return nil -} +func (p *StickyConnPool) Put(cn *Conn) {} -func (p *StickyConnPool) removeUpstream() error { - err := p.pool.Remove(p.cn) +func (p *StickyConnPool) removeUpstream() { + p.pool.Remove(p.cn) p.cn = nil - return err } -func (p *StickyConnPool) Remove(cn *Conn) error { - p.mu.Lock() - defer p.mu.Unlock() - - if p.closed { - return nil - } - return p.removeUpstream() +func (p *StickyConnPool) Remove(cn *Conn) { + p.removeUpstream() } func (p *StickyConnPool) Len() int { @@ -89,7 +74,7 @@ func (p *StickyConnPool) Len() int { return 1 } -func (p *StickyConnPool) FreeLen() int { +func (p *StickyConnPool) IdleLen() int { p.mu.Lock() defer p.mu.Unlock() @@ -111,13 +96,14 @@ func (p *StickyConnPool) Close() error { return ErrClosed } p.closed = true - var err error + if p.cn != nil { if p.reusable { - err = p.putUpstream() + p.putUpstream() } else { - err = p.removeUpstream() + p.removeUpstream() } } - return err + + return nil } diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 68c9a1bef8..49602685fc 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -29,13 +29,13 @@ var _ = Describe("ConnPool", func() { It("should unblock client when conn is removed", func() { // Reserve one connection. - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) // Reserve all other connections. var cns []*pool.Conn for i := 0; i < 9; i++ { - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) cns = append(cns, cn) } @@ -46,12 +46,11 @@ var _ = Describe("ConnPool", func() { defer GinkgoRecover() started <- true - _, _, err := connPool.Get() + _, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) done <- true - err = connPool.Put(cn) - Expect(err).NotTo(HaveOccurred()) + connPool.Put(cn) }() <-started @@ -59,14 +58,13 @@ var _ = Describe("ConnPool", func() { select { case <-done: Fail("Get is not blocked") - default: + case <-time.After(time.Millisecond): // ok } - err = connPool.Remove(cn) - Expect(err).NotTo(HaveOccurred()) + connPool.Remove(cn) - // Check that Ping is unblocked. + // Check that Get is unblocked. select { case <-done: // ok @@ -75,8 +73,7 @@ var _ = Describe("ConnPool", func() { } for _, cn := range cns { - err = connPool.Put(cn) - Expect(err).NotTo(HaveOccurred()) + connPool.Put(cn) } }) }) @@ -107,7 +104,7 @@ var _ = Describe("conns reaper", func() { // add stale connections idleConns = nil for i := 0; i < 3; i++ { - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) cn.SetUsedAt(time.Now().Add(-2 * idleTimeout)) conns = append(conns, cn) @@ -116,17 +113,17 @@ var _ = Describe("conns reaper", func() { // add fresh connections for i := 0; i < 3; i++ { - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) conns = append(conns, cn) } for _, cn := range conns { - Expect(connPool.Put(cn)).NotTo(HaveOccurred()) + connPool.Put(cn) } Expect(connPool.Len()).To(Equal(6)) - Expect(connPool.FreeLen()).To(Equal(6)) + Expect(connPool.IdleLen()).To(Equal(6)) n, err := connPool.ReapStaleConns() Expect(err).NotTo(HaveOccurred()) @@ -136,14 +133,14 @@ var _ = Describe("conns reaper", func() { AfterEach(func() { _ = connPool.Close() Expect(connPool.Len()).To(Equal(0)) - Expect(connPool.FreeLen()).To(Equal(0)) + Expect(connPool.IdleLen()).To(Equal(0)) Expect(len(closedConns)).To(Equal(len(conns))) Expect(closedConns).To(ConsistOf(conns)) }) It("reaps stale connections", func() { Expect(connPool.Len()).To(Equal(3)) - Expect(connPool.FreeLen()).To(Equal(3)) + Expect(connPool.IdleLen()).To(Equal(3)) }) It("does not reap fresh connections", func() { @@ -161,36 +158,34 @@ var _ = Describe("conns reaper", func() { for j := 0; j < 3; j++ { var freeCns []*pool.Conn for i := 0; i < 3; i++ { - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) freeCns = append(freeCns, cn) } Expect(connPool.Len()).To(Equal(3)) - Expect(connPool.FreeLen()).To(Equal(0)) + Expect(connPool.IdleLen()).To(Equal(0)) - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) conns = append(conns, cn) Expect(connPool.Len()).To(Equal(4)) - Expect(connPool.FreeLen()).To(Equal(0)) + Expect(connPool.IdleLen()).To(Equal(0)) - err = connPool.Remove(cn) - Expect(err).NotTo(HaveOccurred()) + connPool.Remove(cn) Expect(connPool.Len()).To(Equal(3)) - Expect(connPool.FreeLen()).To(Equal(0)) + Expect(connPool.IdleLen()).To(Equal(0)) for _, cn := range freeCns { - err := connPool.Put(cn) - Expect(err).NotTo(HaveOccurred()) + connPool.Put(cn) } Expect(connPool.Len()).To(Equal(3)) - Expect(connPool.FreeLen()).To(Equal(3)) + Expect(connPool.IdleLen()).To(Equal(3)) } }) }) @@ -222,18 +217,18 @@ var _ = Describe("race", func() { perform(C, func(id int) { for i := 0; i < N; i++ { - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) if err == nil { - Expect(connPool.Put(cn)).NotTo(HaveOccurred()) + connPool.Put(cn) } } }, func(id int) { for i := 0; i < N; i++ { - cn, _, err := connPool.Get() + cn, err := connPool.Get() Expect(err).NotTo(HaveOccurred()) if err == nil { - Expect(connPool.Remove(cn)).NotTo(HaveOccurred()) + connPool.Remove(cn) } } }) diff --git a/options.go b/options.go index 75648053de..35ce061959 100644 --- a/options.go +++ b/options.go @@ -68,8 +68,7 @@ type Options struct { // Default is 5 minutes. IdleTimeout time.Duration // Frequency of idle checks. - // Default is 1 minute. - // When minus value is set, then idle check is disabled. + // Default is 1 minute. -1 disables idle check. IdleCheckFrequency time.Duration // Enables read only queries on slave nodes. diff --git a/pool_test.go b/pool_test.go index 0ca09adc7c..d7dc9d014a 100644 --- a/pool_test.go +++ b/pool_test.go @@ -30,8 +30,8 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(BeNumerically("<=", 10)) - Expect(pool.FreeLen()).To(BeNumerically("<=", 10)) - Expect(pool.Len()).To(Equal(pool.FreeLen())) + Expect(pool.IdleLen()).To(BeNumerically("<=", 10)) + Expect(pool.Len()).To(Equal(pool.IdleLen())) }) It("respects max size on multi", func() { @@ -55,8 +55,8 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(BeNumerically("<=", 10)) - Expect(pool.FreeLen()).To(BeNumerically("<=", 10)) - Expect(pool.Len()).To(Equal(pool.FreeLen())) + Expect(pool.IdleLen()).To(BeNumerically("<=", 10)) + Expect(pool.Len()).To(Equal(pool.IdleLen())) }) It("respects max size on pipelines", func() { @@ -73,15 +73,15 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(BeNumerically("<=", 10)) - Expect(pool.FreeLen()).To(BeNumerically("<=", 10)) - Expect(pool.Len()).To(Equal(pool.FreeLen())) + Expect(pool.IdleLen()).To(BeNumerically("<=", 10)) + Expect(pool.Len()).To(Equal(pool.IdleLen())) }) It("removes broken connections", func() { - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.SetNetConn(&badConn{}) - Expect(client.Pool().Put(cn)).NotTo(HaveOccurred()) + client.Pool().Put(cn) err = client.Ping().Err() Expect(err).To(MatchError("bad connection")) @@ -92,7 +92,7 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(Equal(1)) - Expect(pool.FreeLen()).To(Equal(1)) + Expect(pool.IdleLen()).To(Equal(1)) stats := pool.Stats() Expect(stats.Hits).To(Equal(uint32(2))) @@ -109,7 +109,7 @@ var _ = Describe("pool", func() { pool := client.Pool() Expect(pool.Len()).To(Equal(1)) - Expect(pool.FreeLen()).To(Equal(1)) + Expect(pool.IdleLen()).To(Equal(1)) stats := pool.Stats() Expect(stats.Hits).To(Equal(uint32(100))) @@ -125,6 +125,7 @@ var _ = Describe("pool", func() { Timeouts: 0, TotalConns: 1, FreeConns: 1, + IdleConns: 1, StaleConns: 0, })) diff --git a/redis.go b/redis.go index 7a606b70ed..beb632e1e9 100644 --- a/redis.go +++ b/redis.go @@ -60,29 +60,30 @@ func (c *baseClient) newConn() (*pool.Conn, error) { return cn, nil } -func (c *baseClient) getConn() (*pool.Conn, bool, error) { - cn, isNew, err := c.connPool.Get() +func (c *baseClient) getConn() (*pool.Conn, error) { + cn, err := c.connPool.Get() if err != nil { - return nil, false, err + return nil, err } if !cn.Inited { - if err := c.initConn(cn); err != nil { - _ = c.connPool.Remove(cn) - return nil, false, err + err := c.initConn(cn) + if err != nil { + c.connPool.Remove(cn) + return nil, err } } - return cn, isNew, nil + return cn, nil } func (c *baseClient) releaseConn(cn *pool.Conn, err error) bool { if internal.IsBadConn(err, false) { - _ = c.connPool.Remove(cn) + c.connPool.Remove(cn) return false } - _ = c.connPool.Put(cn) + c.connPool.Put(cn) return true } @@ -137,7 +138,7 @@ func (c *baseClient) defaultProcess(cmd Cmder) error { time.Sleep(c.retryBackoff(attempt)) } - cn, _, err := c.getConn() + cn, err := c.getConn() if err != nil { cmd.setErr(err) if internal.IsRetryableError(err, true) { @@ -225,7 +226,7 @@ func (c *baseClient) generalProcessPipeline(cmds []Cmder, p pipelineProcessor) e time.Sleep(c.retryBackoff(attempt)) } - cn, _, err := c.getConn() + cn, err := c.getConn() if err != nil { setCmdsErr(cmds, err) return err @@ -234,10 +235,10 @@ func (c *baseClient) generalProcessPipeline(cmds []Cmder, p pipelineProcessor) e canRetry, err := p(cn, cmds) if err == nil || internal.IsRedisError(err) { - _ = c.connPool.Put(cn) + c.connPool.Put(cn) break } - _ = c.connPool.Remove(cn) + c.connPool.Remove(cn) if !canRetry || !internal.IsRetryableError(err, true) { break diff --git a/redis_test.go b/redis_test.go index df2485d394..f46728f945 100644 --- a/redis_test.go +++ b/redis_test.go @@ -145,12 +145,11 @@ var _ = Describe("Client", func() { }) // Put bad connection in the pool. - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.SetNetConn(&badConn{}) - err = client.Pool().Put(cn) - Expect(err).NotTo(HaveOccurred()) + client.Pool().Put(cn) err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) @@ -184,19 +183,18 @@ var _ = Describe("Client", func() { }) It("should update conn.UsedAt on read/write", func() { - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn.UsedAt).NotTo(BeZero()) createdAt := cn.UsedAt() - err = client.Pool().Put(cn) - Expect(err).NotTo(HaveOccurred()) + client.Pool().Put(cn) Expect(cn.UsedAt().Equal(createdAt)).To(BeTrue()) err = client.Ping().Err() Expect(err).NotTo(HaveOccurred()) - cn, _, err = client.Pool().Get() + cn, err = client.Pool().Get() Expect(err).NotTo(HaveOccurred()) Expect(cn).NotTo(BeNil()) Expect(cn.UsedAt().After(createdAt)).To(BeTrue()) diff --git a/ring.go b/ring.go index 362bd0319a..b47a1094e2 100644 --- a/ring.go +++ b/ring.go @@ -525,7 +525,7 @@ func (c *Ring) defaultProcessPipeline(cmds []Cmder) error { continue } - cn, _, err := shard.Client.getConn() + cn, err := shard.Client.getConn() if err != nil { setCmdsErr(cmds, err) continue @@ -533,10 +533,10 @@ func (c *Ring) defaultProcessPipeline(cmds []Cmder) error { canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds) if err == nil || internal.IsRedisError(err) { - _ = shard.Client.connPool.Put(cn) + shard.Client.connPool.Put(cn) continue } - _ = shard.Client.connPool.Remove(cn) + shard.Client.connPool.Remove(cn) if canRetry && internal.IsRetryableError(err, true) { if failedCmdsMap == nil { diff --git a/tx_test.go b/tx_test.go index de597ff064..c70f08ce1d 100644 --- a/tx_test.go +++ b/tx_test.go @@ -124,12 +124,11 @@ var _ = Describe("Tx", func() { It("should recover from bad connection", func() { // Put bad connection in the pool. - cn, _, err := client.Pool().Get() + cn, err := client.Pool().Get() Expect(err).NotTo(HaveOccurred()) cn.SetNetConn(&badConn{}) - err = client.Pool().Put(cn) - Expect(err).NotTo(HaveOccurred()) + client.Pool().Put(cn) do := func() error { err := client.Watch(func(tx *redis.Tx) error { From 4237a34c31b18bc6e9fd69234b3b72fe8753e2ce Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 31 May 2018 10:25:40 +0300 Subject: [PATCH 0451/1746] cluster: fix origin addr check --- cluster.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 5fad373d7b..0c58c85323 100644 --- a/cluster.go +++ b/cluster.go @@ -411,7 +411,7 @@ func newClusterState( var nodes []*clusterNode for i, slotNode := range slot.Nodes { addr := slotNode.Addr - if !isLoopbackOrigin && isLoopbackAddr(addr) { + if !isLoopbackOrigin && useOriginAddr(origin, addr) { addr = origin } @@ -1494,6 +1494,29 @@ func (c *ClusterClient) PSubscribe(channels ...string) *PubSub { return pubsub } +func useOriginAddr(originAddr, nodeAddr string) bool { + nodeHost, nodePort, err := net.SplitHostPort(nodeAddr) + if err != nil { + return false + } + + nodeIP := net.ParseIP(nodeHost) + if nodeIP == nil { + return false + } + + if !nodeIP.IsLoopback() { + return false + } + + _, originPort, err := net.SplitHostPort(originAddr) + if err != nil { + return false + } + + return nodePort == originPort +} + func isLoopbackAddr(addr string) bool { host, _, err := net.SplitHostPort(addr) if err != nil { From 39b4d69170465532d9e9c40b1a0e1b0364bbf7d3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 31 May 2018 13:15:52 +0300 Subject: [PATCH 0452/1746] Export SentinelClient --- sentinel.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/sentinel.go b/sentinel.go index 64248a52d1..3cedf36ee6 100644 --- a/sentinel.go +++ b/sentinel.go @@ -99,25 +99,23 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client { //------------------------------------------------------------------------------ -type sentinelClient struct { - cmdable +type SentinelClient struct { baseClient } -func newSentinel(opt *Options) *sentinelClient { +func NewSentinelClient(opt *Options) *SentinelClient { opt.init() - c := sentinelClient{ + c := &SentinelClient{ baseClient: baseClient{ opt: opt, connPool: newConnPool(opt), }, } c.baseClient.init() - c.cmdable.setProcessor(c.Process) - return &c + return c } -func (c *sentinelClient) PubSub() *PubSub { +func (c *SentinelClient) PubSub() *PubSub { return &PubSub{ opt: c.opt, @@ -128,13 +126,13 @@ func (c *sentinelClient) PubSub() *PubSub { } } -func (c *sentinelClient) GetMasterAddrByName(name string) *StringSliceCmd { +func (c *SentinelClient) GetMasterAddrByName(name string) *StringSliceCmd { cmd := NewStringSliceCmd("SENTINEL", "get-master-addr-by-name", name) c.Process(cmd) return cmd } -func (c *sentinelClient) Sentinels(name string) *SliceCmd { +func (c *SentinelClient) Sentinels(name string) *SliceCmd { cmd := NewSliceCmd("SENTINEL", "sentinels", name) c.Process(cmd) return cmd @@ -151,7 +149,7 @@ type sentinelFailover struct { mu sync.RWMutex masterName string _masterAddr string - sentinel *sentinelClient + sentinel *SentinelClient } func (d *sentinelFailover) Close() error { @@ -205,7 +203,7 @@ func (d *sentinelFailover) masterAddr() (string, error) { } for i, sentinelAddr := range d.sentinelAddrs { - sentinel := newSentinel(&Options{ + sentinel := NewSentinelClient(&Options{ Addr: sentinelAddr, DialTimeout: d.opt.DialTimeout, @@ -219,7 +217,8 @@ func (d *sentinelFailover) masterAddr() (string, error) { masterAddr, err := sentinel.GetMasterAddrByName(d.masterName).Result() if err != nil { - internal.Logf("sentinel: GetMasterAddrByName master=%q failed: %s", d.masterName, err) + internal.Logf("sentinel: GetMasterAddrByName master=%q failed: %s", + d.masterName, err) sentinel.Close() continue } @@ -246,7 +245,7 @@ func (d *sentinelFailover) switchMaster(masterAddr string) { d._masterAddr = masterAddr } -func (d *sentinelFailover) setSentinel(sentinel *sentinelClient) { +func (d *sentinelFailover) setSentinel(sentinel *SentinelClient) { d.discoverSentinels(sentinel) d.sentinel = sentinel go d.listen(sentinel) @@ -268,7 +267,7 @@ func (d *sentinelFailover) _resetSentinel() error { return err } -func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { +func (d *sentinelFailover) discoverSentinels(sentinel *SentinelClient) { sentinels, err := sentinel.Sentinels(d.masterName).Result() if err != nil { internal.Logf("sentinel: Sentinels master=%q failed: %s", d.masterName, err) @@ -292,7 +291,7 @@ func (d *sentinelFailover) discoverSentinels(sentinel *sentinelClient) { } } -func (d *sentinelFailover) listen(sentinel *sentinelClient) { +func (d *sentinelFailover) listen(sentinel *SentinelClient) { var pubsub *PubSub for { if pubsub == nil { From 02692634419948b2dfd20b2057c51b24f3775db9 Mon Sep 17 00:00:00 2001 From: Puneeth Gadangi Date: Thu, 31 May 2018 11:40:56 -0700 Subject: [PATCH 0453/1746] Use cmdSlotAndNode() to route read-only ClusterClient pipeline commands --- cluster.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 0c58c85323..9aa51fb998 100644 --- a/cluster.go +++ b/cluster.go @@ -1207,9 +1207,16 @@ func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, e } cmdsMap := make(map[*clusterNode][]Cmder) + cmdsAreReadOnly := c.cmdsAreReadOnly(cmds) for _, cmd := range cmds { - slot := c.cmdSlot(cmd) - node, err := state.slotMasterNode(slot) + var node *clusterNode + var err error + if cmdsAreReadOnly { + _, node, err = c.cmdSlotAndNode(cmd) + } else { + slot := c.cmdSlot(cmd) + node, err = state.slotMasterNode(slot) + } if err != nil { return nil, err } @@ -1218,6 +1225,16 @@ func (c *ClusterClient) mapCmdsByNode(cmds []Cmder) (map[*clusterNode][]Cmder, e return cmdsMap, nil } +func (c *ClusterClient) cmdsAreReadOnly(cmds []Cmder) bool { + for _, cmd := range cmds { + cmdInfo := c.cmdInfo(cmd.Name()) + if cmdInfo == nil || !cmdInfo.ReadOnly { + return false + } + } + return true +} + func (c *ClusterClient) remapCmds(cmds []Cmder, failedCmds map[*clusterNode][]Cmder) { remappedCmds, err := c.mapCmdsByNode(cmds) if err != nil { From e693227e84e5f5261b78c195127b51f129deb012 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 11 Jun 2018 13:26:24 +0300 Subject: [PATCH 0454/1746] Tweak doc --- options.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/options.go b/options.go index 35ce061959..fd17cbb243 100644 --- a/options.go +++ b/options.go @@ -65,10 +65,11 @@ type Options struct { PoolTimeout time.Duration // Amount of time after which client closes idle connections. // Should be less than server's timeout. - // Default is 5 minutes. + // Default is 5 minutes. -1 disables idle timeout check. IdleTimeout time.Duration - // Frequency of idle checks. - // Default is 1 minute. -1 disables idle check. + // Frequency of idle checks made by idle connections reaper. + // Default is 1 minute. -1 disables idle connections reaper, + // but idle connections are still discarded by the client. IdleCheckFrequency time.Duration // Enables read only queries on slave nodes. From bdf8f069047c9db9845652e85cca2dfd1b84e93f Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 18 Jun 2018 12:55:26 +0300 Subject: [PATCH 0455/1746] cluster: preload commands info --- cluster.go | 2 ++ cluster_test.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index 9aa51fb998..a8115b4ffa 100644 --- a/cluster.go +++ b/cluster.go @@ -653,6 +653,8 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { c.init() _, _ = c.state.Reload() + _, _ = c.cmdsInfoCache.Get() + if opt.IdleCheckFrequency > 0 { go c.reaper(opt.IdleCheckFrequency) } diff --git a/cluster_test.go b/cluster_test.go index db0728d726..80b4d02f23 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -853,8 +853,8 @@ var _ = Describe("ClusterClient timeout", func() { Context("read/write timeout", func() { BeforeEach(func() { opt := redisClusterOptions() - opt.ReadTimeout = 300 * time.Millisecond - opt.WriteTimeout = 300 * time.Millisecond + opt.ReadTimeout = 250 * time.Millisecond + opt.WriteTimeout = 250 * time.Millisecond opt.MaxRedirects = 1 client = cluster.clusterClient(opt) From 7f5b63da851237c27a3d06795129f93ccc7a059e Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 18 Jun 2018 13:07:31 +0300 Subject: [PATCH 0456/1746] Close read-only connections --- internal/error.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/error.go b/internal/error.go index 7b419577ed..e0ff86327e 100644 --- a/internal/error.go +++ b/internal/error.go @@ -19,6 +19,9 @@ func IsRetryableError(err error, retryNetError bool) bool { if strings.HasPrefix(s, "LOADING ") { return true } + if strings.HasPrefix(s, "READONLY ") { + return true + } if strings.HasPrefix(s, "CLUSTERDOWN ") { return true } @@ -38,16 +41,12 @@ func IsNetworkError(err error) bool { return ok } -func IsReadOnlyError(err error) bool { - return strings.HasPrefix(err.Error(), "READONLY ") -} - func IsBadConn(err error, allowTimeout bool) bool { if err == nil { return false } if IsRedisError(err) { - return false + return strings.HasPrefix(err.Error(), "READONLY ") } if allowTimeout { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { From 39bdfc3fa8a23c45274cd0d6bf15cbba6ac07ae3 Mon Sep 17 00:00:00 2001 From: nicktylah Date: Fri, 24 Nov 2017 18:06:13 -0800 Subject: [PATCH 0457/1746] Add basic redis streams support --- command.go | 173 ++++++++++++++++++++++++++++++++++++++++++ commands.go | 131 ++++++++++++++++++++++++++++++++ commands_test.go | 193 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 497 insertions(+) diff --git a/command.go b/command.go index 552c897bb8..11472bec1b 100644 --- a/command.go +++ b/command.go @@ -714,6 +714,179 @@ func (cmd *StringStructMapCmd) readReply(cn *pool.Conn) error { //------------------------------------------------------------------------------ +type XStream struct { + Stream string + Messages []*XMessage +} + +type XMessage struct { + ID string + Values map[string]interface{} +} + +//------------------------------------------------------------------------------ + +type XStreamSliceCmd struct { + baseCmd + + val []*XStream +} + +var _ Cmder = (*XStreamSliceCmd)(nil) + +func NewXStreamSliceCmd(args ...interface{}) *XStreamSliceCmd { + return &XStreamSliceCmd{ + baseCmd: baseCmd{_args: args}, + } +} + +func (cmd *XStreamSliceCmd) Val() []*XStream { + return cmd.val +} + +func (cmd *XStreamSliceCmd) Result() ([]*XStream, error) { + return cmd.val, cmd.err +} + +func (cmd *XStreamSliceCmd) String() string { + return cmdString(cmd, cmd.val) +} + +func (cmd *XStreamSliceCmd) readReply(cn *pool.Conn) error { + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(xStreamSliceParser) + if cmd.err != nil { + return cmd.err + } + cmd.val = v.([]*XStream) + return nil +} + +// Implements proto.MultiBulkParse +func xStreamSliceParser(rd *proto.Reader, n int64) (interface{}, error) { + xx := make([]*XStream, n) + for i := int64(0); i < n; i++ { + v, err := rd.ReadArrayReply(xStreamParser) + if err != nil { + return nil, err + } + xx[i] = v.(*XStream) + } + return xx, nil +} + +// Implements proto.MultiBulkParse +func xStreamParser(rd *proto.Reader, n int64) (interface{}, error) { + if n != 2 { + return nil, fmt.Errorf("got %d, wanted 2", n) + } + + stream, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + v, err := rd.ReadArrayReply(xMessageSliceParser) + if err != nil { + return nil, err + } + + return &XStream{ + Stream: stream, + Messages: v.([]*XMessage), + }, nil +} + +//------------------------------------------------------------------------------ + +type XMessageSliceCmd struct { + baseCmd + + val []*XMessage +} + +var _ Cmder = (*XMessageSliceCmd)(nil) + +func NewXMessageSliceCmd(args ...interface{}) *XMessageSliceCmd { + return &XMessageSliceCmd{ + baseCmd: baseCmd{_args: args}, + } +} + +func (cmd *XMessageSliceCmd) Val() []*XMessage { + return cmd.val +} + +func (cmd *XMessageSliceCmd) Result() ([]*XMessage, error) { + return cmd.val, cmd.err +} + +func (cmd *XMessageSliceCmd) String() string { + return cmdString(cmd, cmd.val) +} + +func (cmd *XMessageSliceCmd) readReply(cn *pool.Conn) error { + var v interface{} + v, cmd.err = cn.Rd.ReadArrayReply(xMessageSliceParser) + if cmd.err != nil { + return cmd.err + } + cmd.val = v.([]*XMessage) + return nil +} + +// Implements proto.MultiBulkParse +func xMessageSliceParser(rd *proto.Reader, n int64) (interface{}, error) { + msgs := make([]*XMessage, n) + for i := int64(0); i < n; i++ { + v, err := rd.ReadArrayReply(xMessageParser) + if err != nil { + return nil, err + } + msgs[i] = v.(*XMessage) + } + return msgs, nil +} + +// Implements proto.MultiBulkParse +func xMessageParser(rd *proto.Reader, n int64) (interface{}, error) { + id, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + v, err := rd.ReadArrayReply(xKeyValueParser) + if err != nil { + return nil, err + } + + return &XMessage{ + ID: id, + Values: v.(map[string]interface{}), + }, nil +} + +// Implements proto.MultiBulkParse +func xKeyValueParser(rd *proto.Reader, n int64) (interface{}, error) { + values := make(map[string]interface{}, n) + for i := int64(0); i < n; i += 2 { + key, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + value, err := rd.ReadStringReply() + if err != nil { + return nil, err + } + + values[key] = value + } + return values, nil +} + +//------------------------------------------------------------------------------ + type ZSliceCmd struct { baseCmd diff --git a/commands.go b/commands.go index c6a88154e5..1debee1e2e 100644 --- a/commands.go +++ b/commands.go @@ -171,6 +171,16 @@ type Cmdable interface { SRem(key string, members ...interface{}) *IntCmd SUnion(keys ...string) *StringSliceCmd SUnionStore(destination string, keys ...string) *IntCmd + XAdd(stream, id string, els map[string]interface{}) *StringCmd + XAddExt(opt *XAddExt) *StringCmd + XLen(key string) *IntCmd + XRange(stream, start, stop string) *XMessageSliceCmd + XRangeN(stream, start, stop string, count int64) *XMessageSliceCmd + XRevRange(stream string, start, stop string) *XMessageSliceCmd + XRevRangeN(stream string, start, stop string, count int64) *XMessageSliceCmd + XRead(streams ...string) *XStreamSliceCmd + XReadN(count int64, streams ...string) *XStreamSliceCmd + XReadExt(opt *XReadExt) *XStreamSliceCmd ZAdd(key string, members ...Z) *IntCmd ZAddNX(key string, members ...Z) *IntCmd ZAddXX(key string, members ...Z) *IntCmd @@ -1282,6 +1292,127 @@ func (c *cmdable) SUnionStore(destination string, keys ...string) *IntCmd { //------------------------------------------------------------------------------ +type XAddExt struct { + Stream string + MaxLen int64 // MAXLEN N + MaxLenApprox int64 // MAXLEN ~ N + ID string + Values map[string]interface{} +} + +func (c *cmdable) XAddExt(opt *XAddExt) *StringCmd { + a := make([]interface{}, 0, 6+len(opt.Values)*2) + a = append(a, "xadd") + a = append(a, opt.Stream) + if opt.MaxLen > 0 { + a = append(a, "maxlen", opt.MaxLen) + } else if opt.MaxLenApprox > 0 { + a = append(a, "maxlen", "~", opt.MaxLenApprox) + } + if opt.ID != "" { + a = append(a, opt.ID) + } else { + a = append(a, "*") + } + for k, v := range opt.Values { + a = append(a, k) + a = append(a, v) + } + + cmd := NewStringCmd(a...) + c.process(cmd) + return cmd +} + +func (c *cmdable) XAdd(stream, id string, values map[string]interface{}) *StringCmd { + return c.XAddExt(&XAddExt{ + Stream: stream, + ID: id, + Values: values, + }) +} + +func (c *cmdable) XLen(key string) *IntCmd { + cmd := NewIntCmd("xlen", key) + c.process(cmd) + return cmd +} + +func (c *cmdable) XRange(stream, start, stop string) *XMessageSliceCmd { + cmd := NewXMessageSliceCmd("xrange", stream, start, stop) + c.process(cmd) + return cmd +} + +func (c *cmdable) XRangeN(stream, start, stop string, count int64) *XMessageSliceCmd { + cmd := NewXMessageSliceCmd("xrange", stream, start, stop, "count", count) + c.process(cmd) + return cmd +} + +func (c *cmdable) XRevRange(stream, start, stop string) *XMessageSliceCmd { + cmd := NewXMessageSliceCmd("xrevrange", stream, start, stop) + c.process(cmd) + return cmd +} + +func (c *cmdable) XRevRangeN(stream, start, stop string, count int64) *XMessageSliceCmd { + cmd := NewXMessageSliceCmd("xrevrange", stream, start, stop, "count", count) + c.process(cmd) + return cmd +} + +type XReadExt struct { + Streams []string + Count int64 + Block time.Duration +} + +func (c *cmdable) XReadExt(opt *XReadExt) *XStreamSliceCmd { + a := make([]interface{}, 0, 5+len(opt.Streams)) + a = append(a, "xread") + if opt != nil { + if opt.Count > 0 { + a = append(a, "count") + a = append(a, opt.Count) + } + if opt.Block > 0 { + a = append(a, "block") + a = append(a, int64(opt.Block/time.Millisecond)) + } + } + a = append(a, "streams") + for _, s := range opt.Streams { + a = append(a, s) + } + + cmd := NewXStreamSliceCmd(a...) + c.process(cmd) + return cmd +} + +func (c *cmdable) XRead(streams ...string) *XStreamSliceCmd { + return c.XReadExt(&XReadExt{ + Streams: streams, + }) +} + +func (c *cmdable) XReadN(count int64, streams ...string) *XStreamSliceCmd { + return c.XReadExt(&XReadExt{ + Streams: streams, + Count: count, + }) +} + +func (c *cmdable) XReadBlock(block time.Duration, streams ...string) *XStreamSliceCmd { + return c.XReadExt(&XReadExt{ + Streams: streams, + Block: block, + }) +} + +//------------------------------------------------------------------------------ + // Z represents sorted set member. type Z struct { Score float64 diff --git a/commands_test.go b/commands_test.go index f4f794f554..896aee90b7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -3018,6 +3018,199 @@ var _ = Describe("Commands", func() { }) + Describe("streams", func() { + createStream := func() { + id, err := client.XAdd("stream", "1-0", map[string]interface{}{ + "uno": "un", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(id).To(Equal("1-0")) + + id, err = client.XAdd("stream", "2-0", map[string]interface{}{ + "dos": "deux", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(id).To(Equal("2-0")) + + id, err = client.XAdd("stream", "3-0", map[string]interface{}{ + "tres": "troix", + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(id).To(Equal("3-0")) + } + + It("should XAdd", func() { + createStream() + + id, err := client.XAdd("stream", "*", map[string]interface{}{ + "quatro": "quatre", + }).Result() + Expect(err).NotTo(HaveOccurred()) + + vals, err := client.XRange("stream", "-", "+").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + {ID: id, Values: map[string]interface{}{"quatro": "quatre"}}, + })) + }) + + It("should XAddExt", func() { + createStream() + + id, err := client.XAddExt(&redis.XAddExt{ + Stream: "stream", + MaxLen: 1, + Values: map[string]interface{}{"quatro": "quatre"}, + }).Result() + Expect(err).NotTo(HaveOccurred()) + + vals, err := client.XRange("stream", "-", "+").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(vals).To(Equal([]*redis.XMessage{ + {ID: id, Values: map[string]interface{}{"quatro": "quatre"}}, + })) + }) + + It("should XLen", func() { + createStream() + + n, err := client.XLen("stream").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(3))) + }) + + It("should XRange", func() { + createStream() + + msgs, err := client.XRange("stream", "-", "+").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + })) + + msgs, err = client.XRange("stream", "2", "+").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + })) + + msgs, err = client.XRange("stream", "-", "2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + })) + }) + + It("should XRangeN", func() { + createStream() + + msgs, err := client.XRangeN("stream", "-", "+", 2).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + })) + + msgs, err = client.XRangeN("stream", "2", "+", 1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + })) + + msgs, err = client.XRangeN("stream", "-", "2", 1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + })) + }) + + It("should XRevRange", func() { + createStream() + + msgs, err := client.XRevRange("stream", "+", "-").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + })) + + msgs, err = client.XRevRange("stream", "+", "2").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + })) + }) + + It("should XRevRangeN", func() { + createStream() + + msgs, err := client.XRevRangeN("stream", "+", "-", 2).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + })) + + msgs, err = client.XRevRangeN("stream", "+", "2", 1).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(msgs).To(Equal([]*redis.XMessage{ + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + })) + }) + + It("should XRead", func() { + createStream() + + res, err := client.XRead("stream", "0").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal([]*redis.XStream{{ + Stream: "stream", + Messages: []*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + {ID: "3-0", Values: map[string]interface{}{"tres": "troix"}}, + }}, + })) + + _, err = client.XRead("stream", "3").Result() + Expect(err).To(Equal(redis.Nil)) + }) + + It("should XReadExt", func() { + createStream() + + res, err := client.XReadExt(&redis.XReadExt{ + Streams: []string{"stream", "0"}, + Count: 2, + Block: 100 * time.Millisecond, + }).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal([]*redis.XStream{{ + Stream: "stream", + Messages: []*redis.XMessage{ + {ID: "1-0", Values: map[string]interface{}{"uno": "un"}}, + {ID: "2-0", Values: map[string]interface{}{"dos": "deux"}}, + }}, + })) + + _, err = client.XReadExt(&redis.XReadExt{ + Streams: []string{"stream", "3"}, + Count: 1, + Block: 100 * time.Millisecond, + }).Result() + Expect(err).To(Equal(redis.Nil)) + }) + }) + Describe("Geo add and radius search", func() { BeforeEach(func() { geoAdd := client.GeoAdd( From 8c513f1b884e3175ff396a819a0e1389df080b0b Mon Sep 17 00:00:00 2001 From: josh-tepper Date: Wed, 27 Jun 2018 19:34:47 -0400 Subject: [PATCH 0458/1746] Estab TLS connections + Handshake should respect DialTimeout --- options.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/options.go b/options.go index fd17cbb243..ebd0adb3da 100644 --- a/options.go +++ b/options.go @@ -85,12 +85,12 @@ func (opt *Options) init() { } if opt.Dialer == nil { opt.Dialer = func() (net.Conn, error) { - conn, err := net.DialTimeout(opt.Network, opt.Addr, opt.DialTimeout) - if opt.TLSConfig == nil || err != nil { - return conn, err + netDialer := &net.Dialer{Timeout: opt.DialTimeout} + if opt.TLSConfig == nil { + return netDialer.Dial(opt.Network, opt.Addr) + } else { + return tls.DialWithDialer(netDialer, opt.Network, opt.Addr, opt.TLSConfig) } - t := tls.Client(conn, opt.TLSConfig) - return t, t.Handshake() } } if opt.PoolSize == 0 { From 1f59be5cc0d808d16c3c2f50f8179d6f9ebdea99 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 29 Jun 2018 10:45:05 +0300 Subject: [PATCH 0459/1746] cluster: add manual setup --- cluster.go | 63 ++++++++++++++++++++++++++++++++----------------- cluster_test.go | 59 +++++++++++++++++++++++++++++++++++++++++++-- example_test.go | 42 +++++++++++++++++++++++++++++++++ main_test.go | 7 +++++- 4 files changed, 146 insertions(+), 25 deletions(-) diff --git a/cluster.go b/cluster.go index a8115b4ffa..d584423cdb 100644 --- a/cluster.go +++ b/cluster.go @@ -30,7 +30,7 @@ type ClusterOptions struct { // The maximum number of retries before giving up. Command is retried // on network errors and MOVED/ASK redirects. - // Default is 8. + // Default is 8 retries. MaxRedirects int // Enables read-only commands on slave nodes. @@ -39,8 +39,14 @@ type ClusterOptions struct { // It automatically enables ReadOnly. RouteByLatency bool // Allows routing read-only commands to the random master or slave node. + // It automatically enables ReadOnly. RouteRandomly bool + // Optional function that is used to load cluster slots information. + // It is useful to manually create cluster of standalone Redis servers + // or load-balance read/write operations between master and slaves. + ClusterSlots func() ([]ClusterSlot, error) + // Following options are copied from Options struct. OnConnect func(*Conn) error @@ -70,7 +76,7 @@ func (opt *ClusterOptions) init() { opt.MaxRedirects = 8 } - if opt.RouteByLatency { + if opt.RouteByLatency || opt.RouteRandomly { opt.ReadOnly = true } @@ -160,10 +166,6 @@ func (n *clusterNode) Close() error { return n.Client.Close() } -func (n *clusterNode) Test() error { - return n.Client.ClusterInfo().Err() -} - func (n *clusterNode) updateLatency() { const probes = 10 @@ -330,7 +332,7 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { v, err := c.nodeCreateGroup.Do(addr, func() (interface{}, error) { node := newClusterNode(c.opt, addr) - return node, node.Test() + return node, nil }) c.mu.Lock() @@ -509,6 +511,10 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode { } func (c *clusterState) IsConsistent() bool { + if c.nodes.opt.ClusterSlots != nil { + return true + } + if len(c.Masters) > len(c.Slaves) { return false } @@ -614,6 +620,14 @@ func (c *clusterStateHolder) Get() (*clusterState, error) { return nil, errors.New("redis: cluster has no state") } +func (c *clusterStateHolder) ReloadOrGet() (*clusterState, error) { + state, err := c.Reload() + if err == nil { + return state, nil + } + return c.Get() +} + //------------------------------------------------------------------------------ // ClusterClient is a Redis Cluster client representing a pool of zero @@ -662,6 +676,12 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } +// ReloadState loads cluster slots information to update cluster topography. +func (c *ClusterClient) ReloadState() error { + _, err := c.state.Reload() + return err +} + func (c *ClusterClient) init() { c.cmdable.setProcessor(c.Process) } @@ -946,12 +966,9 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { // ForEachMaster concurrently calls the fn on each master node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { - state, err := c.state.Reload() + state, err := c.state.ReloadOrGet() if err != nil { - state, err = c.state.Get() - if err != nil { - return err - } + return err } var wg sync.WaitGroup @@ -982,12 +999,9 @@ func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { // ForEachSlave concurrently calls the fn on each slave node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { - state, err := c.state.Reload() + state, err := c.state.ReloadOrGet() if err != nil { - state, err = c.state.Get() - if err != nil { - return err - } + return err } var wg sync.WaitGroup @@ -1018,12 +1032,9 @@ func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { // ForEachNode concurrently calls the fn on each known node in the cluster. // It returns the first error if any. func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { - state, err := c.state.Reload() + state, err := c.state.ReloadOrGet() if err != nil { - state, err = c.state.Get() - if err != nil { - return err - } + return err } var wg sync.WaitGroup @@ -1092,6 +1103,14 @@ func (c *ClusterClient) PoolStats() *PoolStats { } func (c *ClusterClient) loadState() (*clusterState, error) { + if c.opt.ClusterSlots != nil { + slots, err := c.opt.ClusterSlots() + if err != nil { + return nil, err + } + return newClusterState(c.nodes, slots, "") + } + addrs, err := c.nodes.Addrs() if err != nil { return nil, err diff --git a/cluster_test.go b/cluster_test.go index 80b4d02f23..3bedff3240 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -310,7 +310,8 @@ var _ = Describe("ClusterClient", func() { Expect(err).NotTo(HaveOccurred()) } - for _, master := range cluster.masters() { + client.ForEachMaster(func(master *redis.Client) error { + defer GinkgoRecover() Eventually(func() string { return master.Info("keyspace").Val() }, 30*time.Second).Should(Or( @@ -318,7 +319,8 @@ var _ = Describe("ClusterClient", func() { ContainSubstring("keys=29"), ContainSubstring("keys=40"), )) - } + return nil + }) }) It("supports Watch", func() { @@ -750,6 +752,59 @@ var _ = Describe("ClusterClient", func() { assertClusterClient() }) + + Describe("ClusterClient with ClusterSlots", func() { + BeforeEach(func() { + failover = true + + opt = redisClusterOptions() + opt.ClusterSlots = func() ([]redis.ClusterSlot, error) { + slots := []redis.ClusterSlot{{ + Start: 0, + End: 4999, + Nodes: []redis.ClusterNode{{ + Addr: ":" + ringShard1Port, + }}, + }, { + Start: 5000, + End: 9999, + Nodes: []redis.ClusterNode{{ + Addr: ":" + ringShard2Port, + }}, + }, { + Start: 10000, + End: 16383, + Nodes: []redis.ClusterNode{{ + Addr: ":" + ringShard3Port, + }}, + }} + return slots, nil + } + client = cluster.clusterClient(opt) + + err := client.ForEachMaster(func(master *redis.Client) error { + return master.FlushDB().Err() + }) + Expect(err).NotTo(HaveOccurred()) + + err = client.ForEachSlave(func(slave *redis.Client) error { + Eventually(func() int64 { + return client.DBSize().Val() + }, 30*time.Second).Should(Equal(int64(0))) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + failover = false + + err := client.Close() + Expect(err).NotTo(HaveOccurred()) + }) + + assertClusterClient() + }) }) var _ = Describe("ClusterClient without nodes", func() { diff --git a/example_test.go b/example_test.go index 4d18ddb945..eb2bb95e9f 100644 --- a/example_test.go +++ b/example_test.go @@ -71,6 +71,48 @@ func ExampleNewClusterClient() { client.Ping() } +// Following example creates a cluster from 2 master nodes and 2 slave nodes +// without using cluster mode or Redis Sentinel. +func ExampleNewClusterClient_manualSetup() { + loadClusterSlots := func() ([]redis.ClusterSlot, error) { + slots := []redis.ClusterSlot{ + // First node with 1 master and 1 slave. + { + Start: 0, + End: 8191, + Nodes: []redis.ClusterNode{{ + Addr: ":7000", // master + }, { + Addr: ":8000", // 1st slave + }}, + }, + // Second node with 1 master and 1 slave. + { + Start: 8192, + End: 16383, + Nodes: []redis.ClusterNode{{ + Addr: ":7001", // master + }, { + Addr: ":8001", // 1st slave + }}, + }, + } + return slots, nil + } + + client := redis.NewClusterClient(&redis.ClusterOptions{ + ClusterSlots: loadClusterSlots, + RouteRandomly: true, + }) + client.Ping() + + // ReloadState can be used to update cluster topography. + err := client.ReloadState() + if err != nil { + panic(err) + } +} + func ExampleNewRing() { client := redis.NewRing(&redis.RingOptions{ Addrs: map[string]string{ diff --git a/main_test.go b/main_test.go index 4eddc1ee3d..299cd709eb 100644 --- a/main_test.go +++ b/main_test.go @@ -27,6 +27,7 @@ const ( const ( ringShard1Port = "6390" ringShard2Port = "6391" + ringShard3Port = "6392" ) const ( @@ -39,7 +40,7 @@ const ( var ( redisMain *redisProcess - ringShard1, ringShard2 *redisProcess + ringShard1, ringShard2, ringShard3 *redisProcess sentinelMaster, sentinelSlave1, sentinelSlave2, sentinel *redisProcess ) @@ -62,6 +63,9 @@ var _ = BeforeSuite(func() { ringShard2, err = startRedis(ringShard2Port) Expect(err).NotTo(HaveOccurred()) + ringShard3, err = startRedis(ringShard3Port) + Expect(err).NotTo(HaveOccurred()) + sentinelMaster, err = startRedis(sentinelMasterPort) Expect(err).NotTo(HaveOccurred()) @@ -84,6 +88,7 @@ var _ = AfterSuite(func() { Expect(ringShard1.Close()).NotTo(HaveOccurred()) Expect(ringShard2.Close()).NotTo(HaveOccurred()) + Expect(ringShard3.Close()).NotTo(HaveOccurred()) Expect(sentinel.Close()).NotTo(HaveOccurred()) Expect(sentinelSlave1.Close()).NotTo(HaveOccurred()) From bf6400f40955f797590f390e83e4adbf04fb133b Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Fri, 29 Jun 2018 11:14:10 +0300 Subject: [PATCH 0460/1746] Update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9f349764af..f030acc808 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Supports: - [Timeouts](https://godoc.org/github.com/go-redis/redis#Options). - [Redis Sentinel](https://godoc.org/github.com/go-redis/redis#NewFailoverClient). - [Redis Cluster](https://godoc.org/github.com/go-redis/redis#NewClusterClient). +- [Cluster of Redis Servers](https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup) without using cluster mode and Redis Sentinel. - [Ring](https://godoc.org/github.com/go-redis/redis#NewRing). - [Instrumentation](https://godoc.org/github.com/go-redis/redis#ex-package--Instrumentation). - [Cache friendly](https://github.com/go-redis/cache). From ab1a52f0c9e9ebd920caba4492af5af4242705e0 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Thu, 12 Jul 2018 15:57:03 +0300 Subject: [PATCH 0461/1746] Add more docs for Tx --- pipeline.go | 1 + tx.go | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pipeline.go b/pipeline.go index 9349ef553e..ba852283ed 100644 --- a/pipeline.go +++ b/pipeline.go @@ -31,6 +31,7 @@ type Pipeline struct { closed bool } +// Process queues the cmd for later execution. func (c *Pipeline) Process(cmd Cmder) error { c.mu.Lock() c.cmds = append(c.cmds, cmd) diff --git a/tx.go b/tx.go index 6a753b6a09..6a7da99ddd 100644 --- a/tx.go +++ b/tx.go @@ -29,6 +29,10 @@ func (c *Client) newTx() *Tx { return &tx } +// Watch prepares a transcaction and marks the keys to be watched +// for conditional execution if there are any keys. +// +// The transaction is automatically closed when the fn exits. func (c *Client) Watch(fn func(*Tx) error, keys ...string) error { tx := c.newTx() if len(keys) > 0 { @@ -74,6 +78,7 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd { return cmd } +// Pipeline creates a new pipeline. It is more convenient to use Pipelined. func (c *Tx) Pipeline() Pipeliner { pipe := Pipeline{ exec: c.processTxPipeline, @@ -82,23 +87,24 @@ func (c *Tx) Pipeline() Pipeliner { return &pipe } -// Pipelined executes commands queued in the fn in a transaction -// and restores the connection state to normal. +// Pipelined executes commands queued in the fn in a transaction. // // When using WATCH, EXEC will execute commands only if the watched keys // were not modified, allowing for a check-and-set mechanism. // // Exec always returns list of commands. If transaction fails -// TxFailedErr is returned. Otherwise Exec returns error of the first +// TxFailedErr is returned. Otherwise Exec returns an error of the first // failed command or nil. func (c *Tx) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipeline().Pipelined(fn) } +// TxPipelined is an alias for Pipelined. func (c *Tx) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { return c.Pipelined(fn) } +// TxPipeline is an alias for Pipeline. func (c *Tx) TxPipeline() Pipeliner { return c.Pipeline() } From 1932888b442f962c587f5aeadda8a44aa05965f3 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 18 Jul 2018 12:08:43 +0300 Subject: [PATCH 0462/1746] Support XREAD BLOCK 0 --- commands.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 1debee1e2e..efaeb4d029 100644 --- a/commands.go +++ b/commands.go @@ -1376,7 +1376,7 @@ func (c *cmdable) XReadExt(opt *XReadExt) *XStreamSliceCmd { a = append(a, "count") a = append(a, opt.Count) } - if opt.Block > 0 { + if opt.Block >= 0 { a = append(a, "block") a = append(a, int64(opt.Block/time.Millisecond)) } @@ -1394,6 +1394,7 @@ func (c *cmdable) XReadExt(opt *XReadExt) *XStreamSliceCmd { func (c *cmdable) XRead(streams ...string) *XStreamSliceCmd { return c.XReadExt(&XReadExt{ Streams: streams, + Block: -1, }) } @@ -1401,6 +1402,7 @@ func (c *cmdable) XReadN(count int64, streams ...string) *XStreamSliceCmd { return c.XReadExt(&XReadExt{ Streams: streams, Count: count, + Block: -1, }) } From ee41b9092371454ee7c2aa9049ab4d06d5d387fd Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Wed, 18 Jul 2018 15:28:51 +0300 Subject: [PATCH 0463/1746] Improve docs --- cluster.go | 9 ++++++--- example_test.go | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cluster.go b/cluster.go index d584423cdb..982dff42dd 100644 --- a/cluster.go +++ b/cluster.go @@ -42,9 +42,11 @@ type ClusterOptions struct { // It automatically enables ReadOnly. RouteRandomly bool - // Optional function that is used to load cluster slots information. + // Optional function that returns cluster slots information. // It is useful to manually create cluster of standalone Redis servers - // or load-balance read/write operations between master and slaves. + // and load-balance read/write operations between master and slaves. + // It can use service like ZooKeeper to maintain configuration information + // and Cluster.ReloadState to manually trigger state reloading. ClusterSlots func() ([]ClusterSlot, error) // Following options are copied from Options struct. @@ -676,7 +678,8 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient { return c } -// ReloadState loads cluster slots information to update cluster topography. +// ReloadState reloads cluster state. It calls ClusterSlots func +// to get cluster slots information. func (c *ClusterClient) ReloadState() error { _, err := c.state.Reload() return err diff --git a/example_test.go b/example_test.go index eb2bb95e9f..2206e7f59e 100644 --- a/example_test.go +++ b/example_test.go @@ -74,7 +74,10 @@ func ExampleNewClusterClient() { // Following example creates a cluster from 2 master nodes and 2 slave nodes // without using cluster mode or Redis Sentinel. func ExampleNewClusterClient_manualSetup() { - loadClusterSlots := func() ([]redis.ClusterSlot, error) { + // clusterSlots returns cluster slots information. + // It can use service like ZooKeeper to maintain configuration information + // and Cluster.ReloadState to manually trigger state reloading. + clusterSlots := func() ([]redis.ClusterSlot, error) { slots := []redis.ClusterSlot{ // First node with 1 master and 1 slave. { @@ -101,12 +104,13 @@ func ExampleNewClusterClient_manualSetup() { } client := redis.NewClusterClient(&redis.ClusterOptions{ - ClusterSlots: loadClusterSlots, + ClusterSlots: clusterSlots, RouteRandomly: true, }) client.Ping() - // ReloadState can be used to update cluster topography. + // ReloadState reloads cluster state. It calls ClusterSlots func + // to get cluster slots information. err := client.ReloadState() if err != nil { panic(err) From 2a0840b680b10bb94e07f2956b4794f723766d60 Mon Sep 17 00:00:00 2001 From: hyfrey Date: Thu, 19 Jul 2018 17:10:40 +0800 Subject: [PATCH 0464/1746] Hash function and nreplicas used in consistent hash can be set in RingOptions --- ring.go | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/ring.go b/ring.go index b47a1094e2..99783b3ade 100644 --- a/ring.go +++ b/ring.go @@ -16,7 +16,8 @@ import ( "github.com/go-redis/redis/internal/pool" ) -const nreplicas = 100 +// Hash is type of hash function used in consistent hash +type Hash consistenthash.Hash var errRingShardsDown = errors.New("redis: all ring shards are down") @@ -30,6 +31,24 @@ type RingOptions struct { // Shard is considered down after 3 subsequent failed checks. HeartbeatFrequency time.Duration + // Hash function used in consistent hash, will use crc32.ChecksumIEEE + // from hash/crc32 package if not specified + HashFn Hash + + // Number of replicas in consistent hash, default value is 100 + // higher number of replicas will provide less deviation, that is keys will be + // distributed to nodes more evenly. + // + // Following is deviation for common nreplicas: + // -------------------------------------------------------- + // | nreplicas | standard error | 99% confidence interval | + // | 10 | 0.3152 | (0.37, 1.98) | + // | 100 | 0.0997 | (0.76, 1.28) | + // | 1000 | 0.0316 | (0.92, 1.09) | + // -------------------------------------------------------- + // See https://arxiv.org/abs/1406.2294 for reference + Nreplicas int + // Following options are copied from Options struct. OnConnect func(*Conn) error @@ -56,6 +75,10 @@ func (opt *RingOptions) init() { opt.HeartbeatFrequency = 500 * time.Millisecond } + if opt.Nreplicas == 0 { + opt.Nreplicas = 100 + } + switch opt.MinRetryBackoff { case -1: opt.MinRetryBackoff = 0 @@ -133,17 +156,21 @@ func (shard *ringShard) Vote(up bool) bool { //------------------------------------------------------------------------------ type ringShards struct { - mu sync.RWMutex - hash *consistenthash.Map - shards map[string]*ringShard // read only - list []*ringShard // read only - closed bool + mu sync.RWMutex + nreplicas int + hashfn Hash + hash *consistenthash.Map + shards map[string]*ringShard // read only + list []*ringShard // read only + closed bool } -func newRingShards() *ringShards { +func newRingShards(nreplicas int, fn Hash) *ringShards { return &ringShards{ - hash: consistenthash.New(nreplicas, nil), - shards: make(map[string]*ringShard), + nreplicas: nreplicas, + hashfn: fn, + hash: consistenthash.New(nreplicas, consistenthash.Hash(fn)), + shards: make(map[string]*ringShard), } } @@ -238,7 +265,7 @@ func (c *ringShards) Heartbeat(frequency time.Duration) { // rebalance removes dead shards from the Ring. func (c *ringShards) rebalance() { - hash := consistenthash.New(nreplicas, nil) + hash := consistenthash.New(c.nreplicas, consistenthash.Hash(c.hashfn)) for name, shard := range c.shards { if shard.IsUp() { hash.Add(name) @@ -305,7 +332,7 @@ func NewRing(opt *RingOptions) *Ring { ring := &Ring{ opt: opt, - shards: newRingShards(), + shards: newRingShards(opt.Nreplicas, opt.HashFn), } ring.cmdsInfoCache = newCmdsInfoCache(ring.cmdsInfo) From d24417d7517ff6573f437f9f3e691bea0b044053 Mon Sep 17 00:00:00 2001 From: MinJae Kwon Date: Fri, 20 Jul 2018 17:47:53 +0900 Subject: [PATCH 0465/1746] Code highlighting for example codes --- README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index f030acc808..7d05b44661 100644 --- a/README.md +++ b/README.md @@ -87,25 +87,27 @@ Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-exa Some corner cases: - SET key value EX 10 NX - set, err := client.SetNX("key", "value", 10*time.Second).Result() +```go +// SET key value EX 10 NX +set, err := client.SetNX("key", "value", 10*time.Second).Result() - SORT list LIMIT 0 2 ASC - vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() +// SORT list LIMIT 0 2 ASC +vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() - ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2 - vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ - Min: "-inf", - Max: "+inf", - Offset: 0, - Count: 2, - }).Result() +// ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2 +vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + Min: "-inf", + Max: "+inf", + Offset: 0, + Count: 2, +}).Result() - ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM - vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result() +// ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM +vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result() - EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" - vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result() +// EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" +vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result() +``` ## Benchmark From b92dacbfa7229012ae87c520ed28fab2cacf0e5c Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Jul 2018 08:49:48 +0300 Subject: [PATCH 0466/1746] Conn timeout should be higher than read timeout --- cluster.go | 4 ++-- commands.go | 8 ++++---- pubsub.go | 2 +- redis.go | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cluster.go b/cluster.go index 982dff42dd..fe50257741 100644 --- a/cluster.go +++ b/cluster.go @@ -1274,7 +1274,7 @@ func (c *ClusterClient) remapCmds(cmds []Cmder, failedCmds map[*clusterNode][]Cm func (c *ClusterClient) pipelineProcessCmds( node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds map[*clusterNode][]Cmder, ) error { - _ = cn.SetWriteTimeout(c.opt.WriteTimeout) + cn.SetWriteTimeout(c.opt.WriteTimeout) err := writeCmd(cn, cmds...) if err != nil { @@ -1284,7 +1284,7 @@ func (c *ClusterClient) pipelineProcessCmds( } // Set read timeout for all commands. - _ = cn.SetReadTimeout(c.opt.ReadTimeout) + cn.SetReadTimeout(c.opt.ReadTimeout) return c.pipelineReadCmds(cn, cmds, failedCmds) } diff --git a/commands.go b/commands.go index efaeb4d029..7b6c65dfa1 100644 --- a/commands.go +++ b/commands.go @@ -421,7 +421,7 @@ func (c *cmdable) Migrate(host, port, key string, db int64, timeout time.Duratio db, formatMs(timeout), ) - cmd.setReadTimeout(readTimeout(timeout)) + cmd.setReadTimeout(timeout) c.process(cmd) return cmd } @@ -995,7 +995,7 @@ func (c *cmdable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { } args[len(args)-1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) - cmd.setReadTimeout(readTimeout(timeout)) + cmd.setReadTimeout(timeout) c.process(cmd) return cmd } @@ -1008,7 +1008,7 @@ func (c *cmdable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { } args[len(keys)+1] = formatSec(timeout) cmd := NewStringSliceCmd(args...) - cmd.setReadTimeout(readTimeout(timeout)) + cmd.setReadTimeout(timeout) c.process(cmd) return cmd } @@ -1020,7 +1020,7 @@ func (c *cmdable) BRPopLPush(source, destination string, timeout time.Duration) destination, formatSec(timeout), ) - cmd.setReadTimeout(readTimeout(timeout)) + cmd.setReadTimeout(timeout) c.process(cmd) return cmd } diff --git a/pubsub.go b/pubsub.go index b56728f3e6..dbf6d1d1e3 100644 --- a/pubsub.go +++ b/pubsub.go @@ -309,7 +309,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { return nil, err } - cn.SetReadTimeout(timeout) + cn.SetReadTimeout(readTimeout(timeout)) err = c.cmd.readReply(cn) c.releaseConn(cn, err) if err != nil { diff --git a/redis.go b/redis.go index beb632e1e9..ff0651483f 100644 --- a/redis.go +++ b/redis.go @@ -176,9 +176,8 @@ func (c *baseClient) retryBackoff(attempt int) time.Duration { func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration { if timeout := cmd.readTimeout(); timeout != nil { - return *timeout + return readTimeout(*timeout) } - return c.opt.ReadTimeout } From 7c9aa65a40606ea905bc1bb55e7ec69d57bbe986 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Jul 2018 09:27:36 +0300 Subject: [PATCH 0467/1746] Cleanup --- internal/pool/conn.go | 14 ++++++++------ internal/proto/reader.go | 24 ++++++------------------ internal/proto/write_buffer.go | 12 ++++++++++++ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/internal/pool/conn.go b/internal/pool/conn.go index 8af51d9de6..acaf36656d 100644 --- a/internal/pool/conn.go +++ b/internal/pool/conn.go @@ -47,22 +47,24 @@ func (cn *Conn) IsStale(timeout time.Duration) bool { return timeout > 0 && time.Since(cn.UsedAt()) > timeout } -func (cn *Conn) SetReadTimeout(timeout time.Duration) error { +func (cn *Conn) SetReadTimeout(timeout time.Duration) { now := time.Now() cn.SetUsedAt(now) if timeout > 0 { - return cn.netConn.SetReadDeadline(now.Add(timeout)) + cn.netConn.SetReadDeadline(now.Add(timeout)) + } else { + cn.netConn.SetReadDeadline(noDeadline) } - return cn.netConn.SetReadDeadline(noDeadline) } -func (cn *Conn) SetWriteTimeout(timeout time.Duration) error { +func (cn *Conn) SetWriteTimeout(timeout time.Duration) { now := time.Now() cn.SetUsedAt(now) if timeout > 0 { - return cn.netConn.SetWriteDeadline(now.Add(timeout)) + cn.netConn.SetWriteDeadline(now.Add(timeout)) + } else { + cn.netConn.SetWriteDeadline(noDeadline) } - return cn.netConn.SetWriteDeadline(noDeadline) } func (cn *Conn) Write(b []byte) (int, error) { diff --git a/internal/proto/reader.go b/internal/proto/reader.go index d5d6953585..8c28c7b71c 100644 --- a/internal/proto/reader.go +++ b/internal/proto/reader.go @@ -91,11 +91,11 @@ func (r *Reader) ReadReply(m MultiBulkParse) (interface{}, error) { case ErrorReply: return nil, ParseErrorReply(line) case StatusReply: - return parseStatusValue(line), nil + return parseTmpStatusReply(line), nil case IntReply: return util.ParseInt(line[1:], 10, 64) case StringReply: - return r.readTmpBytesValue(line) + return r.readTmpBytesReply(line) case ArrayReply: n, err := parseArrayLen(line) if err != nil { @@ -130,9 +130,9 @@ func (r *Reader) ReadTmpBytesReply() ([]byte, error) { case ErrorReply: return nil, ParseErrorReply(line) case StringReply: - return r.readTmpBytesValue(line) + return r.readTmpBytesReply(line) case StatusReply: - return parseStatusValue(line), nil + return parseTmpStatusReply(line), nil default: return nil, fmt.Errorf("redis: can't parse string reply: %.100q", line) } @@ -229,7 +229,7 @@ func (r *Reader) ReadScanReply() ([]string, uint64, error) { return keys, cursor, err } -func (r *Reader) readTmpBytesValue(line []byte) ([]byte, error) { +func (r *Reader) readTmpBytesReply(line []byte) ([]byte, error) { if isNilReply(line) { return nil, Nil } @@ -294,18 +294,6 @@ func readN(r io.Reader, b []byte, n int) ([]byte, error) { return b, nil } -func formatInt(n int64) string { - return strconv.FormatInt(n, 10) -} - -func formatUint(u uint64) string { - return strconv.FormatUint(u, 10) -} - -func formatFloat(f float64) string { - return strconv.FormatFloat(f, 'f', -1, 64) -} - func isNilReply(b []byte) bool { return len(b) == 3 && (b[0] == StringReply || b[0] == ArrayReply) && @@ -316,7 +304,7 @@ func ParseErrorReply(line []byte) error { return RedisError(string(line[1:])) } -func parseStatusValue(line []byte) []byte { +func parseTmpStatusReply(line []byte) []byte { return line[1:] } diff --git a/internal/proto/write_buffer.go b/internal/proto/write_buffer.go index cc4014fb4f..664f4c3332 100644 --- a/internal/proto/write_buffer.go +++ b/internal/proto/write_buffer.go @@ -99,3 +99,15 @@ func (w *WriteBuffer) AppendBytes(p []byte) { w.b = append(w.b, p...) w.b = append(w.b, '\r', '\n') } + +func formatInt(n int64) string { + return strconv.FormatInt(n, 10) +} + +func formatUint(u uint64) string { + return strconv.FormatUint(u, 10) +} + +func formatFloat(f float64) string { + return strconv.FormatFloat(f, 'f', -1, 64) +} From ce9cfe9417774ec4e2a4e65dab39c261f239d6b6 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Jul 2018 09:46:29 +0300 Subject: [PATCH 0468/1746] Add MemoryUsage --- commands.go | 20 +++++++++++++++++--- commands_test.go | 33 +++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/commands.go b/commands.go index 7b6c65dfa1..763c837050 100644 --- a/commands.go +++ b/commands.go @@ -62,6 +62,7 @@ type Cmdable interface { TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) TxPipeline() Pipeliner + Command() *CommandsInfoCmd ClientGetName() *StringCmd Echo(message interface{}) *StringCmd Ping() *StatusCmd @@ -275,9 +276,9 @@ type Cmdable interface { GeoRadiusByMemberRO(key, member string, query *GeoRadiusQuery) *GeoLocationCmd GeoDist(key string, member1, member2, unit string) *FloatCmd GeoHash(key string, members ...string) *StringSliceCmd - Command() *CommandsInfoCmd ReadOnly() *StatusCmd ReadWrite() *StatusCmd + MemoryUsage(key string, samples ...int) *IntCmd } type StatefulCmdable interface { @@ -355,6 +356,12 @@ func (c *statefulCmdable) SwapDB(index1, index2 int) *StatusCmd { //------------------------------------------------------------------------------ +func (c *cmdable) Command() *CommandsInfoCmd { + cmd := NewCommandsInfoCmd("command") + c.process(cmd) + return cmd +} + func (c *cmdable) Del(keys ...string) *IntCmd { args := make([]interface{}, 1+len(keys)) args[0] = "del" @@ -2301,8 +2308,15 @@ func (c *cmdable) GeoPos(key string, members ...string) *GeoPosCmd { //------------------------------------------------------------------------------ -func (c *cmdable) Command() *CommandsInfoCmd { - cmd := NewCommandsInfoCmd("command") +func (c *cmdable) MemoryUsage(key string, samples ...int) *IntCmd { + args := []interface{}{"memory", "usage", key} + if len(samples) > 0 { + if len(samples) != 1 { + panic("MemoryUsage expects single sample count") + } + args = append(args, "SAMPLES", samples[0]) + } + cmd := NewIntCmd(args...) c.process(cmd) return cmd } diff --git a/commands_test.go b/commands_test.go index 896aee90b7..6bec811628 100644 --- a/commands_test.go +++ b/commands_test.go @@ -244,14 +244,31 @@ var _ = Describe("Commands", func() { Describe("debugging", func() { It("should DebugObject", func() { - debug := client.DebugObject("foo") - Expect(debug.Err()).To(HaveOccurred()) - Expect(debug.Err().Error()).To(Equal("ERR no such key")) - - client.Set("foo", "bar", 0) - debug = client.DebugObject("foo") - Expect(debug.Err()).NotTo(HaveOccurred()) - Expect(debug.Val()).To(ContainSubstring(`serializedlength:4`)) + err := client.DebugObject("foo").Err() + Expect(err).To(MatchError("ERR no such key")) + + err = client.Set("foo", "bar", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + s, err := client.DebugObject("foo").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(s).To(ContainSubstring("serializedlength:4")) + }) + + It("should MemoryUsage", func() { + err := client.MemoryUsage("foo").Err() + Expect(err).To(Equal(redis.Nil)) + + err = client.Set("foo", "bar", 0).Err() + Expect(err).NotTo(HaveOccurred()) + + n, err := client.MemoryUsage("foo").Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(52))) + + n, err = client.MemoryUsage("foo", 0).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(52))) }) }) From 2559f32464f62ec24c2990eb99616bf7058581ed Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Jul 2018 10:50:26 +0300 Subject: [PATCH 0469/1746] cluster: optimize newClusterState --- cluster.go | 76 ++++++++++++++++++++++--------------- cluster_test.go | 67 ++++++++++++++++++++++---------- export_test.go | 2 +- internal/hashtag/hashtag.go | 6 +-- main_test.go | 17 +++++---- ring.go | 2 + 6 files changed, 109 insertions(+), 61 deletions(-) diff --git a/cluster.go b/cluster.go index fe50257741..84a026c235 100644 --- a/cluster.go +++ b/cluster.go @@ -8,7 +8,7 @@ import ( "math" "math/rand" "net" - "strings" + "sort" "sync" "sync/atomic" "time" @@ -387,12 +387,31 @@ func (c *clusterNodes) Random() (*clusterNode, error) { //------------------------------------------------------------------------------ +type clusterSlot struct { + start, end int + nodes []*clusterNode +} + +type clusterSlotSlice []*clusterSlot + +func (p clusterSlotSlice) Len() int { + return len(p) +} + +func (p clusterSlotSlice) Less(i, j int) bool { + return p[i].start < p[j].start +} + +func (p clusterSlotSlice) Swap(i, j int) { + p[i], p[j] = p[j], p[i] +} + type clusterState struct { nodes *clusterNodes Masters []*clusterNode Slaves []*clusterNode - slots [][]*clusterNode + slots []*clusterSlot generation uint32 createdAt time.Time @@ -404,7 +423,7 @@ func newClusterState( c := clusterState{ nodes: nodes, - slots: make([][]*clusterNode, hashtag.SlotNumber), + slots: make([]*clusterSlot, 0, len(slots)), generation: nodes.NextGeneration(), createdAt: time.Now(), @@ -434,11 +453,15 @@ func newClusterState( } } - for i := slot.Start; i <= slot.End; i++ { - c.slots[i] = nodes - } + c.slots = append(c.slots, &clusterSlot{ + start: slot.Start, + end: slot.End, + nodes: nodes, + }) } + sort.Sort(clusterSlotSlice(c.slots)) + time.AfterFunc(time.Minute, func() { nodes.GC(c.generation) }) @@ -506,8 +529,15 @@ func (c *clusterState) slotRandomNode(slot int) *clusterNode { } func (c *clusterState) slotNodes(slot int) []*clusterNode { - if slot >= 0 && slot < len(c.slots) { - return c.slots[slot] + i := sort.Search(len(c.slots), func(i int) bool { + return c.slots[i].end >= slot + }) + if i >= len(c.slots) { + return nil + } + x := c.slots[i] + if slot >= x.start && slot <= x.end { + return x.nodes } return nil } @@ -516,26 +546,7 @@ func (c *clusterState) IsConsistent() bool { if c.nodes.opt.ClusterSlots != nil { return true } - - if len(c.Masters) > len(c.Slaves) { - return false - } - - for _, master := range c.Masters { - s := master.Client.Info("replication").Val() - if !strings.Contains(s, "role:master") { - return false - } - } - - for _, slave := range c.Slaves { - s := slave.Client.Info("replication").Val() - if !strings.Contains(s, "role:slave") { - return false - } - } - - return true + return len(c.Masters) <= len(c.Slaves) } //------------------------------------------------------------------------------ @@ -563,7 +574,7 @@ func (c *clusterStateHolder) Reload() (*clusterState, error) { return nil, err } if !state.IsConsistent() { - c.LazyReload() + time.AfterFunc(time.Second, c.LazyReload) } return state, nil } @@ -843,6 +854,7 @@ func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { } if internal.IsRetryableError(err, true) { + c.state.LazyReload() continue } @@ -929,12 +941,14 @@ func (c *ClusterClient) defaultProcess(cmd Cmder) error { } if internal.IsRetryableError(err, true) { - // Firstly retry the same node. + c.state.LazyReload() + + // First retry the same node. if attempt == 0 { continue } - // Secondly try random node. + // Second try random node. node, err = c.nodes.Random() if err != nil { break diff --git a/cluster_test.go b/cluster_test.go index 3bedff3240..a64a0f6a77 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -51,13 +51,19 @@ func (s *clusterScenario) addrs() []string { func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.ClusterClient { opt.Addrs = s.addrs() client := redis.NewClusterClient(opt) - Eventually(func() bool { + err := eventually(func() error { state, err := client.GetState() if err != nil { - return false + return err + } + if !state.IsConsistent() { + return fmt.Errorf("cluster state is not conistent") } - return state.IsConsistent() - }, 30*time.Second).Should(BeTrue()) + return nil + }, 30*time.Second) + if err != nil { + panic(err) + } return client } @@ -935,18 +941,21 @@ var _ = Describe("ClusterClient timeout", func() { //------------------------------------------------------------------------------ -func BenchmarkRedisClusterPing(b *testing.B) { - if testing.Short() { - b.Skip("skipping in short mode") - } - - cluster := &clusterScenario{ +func newClusterScenario() *clusterScenario { + return &clusterScenario{ ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, nodeIds: make([]string, 6), processes: make(map[string]*redisProcess, 6), clients: make(map[string]*redis.Client, 6), } +} + +func BenchmarkRedisClusterPing(b *testing.B) { + if testing.Short() { + b.Skip("skipping in short mode") + } + cluster := newClusterScenario() if err := startCluster(cluster); err != nil { b.Fatal(err) } @@ -959,7 +968,8 @@ func BenchmarkRedisClusterPing(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Ping().Err(); err != nil { + err := client.Ping().Err() + if err != nil { b.Fatal(err) } } @@ -971,13 +981,7 @@ func BenchmarkRedisClusterSetString(b *testing.B) { b.Skip("skipping in short mode") } - cluster := &clusterScenario{ - ports: []string{"8220", "8221", "8222", "8223", "8224", "8225"}, - nodeIds: make([]string, 6), - processes: make(map[string]*redisProcess, 6), - clients: make(map[string]*redis.Client, 6), - } - + cluster := newClusterScenario() if err := startCluster(cluster); err != nil { b.Fatal(err) } @@ -992,9 +996,34 @@ func BenchmarkRedisClusterSetString(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := client.Set("key", value, 0).Err(); err != nil { + err := client.Set("key", value, 0).Err() + if err != nil { b.Fatal(err) } } }) } + +func BenchmarkRedisClusterReloadState(b *testing.B) { + if testing.Short() { + b.Skip("skipping in short mode") + } + + cluster := newClusterScenario() + if err := startCluster(cluster); err != nil { + b.Fatal(err) + } + defer stopCluster(cluster) + + client := cluster.clusterClient(redisClusterOptions()) + defer client.Close() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + err := client.ReloadState() + if err != nil { + b.Fatal(err) + } + } +} diff --git a/export_test.go b/export_test.go index fcb7fa0d7e..060df10283 100644 --- a/export_test.go +++ b/export_test.go @@ -49,7 +49,7 @@ func (c *ClusterClient) Nodes(key string) ([]*clusterNode, error) { } slot := hashtag.Slot(key) - nodes := state.slots[slot] + nodes := state.slotNodes(slot) if len(nodes) != 2 { return nil, fmt.Errorf("slot=%d does not have enough nodes: %v", slot, nodes) } diff --git a/internal/hashtag/hashtag.go b/internal/hashtag/hashtag.go index 8c7ebbfa64..22f5b3981b 100644 --- a/internal/hashtag/hashtag.go +++ b/internal/hashtag/hashtag.go @@ -5,7 +5,7 @@ import ( "strings" ) -const SlotNumber = 16384 +const slotNumber = 16384 // CRC16 implementation according to CCITT standards. // Copyright 2001-2010 Georges Menie (www.menie.org) @@ -56,7 +56,7 @@ func Key(key string) string { } func RandomSlot() int { - return rand.Intn(SlotNumber) + return rand.Intn(slotNumber) } // hashSlot returns a consistent slot number between 0 and 16383 @@ -66,7 +66,7 @@ func Slot(key string) int { return RandomSlot() } key = Key(key) - return int(crc16sum(key)) % SlotNumber + return int(crc16sum(key)) % slotNumber } func crc16sum(key string) (crc uint16) { diff --git a/main_test.go b/main_test.go index 299cd709eb..e49d954b02 100644 --- a/main_test.go +++ b/main_test.go @@ -8,7 +8,6 @@ import ( "os/exec" "path/filepath" "sync" - "sync/atomic" "testing" "time" @@ -169,24 +168,28 @@ func perform(n int, cbs ...func(int)) { } func eventually(fn func() error, timeout time.Duration) error { - var exit int32 errCh := make(chan error) done := make(chan struct{}) + exit := make(chan struct{}) go func() { - defer GinkgoRecover() - - for atomic.LoadInt32(&exit) == 0 { + for { err := fn() if err == nil { close(done) return } + select { case errCh <- err: default: } - time.Sleep(timeout / 100) + + select { + case <-exit: + return + case <-time.After(timeout / 100): + } } }() @@ -194,7 +197,7 @@ func eventually(fn func() error, timeout time.Duration) error { case <-done: return nil case <-time.After(timeout): - atomic.StoreInt32(&exit, 1) + close(exit) select { case err := <-errCh: return err diff --git a/ring.go b/ring.go index 5cbfb9bf40..8b20d476e2 100644 --- a/ring.go +++ b/ring.go @@ -170,6 +170,8 @@ type ringShards struct { func newRingShards(opt *RingOptions) *ringShards { return &ringShards{ + opt: opt, + hash: newConsistentHash(opt), shards: make(map[string]*ringShard), } From 3143c672b64e5edbf92410f2e14cdc893428cfbf Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Sun, 22 Jul 2018 14:23:04 +0300 Subject: [PATCH 0470/1746] Add changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..b4cf05c00d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## v6.13 + +- Ring got new options called `HashReplicas` and `Hash`. It is recommended to set `HashReplicas = 1000` for better keys distribution between shards. +- Cluster client was optimized to use much less memory when reloading cluster state. + +## v6.12 + +- ClusterClient got new option called `ClusterSlots` which allows to build cluster of normal Redis Servers that don't have cluster mode enabled. See https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup From 4ae24be2876e6489c4984501b167c4117354dc61 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 23 Jul 2018 12:01:13 +0300 Subject: [PATCH 0471/1746] Check cluster state before running the tests --- cluster_test.go | 36 +++++++++++++++++++++++++++++++++--- export_test.go | 4 ---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/cluster_test.go b/cluster_test.go index a64a0f6a77..e94a50993e 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -49,21 +49,51 @@ func (s *clusterScenario) addrs() []string { } func (s *clusterScenario) clusterClient(opt *redis.ClusterOptions) *redis.ClusterClient { + var errBadState = fmt.Errorf("cluster state is not consistent") + opt.Addrs = s.addrs() client := redis.NewClusterClient(opt) + err := eventually(func() error { - state, err := client.GetState() + if opt.ClusterSlots != nil { + return nil + } + + state, err := client.LoadState() if err != nil { return err } + if !state.IsConsistent() { - return fmt.Errorf("cluster state is not conistent") + return errBadState + } + + if len(state.Masters) < 3 { + return errBadState } + for _, master := range state.Masters { + s := master.Client.Info("replication").Val() + if !strings.Contains(s, "role:master") { + return errBadState + } + } + + if len(state.Slaves) < 3 { + return errBadState + } + for _, slave := range state.Slaves { + s := slave.Client.Info("replication").Val() + if !strings.Contains(s, "role:slave") { + return errBadState + } + } + return nil }, 30*time.Second) if err != nil { panic(err) } + return client } @@ -703,7 +733,7 @@ var _ = Describe("ClusterClient", func() { }) Expect(err).NotTo(HaveOccurred()) - state, err := client.GetState() + state, err := client.LoadState() Expect(err).NotTo(HaveOccurred()) Expect(state.IsConsistent()).To(BeTrue()) diff --git a/export_test.go b/export_test.go index 060df10283..e9afda9380 100644 --- a/export_test.go +++ b/export_test.go @@ -21,10 +21,6 @@ func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) return c.receiveMessage(timeout) } -func (c *ClusterClient) GetState() (*clusterState, error) { - return c.state.Get() -} - func (c *ClusterClient) LoadState() (*clusterState, error) { return c.loadState() } From ea9da7c2e83b3e1fb3da7acee9e67009dbfbbec7 Mon Sep 17 00:00:00 2001 From: Vladimir Mihailenco Date: Mon, 23 Jul 2018 15:55:13 +0300 Subject: [PATCH 0472/1746] Rework ReceiveMessage --- CHANGELOG.md | 1 + cluster.go | 8 +- cluster_test.go | 2 +- export_test.go | 5 -- pubsub.go | 217 ++++++++++++++++++++++++++++++------------------ pubsub_test.go | 64 ++++---------- redis.go | 4 +- ring.go | 16 ++++ ring_test.go | 33 ++++---- sentinel.go | 74 ++++++++--------- 10 files changed, 225 insertions(+), 199 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4cf05c00d..b35c4dd104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Ring got new options called `HashReplicas` and `Hash`. It is recommended to set `HashReplicas = 1000` for better keys distribution between shards. - Cluster client was optimized to use much less memory when reloading cluster state. +- ReceiveMessage is re-worked to not use ReceiveTimeout so it does not lose data when timeout occurres. ## v6.12 diff --git a/cluster.go b/cluster.go index 84a026c235..7a1af143e0 100644 --- a/cluster.go +++ b/cluster.go @@ -1500,11 +1500,9 @@ func (c *ClusterClient) txPipelineReadQueued( } func (c *ClusterClient) pubSub(channels []string) *PubSub { - opt := c.opt.clientOptions() - var node *clusterNode - return &PubSub{ - opt: opt, + pubsub := &PubSub{ + opt: c.opt.clientOptions(), newConn: func(channels []string) (*pool.Conn, error) { if node == nil { @@ -1527,6 +1525,8 @@ func (c *ClusterClient) pubSub(channels []string) *PubSub { return node.Client.connPool.CloseConn(cn) }, } + pubsub.init() + return pubsub } // Subscribe subscribes the client to the specified channels. diff --git a/cluster_test.go b/cluster_test.go index e94a50993e..f9c3a90ff3 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -453,7 +453,7 @@ var _ = Describe("ClusterClient", func() { ttl := cmds[(i*2)+1].(*redis.DurationCmd) dur := time.Duration(i+1) * time.Hour - Expect(ttl.Val()).To(BeNumerically("~", dur, 10*time.Second)) + Expect(ttl.Val()).To(BeNumerically("~", dur, 30*time.Second)) } }) diff --git a/export_test.go b/export_test.go index e9afda9380..fab91e2a41 100644 --- a/export_test.go +++ b/export_test.go @@ -3,7 +3,6 @@ package redis import ( "fmt" "net" - "time" "github.com/go-redis/redis/internal/hashtag" "github.com/go-redis/redis/internal/pool" @@ -17,10 +16,6 @@ func (c *PubSub) SetNetConn(netConn net.Conn) { c.cn = pool.NewConn(netConn) } -func (c *PubSub) ReceiveMessageTimeout(timeout time.Duration) (*Message, error) { - return c.receiveMessage(timeout) -} - func (c *ClusterClient) LoadState() (*clusterState, error) { return c.loadState() } diff --git a/pubsub.go b/pubsub.go index dbf6d1d1e3..b76dbc6d00 100644 --- a/pubsub.go +++ b/pubsub.go @@ -2,7 +2,6 @@ package redis import ( "fmt" - "net" "sync" "time" @@ -11,11 +10,11 @@ import ( ) // PubSub implements Pub/Sub commands as described in -// http://redis.io/topics/pubsub. It's NOT safe for concurrent use by -// multiple goroutines. +// http://redis.io/topics/pubsub. Message receiving is NOT safe +// for concurrent use by multiple goroutines. // -// PubSub automatically resubscribes to the channels and patterns -// when Redis becomes unavailable. +// PubSub automatically reconnects to Redis Server and resubscribes +// to the channels in case of network errors. type PubSub struct { opt *Options @@ -27,13 +26,21 @@ type PubSub struct { channels map[string]struct{} patterns map[string]struct{} closed bool + exit chan struct{} cmd *Cmd + pingOnce sync.Once + ping chan struct{} + chOnce sync.Once ch chan *Message } +func (c *PubSub) init() { + c.exit = make(chan struct{}) +} + func (c *PubSub) conn() (*pool.Conn, error) { c.mu.Lock() cn, err := c._conn(nil) @@ -66,31 +73,36 @@ func (c *PubSub) _conn(channels []string) (*pool.Conn, error) { func (c *PubSub) resubscribe(cn *pool.Conn) error { var firstErr error + if len(c.channels) > 0 { - channels := make([]string, len(c.channels)) - i := 0 - for channel := range c.channels { - channels[i] = channel - i++ - } - if err := c._subscribe(cn, "subscribe", channels...); err != nil && firstErr == nil { + channels := mapKeys(c.channels) + err := c._subscribe(cn, "subscribe", channels...) + if err != nil && firstErr == nil { firstErr = err } } + if len(c.patterns) > 0 { - patterns := make([]string, len(c.patterns)) - i := 0 - for pattern := range c.patterns { - patterns[i] = pattern - i++ - } - if err := c._subscribe(cn, "psubscribe", patterns...); err != nil && firstErr == nil { + patterns := mapKeys(c.patterns) + err := c._subscribe(cn, "psubscribe", patterns...) + if err != nil && firstErr == nil { firstErr = err } } + return firstErr } +func mapKeys(m map[string]struct{}) []string { + s := make([]string, len(m)) + i := 0 + for k := range m { + s[i] = k + i++ + } + return s +} + func (c *PubSub) _subscribe(cn *pool.Conn, redisCmd string, channels ...string) error { args := make([]interface{}, 1+len(channels)) args[0] = redisCmd @@ -114,16 +126,30 @@ func (c *PubSub) _releaseConn(cn *pool.Conn, err error) { return } if internal.IsBadConn(err, true) { - _ = c.closeTheCn() + c._reconnect() } } -func (c *PubSub) closeTheCn() error { - err := c.closeConn(c.cn) - c.cn = nil +func (c *PubSub) _closeTheCn() error { + var err error + if c.cn != nil { + err = c.closeConn(c.cn) + c.cn = nil + } return err } +func (c *PubSub) reconnect() { + c.mu.Lock() + c._reconnect() + c.mu.Unlock() +} + +func (c *PubSub) _reconnect() { + _ = c._closeTheCn() + _, _ = c._conn(nil) +} + func (c *PubSub) Close() error { c.mu.Lock() defer c.mu.Unlock() @@ -132,17 +158,18 @@ func (c *PubSub) Close() error { return pool.ErrClosed } c.closed = true + close(c.exit) - if c.cn != nil { - return c.closeTheCn() - } - return nil + err := c._closeTheCn() + return err } // Subscribe the client to the specified channels. It returns // empty subscription if there are no channels. func (c *PubSub) Subscribe(channels ...string) error { c.mu.Lock() + defer c.mu.Unlock() + err := c.subscribe("subscribe", channels...) if c.channels == nil { c.channels = make(map[string]struct{}) @@ -150,7 +177,6 @@ func (c *PubSub) Subscribe(channels ...string) error { for _, channel := range channels { c.channels[channel] = struct{}{} } - c.mu.Unlock() return err } @@ -158,6 +184,8 @@ func (c *PubSub) Subscribe(channels ...string) error { // empty subscription if there are no patterns. func (c *PubSub) PSubscribe(patterns ...string) error { c.mu.Lock() + defer c.mu.Unlock() + err := c.subscribe("psubscribe", patterns...) if c.patterns == nil { c.patterns = make(map[string]struct{}) @@ -165,7 +193,6 @@ func (c *PubSub) PSubscribe(patterns ...string) error { for _, pattern := range patterns { c.patterns[pattern] = struct{}{} } - c.mu.Unlock() return err } @@ -173,11 +200,12 @@ func (c *PubSub) PSubscribe(patterns ...string) error { // them if none is given. func (c *PubSub) Unsubscribe(channels ...string) error { c.mu.Lock() + defer c.mu.Unlock() + err := c.subscribe("unsubscribe", channels...) for _, channel := range channels { delete(c.channels, channel) } - c.mu.Unlock() return err } @@ -185,11 +213,12 @@ func (c *PubSub) Unsubscribe(channels ...string) error { // them if none is given. func (c *PubSub) PUnsubscribe(patterns ...string) error { c.mu.Lock() + defer c.mu.Unlock() + err := c.subscribe("punsubscribe", patterns...) for _, pattern := range patterns { delete(c.patterns, pattern) } - c.mu.Unlock() return err } @@ -298,7 +327,7 @@ func (c *PubSub) newMessage(reply interface{}) (interface{}, error) { // ReceiveTimeout acts like Receive but returns an error if message // is not received in time. This is low-level API and most clients -// should use ReceiveMessage. +// should use ReceiveMessage instead. func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { if c.cmd == nil { c.cmd = NewCmd() @@ -309,7 +338,7 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { return nil, err } - cn.SetReadTimeout(readTimeout(timeout)) + cn.SetReadTimeout(timeout) err = c.cmd.readReply(cn) c.releaseConn(cn, err) if err != nil { @@ -321,48 +350,28 @@ func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) { // Receive returns a message as a Subscription, Message, Pong or error. // See PubSub example for details. This is low-level API and most clients -// should use ReceiveMessage. +// should use ReceiveMessage instead. func (c *PubSub) Receive() (interface{}, error) { return c.ReceiveTimeout(0) } // ReceiveMessage returns a Message or error ignoring Subscription or Pong -// messages. It automatically reconnects to Redis Server and resubscribes -// to channels in case of network errors. +// messages. It periodically sends Ping messages to test connection health. func (c *PubSub) ReceiveMessage() (*Message, error) { - return c.receiveMessage(5 * time.Second) -} - -func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { - var errNum uint + c.pingOnce.Do(c.initPing) for { - msgi, err := c.ReceiveTimeout(timeout) + msg, err := c.Receive() if err != nil { - if !internal.IsNetworkError(err) { - return nil, err - } - - errNum++ - if errNum < 3 { - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - err := c.Ping() - if err != nil { - internal.Logf("PubSub.Ping failed: %s", err) - } - } - } else { - // 3 consequent errors - connection is broken or - // Redis Server is down. - // Sleep to not exceed max number of open connections. - time.Sleep(time.Second) - } - continue + return nil, err } - // Reset error number, because we received a message. - errNum = 0 + // Any message is as good as a ping. + select { + case c.ping <- struct{}{}: + default: + } - switch msg := msgi.(type) { + switch msg := msg.(type) { case *Subscription: // Ignore. case *Pong: @@ -370,30 +379,74 @@ func (c *PubSub) receiveMessage(timeout time.Duration) (*Message, error) { case *Message: return msg, nil default: - return nil, fmt.Errorf("redis: unknown message: %T", msgi) + err := fmt.Errorf("redis: unknown message: %T", msg) + return nil, err } } } // Channel returns a Go channel for concurrently receiving messages. -// The channel is closed with PubSub. Receive or ReceiveMessage APIs -// can not be used after channel is created. +// The channel is closed with PubSub. Receive* APIs can not be used +// after channel is created. func (c *PubSub) Channel() <-chan *Message { - c.chOnce.Do(func() { - c.ch = make(chan *Message, 100) - go func() { - for { - msg, err := c.ReceiveMessage() - if err != nil { - if err == pool.ErrClosed { - break - } - continue + c.chOnce.Do(c.initChannel) + return c.ch +} + +func (c *PubSub) initChannel() { + c.ch = make(chan *Message, 100) + go func() { + var errCount int + for { + msg, err := c.ReceiveMessage() + if err != nil { + if err == pool.ErrClosed { + close(c.ch) + return + } + if errCount > 0 { + time.Sleep(c.retryBackoff(errCount)) } - c.ch <- msg + errCount++ + continue } - close(c.ch) - }() - }) - return c.ch + errCount = 0 + c.ch <- msg + } + }() +} + +func (c *PubSub) initPing() { + const timeout = 5 * time.Second + + c.ping = make(chan struct{}, 10) + go func() { + timer := time.NewTimer(timeout) + timer.Stop() + + var hasPing bool + for { + timer.Reset(timeout) + select { + case <-c.ping: + hasPing = true + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + if hasPing { + hasPing = false + _ = c.Ping() + } else { + c.reconnect() + } + case <-c.exit: + return + } + } + }() +} + +func (c *PubSub) retryBackoff(attempt int) time.Duration { + return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) } diff --git a/pubsub_test.go b/pubsub_test.go index 6a85bd0389..059b4a60b8 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -255,45 +255,6 @@ var _ = Describe("PubSub", func() { Expect(msg.Payload).To(Equal("world")) }) - It("should ReceiveMessage after timeout", func() { - timeout := 100 * time.Millisecond - - pubsub := client.Subscribe("mychannel") - defer pubsub.Close() - - subscr, err := pubsub.ReceiveTimeout(time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(subscr).To(Equal(&redis.Subscription{ - Kind: "subscribe", - Channel: "mychannel", - Count: 1, - })) - - done := make(chan bool, 1) - go func() { - defer GinkgoRecover() - defer func() { - done <- true - }() - - time.Sleep(timeout + 100*time.Millisecond) - n, err := client.Publish("mychannel", "hello").Result() - Expect(err).NotTo(HaveOccurred()) - Expect(n).To(Equal(int64(1))) - }() - - msg, err := pubsub.ReceiveMessageTimeout(timeout) - Expect(err).NotTo(HaveOccurred()) - Expect(msg.Channel).To(Equal("mychannel")) - Expect(msg.Payload).To(Equal("hello")) - - Eventually(done).Should(Receive()) - - stats := client.PoolStats() - Expect(stats.Hits).To(Equal(uint32(1))) - Expect(stats.Misses).To(Equal(uint32(1))) - }) - It("returns an error when subscribe fails", func() { pubsub := client.Subscribe() defer pubsub.Close() @@ -316,24 +277,27 @@ var _ = Describe("PubSub", func() { writeErr: io.EOF, }) - done := make(chan bool, 1) + step := make(chan struct{}, 3) + go func() { defer GinkgoRecover() - defer func() { - done <- true - }() - time.Sleep(100 * time.Millisecond) + Eventually(step).Should(Receive()) err := client.Publish("mychannel", "hello").Err() Expect(err).NotTo(HaveOccurred()) + step <- struct{}{} }() + _, err := pubsub.ReceiveMessage() + Expect(err).To(Equal(io.EOF)) + step <- struct{}{} + msg, err := pubsub.ReceiveMessage() Expect(err).NotTo(HaveOccurred()) Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) - Eventually(done).Should(Receive()) + Eventually(step).Should(Receive()) } It("Subscribe should reconnect on ReceiveMessage error", func() { @@ -380,9 +344,9 @@ var _ = Describe("PubSub", func() { _, err := pubsub.ReceiveMessage() Expect(err).To(HaveOccurred()) - Expect(err).To(SatisfyAny( - MatchError("redis: client is closed"), - MatchError("use of closed network connection"), // Go 1.4 + Expect(err.Error()).To(SatisfyAny( + Equal("redis: client is closed"), + ContainSubstring("use of closed network connection"), )) }() @@ -406,7 +370,7 @@ var _ = Describe("PubSub", func() { defer GinkgoRecover() defer wg.Done() - time.Sleep(2 * timeout) + time.Sleep(timeout) err := pubsub.Subscribe("mychannel") Expect(err).NotTo(HaveOccurred()) @@ -417,7 +381,7 @@ var _ = Describe("PubSub", func() { Expect(err).NotTo(HaveOccurred()) }() - msg, err := pubsub.ReceiveMessageTimeout(timeout) + msg, err := pubsub.ReceiveMessage() Expect(err).NotTo(HaveOccurred()) Expect(msg.Channel).To(Equal("mychannel")) Expect(msg.Payload).To(Equal("hello")) diff --git a/redis.go b/redis.go index ff0651483f..c0f142cc4d 100644 --- a/redis.go +++ b/redis.go @@ -423,7 +423,7 @@ func (c *Client) TxPipeline() Pipeliner { } func (c *Client) pubSub() *PubSub { - return &PubSub{ + pubsub := &PubSub{ opt: c.opt, newConn: func(channels []string) (*pool.Conn, error) { @@ -431,6 +431,8 @@ func (c *Client) pubSub() *PubSub { }, closeConn: c.connPool.CloseConn, } + pubsub.init() + return pubsub } // Subscribe subscribes the client to the specified channels. diff --git a/ring.go b/ring.go index 8b20d476e2..ef85511506 100644 --- a/ring.go +++ b/ring.go @@ -165,6 +165,7 @@ type ringShards struct { hash *consistenthash.Map shards map[string]*ringShard // read only list []*ringShard // read only + len int closed bool } @@ -269,17 +270,27 @@ func (c *ringShards) Heartbeat(frequency time.Duration) { // rebalance removes dead shards from the Ring. func (c *ringShards) rebalance() { hash := newConsistentHash(c.opt) + var shardsNum int for name, shard := range c.shards { if shard.IsUp() { hash.Add(name) + shardsNum++ } } c.mu.Lock() c.hash = hash + c.len = shardsNum c.mu.Unlock() } +func (c *ringShards) Len() int { + c.mu.RLock() + l := c.len + c.mu.RUnlock() + return l +} + func (c *ringShards) Close() error { c.mu.Lock() defer c.mu.Unlock() @@ -398,6 +409,11 @@ func (c *Ring) PoolStats() *PoolStats { return &acc } +// Len returns the current number of shards in the ring. +func (c *Ring) Len() int { + return c.shards.Len() +} + // Subscribe subscribes the client to the specified channels. func (c *Ring) Subscribe(channels ...string) *PubSub { if len(channels) == 0 { diff --git a/ring_test.go b/ring_test.go index 0cad4298bc..1f5bf0d645 100644 --- a/ring_test.go +++ b/ring_test.go @@ -42,8 +42,8 @@ var _ = Describe("Redis Ring", func() { setRingKeys() // Both shards should have some keys now. - Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) - Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + Expect(ringShard1.Info("keyspace").Val()).To(ContainSubstring("keys=57")) + Expect(ringShard2.Info("keyspace").Val()).To(ContainSubstring("keys=43")) }) It("distributes keys when using EVAL", func() { @@ -59,41 +59,36 @@ var _ = Describe("Redis Ring", func() { Expect(err).NotTo(HaveOccurred()) } - Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=57")) - Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + Expect(ringShard1.Info("keyspace").Val()).To(ContainSubstring("keys=57")) + Expect(ringShard2.Info("keyspace").Val()).To(ContainSubstring("keys=43")) }) It("uses single shard when one of the shards is down", func() { // Stop ringShard2. Expect(ringShard2.Close()).NotTo(HaveOccurred()) - // Ring needs 3 * heartbeat time to detect that node is down. - // Give it more to be sure. - time.Sleep(2 * 3 * heartbeat) + Eventually(func() int { + return ring.Len() + }, "30s").Should(Equal(1)) setRingKeys() // RingShard1 should have all keys. - Expect(ringShard1.Info().Val()).To(ContainSubstring("keys=100")) + Expect(ringShard1.Info("keyspace").Val()).To(ContainSubstring("keys=100")) // Start ringShard2. var err error ringShard2, err = startRedis(ringShard2Port) Expect(err).NotTo(HaveOccurred()) - // Wait for ringShard2 to come up. - Eventually(func() error { - return ringShard2.Ping().Err() - }, "1s").ShouldNot(HaveOccurred()) - - // Ring needs heartbeat time to detect that node is up. - // Give it more to be sure. - time.Sleep(heartbeat + heartbeat) + Eventually(func() int { + return ring.Len() + }, "30s").Should(Equal(2)) setRingKeys() // RingShard2 should have its keys. - Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=43")) + Expect(ringShard2.Info("keyspace").Val()).To(ContainSubstring("keys=43")) }) It("supports hash tags", func() { @@ -102,8 +97,8 @@ var _ = Describe("Redis Ring", func() { Expect(err).NotTo(HaveOccurred()) } - Expect(ringShard1.Info().Val()).ToNot(ContainSubstring("keys=")) - Expect(ringShard2.Info().Val()).To(ContainSubstring("keys=100")) + Expect(ringShard1.Info("keyspace").Val()).ToNot(ContainSubstring("keys=")) + Expect(ringShard2.Info("keyspace").Val()).To(ContainSubstring("keys=100")) }) Describe("pipeline", func() { diff --git a/sentinel.go b/sentinel.go index 3cedf36ee6..12c29a7179 100644 --- a/sentinel.go +++ b/sentinel.go @@ -116,7 +116,7 @@ func NewSentinelClient(opt *Options) *SentinelClient { } func (c *SentinelClient) PubSub() *PubSub { - return &PubSub{ + pubsub := &PubSub{ opt: c.opt, newConn: func(channels []string) (*pool.Conn, error) { @@ -124,6 +124,8 @@ func (c *SentinelClient) PubSub() *PubSub { }, closeConn: c.connPool.CloseConn, } + pubsub.init() + return pubsub } func (c *SentinelClient) GetMasterAddrByName(name string) *StringSliceCmd { @@ -180,10 +182,7 @@ func (d *sentinelFailover) MasterAddr() (string, error) { if err != nil { return "", err } - - if d._masterAddr != addr { - d.switchMaster(addr) - } + d._switchMaster(addr) return addr, nil } @@ -194,11 +193,11 @@ func (d *sentinelFailover) masterAddr() (string, error) { addr, err := d.sentinel.GetMasterAddrByName(d.masterName).Result() if err == nil { addr := net.JoinHostPort(addr[0], addr[1]) - internal.Logf("sentinel: master=%q addr=%q", d.masterName, addr) return addr, nil } - internal.Logf("sentinel: GetMasterAddrByName name=%q failed: %s", d.masterName, err) + internal.Logf("sentinel: GetMasterAddrByName name=%q failed: %s", + d.masterName, err) d._resetSentinel() } @@ -234,15 +233,23 @@ func (d *sentinelFailover) masterAddr() (string, error) { return "", errors.New("redis: all sentinels are unreachable") } -func (d *sentinelFailover) switchMaster(masterAddr string) { - internal.Logf( - "sentinel: new master=%q addr=%q", - d.masterName, masterAddr, - ) - _ = d.Pool().Filter(func(cn *pool.Conn) bool { - return cn.RemoteAddr().String() != masterAddr +func (c *sentinelFailover) switchMaster(addr string) { + c.mu.Lock() + c._switchMaster(addr) + c.mu.Unlock() +} + +func (c *sentinelFailover) _switchMaster(addr string) { + if c._masterAddr == addr { + return + } + + internal.Logf("sentinel: new master=%q addr=%q", + c.masterName, addr) + _ = c.Pool().Filter(func(cn *pool.Conn) bool { + return cn.RemoteAddr().String() != addr }) - d._masterAddr = masterAddr + c._masterAddr = addr } func (d *sentinelFailover) setSentinel(sentinel *SentinelClient) { @@ -292,27 +299,25 @@ func (d *sentinelFailover) discoverSentinels(sentinel *SentinelClient) { } func (d *sentinelFailover) listen(sentinel *SentinelClient) { - var pubsub *PubSub - for { - if pubsub == nil { - pubsub = sentinel.PubSub() + pubsub := sentinel.PubSub() + defer pubsub.Close() - if err := pubsub.Subscribe("+switch-master"); err != nil { - internal.Logf("sentinel: Subscribe failed: %s", err) - pubsub.Close() - d.resetSentinel() - return - } - } + err := pubsub.Subscribe("+switch-master") + if err != nil { + internal.Logf("sentinel: Subscribe failed: %s", err) + d.resetSentinel() + return + } + for { msg, err := pubsub.ReceiveMessage() if err != nil { - if err != pool.ErrClosed { - internal.Logf("sentinel: ReceiveMessage failed: %s", err) - pubsub.Close() + if err == pool.ErrClosed { + d.resetSentinel() + return } - d.resetSentinel() - return + internal.Logf("sentinel: ReceiveMessage failed: %s", err) + continue } switch msg.Channel { @@ -323,12 +328,7 @@ func (d *sentinelFailover) listen(sentinel *SentinelClient) { continue } addr := net.JoinHostPort(parts[3], parts[4]) - - d.mu.Lock() - if d._masterAddr != addr { - d.switchMaster(addr) - } - d.mu.Unlock() + d.switchMaster(addr) } } } From bbcb2b79882052692c5ad23279b45e485cd679a4 Mon Sep 17 00:00:00 2001 From: Tianyi Lin Date: Mon, 23 Jul 2018 19:12:20 +0800 Subject: [PATCH 0473/1746] Supports new style syntax of client kill command --- commands.go | 15 +++++++++++++++ commands_test.go | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/commands.go b/commands.go index 763c837050..dddf8acd65 100644 --- a/commands.go +++ b/commands.go @@ -220,6 +220,7 @@ type Cmdable interface { BgRewriteAOF() *StatusCmd BgSave() *StatusCmd ClientKill(ipPort string) *StatusCmd + ClientKillByFilter(keys ...string) *IntCmd ClientList() *StringCmd ClientPause(dur time.Duration) *BoolCmd ConfigGet(parameter string) *SliceCmd @@ -1822,6 +1823,20 @@ func (c *cmdable) ClientKill(ipPort string) *StatusCmd { return cmd } +// ClientKillByFilter is new style synx, while the ClientKill is old +// CLIENT KILL