From ab7bfc03855247f310d0dc5cfc080092e6bf8733 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 25 Feb 2025 11:08:27 -1000 Subject: [PATCH 1/4] Use go-datastore without go-process --- flatfs.go | 29 +++++++++++++---------------- go.mod | 12 +++++------- go.sum | 22 ++++++++++------------ 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/flatfs.go b/flatfs.go index f9512a2..c6b7e67 100644 --- a/flatfs.go +++ b/flatfs.go @@ -20,7 +20,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" - "github.com/jbenet/goprocess" logging "github.com/ipfs/go-log/v2" ) @@ -783,25 +782,23 @@ func (fs *Datastore) Query(ctx context.Context, q query.Query) (query.Results, e // Replicates the logic in ResultsWithChan but actually respects calls // to `Close`. - b := query.NewResultBuilder(q) - b.Process.Go(func(p goprocess.Process) { - err := fs.walkTopLevel(ctx, fs.path, b) + results := query.ResultsWithContext(q, func(qctx context.Context, output chan<- query.Result) { + err := fs.walkTopLevel(ctx, qctx, q, fs.path, output) if err == nil { return } select { - case b.Output <- query.Result{Error: errors.New("walk failed: " + err.Error())}: - case <-p.Closing(): + case output <- query.Result{Error: errors.New("walk failed: " + err.Error())}: + case <-qctx.Done(): } }) - go b.Process.CloseAfterChildren() //nolint // We don't apply _any_ of the query logic ourselves so we'll leave it // all up to the naive query engine. - return query.NaiveQueryApply(q, b.Results()), nil + return query.NaiveQueryApply(q, results), nil } -func (fs *Datastore) walkTopLevel(ctx context.Context, path string, result *query.ResultBuilder) error { +func (fs *Datastore) walkTopLevel(ctx, qctx context.Context, q query.Query, path string, output chan<- query.Result) error { dir, err := os.Open(path) if err != nil { return err @@ -820,7 +817,7 @@ func (fs *Datastore) walkTopLevel(ctx context.Context, path string, result *quer continue } - err = fs.walk(ctx, filepath.Join(path, dir), result) + err = fs.walk(ctx, qctx, q, filepath.Join(path, dir), output) if err != nil { return err } @@ -829,7 +826,7 @@ func (fs *Datastore) walkTopLevel(ctx context.Context, path string, result *quer select { case <-ctx.Done(): return ctx.Err() - case <-result.Process.Closing(): + case <-qctx.Done(): return nil default: } @@ -1148,7 +1145,7 @@ func (fs *Datastore) tempFileOnce() (*os.File, error) { } // only call this on directories. -func (fs *Datastore) walk(ctx context.Context, path string, qrb *query.ResultBuilder) error { +func (fs *Datastore) walk(ctx, qctx context.Context, q query.Query, path string, output chan<- query.Result) error { dir, err := os.Open(path) if err != nil { if os.IsNotExist(err) { @@ -1177,7 +1174,7 @@ func (fs *Datastore) walk(ctx context.Context, path string, qrb *query.ResultBui var result query.Result result.Key = key.String() - if !qrb.Query.KeysOnly { + if !q.KeysOnly { value, err := readFile(filepath.Join(path, fn)) if err != nil { result.Error = err @@ -1187,7 +1184,7 @@ func (fs *Datastore) walk(ctx context.Context, path string, qrb *query.ResultBui result.Value = value result.Size = len(value) } - } else if qrb.Query.ReturnsSizes { + } else if q.ReturnsSizes { var stat os.FileInfo stat, err := os.Stat(filepath.Join(path, fn)) if err != nil { @@ -1198,10 +1195,10 @@ func (fs *Datastore) walk(ctx context.Context, path string, qrb *query.ResultBui } select { - case qrb.Output <- result: + case output <- result: case <-ctx.Done(): return ctx.Err() - case <-qrb.Process.Closing(): + case <-qctx.Done(): return nil } } diff --git a/go.mod b/go.mod index 54c3cb3..343a568 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,17 @@ module github.com/ipfs/go-ds-flatfs +go 1.23 + require ( github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 - github.com/ipfs/go-datastore v0.5.0 + github.com/ipfs/go-datastore v0.7.1-0.20250225204649-1676d105e588 github.com/ipfs/go-log/v2 v2.1.3 - github.com/jbenet/goprocess v0.1.4 ) require ( - github.com/google/uuid v1.1.1 // indirect - github.com/ipfs/go-detect-race v0.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.16.0 // indirect golang.org/x/tools v0.0.0-20210106214847-113979e3529a // indirect ) - -go 1.23 diff --git a/go.sum b/go.sum index a0cb3e8..bbf4ecf 100644 --- a/go.sum +++ b/go.sum @@ -6,18 +6,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU= -github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= -github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/ipfs/go-datastore v0.7.1-0.20250225204649-1676d105e588 h1:pbgZzzDE6/Q7E0vhdbfMpyCW24jOLXkq5HtrCBuruJQ= +github.com/ipfs/go-datastore v0.7.1-0.20250225204649-1676d105e588/go.mod h1:BGN3N+sLTEWQUHMtOjmm2GKccMJnVFPgSLp6kshNqRk= github.com/ipfs/go-log/v2 v2.1.3 h1:1iS3IU7aXRlbgUpN8yTTpJ53NXYjAe37vcI5+5nYrzk= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= -github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= -github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= -github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -32,15 +26,17 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= @@ -83,5 +79,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 4ee312cec7a392bd745928780cfe7d0c832d1c99 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 25 Feb 2025 14:05:27 -1000 Subject: [PATCH 2/4] update go-process --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 343a568..a2ca0ae 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23 require ( github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 - github.com/ipfs/go-datastore v0.7.1-0.20250225204649-1676d105e588 + github.com/ipfs/go-datastore v0.7.1-0.20250226000051-0dbca94435fd github.com/ipfs/go-log/v2 v2.1.3 ) diff --git a/go.sum b/go.sum index bbf4ecf..bb917bd 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/ipfs/go-datastore v0.7.1-0.20250225204649-1676d105e588 h1:pbgZzzDE6/Q7E0vhdbfMpyCW24jOLXkq5HtrCBuruJQ= -github.com/ipfs/go-datastore v0.7.1-0.20250225204649-1676d105e588/go.mod h1:BGN3N+sLTEWQUHMtOjmm2GKccMJnVFPgSLp6kshNqRk= +github.com/ipfs/go-datastore v0.7.1-0.20250226000051-0dbca94435fd h1:FKcZjv1qAZGaBpKDP4Mryct1cH/v81ML+Snp+/zosNQ= +github.com/ipfs/go-datastore v0.7.1-0.20250226000051-0dbca94435fd/go.mod h1:BGN3N+sLTEWQUHMtOjmm2GKccMJnVFPgSLp6kshNqRk= github.com/ipfs/go-log/v2 v2.1.3 h1:1iS3IU7aXRlbgUpN8yTTpJ53NXYjAe37vcI5+5nYrzk= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= From 41b568e799c092a1f8af9e634eb4d33d8a26f444 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Wed, 26 Feb 2025 07:55:03 -1000 Subject: [PATCH 3/4] use tagged version of go-datastore --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a2ca0ae..828e1a1 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23 require ( github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 - github.com/ipfs/go-datastore v0.7.1-0.20250226000051-0dbca94435fd + github.com/ipfs/go-datastore v0.8.0 github.com/ipfs/go-log/v2 v2.1.3 ) diff --git a/go.sum b/go.sum index bb917bd..816f1d5 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/ipfs/go-datastore v0.7.1-0.20250226000051-0dbca94435fd h1:FKcZjv1qAZGaBpKDP4Mryct1cH/v81ML+Snp+/zosNQ= -github.com/ipfs/go-datastore v0.7.1-0.20250226000051-0dbca94435fd/go.mod h1:BGN3N+sLTEWQUHMtOjmm2GKccMJnVFPgSLp6kshNqRk= +github.com/ipfs/go-datastore v0.8.0 h1:n8xKnoSTo/28L5KM+Jfz4kfohD889/afdWcDHve9VuY= +github.com/ipfs/go-datastore v0.8.0/go.mod h1:BGN3N+sLTEWQUHMtOjmm2GKccMJnVFPgSLp6kshNqRk= github.com/ipfs/go-log/v2 v2.1.3 h1:1iS3IU7aXRlbgUpN8yTTpJ53NXYjAe37vcI5+5nYrzk= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= From 204767751e4e419dc33a7ac69bc2b503591d171d Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Wed, 26 Feb 2025 21:14:37 -1000 Subject: [PATCH 4/4] context passed into Query is only for func cancelation, not for lifetime of Results object. Use Results.Close() to cancel results early. --- flatfs.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/flatfs.go b/flatfs.go index c6b7e67..3fe3d6e 100644 --- a/flatfs.go +++ b/flatfs.go @@ -783,13 +783,12 @@ func (fs *Datastore) Query(ctx context.Context, q query.Query) (query.Results, e // Replicates the logic in ResultsWithChan but actually respects calls // to `Close`. results := query.ResultsWithContext(q, func(qctx context.Context, output chan<- query.Result) { - err := fs.walkTopLevel(ctx, qctx, q, fs.path, output) - if err == nil { - return - } - select { - case output <- query.Result{Error: errors.New("walk failed: " + err.Error())}: - case <-qctx.Done(): + err := fs.walkTopLevel(qctx, q, fs.path, output) + if err != nil { + select { + case output <- query.Result{Error: errors.New("walk failed: " + err.Error())}: + case <-qctx.Done(): + } } }) @@ -798,7 +797,7 @@ func (fs *Datastore) Query(ctx context.Context, q query.Query) (query.Results, e return query.NaiveQueryApply(q, results), nil } -func (fs *Datastore) walkTopLevel(ctx, qctx context.Context, q query.Query, path string, output chan<- query.Result) error { +func (fs *Datastore) walkTopLevel(ctx context.Context, q query.Query, path string, output chan<- query.Result) error { dir, err := os.Open(path) if err != nil { return err @@ -817,7 +816,7 @@ func (fs *Datastore) walkTopLevel(ctx, qctx context.Context, q query.Query, path continue } - err = fs.walk(ctx, qctx, q, filepath.Join(path, dir), output) + err = fs.walk(ctx, q, filepath.Join(path, dir), output) if err != nil { return err } @@ -826,8 +825,6 @@ func (fs *Datastore) walkTopLevel(ctx, qctx context.Context, q query.Query, path select { case <-ctx.Done(): return ctx.Err() - case <-qctx.Done(): - return nil default: } } @@ -1145,7 +1142,7 @@ func (fs *Datastore) tempFileOnce() (*os.File, error) { } // only call this on directories. -func (fs *Datastore) walk(ctx, qctx context.Context, q query.Query, path string, output chan<- query.Result) error { +func (fs *Datastore) walk(ctx context.Context, q query.Query, path string, output chan<- query.Result) error { dir, err := os.Open(path) if err != nil { if os.IsNotExist(err) { @@ -1198,8 +1195,6 @@ func (fs *Datastore) walk(ctx, qctx context.Context, q query.Query, path string, case output <- result: case <-ctx.Done(): return ctx.Err() - case <-qctx.Done(): - return nil } } return nil