diff --git a/commonspace/headsync/headstorage/mock_headstorage/mock_headstorage.go b/commonspace/headsync/headstorage/mock_headstorage/mock_headstorage.go index f56b16a7f..25ddaff28 100644 --- a/commonspace/headsync/headstorage/mock_headstorage/mock_headstorage.go +++ b/commonspace/headsync/headstorage/mock_headstorage/mock_headstorage.go @@ -53,21 +53,6 @@ func (mr *MockHeadStorageMockRecorder) AddObserver(observer any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddObserver", reflect.TypeOf((*MockHeadStorage)(nil).AddObserver), observer) } -// MaxLastAddSeq mocks base method. -func (m *MockHeadStorage) MaxLastAddSeq(ctx context.Context) (uint64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MaxLastAddSeq", ctx) - ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MaxLastAddSeq indicates an expected call of MaxLastAddSeq. -func (mr *MockHeadStorageMockRecorder) MaxLastAddSeq(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxLastAddSeq", reflect.TypeOf((*MockHeadStorage)(nil).MaxLastAddSeq), ctx) -} - // DeleteEntry mocks base method. func (m *MockHeadStorage) DeleteEntry(ctx context.Context, id string) error { m.ctrl.T.Helper() @@ -126,6 +111,21 @@ func (mr *MockHeadStorageMockRecorder) IterateEntries(ctx, iterOpts, iter any) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IterateEntries", reflect.TypeOf((*MockHeadStorage)(nil).IterateEntries), ctx, iterOpts, iter) } +// MaxLastAddSeq mocks base method. +func (m *MockHeadStorage) MaxLastAddSeq(ctx context.Context) (uint64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MaxLastAddSeq", ctx) + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MaxLastAddSeq indicates an expected call of MaxLastAddSeq. +func (mr *MockHeadStorageMockRecorder) MaxLastAddSeq(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxLastAddSeq", reflect.TypeOf((*MockHeadStorage)(nil).MaxLastAddSeq), ctx) +} + // UpdateEntry mocks base method. func (m *MockHeadStorage) UpdateEntry(ctx context.Context, update headstorage.HeadsUpdate) error { m.ctrl.T.Helper() diff --git a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go index 92e239b10..9bc972f57 100644 --- a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go +++ b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go @@ -279,6 +279,20 @@ func (mr *MockObjectTreeMockRecorder) IsDerived() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDerived", reflect.TypeOf((*MockObjectTree)(nil).IsDerived)) } +// IterateAfterAddSeq mocks base method. +func (m *MockObjectTree) IterateAfterAddSeq(ctx context.Context, addSeq uint64, convert objecttree.ChangeConvertFunc, iterate objecttree.ChangeIterateFunc) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IterateAfterAddSeq", ctx, addSeq, convert, iterate) + ret0, _ := ret[0].(error) + return ret0 +} + +// IterateAfterAddSeq indicates an expected call of IterateAfterAddSeq. +func (mr *MockObjectTreeMockRecorder) IterateAfterAddSeq(ctx, addSeq, convert, iterate any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IterateAfterAddSeq", reflect.TypeOf((*MockObjectTree)(nil).IterateAfterAddSeq), ctx, addSeq, convert, iterate) +} + // IterateFrom mocks base method. func (m *MockObjectTree) IterateFrom(id string, convert objecttree.ChangeConvertFunc, iterate objecttree.ChangeIterateFunc) error { m.ctrl.T.Helper() @@ -362,6 +376,18 @@ func (mr *MockObjectTreeMockRecorder) Root() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockObjectTree)(nil).Root)) } +// SetDeferredUpdater mocks base method. +func (m *MockObjectTree) SetDeferredUpdater(deferred bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetDeferredUpdater", deferred) +} + +// SetDeferredUpdater indicates an expected call of SetDeferredUpdater. +func (mr *MockObjectTreeMockRecorder) SetDeferredUpdater(deferred any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDeferredUpdater", reflect.TypeOf((*MockObjectTree)(nil).SetDeferredUpdater), deferred) +} + // SetFlusher mocks base method. func (m *MockObjectTree) SetFlusher(flusher objecttree.Flusher) { m.ctrl.T.Helper() @@ -583,32 +609,32 @@ func (mr *MockStorageMockRecorder) Get(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStorage)(nil).Get), ctx, id) } -// GetAfterOrder mocks base method. -func (m *MockStorage) GetAfterOrder(ctx context.Context, orderId string, iter objecttree.StorageIterator) error { +// GetAfterAddSeq mocks base method. +func (m *MockStorage) GetAfterAddSeq(ctx context.Context, addSeq uint64, iter objecttree.StorageIterator) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAfterOrder", ctx, orderId, iter) + ret := m.ctrl.Call(m, "GetAfterAddSeq", ctx, addSeq, iter) ret0, _ := ret[0].(error) return ret0 } -// GetAfterOrder indicates an expected call of GetAfterOrder. -func (mr *MockStorageMockRecorder) GetAfterOrder(ctx, orderId, iter any) *gomock.Call { +// GetAfterAddSeq indicates an expected call of GetAfterAddSeq. +func (mr *MockStorageMockRecorder) GetAfterAddSeq(ctx, addSeq, iter any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAfterOrder", reflect.TypeOf((*MockStorage)(nil).GetAfterOrder), ctx, orderId, iter) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAfterAddSeq", reflect.TypeOf((*MockStorage)(nil).GetAfterAddSeq), ctx, addSeq, iter) } -// GetAfterAddSeq mocks base method. -func (m *MockStorage) GetAfterAddSeq(ctx context.Context, addSeq uint64, iter objecttree.StorageIterator) error { +// GetAfterOrder mocks base method. +func (m *MockStorage) GetAfterOrder(ctx context.Context, orderId string, iter objecttree.StorageIterator) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAfterAddSeq", ctx, addSeq, iter) + ret := m.ctrl.Call(m, "GetAfterOrder", ctx, orderId, iter) ret0, _ := ret[0].(error) return ret0 } -// GetAfterAddSeq indicates an expected call of GetAfterAddSeq. -func (mr *MockStorageMockRecorder) GetAfterAddSeq(ctx, addSeq, iter any) *gomock.Call { +// GetAfterOrder indicates an expected call of GetAfterOrder. +func (mr *MockStorageMockRecorder) GetAfterOrder(ctx, orderId, iter any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAfterAddSeq", reflect.TypeOf((*MockStorage)(nil).GetAfterAddSeq), ctx, addSeq, iter) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAfterOrder", reflect.TypeOf((*MockStorage)(nil).GetAfterOrder), ctx, orderId, iter) } // Has mocks base method. diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index f0b9cb7cd..a9cc07822 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -85,6 +85,7 @@ type ReadableObjectTree interface { Debug(parser DescriptionParser) (DebugInfo, error) IterateRoot(convert ChangeConvertFunc, iterate ChangeIterateFunc) error IterateFrom(id string, convert ChangeConvertFunc, iterate ChangeIterateFunc) error + IterateAfterAddSeq(ctx context.Context, addSeq uint64, convert ChangeConvertFunc, iterate ChangeIterateFunc) error } type ObjectTree interface { @@ -106,6 +107,7 @@ type ObjectTree interface { Delete() error Close() error SetFlusher(flusher Flusher) + SetDeferredUpdater(deferred bool) TryClose(objectTTL time.Duration) (bool, error) } @@ -122,9 +124,10 @@ type objectTree struct { root *Change tree *Tree - keys map[string]crypto.SymKey - currentReadKey crypto.SymKey - isDeleted bool + keys map[string]crypto.SymKey + currentReadKey crypto.SymKey + isDeleted bool + deferredUpdater bool // buffers difSnapshotBuf []*treechangeproto.RawTreeChangeWithId @@ -204,6 +207,10 @@ func (ot *objectTree) SetFlusher(flusher Flusher) { ot.flusher = flusher } +func (ot *objectTree) SetDeferredUpdater(deferred bool) { + ot.deferredUpdater = deferred +} + func (ot *objectTree) UnmarshalledHeader() *Change { return ot.root } @@ -412,7 +419,7 @@ func (ot *objectTree) AddRawChangesWithUpdater(ctx context.Context, changes RawC } } - if updater != nil { + if updater != nil && !ot.deferredUpdater { err = updater(ot, addResult.Mode) if err != nil { rollback() @@ -425,6 +432,13 @@ func (ot *objectTree) AddRawChangesWithUpdater(ctx context.Context, changes RawC rollback() return } + + if updater != nil && ot.deferredUpdater { + err = updater(ot, addResult.Mode) + if err != nil { + return + } + } ot.flusher.Flush(ot) return } @@ -701,6 +715,48 @@ func (ot *objectTree) IterateFrom(id string, convert ChangeConvertFunc, iterate return } +func (ot *objectTree) IterateAfterAddSeq(ctx context.Context, addSeq uint64, convert ChangeConvertFunc, iterate ChangeIterateFunc) (err error) { + if ot.isDeleted { + return ErrDeleted + } + var buf []byte + return ot.storage.GetAfterAddSeq(ctx, addSeq, func(ctx context.Context, sc StorageChange) (bool, error) { + raw := sc.RawTreeChangeWithId() + ch, uErr := ot.changeBuilder.Unmarshall(raw, false) + if uErr != nil { + return false, uErr + } + ch.OrderId = sc.OrderId + ch.SnapshotCounter = sc.SnapshotCounter + ch.AddSeq = sc.AddSeq + + if convert != nil && ch.Id != ot.id { + if ch.ReadKeyId != "" { + readKey, exists := ot.keys[ch.ReadKeyId] + if !exists { + return false, list.ErrNoReadKey + } + if ch.Data == nil { + return false, fmt.Errorf("no data in change %s", ch.Id) + } + buf, uErr = readKey.DecryptReuse(buf, ch.Data) + if uErr != nil { + return false, uErr + } + } else { + buf = ch.Data + } + model, cErr := convert(ch, buf) + if cErr != nil { + return false, cErr + } + ch.Model = model + ch.Data = nil + } + return iterate(ch), nil + }) +} + func (ot *objectTree) HasChanges(chs ...string) bool { if ot.isDeleted { return false diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 15e8c3dcf..fbaebc7ed 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -32,6 +32,7 @@ type HistoryTreeParams struct { AclList list.AclList Heads []string IncludeBeforeId bool + BuildEmptyData bool } type objectTreeDeps struct { @@ -213,7 +214,13 @@ func BuildNonVerifiableHistoryTree(params HistoryTreeParams) (HistoryTree, error } root := rootChange.RawTreeChangeWithId() // Use real key storage to preserve actual identities, but skip verification - changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(crypto.NewKeyStorage(), root)} + var cb ChangeBuilder + if params.BuildEmptyData { + cb = NewEmptyDataChangeBuilder(crypto.NewKeyStorage(), root) + } else { + cb = NewChangeBuilder(crypto.NewKeyStorage(), root) + } + changeBuilder := &nonVerifiableChangeBuilder{cb} treeBuilder := newTreeBuilder(params.Storage, changeBuilder) deps := objectTreeDeps{ changeBuilder: changeBuilder, @@ -231,7 +238,13 @@ func BuildHistoryTree(params HistoryTreeParams) (HistoryTree, error) { if err != nil { return nil, err } - deps := defaultObjectTreeDeps(rootChange.RawTreeChangeWithId(), params.Storage, params.AclList) + root := rootChange.RawTreeChangeWithId() + var deps objectTreeDeps + if params.BuildEmptyData { + deps = emptyDataTreeDeps(root, params.Storage, params.AclList) + } else { + deps = defaultObjectTreeDeps(root, params.Storage, params.AclList) + } return buildHistoryTree(deps, params) } diff --git a/commonspace/object/tree/objecttree/treemigrator.go b/commonspace/object/tree/objecttree/treemigrator.go index d38aef43d..ec9294653 100644 --- a/commonspace/object/tree/objecttree/treemigrator.go +++ b/commonspace/object/tree/objecttree/treemigrator.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" anystore "github.com/anyproto/any-store" @@ -85,6 +86,10 @@ func (tm *TreeMigrator) MigrateTreeStorage(ctx context.Context, storage treeStor return fmt.Errorf("migration: failed to start old storage: %w", err) } } + // Set up AddSeq counter so storage.AddAll can assign sequence numbers + if setter, ok := newStorage.(interface{ SetAddSeq(seq *atomic.Uint64) }); ok { + setter.SetAddSeq(&atomic.Uint64{}) + } objTree, err := BuildMigratableObjectTree(newStorage, tm.aclList) if err != nil { return fmt.Errorf("migration: failed to build object tree: %w", err) diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 966741d82..da0c4d0a3 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -346,6 +346,20 @@ func (mr *MockSyncTreeMockRecorder) IsDerived() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDerived", reflect.TypeOf((*MockSyncTree)(nil).IsDerived)) } +// IterateAfterAddSeq mocks base method. +func (m *MockSyncTree) IterateAfterAddSeq(ctx context.Context, addSeq uint64, convert objecttree.ChangeConvertFunc, iterate objecttree.ChangeIterateFunc) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IterateAfterAddSeq", ctx, addSeq, convert, iterate) + ret0, _ := ret[0].(error) + return ret0 +} + +// IterateAfterAddSeq indicates an expected call of IterateAfterAddSeq. +func (mr *MockSyncTreeMockRecorder) IterateAfterAddSeq(ctx, addSeq, convert, iterate any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IterateAfterAddSeq", reflect.TypeOf((*MockSyncTree)(nil).IterateAfterAddSeq), ctx, addSeq, convert, iterate) +} + // IterateFrom mocks base method. func (m *MockSyncTree) IterateFrom(id string, convert objecttree.ChangeConvertFunc, iterate objecttree.ChangeIterateFunc) error { m.ctrl.T.Helper() @@ -443,6 +457,18 @@ func (mr *MockSyncTreeMockRecorder) Root() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockSyncTree)(nil).Root)) } +// SetDeferredUpdater mocks base method. +func (m *MockSyncTree) SetDeferredUpdater(deferred bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetDeferredUpdater", deferred) +} + +// SetDeferredUpdater indicates an expected call of SetDeferredUpdater. +func (mr *MockSyncTreeMockRecorder) SetDeferredUpdater(deferred any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDeferredUpdater", reflect.TypeOf((*MockSyncTree)(nil).SetDeferredUpdater), deferred) +} + // SetFlusher mocks base method. func (m *MockSyncTree) SetFlusher(flusher objecttree.Flusher) { m.ctrl.T.Helper() diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index fea6696bf..dd597d502 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -35,6 +35,7 @@ type HeadNotifiable interface { type ListenerSetter interface { SetListener(listener updatelistener.UpdateListener) + SetDeferredUpdater(deferred bool) } type peerSendableObjectTree interface { @@ -177,6 +178,17 @@ func (s *syncTree) IterateRoot(convert objecttree.ChangeConvertFunc, iterate obj return s.ObjectTree.IterateRoot(convert, iterate) } +func (s *syncTree) IterateAfterAddSeq(ctx context.Context, addSeq uint64, convert objecttree.ChangeConvertFunc, iterate objecttree.ChangeIterateFunc) (err error) { + if err = s.checkAlive(); err != nil { + return + } + return s.ObjectTree.IterateAfterAddSeq(ctx, addSeq, convert, iterate) +} + +func (s *syncTree) SetDeferredUpdater(deferred bool) { + s.ObjectTree.SetDeferredUpdater(deferred) +} + func (s *syncTree) AddContent(ctx context.Context, content objecttree.SignableChangeContent) (res objecttree.AddResult, err error) { return s.AddContentWithValidator(ctx, content, nil) } diff --git a/commonspace/objecttreebuilder/treebuilder.go b/commonspace/objecttreebuilder/treebuilder.go index bb8e0ea39..a414f25a3 100644 --- a/commonspace/objecttreebuilder/treebuilder.go +++ b/commonspace/objecttreebuilder/treebuilder.go @@ -38,8 +38,9 @@ var log = logger.NewNamed(CName) var ErrSpaceClosed = errors.New("space is closed") type HistoryTreeOpts struct { - Heads []string - Include bool + Heads []string + Include bool + BuildEmptyData bool } type TreeBuilder interface { @@ -176,6 +177,7 @@ func (t *treeBuilder) BuildHistoryTree(ctx context.Context, id string, opts Hist AclList: t.aclList, Heads: opts.Heads, IncludeBeforeId: opts.Include, + BuildEmptyData: opts.BuildEmptyData, } params.Storage, err = t.spaceStorage.TreeStorage(ctx, id) if err != nil {