Just stop being clever to save 4 bytes per keystone

api/tbcapi/tbcapi.go

@@ -302,7 +302,7 @@ type BlockKeystoneByL2KeystoneAbrevHashResponse struct {
 }
 
 type KeystonesByHeightRequest struct {
-	Height uint64 `json:"height"`
+	Height uint32 `json:"height"`
 	Depth  int    `json:"depth"`
 }
 

cmd/hemictl/hemictl.go

@@ -762,7 +762,7 @@ func tbcdb(pctx context.Context, flags []string) error {
 			return errors.New("height: must be set")
 		}
 
-		h, err := strconv.ParseUint(height, 10, 64)
+		h, err := strconv.ParseUint(height, 10, 32)
 		if err != nil {
 			return fmt.Errorf("parse height: %w", err)
 		}
@@ -777,7 +777,7 @@ func tbcdb(pctx context.Context, flags []string) error {
 			return fmt.Errorf("parse depth: %w", err)
 		}
 
-		kssList, err := s.KeystonesByHeight(ctx, h, int(d))
+		kssList, err := s.KeystonesByHeight(ctx, uint32(h), int(d))
 		if err != nil {
 			return fmt.Errorf("retrieve keystones: %w", err)
 		}

database/tbcd/database.go

@@ -132,12 +132,12 @@ type Database interface {
 	BlockKeystoneUpdate(ctx context.Context, direction int, keystones map[chainhash.Hash]Keystone, keystoneIndexHash chainhash.Hash) error
 	BlockKeystoneByL2KeystoneAbrevHash(ctx context.Context, abrevhash chainhash.Hash) (*Keystone, error)
 	BlockHeaderByKeystoneIndex(ctx context.Context) (*BlockHeader, error)
-	KeystonesByHeight(ctx context.Context, height uint64, depth int) ([]Keystone, error)
+	KeystonesByHeight(ctx context.Context, height uint32, depth int) ([]Keystone, error)
 }
 
 type Keystone struct {
 	BlockHash           chainhash.Hash                 // Block that contains abbreviated keystone
-	BlockHeight         uint64                         // Block height
+	BlockHeight         uint32                         // Block height
 	AbbreviatedKeystone [hemi.L2KeystoneAbrevSize]byte // Abbreviated keystone
 }
 

database/tbcd/level/level.go

@@ -55,7 +55,7 @@ const (
 
 	heighthashSize         = 8 + 1 + chainhash.HashSize
 	blockheaderSize        = 120
-	keystoneSize           = chainhash.HashSize + hemi.L2KeystoneAbrevSize
+	keystoneSize           = 4 + chainhash.HashSize + hemi.L2KeystoneAbrevSize
 	keystoneHeightHashSize = 1 + 4 + chainhash.HashSize // h uint32(height) block_hash
 )
 
@@ -462,33 +462,6 @@ func (l *ldb) BlockKeystoneByL2KeystoneAbrevHash(ctx context.Context, abrevhash
 		return nil, fmt.Errorf("l2 keystone: %w", err)
 	}
 	ks := decodeKeystone(eks)
-
-	// Recreate height from index.
-	i := kssDB.NewIterator(keystoneHeightHashRange(abrevhash), nil)
-	defer func() {
-		i.Release()
-	}()
-
-	// seek for index with expected hash but increasing height
-	nextKey := encodeKeystoneHeightHash(0, abrevhash)
-	for j := 1; i.Seek(nextKey[:]); j++ {
-		// check if retrieved key indeed has the expected hash
-		if bytes.Equal(i.Key(), nextKey[:]) {
-			break
-		}
-		nextKey = encodeKeystoneHeightHash(uint32(j), abrevhash)
-	}
-
-	// check if we reached the end and didn't find a matching key
-	if !bytes.Equal(i.Key(), nextKey[:]) {
-		return nil, database.NotFoundError(fmt.Sprintf("height not found: %v", ks.BlockHash))
-	}
-	if i.Error() != nil {
-		return nil, fmt.Errorf("range: %w", i.Error())
-	}
-	height, _ := decodeKeystoneHeightHash(i.Key())
-	ks.BlockHeight = uint64(height)
-
 	return &ks, nil
 }
 
@@ -1967,8 +1940,11 @@ func (l *ldb) BlockTxUpdate(ctx context.Context, direction int, txs map[tbcd.TxK
 // [blockhash,abbreviated keystone] or [32+76] bytes. The abbreviated keystone
 // hash is the leveldb table key.
 func encodeKeystone(ks tbcd.Keystone) (eks [keystoneSize]byte) {
-	copy(eks[0:32], ks.BlockHash[:])
-	copy(eks[32:], ks.AbbreviatedKeystone[:])
+	var h [4]byte
+	binary.BigEndian.PutUint32(h[:], ks.BlockHeight)
+	copy(eks[0:4], h[:])
+	copy(eks[4:4+32], ks.BlockHash[:])
+	copy(eks[36:], ks.AbbreviatedKeystone[:])
 	return
 }
 
@@ -1979,13 +1955,14 @@ func encodeKeystoneToSlice(ks tbcd.Keystone) []byte {
 
 // decodeKeystone reverse the process of encodeKeystone.
 func decodeKeystone(eks []byte) (ks tbcd.Keystone) {
-	bh, err := chainhash.NewHash(eks[0:32])
+	ks.BlockHeight = binary.BigEndian.Uint32(eks[0:4])
+	bh, err := chainhash.NewHash(eks[4 : 4+32])
 	if err != nil {
 		panic(err) // Can't happen
 	}
 	ks.BlockHash = *bh
 	// copy the values to prevent slicing reentrancy problems.
-	copy(ks.AbbreviatedKeystone[:], eks[32:])
+	copy(ks.AbbreviatedKeystone[:], eks[36:])
 	return ks
 }
 
@@ -2019,12 +1996,6 @@ func decodeKeystoneHeightHash(v []byte) (height uint32, hash chainhash.Hash) {
 	return
 }
 
-func keystoneHeightHashRange(hash chainhash.Hash) *util.Range {
-	limit := encodeKeystoneHeightHashSlice(0, hash)
-	limit[0]++
-	return &util.Range{Start: encodeKeystoneHeightHashSlice(0, hash), Limit: limit}
-}
-
 func keystoneHeightRange(height uint32, depth int) *util.Range {
 	// Casting is a bit awkward here but I am not sure if we can make this
 	// look better somehow.
@@ -2040,16 +2011,13 @@ func keystoneHeightRange(height uint32, depth int) *util.Range {
 	}
 }
 
-func (l *ldb) KeystonesByHeight(ctx context.Context, height uint64, depth int) ([]tbcd.Keystone, error) {
+func (l *ldb) KeystonesByHeight(ctx context.Context, height uint32, depth int) ([]tbcd.Keystone, error) {
 	log.Tracef("KeystonesByHeight")
 	defer log.Tracef("KeystonesByHeight exit")
 
 	if depth == 0 {
 		return nil, errors.New("depth must not be 0")
 	}
-	if height > math.MaxUint32 {
-		return nil, errors.New("overflow for codeql")
-	}
 	if int64(height)+int64(depth) > math.MaxUint32 {
 		return nil, errors.New("the overflow that matters")
 	}
@@ -2058,19 +2026,18 @@ func (l *ldb) KeystonesByHeight(ctx context.Context, height uint64, depth int) (
 	}
 
 	kssDB := l.pool[level.KeystonesDB]
-	i := kssDB.NewIterator(keystoneHeightRange(uint32(height), depth), nil)
+	i := kssDB.NewIterator(keystoneHeightRange(height, depth), nil)
 	defer func() { i.Release() }()
 
 	kssList := make([]tbcd.Keystone, 0, 16)
 	for i.Next() {
-		h, hash := decodeKeystoneHeightHash(i.Key())
+		_, hash := decodeKeystoneHeightHash(i.Key())
 		eks, err := kssDB.Get(hash[:], nil)
 		if err != nil {
 			// mismatch between heighthash and hash indexes
 			panic(fmt.Sprintf("data corruption: %v", err))
 		}
 		deks := decodeKeystone(eks)
-		deks.BlockHeight = uint64(h)
 		kssList = append(kssList, deks)
 	}
 	if len(kssList) == 0 {
@@ -2109,15 +2076,15 @@ func (l *ldb) BlockKeystoneUpdate(ctx context.Context, direction int, keystones
 				// previously found block.
 				if ks.BlockHash.IsEqual(&v.BlockHash) {
 					kssBatch.Delete(k[:])
-					kssBatch.Delete(encodeKeystoneHeightHashSlice(uint32(v.BlockHeight), k))
+					kssBatch.Delete(encodeKeystoneHeightHashSlice(v.BlockHeight, k))
 				}
 			}
 		case 1:
 			has, _ := kssTx.Has(k[:], nil)
 			if !has {
 				// Only store unknown keystones and indexes
 				kssBatch.Put(k[:], encodeKeystoneToSlice(v))
-				kssBatch.Put(encodeKeystoneHeightHashSlice(uint32(v.BlockHeight), k), nil)
+				kssBatch.Put(encodeKeystoneHeightHashSlice(v.BlockHeight, k), nil)
 			}
 		}
 

database/tbcd/level/level_test.go

@@ -125,14 +125,14 @@ func TestMD(t *testing.T) {
 	}
 }
 
-func makeKssMap(from uint64, kssList []hemi.L2Keystone, blockHashSeed string) map[chainhash.Hash]tbcd.Keystone {
+func makeKssMap(from uint32, kssList []hemi.L2Keystone, blockHashSeed string) map[chainhash.Hash]tbcd.Keystone {
 	kssMap := make(map[chainhash.Hash]tbcd.Keystone)
 	for i, l2Keystone := range kssList {
 		abrvKs := hemi.L2KeystoneAbbreviate(l2Keystone).Serialize()
 		kssMap[*hemi.L2KeystoneAbbreviate(l2Keystone).Hash()] = tbcd.Keystone{
 			BlockHash:           chainhash.Hash(testutil.FillBytes(blockHashSeed, 32)),
 			AbbreviatedKeystone: abrvKs,
-			BlockHeight:         from + uint64(i),
+			BlockHeight:         from + uint32(i),
 		}
 	}
 	return kssMap
@@ -224,7 +224,7 @@ func TestHeightHashIndexing(t *testing.T) {
 			kssMap[*hemi.L2KeystoneAbbreviate(ks).Hash()] = tbcd.Keystone{
 				BlockHash:           chainhash.Hash(testutil.FillBytes("blockhash", 32)),
 				AbbreviatedKeystone: abrvKs,
-				BlockHeight:         uint64(i + 1),
+				BlockHeight:         uint32(i + 1),
 			}
 			l2Block += 25
 		}
@@ -267,7 +267,7 @@ func TestHeightHashIndexing(t *testing.T) {
 
 	// check each height
 	for n := range blockNum {
-		kssList, err := db.KeystonesByHeight(ctx, uint64(n+1), 1)
+		kssList, err := db.KeystonesByHeight(ctx, uint32(n+1), 1)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -277,7 +277,7 @@ func TestHeightHashIndexing(t *testing.T) {
 		}
 
 		for _, k := range kssList {
-			if k.BlockHeight != uint64(n+1) {
+			if k.BlockHeight != uint32(n+1) {
 				t.Fatalf("keystone height mismatch %v, expected %v", k.BlockHeight, n+1)
 			}
 		}

service/tbc/crawler.go

@@ -1411,7 +1411,7 @@ func processKeystones(block *btcutil.Block, direction int, kssCache map[chainhas
 			abvKss := aPoPTx.L2Keystone.Serialize()
 			kssCache[*aPoPTx.L2Keystone.Hash()] = tbcd.Keystone{
 				BlockHash:           blockHash,
-				BlockHeight:         blockHeight,
+				BlockHeight:         uint32(blockHeight),
 				AbbreviatedKeystone: abvKss,
 			}
 		}

service/tbc/tbc.go

@@ -1736,7 +1736,7 @@ func (s *Server) BlockByHash(ctx context.Context, hash chainhash.Hash) (*btcutil
 }
 
 // KeystonesByHeight returns any keystones found at a given height and up to a depth.
-func (s *Server) KeystonesByHeight(ctx context.Context, height uint64, depth int) ([]tbcd.Keystone, error) {
+func (s *Server) KeystonesByHeight(ctx context.Context, height uint32, depth int) ([]tbcd.Keystone, error) {
 	log.Tracef("KeystonesByHeight")
 	defer log.Tracef("KeystonesByHeight exit")
 

service/tbc/tbcfork_test.go

@@ -1717,7 +1717,7 @@ func TestKeystoneIndexNoFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err := s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err := s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1765,7 +1765,7 @@ func TestKeystoneIndexNoFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1789,7 +1789,7 @@ func TestKeystoneIndexNoFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b3.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b3.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1858,7 +1858,7 @@ func TestKeystoneIndexNoFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2391,7 +2391,7 @@ func TestKeystoneIndexFork(t *testing.T) {
 		t.Fatalf("wrong blockhash for stored keystone: %v", kss1Hash)
 	}
 	// check if keystone stored using heighthash index
-	hk, err := s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err := s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2440,7 +2440,7 @@ func TestKeystoneIndexFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2464,7 +2464,7 @@ func TestKeystoneIndexFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b3.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b3.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2576,7 +2576,7 @@ func TestKeystoneIndexFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2664,7 +2664,7 @@ func TestKeystoneIndexFork(t *testing.T) {
 	}
 
 	// check if keystone stored using heighthash index
-	hk, err = s.db.KeystonesByHeight(ctx, uint64(b2.Height()), 1)
+	hk, err = s.db.KeystonesByHeight(ctx, uint32(b2.Height()), 1)
 	if err != nil {
 		t.Fatal(err)
 	}