feat: prototype embedded repo migrations

this needs more testing and work, but we will do that
after autoconfig is wired in future commits

cmd/ipfs/kubo/daemon.go

@@ -277,7 +277,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 	}
 
 	var cacheMigrations, pinMigrations bool
-	var fetcher migrations.Fetcher
+	var externalMigrationFetcher migrations.Fetcher
 
 	// acquire the repo lock _before_ constructing a node. we need to make
 	// sure we are permitted to access the resources (datastore, etc.)
@@ -294,65 +294,78 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		}
 
 		if !domigrate {
-			fmt.Println("Not running migrations of fs-repo now.")
-			fmt.Println("Please get fs-repo-migrations from https://dist.ipfs.tech")
+			fmt.Println("Not running migrations of fs-repo now. Re-run daemon with --migrate or see 'ipfs repo migrate --help'")
 			return errors.New("fs-repo requires migration")
 		}
 
-		// Read Migration section of IPFS config
-		configFileOpt, _ := req.Options[commands.ConfigFileOption].(string)
-		migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot, configFileOpt)
+		// Try embedded migrations first
+		err = migrations.RunEmbeddedMigrations(cctx.Context(), fsrepo.RepoVersion, cctx.ConfigRoot, false)
 		if err != nil {
-			return err
-		}
-
-		// Define function to create IPFS fetcher.  Do not supply an
-		// already-constructed IPFS fetcher, because this may be expensive and
-		// not needed according to migration config. Instead, supply a function
-		// to construct the particular IPFS fetcher implementation used here,
-		// which is called only if an IPFS fetcher is needed.
-		newIpfsFetcher := func(distPath string) migrations.Fetcher {
-			return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot, configFileOpt)
-		}
+			fmt.Println("Embedded Kubo migrations failed, falling back to external binary migrations:")
+			fmt.Printf("  %s\n", err)
 
-		// Fetch migrations from current distribution, or location from environ
-		fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)
+			// Read Migration section of IPFS config for external migration fallback
+			// Note: External migrations only apply to repo versions <17. From repo version 17+,
+			// all migrations are embedded in Kubo and don't require external binaries.
+			configFileOpt, _ := req.Options[commands.ConfigFileOption].(string)
+			migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot, configFileOpt)
+			if err != nil {
+				return err
+			}
 
-		// Create fetchers according to migrationCfg.DownloadSources
-		fetcher, err = migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher)
-		if err != nil {
-			return err
-		}
-		defer fetcher.Close()
+			// Define function to create IPFS fetcher.  Do not supply an
+			// already-constructed IPFS fetcher, because this may be expensive and
+			// not needed according to migration config. Instead, supply a function
+			// to construct the particular IPFS fetcher implementation used here,
+			// which is called only if an IPFS fetcher is needed.
+			newIpfsFetcher := func(distPath string) migrations.Fetcher {
+				return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot, configFileOpt)
+			}
 
-		if migrationCfg.Keep == "cache" {
-			cacheMigrations = true
-		} else if migrationCfg.Keep == "pin" {
-			pinMigrations = true
-		}
+			// Fetch migrations from current distribution, or location from environ
+			fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)
 
-		if cacheMigrations || pinMigrations {
-			// Create temp directory to store downloaded migration archives
-			migrations.DownloadDirectory, err = os.MkdirTemp("", "migrations")
+			// Create fetchers according to migrationCfg.DownloadSources
+			fetcher, err := migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher)
 			if err != nil {
 				return err
 			}
-			// Defer cleanup of download directory so that it gets cleaned up
-			// if daemon returns early due to error
-			defer func() {
-				if migrations.DownloadDirectory != "" {
-					os.RemoveAll(migrations.DownloadDirectory)
+			defer fetcher.Close()
+
+			if migrationCfg.Keep == "cache" {
+				cacheMigrations = true
+			} else if migrationCfg.Keep == "pin" {
+				pinMigrations = true
+			}
+
+			if cacheMigrations || pinMigrations {
+				// Create temp directory to store downloaded migration archives
+				migrations.DownloadDirectory, err = os.MkdirTemp("", "migrations")
+				if err != nil {
+					return err
 				}
-			}()
-		}
+				// Defer cleanup of download directory so that it gets cleaned up
+				// if daemon returns early due to error
+				defer func() {
+					if migrations.DownloadDirectory != "" {
+						os.RemoveAll(migrations.DownloadDirectory)
+					}
+				}()
+			}
 
-		err = migrations.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", false)
-		if err != nil {
-			fmt.Println("The migrations of fs-repo failed:")
-			fmt.Printf("  %s\n", err)
-			fmt.Println("If you think this is a bug, please file an issue and include this whole log output.")
-			fmt.Println("  https://github.com/ipfs/fs-repo-migrations")
-			return err
+			// Fall back to external migration system
+			err = migrations.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", false)
+			if err != nil {
+				fmt.Println("The legacy migrations of fs-repo failed:")
+				fmt.Printf("  %s\n", err)
+				fmt.Println("If you think this is a bug, please file an issue and include this whole log output.")
+				fmt.Println("  https://github.com/ipfs/kubo")
+				return err
+			}
+
+			// External migrations completed successfully.
+			// Store migration data for later processing after node is created.
+			externalMigrationFetcher = fetcher
 		}
 
 		repo, err = fsrepo.Open(cctx.ConfigRoot)
@@ -566,9 +579,9 @@ take effect.
 		return err
 	}
 
-	// Add any files downloaded by migration.
-	if cacheMigrations || pinMigrations {
-		err = addMigrations(cctx.Context(), node, fetcher, pinMigrations)
+	// Add any files downloaded by external migrations (embedded migrations don't download files)
+	if externalMigrationFetcher != nil && (cacheMigrations || pinMigrations) {
+		err = addMigrations(cctx.Context(), node, externalMigrationFetcher, pinMigrations)
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Could not add migration to IPFS:", err)
 		}
@@ -577,10 +590,10 @@ take effect.
 		os.RemoveAll(migrations.DownloadDirectory)
 		migrations.DownloadDirectory = ""
 	}
-	if fetcher != nil {
+	if externalMigrationFetcher != nil {
 		// If there is an error closing the IpfsFetcher, then print error, but
 		// do not fail because of it.
-		err = fetcher.Close()
+		err = externalMigrationFetcher.Close()
 		if err != nil {
 			log.Errorf("error closing IPFS fetcher: %s", err)
 		}

config/autoconfig.go

@@ -15,12 +15,13 @@ type AutoConfig struct {
 	// LastUpdate is the timestamp of when the autoconfig was last successfully updated
 	LastUpdate *time.Time `json:",omitempty"`
 
-	// LastVersion stores the ETag or Last-Modified value from the last successful fetch
+	// LastVersion stores a string representation of AutoConfigVersion from the autoconfig.json file (if present),
+	// or falls back to ETag or Last-Modified header value from the last successful fetch
 	LastVersion string `json:",omitempty"`
 
-	// Interval is how often to check for updates
+	// CheckInterval is how often to check for updates
 	// Default: 24h
-	Interval *OptionalDuration `json:",omitempty"`
+	CheckInterval *OptionalDuration `json:",omitempty"`
 
 	// TLSInsecureSkipVerify allows skipping TLS verification (for testing only)
 	// Default: false

config/autoconfig_test.go

@@ -22,12 +22,12 @@ func TestAutoConfigDefaults(t *testing.T) {
 		t.Error("expected AutoConfig.Enabled to be true by default")
 	}
 
-	// Test default interval
-	if cfg.AutoConfig.Interval == nil {
+	// Test default check interval
+	if cfg.AutoConfig.CheckInterval == nil {
 		// This is expected - nil means use default
 		duration := (*OptionalDuration)(nil).WithDefault(DefaultAutoConfigInterval)
 		if duration != 24*time.Hour {
-			t.Errorf("expected default interval to be 24h, got %v", duration)
+			t.Errorf("expected default check interval to be 24h, got %v", duration)
 		}
 	}
 }

config/bootstrap_peers.go

@@ -2,47 +2,18 @@ package config
 
 import (
 	"errors"
-	"fmt"
 
 	peer "github.com/libp2p/go-libp2p/core/peer"
 	ma "github.com/multiformats/go-multiaddr"
 )
 
-// DefaultBootstrapAddresses are the hardcoded bootstrap addresses
-// for IPFS. they are nodes run by the IPFS team. docs on these later.
-// As with all p2p networks, bootstrap is an important security concern.
-//
-// NOTE: This is here -- and not inside cmd/ipfs/init.go -- because of an
-// import dependency issue. TODO: move this into a config/default/ package.
-var DefaultBootstrapAddresses = []string{
-	"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
-	"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", // rust-libp2p-server
-	"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
-	"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
-	"/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", // js-libp2p-amino-dht-bootstrapper
-	"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",           // mars.i.ipfs.io
-	"/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",   // mars.i.ipfs.io
-}
-
 // ErrInvalidPeerAddr signals an address is not a valid peer address.
 var ErrInvalidPeerAddr = errors.New("invalid peer address")
 
 func (c *Config) BootstrapPeers() ([]peer.AddrInfo, error) {
 	return ParseBootstrapPeers(c.Bootstrap)
 }
 
-// DefaultBootstrapPeers returns the (parsed) set of default bootstrap peers.
-// if it fails, it returns a meaningful error for the user.
-// This is here (and not inside cmd/ipfs/init) because of module dependency problems.
-func DefaultBootstrapPeers() ([]peer.AddrInfo, error) {
-	ps, err := ParseBootstrapPeers(DefaultBootstrapAddresses)
-	if err != nil {
-		return nil, fmt.Errorf(`failed to parse hardcoded bootstrap peers: %w
-This is a problem with the ipfs codebase. Please report it to the dev team`, err)
-	}
-	return ps, nil
-}
-
 func (c *Config) SetBootstrapPeers(bps []peer.AddrInfo) {
 	c.Bootstrap = BootstrapPeerStrings(bps)
 }

config/profile.go

@@ -97,11 +97,12 @@ Inverse profile of the test profile.`,
 		Transform: func(c *Config) error {
 			c.Addresses = addressesConfig()
 
-			bootstrapPeers, err := DefaultBootstrapPeers()
-			if err != nil {
-				return err
+			// Use AutoConfig system for bootstrap peers
+			c.Bootstrap = []string{"auto"}
+			c.AutoConfig.Enabled = True
+			if c.AutoConfig.URL == "" {
+				c.AutoConfig.URL = DefaultAutoConfigURL
 			}
-			c.Bootstrap = appendSingle(c.Bootstrap, BootstrapPeerStrings(bootstrapPeers))
 
 			c.Swarm.DisableNatPortMap = false
 			c.Discovery.MDNS.Enabled = true

core/commands/bootstrap.go

@@ -41,10 +41,6 @@ Running 'ipfs bootstrap' with no arguments will run 'ipfs bootstrap list'.
 	},
 }
 
-const (
-	defaultOptionName = "default"
-)
-
 var bootstrapAddCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Add peers to the bootstrap list.",
@@ -57,24 +53,16 @@ in the bootstrap list).
 		cmds.StringArg("peer", false, true, peerOptionDesc).EnableStdin(),
 	},
 
-	Options: []cmds.Option{
-		cmds.BoolOption(defaultOptionName, "Add default bootstrap nodes. (Deprecated, use 'default' subcommand instead)"),
-	},
 	Subcommands: map[string]*cmds.Command{
 		"default": bootstrapAddDefaultCmd,
+		"auto":    bootstrapAddDefaultCmd, // Alias for "default"
 	},
 
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
-		deflt, _ := req.Options[defaultOptionName].(bool)
-
-		inputPeers := config.DefaultBootstrapAddresses
-		if !deflt {
-			if err := req.ParseBodyArgs(); err != nil {
-				return err
-			}
-
-			inputPeers = req.Arguments
+		if err := req.ParseBodyArgs(); err != nil {
+			return err
 		}
+		inputPeers := req.Arguments
 
 		if len(inputPeers) == 0 {
 			return errors.New("no bootstrap peers to add")
@@ -133,7 +121,11 @@ in the bootstrap list).`,
 			return err
 		}
 
-		added, err := bootstrapAdd(r, cfg, config.DefaultBootstrapAddresses)
+		if !cfg.AutoConfig.Enabled.WithDefault(true) {
+			return errors.New("cannot add default bootstrap peers: AutoConfig is disabled. Enable AutoConfig or add specific peer addresses")
+		}
+
+		added, err := bootstrapAdd(r, cfg, []string{"auto"})
 		if err != nil {
 			return err
 		}