split data store by addrhash

AkihiroSuda · AkihiroSuda · commit 9152f2044c6b · 2021-02-17T16:43:28.000+09:00
See docs/dir.md

Signed-off-by: Akihiro Suda &lt;akihiro.suda.cz@hco.ntt.co.jp&gt;
diff --git a/client.go b/client.go
@@ -19,11 +19,13 @@ package main
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/namespaces"
+	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sys/unix"
@@ -61,3 +63,36 @@ func isSocketAccessible(s string) error {
 	// set AT_EACCESS to allow running nerdctl as a setuid binary
 	return unix.Faccessat(-1, abs, unix.R_OK|unix.W_OK, unix.AT_EACCESS)
 }
+
+// getDataStore returns a string like "/var/lib/nerdctl/1935db59".
+// "1935db9" is from `$(echo -n "/run/containerd/containerd.sock" | sha256sum | cut -c1-8)``
+func getDataStore(clicontext *cli.Context) (string, error) {
+	dataRoot := clicontext.String("data-root")
+	if err := os.MkdirAll(dataRoot, 0700); err != nil {
+		return "", err
+	}
+	addrHash, err := getAddrHash(clicontext.String("address"))
+	if err != nil {
+		return "", err
+	}
+	dataStore := filepath.Join(dataRoot, addrHash)
+	if err := os.MkdirAll(dataStore, 0700); err != nil {
+		return "", err
+	}
+	return dataStore, nil
+}
+
+func getAddrHash(addr string) (string, error) {
+	const addrHashLen = 8
+
+	addr = strings.TrimPrefix(addr, "unix://")
+	var err error
+	addr, err = filepath.EvalSymlinks(addr)
+	if err != nil {
+		return "", err
+	}
+
+	d := digest.SHA256.FromString(addr)
+	h := d.Encoded()[0:addrHashLen]
+	return h, nil
+}
diff --git a/docs/dir.md b/docs/dir.md
@@ -0,0 +1,52 @@
+# nerdctl directory layout
+
+## Data
+### `<DATAROOT>`
+e.g. `/var/lib/nerdctl`
+
+Can be overriden with `nerdctl --data-root=<DATAROOT>` flag.
+
+The directory is solely managed by nerdctl, not by containerd.
+The directory has nothing to do with containerd data root `/var/lib/containerd`.
+
+### `<DATAROOT>/<ADDRHASH>`
+e.g. `/var/lib/nerdctl/1935db59`
+
+`1935db9` is from `$(echo -n "/run/containerd/containerd.sock" | sha256sum | cut -c1-8)`
+
+This directory is also called "data store" in the implementation.
+
+### `<DATAROOT>/<ADDRHASH>/containers/<NAMESPACE>/<CID>`
+e.g. `/var/lib/nerdctl/1935db59/containers/default/c4ed811cc361d26faffdee8d696ddbc45a9d93c571b5b3c54d3da01cb29caeb1`
+
+Files:
+- `resolv.conf`: mounted to the container as `/etc/resolv.conf`
+- `<CID>-json.log`: used by `nerdctl logs`
+- `oci-hook.*.log`: logs of the OCI hook
+
+### `<DATAROOT>/<ADDRHASH>/names/<NAMESPACE>`
+e.g. `/var/lib/nerdctl/1935db59/names/default`
+
+Files:
+- `<NAME>`: contains the container ID (CID). Represents that the name is taken by that container. 
+
+Files must be operated with a `LOCK_EX` lock against the `<DATAROOT>/<ADDRHASH>/names/<NAMESPACE>` directory.
+
+### `<DATAROOT>/<ADDRHASH>/etchosts/<NAMESPACE>/<CID>`
+e.g. `/var/lib/nerdctl/1935db59/etchosts/default/c4ed811cc361d26faffdee8d696ddbc45a9d93c571b5b3c54d3da01cb29caeb1`
+
+Files:
+- `hosts`: mounted to the container as `/etc/hosts`
+- `meta.json`: metadata
+
+Files must be operated with a `LOCK_EX` lock against the `<DATAROOT>/<ADDRHASH>/etchosts` directory.
+
+## CNI
+
+### `<NETCONFPATH>`
+e.g. `/etc/cni/net.d`
+
+Can be overriden with `nerdctl --cni-netconfpath=<NETCONFPATH>` flag and environment variable `$NETCONFPATH`.
+
+Files:
+- `nerdctl-<NWNAME>.conflist`: CNI conf list created by nerdctl
diff --git a/internal_oci_hook.go b/internal_oci_hook.go
@@ -34,8 +34,12 @@ func internalOCIHookAction(clicontext *cli.Context) error {
 	if event == "" {
 		return errors.New("event type needs to be passed")
 	}
+	dataStore, err := getDataStore(clicontext)
+	if err != nil {
+		return err
+	}
 	return ocihook.Run(clicontext.App.Reader, clicontext.App.ErrWriter, event,
-		clicontext.String("data-root"),
+		dataStore,
 		clicontext.String("cni-path"),
 		clicontext.String("cni-netconfpath"),
 	)
diff --git a/logs.go b/logs.go
@@ -40,7 +40,10 @@ func logsAction(clicontext *cli.Context) error {
 		return errors.Errorf("requires exactly 1 argument")
 	}
 
-	dataRoot := clicontext.String("data-root")
+	dataStore, err := getDataStore(clicontext)
+	if err != nil {
+		return err
+	}
 
 	ns := clicontext.String("namespace")
 	switch ns {
@@ -60,7 +63,7 @@ func logsAction(clicontext *cli.Context) error {
 			if found.MatchCount > 1 {
 				return errors.Errorf("ambiguous ID %q", found.Req)
 			}
-			logJSONFilePath := jsonfile.Path(dataRoot, ns, found.Container.ID())
+			logJSONFilePath := jsonfile.Path(dataStore, ns, found.Container.ID())
 			f, err := os.Open(logJSONFilePath)
 			if err != nil {
 				return errors.Wrapf(err, "failed to open %q, container is not created with `nerdctl run -d`?", logJSONFilePath)
diff --git a/pkg/dnsutil/hostsstore/hostsstore.go b/pkg/dnsutil/hostsstore/hostsstore.go
@@ -15,7 +15,7 @@
    limitations under the License.
 */
 
-// Package hoststore provides the interface for /var/lib/nerdctl/etchosts .
+// Package hoststore provides the interface for /var/lib/nerdctl/<ADDRHASH>/etchosts .
 // Prioritize simplicity over scalability.
 package hostsstore
 
@@ -32,18 +32,18 @@ import (
 )
 
 const (
-	// hostsDirBasename is the base name of /var/lib/nerdctl/etchosts
+	// hostsDirBasename is the base name of /var/lib/nerdctl/<ADDRHASH>/etchosts
 	hostsDirBasename = "etchosts"
-	// metaJSON is stored as /var/lib/nerdctl/etchosts/<NS>/<ID>/meta.json
+	// metaJSON is stored as /var/lib/nerdctl/<ADDRHASH>/etchosts/<NS>/<ID>/meta.json
 	metaJSON = "meta.json"
 )
 
-// HostsPath returns "/var/lib/nerdctl/etchosts/<NS>/<ID>/hosts"
-func HostsPath(dataRoot, ns, id string) string {
-	if dataRoot == "" || ns == "" || id == "" {
+// HostsPath returns "/var/lib/nerdctl/<ADDRHASH>/etchosts/<NS>/<ID>/hosts"
+func HostsPath(dataStore, ns, id string) string {
+	if dataStore == "" || ns == "" || id == "" {
 		panic(errdefs.ErrInvalidArgument)
 	}
-	return filepath.Join(dataRoot, hostsDirBasename, ns, id, "hosts")
+	return filepath.Join(dataStore, hostsDirBasename, ns, id, "hosts")
 }
 
 // ensureFile ensures a file with permission 0644.
@@ -66,23 +66,23 @@ func ensureFile(path string) error {
 
 // EnsureHostsFile is used for creating mount-bindable /etc/hosts file.
 // The file is initialized with no content.
-func EnsureHostsFile(dataRoot, ns, id string) (string, error) {
-	lockDir := filepath.Join(dataRoot, hostsDirBasename)
+func EnsureHostsFile(dataStore, ns, id string) (string, error) {
+	lockDir := filepath.Join(dataStore, hostsDirBasename)
 	if err := os.MkdirAll(lockDir, 0700); err != nil {
 		return "", err
 	}
-	path := HostsPath(dataRoot, ns, id)
+	path := HostsPath(dataStore, ns, id)
 	fn := func() error {
 		return ensureFile(path)
 	}
 	err := lockutil.WithDirLock(lockDir, fn)
 	return path, err
 }
 
-func NewStore(dataRoot string) (Store, error) {
+func NewStore(dataStore string) (Store, error) {
 	store := &store{
-		dataRoot: dataRoot,
-		hostsD:   filepath.Join(dataRoot, hostsDirBasename),
+		dataStore: dataStore,
+		hostsD:    filepath.Join(dataStore, hostsDirBasename),
 	}
 	return store, os.MkdirAll(store.hostsD, 0700)
 }
@@ -101,15 +101,15 @@ type Store interface {
 }
 
 type store struct {
-	// dataRoot is /var/lib/nerdctl
-	dataRoot string
-	// hostsD is /var/lib/nerdctl/etchosts
+	// dataStore is /var/lib/nerdctl/<ADDRHASH>
+	dataStore string
+	// hostsD is /var/lib/nerdctl/<ADDRHASH>/etchosts
 	hostsD string
 }
 
 func (x *store) Acquire(meta Meta) error {
 	fn := func() error {
-		hostsPath := HostsPath(x.dataRoot, meta.Namespace, meta.ID)
+		hostsPath := HostsPath(x.dataStore, meta.Namespace, meta.ID)
 		if err := ensureFile(hostsPath); err != nil {
 			return err
 		}
diff --git a/pkg/dnsutil/hostsstore/updater.go b/pkg/dnsutil/hostsstore/updater.go
@@ -31,7 +31,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// newUpdater creates an updater for hostsD (/var/lib/nerdctl/hosts.d)
+// newUpdater creates an updater for hostsD (/var/lib/nerdctl/<ADDRHASH>/etchosts)
 func newUpdater(hostsD string) *updater {
 	u := &updater{
 		hostsD:      hostsD,
@@ -43,9 +43,9 @@ func newUpdater(hostsD string) *updater {
 
 // updater is the struct for updater.update()
 type updater struct {
-	hostsD      string           // "/varlib/nerdctl/hosts.d"
+	hostsD      string           // "/var/lib/nerdctl/<ADDRHASH>/etchosts"
 	metaByIPStr map[string]*Meta // key: IP string
-	metaByDir   map[string]*Meta // key: "/var/lib/nerdctl/hosts.d/<NS>/<ID>"
+	metaByDir   map[string]*Meta // key: "/var/lib/nerdctl/<ADDRHASH>/etchosts/<NS>/<ID>"
 }
 
 // update updates the hostsD tree.
diff --git a/pkg/labels/labels.go b/pkg/labels/labels.go
@@ -33,7 +33,7 @@ const (
 	// Hostname
 	Hostname = Prefix + "hostname"
 
-	// StateDir is "/var/lib/nerdctl/c/<NAMESPACE>/<ID>"
+	// StateDir is "/var/lib/nerdctl/<ADDRHASH>/containers/<NAMESPACE>/<ID>"
 	StateDir = Prefix + "state-dir"
 
 	// Networks is a JSON-marshalled string of []string, e.g. []string{"bridge"}.
diff --git a/pkg/logging/jsonfile/jsonfile.go b/pkg/logging/jsonfile/jsonfile.go
@@ -35,9 +35,9 @@ type Entry struct {
 	Time   time.Time `json:"time"`             // e.g. "2020-12-11T20:29:41.939902251Z"
 }
 
-func Path(dataRoot, ns, id string) string {
+func Path(dataStore, ns, id string) string {
 	// the file name corresponds to Docker
-	return filepath.Join(dataRoot, "containers", ns, id, id+"-json.log")
+	return filepath.Join(dataStore, "containers", ns, id, id+"-json.log")
 }
 
 func Encode(w io.Writer, stdout, stderr io.Reader) error {
diff --git a/pkg/logging/logging.go b/pkg/logging/logging.go
@@ -42,15 +42,15 @@ func Main(argv2 string) error {
 	return nil
 }
 
-func getLoggerFunc(dataRoot string) (logging.LoggerFunc, error) {
-	if dataRoot == "" {
-		return nil, errors.New("got empty data-root")
+func getLoggerFunc(dataStore string) (logging.LoggerFunc, error) {
+	if dataStore == "" {
+		return nil, errors.New("got empty data store")
 	}
 	return func(_ context.Context, config *logging.Config, ready func() error) error {
 		if config.Namespace == "" || config.ID == "" {
 			return errors.New("got invalid config")
 		}
-		logJSONFilePath := jsonfile.Path(dataRoot, config.Namespace, config.ID)
+		logJSONFilePath := jsonfile.Path(dataStore, config.Namespace, config.ID)
 		if err := os.MkdirAll(filepath.Dir(logJSONFilePath), 0700); err != nil {
 			return err
 		}
diff --git a/pkg/namestore/namestore.go b/pkg/namestore/namestore.go
@@ -27,8 +27,8 @@ import (
 	"github.com/pkg/errors"
 )
 
-func New(dataRoot, ns string) (NameStore, error) {
-	dir := filepath.Join(dataRoot, "names", ns)
+func New(dataStore, ns string) (NameStore, error) {
+	dir := filepath.Join(dataStore, "names", ns)
 	if err := os.MkdirAll(dir, 0700); err != nil {
 		return nil, err
 	}
diff --git a/pkg/ocihook/ocihook.go b/pkg/ocihook/ocihook.go
@@ -38,8 +38,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func Run(stdin io.Reader, stderr io.Writer, event, dataRoot, cniPath, cniNetconfPath string) error {
-	if stdin == nil || event == "" || dataRoot == "" || cniPath == "" || cniNetconfPath == "" {
+func Run(stdin io.Reader, stderr io.Writer, event, dataStore, cniPath, cniNetconfPath string) error {
+	if stdin == nil || event == "" || dataStore == "" || cniPath == "" || cniNetconfPath == "" {
 		return errors.New("got insufficient args")
 	}
 
@@ -63,7 +63,7 @@ func Run(stdin io.Reader, stderr io.Writer, event, dataRoot, cniPath, cniNetconf
 		logrus.SetOutput(io.MultiWriter(stderr, logFile))
 	}
 
-	opts, err := newHandlerOpts(&state, dataRoot, cniPath, cniNetconfPath)
+	opts, err := newHandlerOpts(&state, dataStore, cniPath, cniNetconfPath)
 	if err != nil {
 		return err
 	}
@@ -78,10 +78,10 @@ func Run(stdin io.Reader, stderr io.Writer, event, dataRoot, cniPath, cniNetconf
 	}
 }
 
-func newHandlerOpts(state *specs.State, dataRoot, cniPath, cniNetconfPath string) (*handlerOpts, error) {
+func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath string) (*handlerOpts, error) {
 	o := &handlerOpts{
-		state:    state,
-		dataRoot: dataRoot,
+		state:     state,
+		dataStore: dataStore,
 	}
 	hs, err := loadSpec(o.state.Bundle)
 	if err != nil {
@@ -147,13 +147,13 @@ func newHandlerOpts(state *specs.State, dataRoot, cniPath, cniNetconfPath string
 }
 
 type handlerOpts struct {
-	state    *specs.State
-	dataRoot string
-	fullID   string
-	rootfs   string
-	ports    []gocni.PortMapping
-	cni      gocni.CNI // TODO: support multi network
-	cniName  string    // TODO: support multi network
+	state     *specs.State
+	dataStore string
+	fullID    string
+	rootfs    string
+	ports     []gocni.PortMapping
+	cni       gocni.CNI // TODO: support multi network
+	cniName   string    // TODO: support multi network
 }
 
 // hookSpec is from https://github.com/containerd/containerd/blob/v1.4.3/cmd/containerd/command/oci-hook.go#L59-L64
@@ -216,7 +216,7 @@ func onCreateRuntime(opts *handlerOpts) error {
 		if err != nil {
 			return errors.Wrap(err, "failed to call cni.Setup")
 		}
-		hs, err := hostsstore.NewStore(opts.dataRoot)
+		hs, err := hostsstore.NewStore(opts.dataStore)
 		if err != nil {
 			return err
 		}
@@ -247,7 +247,7 @@ func onPostStop(opts *handlerOpts) error {
 			logrus.WithError(err).Errorf("failed to call cni.Remove")
 			return err
 		}
-		hs, err := hostsstore.NewStore(opts.dataRoot)
+		hs, err := hostsstore.NewStore(opts.dataStore)
 		if err != nil {
 			return err
 		}
diff --git a/rm.go b/rm.go
@@ -58,17 +58,22 @@ func rmAction(clicontext *cli.Context) error {
 	}
 	defer cancel()
 
+	dataStore, err := getDataStore(clicontext)
+	if err != nil {
+		return err
+	}
+
 	force := clicontext.Bool("force")
 
-	containerNameStore, err := namestore.New(clicontext.String("data-root"), clicontext.String("namespace"))
+	containerNameStore, err := namestore.New(dataStore, clicontext.String("namespace"))
 	if err != nil {
 		return err
 	}
 
 	walker := &containerwalker.ContainerWalker{
 		Client: client,
 		OnFound: func(ctx context.Context, found containerwalker.Found) error {
-			stateDir, err := getContainerStateDirPath(clicontext, found.Container.ID())
+			stateDir, err := getContainerStateDirPath(clicontext, dataStore, found.Container.ID())
 			if err != nil {
 				return err
 			}
diff --git a/run.go b/run.go

Original file line number	Diff line number	Diff line change
`@@ -34,8 +34,12 @@ func internalOCIHookAction(clicontext *cli.Context) error {`
`34`	`34`	`if event == "" {`
`35`	`35`	`return errors.New("event type needs to be passed")`
`36`	`36`	`}`
	`37`	`+ dataStore, err := getDataStore(clicontext)`
	`38`	`+ if err != nil {`
	`39`	`+ return err`
	`40`	`+ }`
`37`	`41`	`return ocihook.Run(clicontext.App.Reader, clicontext.App.ErrWriter, event,`
`38`		`- clicontext.String("data-root"),`
	`42`	`+ dataStore,`
`39`	`43`	`clicontext.String("cni-path"),`
`40`	`44`	`clicontext.String("cni-netconfpath"),`
`41`	`45`	`)`
Original file line number	Diff line number	Diff line change
`@@ -35,9 +35,9 @@ type Entry struct {`
`35`	`35`	Time time.Time `json:"time"` // e.g. "2020-12-11T20:29:41.939902251Z"
`36`	`36`	`}`
`37`	`37`
`38`		`-func Path(dataRoot, ns, id string) string {`
	`38`	`+func Path(dataStore, ns, id string) string {`
`39`	`39`	`// the file name corresponds to Docker`
`40`		`- return filepath.Join(dataRoot, "containers", ns, id, id+"-json.log")`
	`40`	`+ return filepath.Join(dataStore, "containers", ns, id, id+"-json.log")`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`func Encode(w io.Writer, stdout, stderr io.Reader) error {`
Original file line number	Diff line number	Diff line change
`@@ -42,15 +42,15 @@ func Main(argv2 string) error {`
`42`	`42`	`return nil`
`43`	`43`	`}`
`44`	`44`
`45`		`-func getLoggerFunc(dataRoot string) (logging.LoggerFunc, error) {`
`46`		`- if dataRoot == "" {`
`47`		`- return nil, errors.New("got empty data-root")`
	`45`	`+func getLoggerFunc(dataStore string) (logging.LoggerFunc, error) {`
	`46`	`+ if dataStore == "" {`
	`47`	`+ return nil, errors.New("got empty data store")`
`48`	`48`	`}`
`49`	`49`	`return func(_ context.Context, config *logging.Config, ready func() error) error {`
`50`	`50`	`if config.Namespace == "" \|\| config.ID == "" {`
`51`	`51`	`return errors.New("got invalid config")`
`52`	`52`	`}`
`53`		`- logJSONFilePath := jsonfile.Path(dataRoot, config.Namespace, config.ID)`
	`53`	`+ logJSONFilePath := jsonfile.Path(dataStore, config.Namespace, config.ID)`
`54`	`54`	`if err := os.MkdirAll(filepath.Dir(logJSONFilePath), 0700); err != nil {`
`55`	`55`	`return err`
`56`	`56`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,8 +27,8 @@ import (`
`27`	`27`	`"github.com/pkg/errors"`
`28`	`28`	`)`
`29`	`29`
`30`		`-func New(dataRoot, ns string) (NameStore, error) {`
`31`		`- dir := filepath.Join(dataRoot, "names", ns)`
	`30`	`+func New(dataStore, ns string) (NameStore, error) {`
	`31`	`+ dir := filepath.Join(dataStore, "names", ns)`
`32`	`32`	`if err := os.MkdirAll(dir, 0700); err != nil {`
`33`	`33`	`return nil, err`
`34`	`34`	`}`