diff --git a/random/files/doc.go b/random/files/doc.go new file mode 100644 index 0000000..4b910d7 --- /dev/null +++ b/random/files/doc.go @@ -0,0 +1,5 @@ +// Package files provides functionality for creating filesystem hierarchies of +// random files an directories. This is useful for testing that needs to +// operate on directory trees. Random results are reproducible by reusing the +// same seed. Random values are not cryptographically secure. +package files diff --git a/random/files/files.go b/random/files/files.go new file mode 100644 index 0000000..1ac1dc5 --- /dev/null +++ b/random/files/files.go @@ -0,0 +1,193 @@ +package files + +import ( + "errors" + "fmt" + "io" + "math/rand" + "os" + "path/filepath" + + "github.com/ipfs/go-test/random" +) + +const ( + fileNameSize = 16 + fileNameAlpha = "abcdefghijklmnopqrstuvwxyz01234567890-_" +) + +type Config struct { + // Depth is the depth of the directory tree including the root directory. + Depth int + // Dirs is the number of subdirectories at each depth. + Dirs int + // Files is the number of files at each depth. + Files int + // FileSize sets the number of random bytes in each file. + FileSize int64 + // Where to write display output, such as os.Stdout. Default is nil. + Out io.Writer + // RandomDirss specifies whether or not to randomize the number of + // subdirectoriess from 1 to the value configured by Dirs. + RandomDirs bool + // RandomFiles specifies whether or not to randomize the number of files + // from 1 to the value configured by Files. + RandomFiles bool + // RandomSize specifies whether or not to randomize the file size from 1 to + // the value configured by FileSize. + RandomSize bool + // Seed sets the seen for the random number generator when set to a + // non-zero value. + Seed int64 +} + +func DefaultConfig() Config { + return Config{ + Depth: 2, + Dirs: 5, + Files: 10, + FileSize: 4096, + RandomDirs: false, + RandomFiles: false, + RandomSize: true, + } +} + +func validateConfig(cfg *Config) error { + if cfg.Depth < 1 || cfg.Depth > 64 { + return errors.New("depth out of range, must be between 1 and 64") + } + if cfg.Dirs < 0 || cfg.Dirs > 64 { + return errors.New("dirs out of range, must be between 0 and 64") + } + if cfg.Files < 0 || cfg.Files > 64 { + return errors.New("files out of range, must be between 0 and 64") + } + if cfg.FileSize < 0 { + return errors.New("file size out of range, must be 0 or greater") + } + if cfg.Depth > 1 && cfg.Dirs < 1 { + return errors.New("dirs must be at least 1 for depth > 1") + } + + return nil +} + +func Create(cfg Config, paths ...string) error { + if len(paths) == 0 { + return errors.New("must provide at least 1 root directory path") + } + err := validateConfig(&cfg) + if err != nil { + return err + } + + var rnd *rand.Rand + if cfg.Seed == 0 { + rnd = random.NewRand() + } else { + rnd = random.NewSeededRand(cfg.Seed) + } + + for _, root := range paths { + err := os.MkdirAll(root, 0755) + if err != nil { + return err + } + + err = writeTree(rnd, root, 1, &cfg) + if err != nil { + return err + } + + } + + return nil +} + +func writeTree(rnd *rand.Rand, root string, depth int, cfg *Config) error { + nFiles := cfg.Files + if nFiles != 0 { + if cfg.RandomFiles && nFiles > 1 { + nFiles = rnd.Intn(nFiles) + 1 + } + + for i := 0; i < nFiles; i++ { + if err := writeFile(rnd, root, cfg); err != nil { + return err + } + } + } + + return writeSubdirs(rnd, root, depth, cfg) +} + +func writeSubdirs(rnd *rand.Rand, root string, depth int, cfg *Config) error { + if depth == cfg.Depth { + return nil + } + depth++ + + nDirs := cfg.Dirs + if cfg.RandomDirs && nDirs > 1 { + nDirs = rnd.Intn(nDirs) + 1 + } + + for i := 0; i < nDirs; i++ { + if err := writeSubdir(rnd, root, depth, cfg); err != nil { + return err + } + } + + return nil +} + +func writeSubdir(rnd *rand.Rand, root string, depth int, cfg *Config) error { + name := randomFilename(rnd) + root = filepath.Join(root, name) + if err := os.MkdirAll(root, 0755); err != nil { + return err + } + + if cfg.Out != nil { + fmt.Fprintln(cfg.Out, root+"/") + } + + return writeTree(rnd, root, depth, cfg) +} + +func randomFilename(rnd *rand.Rand) string { + n := rnd.Intn(fileNameSize-4) + 4 + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = fileNameAlpha[rnd.Intn(len(fileNameAlpha))] + } + return string(b) +} + +func writeFile(rnd *rand.Rand, root string, cfg *Config) error { + name := randomFilename(rnd) + filePath := filepath.Join(root, name) + f, err := os.Create(filePath) + if err != nil { + return err + } + + if cfg.FileSize > 0 { + fileSize := cfg.FileSize + if cfg.RandomSize && fileSize > 1 { + fileSize = rnd.Int63n(fileSize) + 1 + } + + if _, err := io.CopyN(f, rnd, fileSize); err != nil { + f.Close() + return err + } + } + + if cfg.Out != nil { + fmt.Fprintln(cfg.Out, filePath) + } + + return f.Close() +} diff --git a/random/files/files_test.go b/random/files/files_test.go new file mode 100644 index 0000000..0e33a26 --- /dev/null +++ b/random/files/files_test.go @@ -0,0 +1,87 @@ +package files_test + +import ( + "bufio" + "bytes" + "os" + "testing" + + "github.com/ipfs/go-test/random/files" + "github.com/stretchr/testify/require" +) + +func TestRandomFiles(t *testing.T) { + var b bytes.Buffer + cfg := files.DefaultConfig() + cfg.Depth = 2 + cfg.Dirs = 5 + cfg.Files = 3 + cfg.Out = &b + + roots := []string{"foo"} + err := files.Create(cfg, roots...) + require.NoError(t, err) + t.Cleanup(func() { + for _, root := range roots { + os.RemoveAll(root) + } + }) + + t.Logf("Created file hierarchy:\n%s", b.String()) + + var lines int + scanner := bufio.NewScanner(&b) + for scanner.Scan() { + lines++ + } + require.NoError(t, scanner.Err()) + + subdirs := 0 + if cfg.Depth > 1 { + dirsAtDepth := cfg.Dirs + subdirs += dirsAtDepth + for i := 0; i < cfg.Depth-2; i++ { + dirsAtDepth *= cfg.Dirs + subdirs += dirsAtDepth + } + } + linesPerSubdir := cfg.Files + 1 + expect := ((subdirs * linesPerSubdir) + cfg.Files) * len(roots) + require.Equal(t, expect, lines) +} + +func TestRandomFilesValidation(t *testing.T) { + cfg := files.DefaultConfig() + err := files.Create(cfg) + require.Error(t, err) + + cfg.Depth = 0 + require.Error(t, files.Create(cfg, "foo")) + cfg.Depth = 65 + require.Error(t, files.Create(cfg, "foo")) + + cfg = files.DefaultConfig() + + cfg.Dirs = -1 + require.Error(t, files.Create(cfg, "foo")) + cfg.Dirs = 65 + require.Error(t, files.Create(cfg, "foo")) + + cfg = files.DefaultConfig() + + cfg.Files = -1 + require.Error(t, files.Create(cfg, "foo")) + cfg.Files = 65 + require.Error(t, files.Create(cfg, "foo")) + + cfg = files.DefaultConfig() + + cfg.FileSize = -1 + require.Error(t, files.Create(cfg, "foo")) + + cfg = files.DefaultConfig() + + cfg.Depth = 2 + cfg.Dirs = 0 + require.Error(t, files.Create(cfg, "foo")) +} diff --git a/random/files/random-files/README.md b/random/files/random-files/README.md new file mode 100644 index 0000000..eeb2087 --- /dev/null +++ b/random/files/random-files/README.md @@ -0,0 +1,104 @@ +# random-files - create random filesystem hierarchies + +random-files creates random filesystem hierarchies for testing + +## Install + +``` +go install github.com/ipfs/go-test/random/files/random-files +``` + +## Usage + +```sh +> random-files -help +usage: ./random-files [options] ... +Write a random filesystem hierarchy to each + +Options: + -depth int + depth of the directory tree including the root directory (default 2) + -dirs int + number of subdirectories at each depth (default 5) + -files int + number of files at each depth (default 10) + -filesize int + file fize, or the max file size id RandomSize is true (default 4096) + -q do not print files and directories + -random-dirs + randomize number of subdirectories, from 1 to -Dirs + -random-files + randomize number of files, from 1 to -Files + -random-size + randomize file size, from 1 to -FileSize (default true) + -seed int + random seed, 0 for current time +``` + +## Examples + +```sh +> random-files -depth=2 -files=3 -seed=1701 foo +foo/rwd67uvnj9yz- +foo/7vovyvr9 +foo/fjv0w0 +foo/gyubi50rec5/ +foo/gyubi50rec5/vr6x-ce4uupj +foo/gyubi50rec5/ob9ud0e8lt_2e +foo/gyubi50rec5/11gip6zea +foo/nzu5j29-sh-ku4/ +foo/nzu5j29-sh-ku4/vcs1629n +foo/nzu5j29-sh-ku4/rky_i_qsxrp +foo/nzu5j29-sh-ku4/xr1usy5ic0 +foo/w30dzrx2w4_d/ +foo/w30dzrx2w4_d/7ued6 +foo/w30dzrx2w4_d/r1d3j +foo/w30dzrx2w4_d/av7d09i-av +foo/s6ha-58/ +foo/s6ha-58/nukjsxg7t +foo/s6ha-58/7of_84 +foo/s6ha-58/h0jgq8mu1n7u +foo/tq_8/ +foo/tq_8/sx-a2jgmz_mk6 +foo/tq_8/9hzrksz8 +foo/tq_8/8b5swu +``` + +It made: + +```sh +> tree foo +foo +├── 7vovyvr9 +├── fjv0w0 +├── gyubi50rec5 +│   ├── 11gip6zea +│   ├── ob9ud0e8lt_2e +│   └── vr6x-ce4uupj +├── nzu5j29-sh-ku4 +│   ├── rky_i_qsxrp +│   ├── vcs1629n +│   └── xr1usy5ic0 +├── rwd67uvnj9yz- +├── s6ha-58 +│   ├── 7of_84 +│   ├── h0jgq8mu1n7u +│   └── nukjsxg7t +├── tq_8 +│   ├── 8b5swu +│   ├── 9hzrksz8 +│   └── sx-a2jgmz_mk6 +└── w30dzrx2w4_d + ├── 7ued6 + ├── av7d09i-av + └── r1d3j + +6 directories, 18 files +``` + +Note: Specifying the same seed will produce the same results. + + +### Acknowledgments + +Credit to Juan Benet as the author of [`go-random-files`](https://github.com/jbenet/go-random-files) from which this code was derived. diff --git a/random/files/random-files/main.go b/random/files/random-files/main.go new file mode 100644 index 0000000..cb3ea43 --- /dev/null +++ b/random/files/random-files/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/ipfs/go-test/random/files" +) + +func main() { + var usage = `usage: %s [options] ... +Write a random filesystem hierarchy to each + +Options: +` + flag.Usage = func() { + fmt.Fprintf(os.Stderr, usage, os.Args[0]) + flag.PrintDefaults() + } + + var ( + quiet bool + paths []string + ) + + cfg := files.DefaultConfig() + + flag.BoolVar(&quiet, "q", false, "do not print files and directories") + flag.IntVar(&cfg.Depth, "depth", cfg.Depth, "depth of the directory tree including the root directory") + flag.Int64Var(&cfg.FileSize, "filesize", cfg.FileSize, "bytes of random data in each file") + flag.IntVar(&cfg.Dirs, "dirs", cfg.Dirs, "number of subdirectories at each depth") + flag.IntVar(&cfg.Files, "files", cfg.Files, "number of files at each depth") + flag.BoolVar(&cfg.RandomDirs, "random-dirs", cfg.RandomDirs, "randomize number of subdirectories, from 1 to -Dirs") + flag.BoolVar(&cfg.RandomFiles, "random-files", cfg.RandomFiles, "randomize number of files, from 1 to -Files") + flag.BoolVar(&cfg.RandomSize, "random-size", cfg.RandomSize, "randomize file size, from 1 to -FileSize") + flag.Int64Var(&cfg.Seed, "seed", cfg.Seed, "random seed, 0 for current time") + flag.Parse() + + paths = flag.Args() + + if !quiet { + cfg.Out = os.Stdout + } + + err := files.Create(cfg, paths...) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +}