From 180a934f12c972d348aa982c49ca396e6fa49d45 Mon Sep 17 00:00:00 2001 From: philipz Date: Thu, 24 Jul 2014 23:43:34 +0800 Subject: [PATCH 1/8] Fix myapp-nginx.tmpl filename --- docs/templates-interation-example.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templates-interation-example.md b/docs/templates-interation-example.md index 734ddfc02..0cf15cfea 100644 --- a/docs/templates-interation-example.md +++ b/docs/templates-interation-example.md @@ -29,7 +29,7 @@ reload_cmd = "/usr/sbin/service nginx reload" ## Create a source template -`/etc/confd/templates/nginx.tmpl` +`/etc/confd/templates/myapp-nginx.tmpl` ``` upstream myapp { From 7d77798b61030eb876f9416eed9b2843e432882e Mon Sep 17 00:00:00 2001 From: Bill Hathaway Date: Wed, 30 Jul 2014 13:15:53 -0700 Subject: [PATCH 2/8] sort results from etcd so keys are always in the same order --- backends/etcd/etcdutil/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/etcd/etcdutil/client.go b/backends/etcd/etcdutil/client.go index f87596711..22ba308af 100644 --- a/backends/etcd/etcdutil/client.go +++ b/backends/etcd/etcdutil/client.go @@ -47,7 +47,7 @@ func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, func (c *Client) GetValues(keys []string) (map[string]interface{}, error) { vars := make(map[string]interface{}) for _, key := range keys { - resp, err := c.client.Get(key, false, true) + resp, err := c.client.Get(key, true, true) if err != nil { return vars, err } From f92a4c8bd0cd55103c27d2a319671673157b8f05 Mon Sep 17 00:00:00 2001 From: Jason Martin Date: Tue, 5 Aug 2014 15:11:32 -0700 Subject: [PATCH 3/8] Update consul-getting-started.md Fix doc for calling consul backend. --- docs/consul-getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/consul-getting-started.md b/docs/consul-getting-started.md index f4bd9e9e6..95d7d2727 100644 --- a/docs/consul-getting-started.md +++ b/docs/consul-getting-started.md @@ -57,7 +57,7 @@ confd supports two modes of operation, daemon and onetime mode. In daemon mode, Assuming your consul server is running at http://127.0.0.1:8500 you can run the following command to process the `~/confd/conf.d/myconfig.toml` template resource: ``` -confd -verbose -onetime -consul -consul-addr 127.0.0.1:8500 -confdir ~/confd +confd -verbose -onetime -backend consul -node 127.0.0.1:8500 -confdir ~/confd ``` Output: ``` From 7f5790fe1982d412bc87f29b038a252fe4d846c3 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Fri, 8 Aug 2014 08:11:42 -0700 Subject: [PATCH 4/8] Update deps --- Godeps/Godeps.json | 13 +- .../src/github.com/BurntSushi/toml/.gitignore | 1 + .../github.com/BurntSushi/toml/.travis.yml | 12 + .../src/github.com/BurntSushi/toml/Makefile | 7 +- .../src/github.com/BurntSushi/toml/README.md | 14 +- .../toml-test-decoder}/COPYING | 0 .../toml-test-decoder}/README.md | 0 .../toml-test-decoder}/main.go | 2 + .../toml-test-encoder}/COPYING | 0 .../{ => cmd}/toml-test-encoder/README.md | 0 .../toml/{ => cmd}/toml-test-encoder/main.go | 2 + .../BurntSushi/toml/{ => cmd}/tomlv/COPYING | 0 .../BurntSushi/toml/{ => cmd}/tomlv/README.md | 2 +- .../BurntSushi/toml/{ => cmd}/tomlv/main.go | 1 + .../src/github.com/BurntSushi/toml/decode.go | 292 +++----- .../github.com/BurntSushi/toml/decode_meta.go | 99 +++ .../github.com/BurntSushi/toml/decode_test.go | 261 ++++--- .../src/github.com/BurntSushi/toml/doc.go | 27 +- .../src/github.com/BurntSushi/toml/encode.go | 644 ++++++++---------- .../github.com/BurntSushi/toml/encode_test.go | 316 +++++++-- .../BurntSushi/toml/encoding_types.go | 9 + .../BurntSushi/toml/encoding_types_1.1.go | 7 + .../src/github.com/BurntSushi/toml/lex.go | 29 +- .../github.com/BurntSushi/toml/lex_test.go | 59 -- .../github.com/BurntSushi/toml/out_test.go | 19 - .../github.com/BurntSushi/toml/parse_test.go | 73 -- .../github.com/BurntSushi/toml/type_check.go | 7 + .../src/github.com/armon/consul-kv/README.md | 6 +- .../src/github.com/armon/consul-kv/client.go | 3 +- .../github.com/coreos/go-etcd/etcd/client.go | 13 +- .../github.com/coreos/go-etcd/etcd/debug.go | 29 +- .../coreos/go-etcd/etcd/debug_test.go | 28 + .../coreos/go-etcd/etcd/requests.go | 68 +- 33 files changed, 1137 insertions(+), 906 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml rename Godeps/_workspace/src/github.com/BurntSushi/toml/{toml-test-encoder => cmd/toml-test-decoder}/COPYING (100%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{toml-test-go => cmd/toml-test-decoder}/README.md (100%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{toml-test-go => cmd/toml-test-decoder}/main.go (92%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{toml-test-go => cmd/toml-test-encoder}/COPYING (100%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{ => cmd}/toml-test-encoder/README.md (100%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{ => cmd}/toml-test-encoder/main.go (94%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{ => cmd}/tomlv/COPYING (100%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{ => cmd}/tomlv/README.md (91%) rename Godeps/_workspace/src/github.com/BurntSushi/toml/{ => cmd}/tomlv/main.go (93%) create mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go delete mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/lex_test.go delete mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/out_test.go delete mode 100644 Godeps/_workspace/src/github.com/BurntSushi/toml/parse_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 8caa878aa..99fd86242 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,19 +1,22 @@ { "ImportPath": "github.com/kelseyhightower/confd", - "GoVersion": "go1.2.1", + "GoVersion": "go1.3", + "Packages": [ + "." + ], "Deps": [ { "ImportPath": "github.com/BurntSushi/toml", - "Rev": "890ff2dcee0c5b7fc3c7453325d8146aaf1850c1" + "Rev": "2ceedfee35ad3848e49308ab0c9a4f640cfb5fb2" }, { "ImportPath": "github.com/armon/consul-kv", - "Rev": "d9bed0798395bbc5b77990c38e6c4ba542576a67" + "Rev": "3898520ef0484e72663aa4f8cda0396a61e15e7f" }, { "ImportPath": "github.com/coreos/go-etcd/etcd", - "Comment": "v0.2.0-rc1-106-g26f4504", - "Rev": "26f4504e71d6abefd467eb44a02ff15b933b6eb6" + "Comment": "v0.2.0-rc1-120-g23142f6", + "Rev": "23142f6773a676cc2cae8dd0cb90b2ea761c853f" } ] } diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore b/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore index 55e90a1e5..0cd380037 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/.gitignore @@ -2,3 +2,4 @@ TAGS tags .*.swp tomlcheck/tomlcheck +toml.test diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml b/Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml new file mode 100644 index 000000000..43caf6d02 --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/.travis.yml @@ -0,0 +1,12 @@ +language: go +go: + - 1.1 + - 1.2 + - tip +install: + - go install ./... + - go get github.com/BurntSushi/toml-test +script: + - export PATH="$PATH:$HOME/gopath/bin" + - make test + diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile b/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile index e6adc3e9e..3600848d3 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/Makefile @@ -1,5 +1,10 @@ install: - go install + go install ./... + +test: install + go test -v + toml-test toml-test-decoder + toml-test -encoder toml-test-encoder fmt: gofmt -w *.go */*.go diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md index 49f43f34e..380bb36bb 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/README.md @@ -1,4 +1,4 @@ -# TOML parser and encoder for Go with reflection +## TOML parser and encoder for Go with reflection TOML stands for Tom's Obvious, Minimal Language. This Go package provides a reflection interface similar to Go's standard library `json` and `xml` @@ -22,18 +22,20 @@ go get github.com/BurntSushi/toml Try the toml validator: ```bash -go get github.com/BurntSushi/toml/tomlv +go get github.com/BurntSushi/toml/cmd/tomlv tomlv some-toml-file.toml ``` +[![Build status](https://api.travis-ci.org/BurntSushi/toml.png)](https://travis-ci.org/BurntSushi/toml) -## Testing + +### Testing This package passes all tests in [toml-test](https://github.com/BurntSushi/toml-test) for both the decoder and the encoder. -## Examples +### Examples This package works similarly to how the Go standard library handles `XML` and `JSON`. Namely, data is loaded into Go values via reflection. @@ -83,7 +85,7 @@ type TOML struct { } ``` -## Using the `encoding.TextUnmarshaler` interface +### Using the `encoding.TextUnmarshaler` interface Here's an example that automatically parses duration strings into `time.Duration` values: @@ -133,7 +135,7 @@ func (d *duration) UnmarshalText(text []byte) error { } ``` -## More complex usage +### More complex usage Here's an example of how to load the example from the official spec page: diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING similarity index 100% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/COPYING rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md similarity index 100% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/README.md rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/main.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go similarity index 92% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/main.go rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go index eb0f95b8d..14e755700 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/main.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go @@ -1,3 +1,5 @@ +// Command toml-test-decoder satisfies the toml-test interface for testing +// TOML decoders. Namely, it accepts TOML on stdin and outputs JSON on stdout. package main import ( diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING similarity index 100% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-go/COPYING rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md similarity index 100% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/README.md rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/main.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go similarity index 94% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/main.go rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go index b7897e79d..092cc6844 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/toml-test-encoder/main.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go @@ -1,3 +1,5 @@ +// Command toml-test-encoder satisfies the toml-test interface for testing +// TOML encoders. Namely, it accepts JSON on stdin and outputs TOML on stdout. package main import ( diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/COPYING b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/COPYING similarity index 100% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/COPYING rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/COPYING diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/README.md b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md similarity index 91% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/README.md rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md index bcc3f47b1..5df0dc32b 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/README.md +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/README.md @@ -3,7 +3,7 @@ If Go is installed, it's simple to try it out: ```bash -go get github.com/BurntSushi/toml/tomlv +go get github.com/BurntSushi/toml/cmd/tomlv tomlv some-toml-file.toml ``` diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/main.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go similarity index 93% rename from Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/main.go rename to Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go index a52086ef5..c7d689a7e 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/tomlv/main.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/cmd/tomlv/main.go @@ -1,3 +1,4 @@ +// Command tomlv validates TOML documents and prints each key's type. package main import ( diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go index 312b06b2c..b6d75d042 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode.go @@ -24,18 +24,34 @@ var e = fmt.Errorf // N.B. Primitive values are still parsed, so using them will only avoid // the overhead of reflection. They can be useful when you don't know the // exact type of TOML data until run time. -type Primitive interface{} +type Primitive struct { + undecoded interface{} + context Key +} + +// DEPRECATED! +// +// Use MetaData.PrimitiveDecode instead. +func PrimitiveDecode(primValue Primitive, v interface{}) error { + md := MetaData{decoded: make(map[string]bool)} + return md.unify(primValue.undecoded, rvalue(v)) +} // PrimitiveDecode is just like the other `Decode*` functions, except it // decodes a TOML value that has already been parsed. Valid primitive values // can *only* be obtained from values filled by the decoder functions, -// including `PrimitiveDecode`. (i.e., `v` may contain more `Primitive` +// including this method. (i.e., `v` may contain more `Primitive` // values.) // // Meta data for primitive values is included in the meta data returned by -// the `Decode*` functions. -func PrimitiveDecode(primValue Primitive, v interface{}) error { - return unify(primValue, rvalue(v)) +// the `Decode*` functions with one exception: keys returned by the Undecoded +// method will only reflect keys that were decoded. Namely, any keys hidden +// behind a Primitive will be considered undecoded. Executing this method will +// update the undecoded keys in the meta data. (See the example.) +func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { + md.context = primValue.context + defer func() { md.context = nil }() + return md.unify(primValue.undecoded, rvalue(v)) } // Decode will decode the contents of `data` in TOML format into a pointer @@ -55,48 +71,8 @@ func PrimitiveDecode(primValue Primitive, v interface{}) error { // An exception to the above rules is if a type implements the // encoding.TextUnmarshaler interface. In this case, any primitive TOML value // (floats, strings, integers, booleans and datetimes) will be converted to -// a byte string and given to the value's UnmarshalText method. Here's an -// example for parsing durations: -// -// type duration struct { -// time.Duration -// } -// -// func (d *duration) UnmarshalText(text []byte) error { -// var err error -// d.Duration, err = time.ParseDuration(string(text)) -// return err -// } -// -// func ExampleUnmarshaler() { -// blob := ` -// [[song]] -// name = "Thunder Road" -// duration = "4m49s" -// -// [[song]] -// name = "Stairway to Heaven" -// duration = "8m03s" -// ` -// type song struct { -// Name string -// Duration duration -// } -// type songs struct { -// Song []song -// } -// var favorites songs -// if _, err := Decode(blob, &favorites); err != nil { -// log.Fatal(err) -// } -// -// for _, s := range favorites.Song { -// fmt.Printf("%s (%s)\n", s.Name, s.Duration) -// } -// // Output: -// // Thunder Road (4m49s) -// // Stairway to Heaven (8m3s) -// } +// a byte string and given to the value's UnmarshalText method. See the +// Unmarshaler example for a demonstration with time duration strings. // // Key mapping // @@ -109,7 +85,8 @@ func PrimitiveDecode(primValue Primitive, v interface{}) error { // The mapping between TOML values and Go values is loose. That is, there // may exist TOML values that cannot be placed into your representation, and // there may be parts of your representation that do not correspond to -// TOML values. +// TOML values. This loose mapping can be made stricter by using the IsDefined +// and/or Undecoded methods on the MetaData returned. // // This decoder will not handle cyclic types. If a cyclic type is passed, // `Decode` will not terminate. @@ -118,7 +95,11 @@ func Decode(data string, v interface{}) (MetaData, error) { if err != nil { return MetaData{}, err } - return MetaData{p.mapping, p.types, p.ordered}, unify(p.mapping, rvalue(v)) + md := MetaData{ + p.mapping, p.types, p.ordered, + make(map[string]bool, len(p.ordered)), nil, + } + return md, md.unify(p.mapping, rvalue(v)) } // DecodeFile is just like Decode, except it will automatically read the @@ -146,15 +127,31 @@ func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { // // Any type mismatch produces an error. Finding a type that we don't know // how to handle produces an unsupported type error. -func unify(data interface{}, rv reflect.Value) error { +func (md *MetaData) unify(data interface{}, rv reflect.Value) error { // Special case. Look for a `Primitive` value. if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { - return unifyAnything(data, rv) + // Save the undecoded data and the key context into the primitive + // value. + context := make(Key, len(md.context)) + copy(context, md.context) + rv.Set(reflect.ValueOf(Primitive{ + undecoded: data, + context: context, + })) + return nil + } + + // Special case. Handle time.Time values specifically. + // TODO: Remove this code when we decide to drop support for Go 1.1. + // This isn't necessary in Go 1.2 because time.Time satisfies the encoding + // interfaces. + if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { + return md.unifyDatetime(data, rv) } // Special case. Look for a value satisfying the TextUnmarshaler interface. if v, ok := rv.Interface().(TextUnmarshaler); ok { - return unifyText(data, v) + return md.unifyText(data, v) } // BUG(burntsushi) // The behavior here is incorrect whenever a Go type satisfies the @@ -168,44 +165,44 @@ func unify(data interface{}, rv reflect.Value) error { // laziness if k >= reflect.Int && k <= reflect.Uint64 { - return unifyInt(data, rv) + return md.unifyInt(data, rv) } switch k { case reflect.Ptr: elem := reflect.New(rv.Type().Elem()) - err := unify(data, reflect.Indirect(elem)) + err := md.unify(data, reflect.Indirect(elem)) if err != nil { return err } rv.Set(elem) return nil case reflect.Struct: - return unifyStruct(data, rv) + return md.unifyStruct(data, rv) case reflect.Map: - return unifyMap(data, rv) + return md.unifyMap(data, rv) case reflect.Array: - return unifyArray(data, rv) + return md.unifyArray(data, rv) case reflect.Slice: - return unifySlice(data, rv) + return md.unifySlice(data, rv) case reflect.String: - return unifyString(data, rv) + return md.unifyString(data, rv) case reflect.Bool: - return unifyBool(data, rv) + return md.unifyBool(data, rv) case reflect.Interface: // we only support empty interfaces. if rv.NumMethod() > 0 { return e("Unsupported type '%s'.", rv.Kind()) } - return unifyAnything(data, rv) + return md.unifyAnything(data, rv) case reflect.Float32: fallthrough case reflect.Float64: - return unifyFloat64(data, rv) + return md.unifyFloat64(data, rv) } return e("Unsupported type '%s'.", rv.Kind()) } -func unifyStruct(mapping interface{}, rv reflect.Value) error { +func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { tmap, ok := mapping.(map[string]interface{}) if !ok { return mismatch(rv, "map", mapping) @@ -227,21 +224,16 @@ func unifyStruct(mapping interface{}, rv reflect.Value) error { if f != nil { subv := rv for _, i := range f.index { - if subv.Kind() == reflect.Ptr { - if subv.IsNil() { - subv.Set(reflect.New(subv.Type().Elem())) - } - subv = subv.Elem() - } - subv = subv.Field(i) + subv = indirect(subv.Field(i)) } - sf := indirect(subv) - - if isUnifiable(sf) { - if err := unify(datum, sf); err != nil { + if isUnifiable(subv) { + md.decoded[md.context.add(key).String()] = true + md.context = append(md.context, key) + if err := md.unify(datum, subv); err != nil { return e("Type mismatch for '%s.%s': %s", rv.Type().String(), f.name, err) } + md.context = md.context[0 : len(md.context)-1] } else if f.name != "" { // Bad user! No soup for you! return e("Field '%s.%s' is unexported, and therefore cannot "+ @@ -252,7 +244,7 @@ func unifyStruct(mapping interface{}, rv reflect.Value) error { return nil } -func unifyMap(mapping interface{}, rv reflect.Value) error { +func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { tmap, ok := mapping.(map[string]interface{}) if !ok { return badtype("map", mapping) @@ -261,11 +253,15 @@ func unifyMap(mapping interface{}, rv reflect.Value) error { rv.Set(reflect.MakeMap(rv.Type())) } for k, v := range tmap { + md.decoded[md.context.add(k).String()] = true + md.context = append(md.context, k) + rvkey := indirect(reflect.New(rv.Type().Key())) rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) - if err := unify(v, rvval); err != nil { + if err := md.unify(v, rvval); err != nil { return err } + md.context = md.context[0 : len(md.context)-1] rvkey.SetString(k) rv.SetMapIndex(rvkey, rvval) @@ -273,7 +269,7 @@ func unifyMap(mapping interface{}, rv reflect.Value) error { return nil } -func unifyArray(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { datav := reflect.ValueOf(data) if datav.Kind() != reflect.Slice { return badtype("slice", data) @@ -283,17 +279,10 @@ func unifyArray(data interface{}, rv reflect.Value) error { return e("expected array length %d; got TOML array of length %d", rv.Len(), sliceLen) } - for i := 0; i < sliceLen; i++ { - v := datav.Index(i).Interface() - sliceval := indirect(rv.Index(i)) - if err := unify(v, sliceval); err != nil { - return err - } - } - return nil + return md.unifySliceArray(datav, rv) } -func unifySlice(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { datav := reflect.ValueOf(data) if datav.Kind() != reflect.Slice { return badtype("slice", data) @@ -302,17 +291,22 @@ func unifySlice(data interface{}, rv reflect.Value) error { if rv.IsNil() { rv.Set(reflect.MakeSlice(rv.Type(), sliceLen, sliceLen)) } + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { + sliceLen := data.Len() for i := 0; i < sliceLen; i++ { - v := datav.Index(i).Interface() + v := data.Index(i).Interface() sliceval := indirect(rv.Index(i)) - if err := unify(v, sliceval); err != nil { + if err := md.unify(v, sliceval); err != nil { return err } } return nil } -func unifyDatetime(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { if _, ok := data.(time.Time); ok { rv.Set(reflect.ValueOf(data)) return nil @@ -320,7 +314,7 @@ func unifyDatetime(data interface{}, rv reflect.Value) error { return badtype("time.Time", data) } -func unifyString(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { if s, ok := data.(string); ok { rv.SetString(s) return nil @@ -328,7 +322,7 @@ func unifyString(data interface{}, rv reflect.Value) error { return badtype("string", data) } -func unifyFloat64(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { if num, ok := data.(float64); ok { switch rv.Kind() { case reflect.Float32: @@ -343,7 +337,7 @@ func unifyFloat64(data interface{}, rv reflect.Value) error { return badtype("float", data) } -func unifyInt(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { if num, ok := data.(int64); ok { if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { switch rv.Kind() { @@ -381,7 +375,7 @@ func unifyInt(data interface{}, rv reflect.Value) error { return e("Value '%d' is out of range for uint32.", num) } } - rv.SetInt(num) + rv.SetUint(unum) } else { panic("unreachable") } @@ -390,7 +384,7 @@ func unifyInt(data interface{}, rv reflect.Value) error { return badtype("integer", data) } -func unifyBool(data interface{}, rv reflect.Value) error { +func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { if b, ok := data.(bool); ok { rv.SetBool(b) return nil @@ -398,13 +392,12 @@ func unifyBool(data interface{}, rv reflect.Value) error { return badtype("boolean", data) } -func unifyAnything(data interface{}, rv reflect.Value) error { - // too awesome to fail +func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { rv.Set(reflect.ValueOf(data)) return nil } -func unifyText(data interface{}, v TextUnmarshaler) error { +func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { var s string switch sdata := data.(type) { case TextMarshaler: @@ -469,114 +462,11 @@ func isUnifiable(rv reflect.Value) bool { return false } -func tstring(rv reflect.Value) string { - return rv.Type().String() -} - func badtype(expected string, data interface{}) error { return e("Expected %s but found '%T'.", expected, data) } func mismatch(user reflect.Value, expected string, data interface{}) error { return e("Type mismatch for %s. Expected %s but found '%T'.", - tstring(user), expected, data) -} - -func insensitiveGet( - tmap map[string]interface{}, kname string) (interface{}, bool) { - - if datum, ok := tmap[kname]; ok { - return datum, true - } - for k, v := range tmap { - if strings.EqualFold(kname, k) { - return v, true - } - } - return nil, false -} - -// MetaData allows access to meta information about TOML data that may not -// be inferrable via reflection. In particular, whether a key has been defined -// and the TOML type of a key. -type MetaData struct { - mapping map[string]interface{} - types map[string]tomlType - keys []Key -} - -// IsDefined returns true if the key given exists in the TOML data. The key -// should be specified hierarchially. e.g., -// -// // access the TOML key 'a.b.c' -// IsDefined("a", "b", "c") -// -// IsDefined will return false if an empty key given. Keys are case sensitive. -func (md MetaData) IsDefined(key ...string) bool { - var hashOrVal interface{} - var hash map[string]interface{} - var ok bool - - if len(key) == 0 { - return false - } - - hashOrVal = md.mapping - for _, k := range key { - if hash, ok = hashOrVal.(map[string]interface{}); !ok { - return false - } - if hashOrVal, ok = hash[k]; !ok { - return false - } - } - return true -} - -// Type returns a string representation of the type of the key specified. -// -// Type will return the empty string if given an empty key or a key that -// does not exist. Keys are case sensitive. -func (md MetaData) Type(key ...string) string { - fullkey := strings.Join(key, ".") - if typ, ok := md.types[fullkey]; ok { - return typ.typeString() - } - return "" -} - -// Key is the type of any TOML key, including key groups. Use (MetaData).Keys -// to get values of this type. -type Key []string - -func (k Key) String() string { - return strings.Join(k, ".") -} - -func (k Key) add(piece string) Key { - newKey := make(Key, len(k)) - copy(newKey, k) - return append(newKey, piece) -} - -// Keys returns a slice of every key in the TOML data, including key groups. -// Each key is itself a slice, where the first element is the top of the -// hierarchy and the last is the most specific. -// -// The list will have the same order as the keys appeared in the TOML data. -// -// All keys returned are non-empty. -func (md MetaData) Keys() []Key { - return md.keys -} - -func allKeys(m map[string]interface{}, context Key) []Key { - keys := make([]Key, 0, len(m)) - for k, v := range m { - keys = append(keys, context.add(k)) - if t, ok := v.(map[string]interface{}); ok { - keys = append(keys, allKeys(t, context.add(k))...) - } - } - return keys + user.Type().String(), expected, data) } diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go new file mode 100644 index 000000000..c8114453b --- /dev/null +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_meta.go @@ -0,0 +1,99 @@ +package toml + +import "strings" + +// MetaData allows access to meta information about TOML data that may not +// be inferrable via reflection. In particular, whether a key has been defined +// and the TOML type of a key. +type MetaData struct { + mapping map[string]interface{} + types map[string]tomlType + keys []Key + decoded map[string]bool + context Key // Used only during decoding. +} + +// IsDefined returns true if the key given exists in the TOML data. The key +// should be specified hierarchially. e.g., +// +// // access the TOML key 'a.b.c' +// IsDefined("a", "b", "c") +// +// IsDefined will return false if an empty key given. Keys are case sensitive. +func (md *MetaData) IsDefined(key ...string) bool { + if len(key) == 0 { + return false + } + + var hash map[string]interface{} + var ok bool + var hashOrVal interface{} = md.mapping + for _, k := range key { + if hash, ok = hashOrVal.(map[string]interface{}); !ok { + return false + } + if hashOrVal, ok = hash[k]; !ok { + return false + } + } + return true +} + +// Type returns a string representation of the type of the key specified. +// +// Type will return the empty string if given an empty key or a key that +// does not exist. Keys are case sensitive. +func (md *MetaData) Type(key ...string) string { + fullkey := strings.Join(key, ".") + if typ, ok := md.types[fullkey]; ok { + return typ.typeString() + } + return "" +} + +// Key is the type of any TOML key, including key groups. Use (MetaData).Keys +// to get values of this type. +type Key []string + +func (k Key) String() string { + return strings.Join(k, ".") +} + +func (k Key) add(piece string) Key { + newKey := make(Key, len(k)+1) + copy(newKey, k) + newKey[len(k)] = piece + return newKey +} + +// Keys returns a slice of every key in the TOML data, including key groups. +// Each key is itself a slice, where the first element is the top of the +// hierarchy and the last is the most specific. +// +// The list will have the same order as the keys appeared in the TOML data. +// +// All keys returned are non-empty. +func (md *MetaData) Keys() []Key { + return md.keys +} + +// Undecoded returns all keys that have not been decoded in the order in which +// they appear in the original TOML document. +// +// This includes keys that haven't been decoded because of a Primitive value. +// Once the Primitive value is decoded, the keys will be considered decoded. +// +// Also note that decoding into an empty interface will result in no decoding, +// and so no keys will be considered decoded. +// +// In this sense, the Undecoded keys correspond to keys in the TOML document +// that do not have a concrete type in your representation. +func (md *MetaData) Undecoded() []Key { + undecoded := make([]Key, 0, len(md.keys)) + for _, key := range md.keys { + if !md.decoded[key.String()] { + undecoded = append(undecoded, key) + } + } + return undecoded +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go index 0eae37195..b940333dc 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/decode_test.go @@ -1,7 +1,6 @@ package toml import ( - "encoding/json" "fmt" "log" "reflect" @@ -13,9 +12,9 @@ func init() { log.SetFlags(0) } -var testSimple = ` +func TestDecodeSimple(t *testing.T) { + var testSimple = ` age = 250 - andrew = "gallant" kait = "brady" now = 1987-07-05T05:45:00Z @@ -26,46 +25,60 @@ colors = [ ["cyan", "magenta", "yellow", "black"], ] -[Annoying.Cats] -plato = "smelly" -cauchy = "stupido" - +[My.Cats] +plato = "cat 1" +cauchy = "cat 2" ` -type kitties struct { - Plato string - Cauchy string -} - -type simple struct { - Age int - Colors [][]string - Pi float64 - YesOrNo bool - Now time.Time - Andrew string - Kait string - Annoying map[string]kitties -} + type cats struct { + Plato string + Cauchy string + } + type simple struct { + Age int + Colors [][]string + Pi float64 + YesOrNo bool + Now time.Time + Andrew string + Kait string + My map[string]cats + } -func TestDecode(t *testing.T) { var val simple - - md, err := Decode(testSimple, &val) + _, err := Decode(testSimple, &val) if err != nil { t.Fatal(err) } - testf("Is 'Annoying.Cats.plato' defined? %v\n", - md.IsDefined("Annoying", "Cats", "plato")) - testf("Is 'Cats.Stinky' defined? %v\n", md.IsDefined("Cats", "Stinky")) - testf("Type of 'colors'? %s\n\n", md.Type("colors")) - - testf("%v\n", val) + now, err := time.Parse("2006-01-02T15:04:05", "1987-07-05T05:45:00") + if err != nil { + panic(err) + } + var answer = simple{ + Age: 250, + Andrew: "gallant", + Kait: "brady", + Now: now, + YesOrNo: true, + Pi: 3.14, + Colors: [][]string{ + {"red", "green", "blue"}, + {"cyan", "magenta", "yellow", "black"}, + }, + My: map[string]cats{ + "Cats": cats{Plato: "cat 1", Cauchy: "cat 2"}, + }, + } + if !reflect.DeepEqual(val, answer) { + t.Fatalf("Expected\n-----\n%#v\n-----\nbut got\n-----\n%#v\n", + answer, val) + } } func TestDecodeEmbedded(t *testing.T) { type Dog struct{ Name string } + type Age int tests := map[string]struct { input string @@ -87,6 +100,11 @@ func TestDecodeEmbedded(t *testing.T) { decodeInto: &struct{ *Dog }{}, wantDecoded: &struct{ *Dog }{nil}, }, + "embedded int": { + input: `Age = -5`, + decodeInto: &struct{ Age }{}, + wantDecoded: &struct{ Age }{-5}, + }, } for label, test := range tests { @@ -94,24 +112,15 @@ func TestDecodeEmbedded(t *testing.T) { if err != nil { t.Fatal(err) } - - want, got := jsonstr(test.wantDecoded), jsonstr(test.decodeInto) - if want != got { - t.Errorf("%s: want decoded == %+v, got %+v", label, want, got) + if !reflect.DeepEqual(test.wantDecoded, test.decodeInto) { + t.Errorf("%s: want decoded == %+v, got %+v", + label, test.wantDecoded, test.decodeInto) } } } -// jsonstr allows comparison of deeply nested structs with pointer members. -func jsonstr(o interface{}) string { - s, err := json.MarshalIndent(o, "", " ") - if err != nil { - panic(err.Error()) - } - return string(s) -} - -var tomlTableArrays = ` +func TestTableArrays(t *testing.T) { + var tomlTableArrays = ` [[albums]] name = "Born to Run" @@ -131,20 +140,19 @@ name = "Born in the USA" name = "Dancing in the Dark" ` -type Music struct { - Albums []Album -} + type Song struct { + Name string + } -type Album struct { - Name string - Songs []Song -} + type Album struct { + Name string + Songs []Song + } -type Song struct { - Name string -} + type Music struct { + Albums []Album + } -func TestTableArrays(t *testing.T) { expected := Music{[]Album{ {"Born to Run", []Song{{"Jungleland"}, {"Meeting Across the River"}}}, {"Born in the USA", []Song{{"Glory Days"}, {"Dancing in the Dark"}}}, @@ -163,7 +171,8 @@ func TestTableArrays(t *testing.T) { // but implementations change. // Probably still missing demonstrations of some ugly corner cases regarding // case insensitive matching and multiple fields. -var caseToml = ` +func TestCase(t *testing.T) { + var caseToml = ` tOpString = "string" tOpInt = 1 tOpFloat = 1.1 @@ -177,29 +186,28 @@ once = "just once" nEstedString = "another string" ` -type Insensitive struct { - TopString string - TopInt int - TopFloat float64 - TopBool bool - TopDate time.Time - TopArray []string - Match string - MatcH string - Once string - OncE string - Nest InsensitiveNest -} + type InsensitiveEd struct { + NestedString string + } -type InsensitiveNest struct { - Ed InsensitiveEd -} + type InsensitiveNest struct { + Ed InsensitiveEd + } -type InsensitiveEd struct { - NestedString string -} + type Insensitive struct { + TopString string + TopInt int + TopFloat float64 + TopBool bool + TopDate time.Time + TopArray []string + Match string + MatcH string + Once string + OncE string + Nest InsensitiveNest + } -func TestCase(t *testing.T) { tme, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5]) if err != nil { panic(err) @@ -220,8 +228,7 @@ func TestCase(t *testing.T) { }, } var got Insensitive - _, err = Decode(caseToml, &got) - if err != nil { + if _, err := Decode(caseToml, &got); err != nil { t.Fatal(err) } if !reflect.DeepEqual(expected, got) { @@ -283,19 +290,21 @@ type sphere struct { Radius float64 } -func TestDecodeArrays(t *testing.T) { +func TestDecodeSimpleArray(t *testing.T) { var s1 sphere if _, err := Decode(`center = [0.0, 1.5, 0.0]`, &s1); err != nil { t.Fatal(err) } +} - var s2 sphere - if _, err := Decode(`center = [0.1, 2.3]`, &s2); err == nil { +func TestDecodeArrayWrongSize(t *testing.T) { + var s1 sphere + if _, err := Decode(`center = [0.1, 2.3]`, &s1); err == nil { t.Fatal("Expected array type mismatch error") } } -func TestDecodeSmallInt(t *testing.T) { +func TestDecodeLargeIntoSmallInt(t *testing.T) { type table struct { Value int8 } @@ -305,7 +314,42 @@ func TestDecodeSmallInt(t *testing.T) { } } -func ExamplePrimitiveDecode() { +func TestDecodeSizedInts(t *testing.T) { + type table struct { + U8 uint8 + U16 uint16 + U32 uint32 + U64 uint64 + U uint + I8 int8 + I16 int16 + I32 int32 + I64 int64 + I int + } + answer := table{1, 1, 1, 1, 1, -1, -1, -1, -1, -1} + toml := ` + u8 = 1 + u16 = 1 + u32 = 1 + u64 = 1 + u = 1 + i8 = -1 + i16 = -1 + i32 = -1 + i64 = -1 + i = -1 + ` + var tab table + if _, err := Decode(toml, &tab); err != nil { + t.Fatal(err.Error()) + } + if answer != tab { + t.Fatalf("Expected %#v but got %#v", answer, tab) + } +} + +func ExampleMetaData_PrimitiveDecode() { var md MetaData var err error @@ -325,7 +369,6 @@ albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"] Started int Albums []string } - type classics struct { Ranking []string Bands map[string]Primitive @@ -348,16 +391,20 @@ albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"] primValue := music.Bands[artist] var aBand band - if err = PrimitiveDecode(primValue, &aBand); err != nil { + if err = md.PrimitiveDecode(primValue, &aBand); err != nil { log.Fatal(err) } fmt.Printf("%s started in %d.\n", artist, aBand.Started) } + // Check to see if there were any fields left undecoded. + // Note that this won't be empty before decoding the Primitive value! + fmt.Printf("Undecoded: %q\n", md.Undecoded()) // Output: // Is `bands.Springsteen` defined? true // Springsteen started in 1973. // J Geils started in 1970. + // Undecoded: [] } func ExampleDecode() { @@ -423,8 +470,9 @@ func (d *duration) UnmarshalText(text []byte) error { return err } -// Example Unmarshaler blah blah. -func ExampleUnmarshaler() { +// Example Unmarshaler shows how to decode TOML strings into your own +// custom data type. +func Example_unmarshaler() { blob := ` [[song]] name = "Thunder Road" @@ -446,6 +494,18 @@ duration = "8m03s" log.Fatal(err) } + // Code to implement the TextUnmarshaler interface for `duration`: + // + // type duration struct { + // time.Duration + // } + // + // func (d *duration) UnmarshalText(text []byte) error { + // var err error + // d.Duration, err = time.ParseDuration(string(text)) + // return err + // } + for _, s := range favorites.Song { fmt.Printf("%s (%s)\n", s.Name, s.Duration) } @@ -453,3 +513,28 @@ duration = "8m03s" // Thunder Road (4m49s) // Stairway to Heaven (8m3s) } + +// Example StrictDecoding shows how to detect whether there are keys in the +// TOML document that weren't decoded into the value given. This is useful +// for returning an error to the user if they've included extraneous fields +// in their configuration. +func Example_strictDecoding() { + var blob = ` +key1 = "value1" +key2 = "value2" +key3 = "value3" +` + type config struct { + Key1 string + Key3 string + } + + var conf config + md, err := Decode(blob, &conf) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Undecoded keys: %q\n", md.Undecoded()) + // Output: + // Undecoded keys: ["key2"] +} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go index 1c2d7dffc..fe2680004 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/doc.go @@ -1,10 +1,27 @@ /* -Package toml provides facilities for decoding TOML configuration files -via reflection. +Package toml provides facilities for decoding and encoding TOML configuration +files via reflection. There is also support for delaying decoding with +the Primitive type, and querying the set of keys in a TOML document with the +MetaData type. -Specification: https://github.com/mojombo/toml +The specification implemented: https://github.com/mojombo/toml -Use github.com/BurntSushi/toml/tomlv to check whether a file is valid -TOML or not, with helpful error messages. +The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify +whether a file is a valid TOML document. It can also be used to print the +type of each key in a TOML document. + +Testing + +There are two important types of tests used for this package. The first is +contained inside '*_test.go' files and uses the standard Go unit testing +framework. These tests are primarily devoted to holistically testing the +decoder and encoder. + +The second type of testing is used to verify the implementation's adherence +to the TOML specification. These tests have been factored into their own +project: https://github.com/BurntSushi/toml-test + +The reason the tests are in a separate project is so that they can be used by +any implementation of TOML. Namely, it is language agnostic. */ package toml diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go index bada06a6e..361871347 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode.go @@ -12,23 +12,47 @@ import ( "time" ) +type tomlEncodeError struct{ error } + var ( - ErrArrayMixedElementTypes = errors.New( + errArrayMixedElementTypes = errors.New( "can't encode array with mixed element types") - ErrArrayNilElement = errors.New( + errArrayNilElement = errors.New( "can't encode array with nil element") + errNonString = errors.New( + "can't encode a map with non-string key type") + errAnonNonStruct = errors.New( + "can't encode an anonymous field that is not a struct") + errArrayNoTable = errors.New( + "TOML array element can't contain a table") + errNoKey = errors.New( + "top-level values must be a Go map or struct") + errAnything = errors.New("") // used in testing +) + +var quotedReplacer = strings.NewReplacer( + "\t", "\\t", + "\n", "\\n", + "\r", "\\r", + "\"", "\\\"", + "\\", "\\\\", ) +// Encoder controls the encoding of Go values to a TOML document to some +// io.Writer. +// +// The indentation level can be controlled with the Indent field. type Encoder struct { // A single indentation level. By default it is two spaces. Indent string - w *bufio.Writer - // hasWritten is whether we have written any output to w yet. hasWritten bool + w *bufio.Writer } +// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer +// given. By default, a single indentation level is 2 spaces. func NewEncoder(w io.Writer) *Encoder { return &Encoder{ w: bufio.NewWriter(w), @@ -36,44 +60,60 @@ func NewEncoder(w io.Writer) *Encoder { } } +// Encode writes a TOML representation of the Go value to the underlying +// io.Writer. If the value given cannot be encoded to a valid TOML document, +// then an error is returned. +// +// The mapping between Go values and TOML values should be precisely the same +// as for the Decode* functions. Similarly, the TextMarshaler interface is +// supported by encoding the resulting bytes as strings. (If you want to write +// arbitrary binary data then you will need to use something like base64 since +// TOML does not have any binary types.) +// +// When encoding TOML hashes (i.e., Go maps or structs), keys without any +// sub-hashes are encoded first. +// +// If a Go map is encoded, then its keys are sorted alphabetically for +// deterministic output. More control over this behavior may be provided if +// there is demand for it. +// +// Encoding Go values without a corresponding TOML representation---like map +// types with non-string keys---will cause an error to be returned. Similarly +// for mixed arrays/slices, arrays/slices with nil elements, embedded +// non-struct types and nested slices containing maps or structs. +// (e.g., [][]map[string]string is not allowed but []map[string]string is OK +// and so is []map[string][]string.) func (enc *Encoder) Encode(v interface{}) error { rv := eindirect(reflect.ValueOf(v)) - if err := enc.encode(Key([]string{}), rv); err != nil { + if err := enc.safeEncode(Key([]string{}), rv); err != nil { return err } return enc.w.Flush() } -func (enc *Encoder) encode(key Key, rv reflect.Value) error { - // Extra special case. Time needs to be in ISO8601 format. - switch rv.Interface().(type) { - case time.Time: - if enc.hasWritten { - _, err := enc.w.Write([]byte{'\n'}) - if err != nil { - return err +func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { + defer func() { + if r := recover(); r != nil { + if terr, ok := r.(tomlEncodeError); ok { + err = terr.error + return } + panic(r) } - err := enc.eKeyEq(key) - if err != nil { - return err - } - return enc.eElement(rv) - } + }() + enc.encode(key, rv) + return nil +} +func (enc *Encoder) encode(key Key, rv reflect.Value) { + // Special case. Time needs to be in ISO8601 format. // Special case. If we can marshal the type to text, then we used that. - if _, ok := rv.Interface().(TextMarshaler); ok { - if enc.hasWritten { - _, err := enc.w.Write([]byte{'\n'}) - if err != nil { - return err - } - } - err := enc.eKeyEq(key) - if err != nil { - return err - } - return enc.eElement(rv) + // Basically, this prevents the encoder for handling these types as + // generic structs (or whatever the underlying type of a TextMarshaler is). + switch rv.Interface().(type) { + case time.Time, TextMarshaler: + enc.keyEqElement(key, rv) + return } k := rv.Kind() @@ -81,244 +121,150 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) error { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64, - reflect.String, reflect.Bool: - err := enc.eKeyEq(key) - if err != nil { - return err - } - return enc.eElement(rv) + reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: + enc.keyEqElement(key, rv) case reflect.Array, reflect.Slice: - return enc.eArrayOrSlice(key, rv) + if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { + enc.eArrayOfTables(key, rv) + } else { + enc.keyEqElement(key, rv) + } case reflect.Interface: if rv.IsNil() { - return nil + return } - return enc.encode(key, rv.Elem()) + enc.encode(key, rv.Elem()) case reflect.Map: if rv.IsNil() { - return nil + return } - return enc.eTable(key, rv) + enc.eTable(key, rv) case reflect.Ptr: if rv.IsNil() { - return nil + return } - return enc.encode(key, rv.Elem()) + enc.encode(key, rv.Elem()) case reflect.Struct: - return enc.eTable(key, rv) + enc.eTable(key, rv) + default: + panic(e("Unsupported type for key '%s': %s", key, k)) } - return e("Unsupported type for key '%s': %s", key, k) } // eElement encodes any value that can be an array element (primitives and // arrays). -func (enc *Encoder) eElement(rv reflect.Value) error { - ws := func(s string) error { - _, err := io.WriteString(enc.w, s) - return err - } - // By the TOML spec, all floats must have a decimal with at least one - // number on either side. - floatAddDecimal := func(fstr string) string { - if !strings.Contains(fstr, ".") { - return fstr + ".0" - } - return fstr - } - - // Extra special case. Time needs to be in ISO8601 format. - switch s := rv.Interface().(type) { +func (enc *Encoder) eElement(rv reflect.Value) { + switch v := rv.Interface().(type) { case time.Time: - return ws(s.UTC().Format("2006-01-02T03:04:05Z")) - } - - // Special case. Use text marshaler if it's available for this value. - if v, ok := rv.Interface().(TextMarshaler); ok { - s, err := v.MarshalText() - if err != nil { - return err + // Special case time.Time as a primitive. Has to come before + // TextMarshaler below because time.Time implements + // encoding.TextMarshaler, but we need to always use UTC. + enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) + return + case TextMarshaler: + // Special case. Use text marshaler if it's available for this value. + if s, err := v.MarshalText(); err != nil { + encPanic(err) + } else { + enc.writeQuoted(string(s)) } - return ws(string(s)) + return } - - var err error - k := rv.Kind() - switch k { + switch rv.Kind() { case reflect.Bool: - err = ws(strconv.FormatBool(rv.Bool())) + enc.wf(strconv.FormatBool(rv.Bool())) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - err = ws(strconv.FormatInt(rv.Int(), 10)) + enc.wf(strconv.FormatInt(rv.Int(), 10)) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - err = ws(strconv.FormatUint(rv.Uint(), 10)) + enc.wf(strconv.FormatUint(rv.Uint(), 10)) case reflect.Float32: - err = ws(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) case reflect.Float64: - err = ws(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) case reflect.Array, reflect.Slice: - return enc.eArrayOrSliceElement(rv) + enc.eArrayOrSliceElement(rv) case reflect.Interface: - return enc.eElement(rv.Elem()) + enc.eElement(rv.Elem()) case reflect.String: - s := rv.String() - s = strings.NewReplacer( - "\t", "\\t", - "\n", "\\n", - "\r", "\\r", - "\"", "\\\"", - "\\", "\\\\", - ).Replace(s) - err = ws("\"" + s + "\"") + enc.writeQuoted(rv.String()) default: - return e("Unexpected primitive type: %s", k) + panic(e("Unexpected primitive type: %s", rv.Kind())) } - return err } -func (enc *Encoder) eArrayOrSlice(key Key, rv reflect.Value) error { - // Determine whether this is an array of tables or of primitives. - elemV := reflect.ValueOf(nil) - if rv.Len() > 0 { - elemV = rv.Index(0) - } - isTableType, err := isTOMLTableType(rv.Type().Elem(), elemV) - if err != nil { - return err +// By the TOML spec, all floats must have a decimal with at least one +// number on either side. +func floatAddDecimal(fstr string) string { + if !strings.Contains(fstr, ".") { + return fstr + ".0" } - - if len(key) > 0 && isTableType { - return enc.eArrayOfTables(key, rv) - } - - err = enc.eKeyEq(key) - if err != nil { - return err - } - return enc.eArrayOrSliceElement(rv) + return fstr } -func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) error { - if _, err := enc.w.Write([]byte{'['}); err != nil { - return err - } +func (enc *Encoder) writeQuoted(s string) { + enc.wf("\"%s\"", quotedReplacer.Replace(s)) +} +func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { length := rv.Len() - if length > 0 { - arrayElemType, isNil := tomlTypeName(rv.Index(0)) - if isNil { - return ErrArrayNilElement - } - - for i := 0; i < length; i++ { - elem := rv.Index(i) - - // Ensure that the array's elements each have the same TOML type. - elemType, isNil := tomlTypeName(elem) - if isNil { - return ErrArrayNilElement - } - if elemType != arrayElemType { - return ErrArrayMixedElementTypes - } - - if err := enc.eElement(elem); err != nil { - return err - } - if i != length-1 { - if _, err := enc.w.Write([]byte(", ")); err != nil { - return err - } - } + enc.wf("[") + for i := 0; i < length; i++ { + elem := rv.Index(i) + enc.eElement(elem) + if i != length-1 { + enc.wf(", ") } } - - if _, err := enc.w.Write([]byte{']'}); err != nil { - return err - } - return nil + enc.wf("]") } -func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) error { - if enc.hasWritten { - _, err := enc.w.Write([]byte{'\n'}) - if err != nil { - return err - } +func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) } - + panicIfInvalidKey(key, true) for i := 0; i < rv.Len(); i++ { trv := rv.Index(i) if isNil(trv) { continue } - - _, err := fmt.Fprintf(enc.w, "%s[[%s]]\n", - strings.Repeat(enc.Indent, len(key)-1), key.String()) - if err != nil { - return err - } - - err = enc.eMapOrStruct(key, trv) - if err != nil { - return err - } - - if i != rv.Len()-1 { - if _, err := enc.w.Write([]byte("\n\n")); err != nil { - return err - } - } - enc.hasWritten = true + enc.newline() + enc.wf("%s[[%s]]", enc.indentStr(key), key.String()) + enc.newline() + enc.eMapOrStruct(key, trv) } - return nil } -func isStructOrMap(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.Interface, reflect.Ptr: - return isStructOrMap(rv.Elem()) - case reflect.Map, reflect.Struct: - return true - default: - return false - } -} - -func (enc *Encoder) eTable(key Key, rv reflect.Value) error { - if enc.hasWritten { - _, err := enc.w.Write([]byte{'\n'}) - if err != nil { - return err - } +func (enc *Encoder) eTable(key Key, rv reflect.Value) { + if len(key) == 1 { + // Output an extra new line between top-level tables. + // (The newline isn't written if nothing else has been written though.) + enc.newline() } if len(key) > 0 { - _, err := fmt.Fprintf(enc.w, "%s[%s]\n", - strings.Repeat(enc.Indent, len(key)-1), key.String()) - if err != nil { - return err - } + panicIfInvalidKey(key, true) + enc.wf("%s[%s]", enc.indentStr(key), key.String()) + enc.newline() } - return enc.eMapOrStruct(key, rv) + enc.eMapOrStruct(key, rv) } -func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) error { - switch rv.Kind() { +func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { + switch rv := eindirect(rv); rv.Kind() { case reflect.Map: - return enc.eMap(key, rv) + enc.eMap(key, rv) case reflect.Struct: - return enc.eStruct(key, rv) - case reflect.Ptr, reflect.Interface: - return enc.eMapOrStruct(key, rv.Elem()) + enc.eStruct(key, rv) default: panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) } } -func (enc *Encoder) eMap(key Key, rv reflect.Value) error { +func (enc *Encoder) eMap(key Key, rv reflect.Value) { rt := rv.Type() if rt.Key().Kind() != reflect.String { - return errors.New("can't encode a map with non-string key type") + encPanic(errNonString) } // Sort keys so that we have deterministic output. And write keys directly @@ -326,86 +272,61 @@ func (enc *Encoder) eMap(key Key, rv reflect.Value) error { var mapKeysDirect, mapKeysSub []string for _, mapKey := range rv.MapKeys() { k := mapKey.String() - mrv := rv.MapIndex(mapKey) - if isStructOrMap(mrv) { + if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { mapKeysSub = append(mapKeysSub, k) } else { mapKeysDirect = append(mapKeysDirect, k) } } - var writeMapKeys = func(mapKeys []string) error { + var writeMapKeys = func(mapKeys []string) { sort.Strings(mapKeys) - for i, mapKey := range mapKeys { + for _, mapKey := range mapKeys { mrv := rv.MapIndex(reflect.ValueOf(mapKey)) if isNil(mrv) { // Don't write anything for nil fields. continue } - if err := enc.encode(key.add(mapKey), mrv); err != nil { - return err - } - - if i != len(mapKeys)-1 { - if _, err := enc.w.Write([]byte{'\n'}); err != nil { - return err - } - } - enc.hasWritten = true + enc.encode(key.add(mapKey), mrv) } - - return nil - } - - err := writeMapKeys(mapKeysDirect) - if err != nil { - return err } - err = writeMapKeys(mapKeysSub) - if err != nil { - return err - } - return nil + writeMapKeys(mapKeysDirect) + writeMapKeys(mapKeysSub) } -func (enc *Encoder) eStruct(key Key, rv reflect.Value) error { +func (enc *Encoder) eStruct(key Key, rv reflect.Value) { // Write keys for fields directly under this key first, because if we write // a field that creates a new table, then all keys under it will be in that // table (not the one we're writing here). rt := rv.Type() var fieldsDirect, fieldsSub [][]int - var addFields func(rt reflect.Type, rv reflect.Value, start []int) error - addFields = func(rt reflect.Type, rv reflect.Value, start []int) error { + var addFields func(rt reflect.Type, rv reflect.Value, start []int) + addFields = func(rt reflect.Type, rv reflect.Value, start []int) { for i := 0; i < rt.NumField(); i++ { f := rt.Field(i) + // skip unexporded fields + if f.PkgPath != "" { + continue + } frv := rv.Field(i) if f.Anonymous { + frv := eindirect(frv) t := frv.Type() - if t.Kind() == reflect.Ptr { - t = t.Elem() - frv = frv.Elem() - } if t.Kind() != reflect.Struct { - return errors.New( - "can't encode an anonymous field that is not a struct") + encPanic(errAnonNonStruct) } - if err := addFields(t, frv, f.Index); err != nil { - return err - } - } else if isStructOrMap(frv) { + addFields(t, frv, f.Index) + } else if typeIsHash(tomlTypeOfGo(frv)) { fieldsSub = append(fieldsSub, append(start, f.Index...)) } else { fieldsDirect = append(fieldsDirect, append(start, f.Index...)) } } - return nil - } - if err := addFields(rt, rv, nil); err != nil { - return err } + addFields(rt, rv, nil) - var writeFields = func(fields [][]int) error { - for i, fieldIndex := range fields { + var writeFields = func(fields [][]int) { + for _, fieldIndex := range fields { sft := rt.FieldByIndex(fieldIndex) sf := rv.FieldByIndex(fieldIndex) if isNil(sf) { @@ -420,111 +341,132 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) error { if keyName == "" { keyName = sft.Name } - - if err := enc.encode(key.add(keyName), sf); err != nil { - return err - } - - if i != len(fields)-1 { - if _, err := enc.w.Write([]byte{'\n'}); err != nil { - return err - } - } - enc.hasWritten = true - } - return nil - } - - err := writeFields(fieldsDirect) - if err != nil { - return err - } - if len(fieldsDirect) > 0 && len(fieldsSub) > 0 { - _, err = enc.w.Write([]byte{'\n'}) - if err != nil { - return err + enc.encode(key.add(keyName), sf) } } - err = writeFields(fieldsSub) - if err != nil { - return err - } - return nil + writeFields(fieldsDirect) + writeFields(fieldsSub) } // tomlTypeName returns the TOML type name of the Go value's type. It is used to // determine whether the types of array elements are mixed (which is forbidden). // If the Go value is nil, then it is illegal for it to be an array element, and // valueIsNil is returned as true. -func tomlTypeName(rv reflect.Value) (typeName string, valueIsNil bool) { - if isNil(rv) { - return "", true + +// Returns the TOML type of a Go value. The type may be `nil`, which means +// no concrete TOML type could be found. +func tomlTypeOfGo(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() { + return nil } - k := rv.Kind() - switch k { + switch rv.Kind() { case reflect.Bool: - return "bool", false + return tomlBool case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return "integer", false + return tomlInteger case reflect.Float32, reflect.Float64: - return "float", false + return tomlFloat case reflect.Array, reflect.Slice: - return "array", false + if typeEqual(tomlHash, tomlArrayType(rv)) { + return tomlArrayHash + } else { + return tomlArray + } case reflect.Ptr, reflect.Interface: - return tomlTypeName(rv.Elem()) + return tomlTypeOfGo(rv.Elem()) case reflect.String: - return "string", false - case reflect.Map, reflect.Struct: - return "table", false + return tomlString + case reflect.Map: + return tomlHash + case reflect.Struct: + switch rv.Interface().(type) { + case time.Time: + return tomlDatetime + case TextMarshaler: + return tomlString + default: + return tomlHash + } default: - panic("unexpected reflect.Kind: " + k.String()) + panic("unexpected reflect.Kind: " + rv.Kind().String()) } } -// isTOMLTableType returns whether this type and value represents a TOML table -// type (true) or element type (false). Both rt and rv are needed to determine -// this, in case the Go type is interface{} or in case rv is nil. If there is -// some other impossible situation detected, an error is returned. -func isTOMLTableType(rt reflect.Type, rv reflect.Value) (bool, error) { - k := rt.Kind() - switch k { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, - reflect.Uint64, - reflect.Float32, reflect.Float64, - reflect.String, reflect.Bool: - return false, nil - case reflect.Array, reflect.Slice: - // Make sure that these eventually contain an underlying non-table type - // element. - elemV := reflect.ValueOf(nil) - if rv.Len() > 0 { - elemV = rv.Index(0) - } - hasUnderlyingTableType, err := isTOMLTableType(rt.Elem(), elemV) - if err != nil { - return false, err - } - if hasUnderlyingTableType { - return true, errors.New("TOML array element can't contain a table") - } - return false, nil - case reflect.Ptr: - return isTOMLTableType(rt.Elem(), rv.Elem()) - case reflect.Interface: - if rv.Kind() == reflect.Interface { - return false, nil +// tomlArrayType returns the element type of a TOML array. The type returned +// may be nil if it cannot be determined (e.g., a nil slice or a zero length +// slize). This function may also panic if it finds a type that cannot be +// expressed in TOML (such as nil elements, heterogeneous arrays or directly +// nested arrays of tables). +func tomlArrayType(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { + return nil + } + firstType := tomlTypeOfGo(rv.Index(0)) + if firstType == nil { + encPanic(errArrayNilElement) + } + + rvlen := rv.Len() + for i := 1; i < rvlen; i++ { + elem := rv.Index(i) + switch elemType := tomlTypeOfGo(elem); { + case elemType == nil: + encPanic(errArrayNilElement) + case !typeEqual(firstType, elemType): + encPanic(errArrayMixedElementTypes) } - if rv.IsValid() { - return isTOMLTableType(rv.Type(), rv) + } + // If we have a nested array, then we must make sure that the nested + // array contains ONLY primitives. + // This checks arbitrarily nested arrays. + if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { + nest := tomlArrayType(eindirect(rv.Index(0))) + if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { + encPanic(errArrayNoTable) } - return false, nil - case reflect.Map, reflect.Struct: - return true, nil + } + return firstType +} + +func (enc *Encoder) newline() { + if enc.hasWritten { + enc.wf("\n") + } +} + +func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + panicIfInvalidKey(key, false) + enc.wf("%s%s = ", enc.indentStr(key), key[len(key)-1]) + enc.eElement(val) + enc.newline() +} + +func (enc *Encoder) wf(format string, v ...interface{}) { + if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { + encPanic(err) + } + enc.hasWritten = true +} + +func (enc *Encoder) indentStr(key Key) string { + return strings.Repeat(enc.Indent, len(key)-1) +} + +func encPanic(err error) { + panic(tomlEncodeError{err}) +} + +func eindirect(v reflect.Value) reflect.Value { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + return eindirect(v.Elem()) default: - panic("unexpected reflect.Kind: " + k.String()) + return v } } @@ -537,21 +479,37 @@ func isNil(rv reflect.Value) bool { } } -func (enc *Encoder) eKeyEq(key Key) error { - _, err := io.WriteString(enc.w, strings.Repeat(enc.Indent, len(key)-1)) - if err != nil { - return err +func panicIfInvalidKey(key Key, hash bool) { + if hash { + for _, k := range key { + if !isValidTableName(k) { + encPanic(e("Key '%s' is not a valid table name. Table names "+ + "cannot contain '[', ']' or '.'.", key.String())) + } + } + } else { + if !isValidKeyName(key[len(key)-1]) { + encPanic(e("Key '%s' is not a name. Key names "+ + "cannot contain whitespace.", key.String())) + } } - _, err = io.WriteString(enc.w, key[len(key)-1]+" = ") - if err != nil { - return err +} + +func isValidTableName(s string) bool { + if len(s) == 0 { + return false } - return nil + for _, r := range s { + if r == '[' || r == ']' || r == '.' { + return false + } + } + return true } -func eindirect(v reflect.Value) reflect.Value { - if v.Kind() != reflect.Ptr { - return v +func isValidKeyName(s string) bool { + if len(s) == 0 { + return false } - return eindirect(reflect.Indirect(v)) + return true } diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go index 27cea0fdf..74a5ee5d2 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encode_test.go @@ -2,13 +2,72 @@ package toml import ( "bytes" + "fmt" + "log" + "net" "testing" + "time" ) +func TestEncodeRoundTrip(t *testing.T) { + type Config struct { + Age int + Cats []string + Pi float64 + Perfection []int + DOB time.Time + Ipaddress net.IP + } + + var inputs = Config{ + 13, + []string{"one", "two", "three"}, + 3.145, + []int{11, 2, 3, 4}, + time.Now(), + net.ParseIP("192.168.59.254"), + } + + var firstBuffer bytes.Buffer + e := NewEncoder(&firstBuffer) + err := e.Encode(inputs) + if err != nil { + t.Fatal(err) + } + var outputs Config + if _, err := Decode(firstBuffer.String(), &outputs); err != nil { + log.Printf("Could not decode:\n-----\n%s\n-----\n", + firstBuffer.String()) + t.Fatal(err) + } + + // could test each value individually, but I'm lazy + var secondBuffer bytes.Buffer + e2 := NewEncoder(&secondBuffer) + err = e2.Encode(outputs) + if err != nil { + t.Fatal(err) + } + if firstBuffer.String() != secondBuffer.String() { + t.Error( + firstBuffer.String(), + "\n\n is not identical to\n\n", + secondBuffer.String()) + } +} + // XXX(burntsushi) // I think these tests probably should be removed. They are good, but they // ought to be obsolete by toml-test. func TestEncode(t *testing.T) { + type Embedded struct { + Int int `toml:"_int"` + } + type NonStruct int + + date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600)) + dateStr := "2014-05-11T19:30:40Z" + tests := map[string]struct { input interface{} wantOutput string @@ -19,7 +78,7 @@ func TestEncode(t *testing.T) { BoolTrue bool BoolFalse bool }{true, false}, - wantOutput: "BoolTrue = true\nBoolFalse = false", + wantOutput: "BoolTrue = true\nBoolFalse = false\n", }, "int fields": { input: struct { @@ -29,7 +88,7 @@ func TestEncode(t *testing.T) { Int32 int32 Int64 int64 }{1, 2, 3, 4, 5}, - wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5", + wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n", }, "uint fields": { input: struct { @@ -40,31 +99,58 @@ func TestEncode(t *testing.T) { Uint64 uint64 }{1, 2, 3, 4, 5}, wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" + - "\nUint64 = 5", + "\nUint64 = 5\n", }, "float fields": { input: struct { Float32 float32 Float64 float64 }{1.5, 2.5}, - wantOutput: "Float32 = 1.5\nFloat64 = 2.5", + wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n", }, "string field": { input: struct{ String string }{"foo"}, - wantOutput: `String = "foo"`, + wantOutput: "String = \"foo\"\n", + }, + "string field and unexported field": { + input: struct { + String string + unexported int + }{"foo", 0}, + wantOutput: "String = \"foo\"\n", + }, + "datetime field in UTC": { + input: struct{ Date time.Time }{date}, + wantOutput: fmt.Sprintf("Date = %s\n", dateStr), + }, + "datetime field as primitive": { + // Using a map here to fail if isStructOrMap() returns true for + // time.Time. + input: map[string]interface{}{ + "Date": date, + "Int": 1, + }, + wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr), }, "array fields": { input: struct { IntArray0 [0]int IntArray3 [3]int }{[0]int{}, [3]int{1, 2, 3}}, - wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]", + wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n", }, "slice fields": { input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{ nil, []int{}, []int{1, 2, 3}, }, - wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]", + wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n", + }, + "datetime slices": { + input: struct{ DatetimeSlice []time.Time }{ + []time.Time{date, date}, + }, + wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n", + dateStr, dateStr), }, "nested arrays and slices": { input: struct { @@ -105,15 +191,20 @@ ArrayOfSlices = [[1, 2], [3, 4]] SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] SliceOfMixedArrays = [[1, 2], ["a", "b"]] -ArrayOfMixedSlices = [[1, 2], ["a", "b"]]`, +ArrayOfMixedSlices = [[1, 2], ["a", "b"]] +`, + }, + "empty slice": { + input: struct{ Empty []interface{} }{[]interface{}{}}, + wantOutput: "Empty = []\n", }, "(error) slice with element type mismatch (string and integer)": { input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}}, - wantError: ErrArrayMixedElementTypes, + wantError: errArrayMixedElementTypes, }, "(error) slice with element type mismatch (integer and float)": { input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}}, - wantError: ErrArrayMixedElementTypes, + wantError: errArrayMixedElementTypes, }, "slice with elems of differing Go types, same TOML types": { input: struct { @@ -127,64 +218,64 @@ ArrayOfMixedSlices = [[1, 2], ["a", "b"]]`, []interface{}{float32(1.5), float64(2.5)}, }, wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" + - "MixedFloats = [1.5, 2.5]", + "MixedFloats = [1.5, 2.5]\n", }, "(error) slice w/ element type mismatch (one is nested array)": { input: struct{ Mixed []interface{} }{ []interface{}{1, []interface{}{2}}, }, - wantError: ErrArrayMixedElementTypes, + wantError: errArrayMixedElementTypes, }, "(error) slice with 1 nil element": { input: struct{ NilElement1 []interface{} }{[]interface{}{nil}}, - wantError: ErrArrayNilElement, + wantError: errArrayNilElement, }, "(error) slice with 1 nil element (and other non-nil elements)": { input: struct{ NilElement []interface{} }{ []interface{}{1, nil}, }, - wantError: ErrArrayNilElement, + wantError: errArrayNilElement, }, "simple map": { input: map[string]int{"a": 1, "b": 2}, - wantOutput: "a = 1\nb = 2", + wantOutput: "a = 1\nb = 2\n", }, "map with interface{} value type": { input: map[string]interface{}{"a": 1, "b": "c"}, - wantOutput: "a = 1\nb = \"c\"", + wantOutput: "a = 1\nb = \"c\"\n", }, "map with interface{} value type, some of which are structs": { input: map[string]interface{}{ "a": struct{ Int int }{2}, "b": 1, }, - wantOutput: "b = 1\n[a]\n Int = 2", + wantOutput: "b = 1\n\n[a]\n Int = 2\n", }, "nested map": { input: map[string]map[string]int{ "a": {"b": 1}, "c": {"d": 2}, }, - wantOutput: "[a]\n b = 1\n\n[c]\n d = 2", + wantOutput: "[a]\n b = 1\n\n[c]\n d = 2\n", }, "nested struct": { input: struct{ Struct struct{ Int int } }{ struct{ Int int }{1}, }, - wantOutput: "[Struct]\n Int = 1", + wantOutput: "[Struct]\n Int = 1\n", }, "nested struct and non-struct field": { input: struct { Struct struct{ Int int } Bool bool }{struct{ Int int }{1}, true}, - wantOutput: "Bool = true\n\n[Struct]\n Int = 1", + wantOutput: "Bool = true\n\n[Struct]\n Int = 1\n", }, "2 nested structs": { input: struct{ Struct1, Struct2 struct{ Int int } }{ struct{ Int int }{1}, struct{ Int int }{2}, }, - wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2", + wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2\n", }, "deeply nested structs": { input: struct { @@ -223,60 +314,193 @@ ArrayOfMixedSlices = [[1, 2], ["a", "b"]]`, Int int `toml:"_int"` }{1}, true, }, - wantOutput: "_bool = true\n\n[_struct]\n _int = 1", + wantOutput: "_bool = true\n\n[_struct]\n _int = 1\n", }, "embedded struct": { input: struct{ Embedded }{Embedded{1}}, - wantOutput: "_int = 1", + wantOutput: "_int = 1\n", }, "embedded *struct": { input: struct{ *Embedded }{&Embedded{1}}, - wantOutput: "_int = 1", + wantOutput: "_int = 1\n", }, "nested embedded struct": { input: struct { Struct struct{ Embedded } `toml:"_struct"` }{struct{ Embedded }{Embedded{1}}}, - wantOutput: "[_struct]\n _int = 1", + wantOutput: "[_struct]\n _int = 1\n", }, "nested embedded *struct": { input: struct { Struct struct{ *Embedded } `toml:"_struct"` }{struct{ *Embedded }{&Embedded{1}}}, - wantOutput: "[_struct]\n _int = 1", + wantOutput: "[_struct]\n _int = 1\n", }, "array of tables": { input: struct { Structs []*struct{ Int int } `toml:"struct"` }{ - []*struct{ Int int }{ - {1}, nil, {3}, + []*struct{ Int int }{{1}, {3}}, + }, + wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3\n", + }, + "array of tables order": { + input: map[string]interface{}{ + "map": map[string]interface{}{ + "zero": 5, + "arr": []map[string]int{ + map[string]int{ + "friend": 5, + }, + }, }, }, - wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3", + wantOutput: "[map]\n zero = 5\n\n [[map.arr]]\n friend = 5\n", + }, + "(error) top-level slice": { + input: []struct{ Int int }{{1}, {2}, {3}}, + wantError: errNoKey, + }, + "(error) slice of slice": { + input: struct { + Slices [][]struct{ Int int } + }{ + [][]struct{ Int int }{{{1}}, {{2}}, {{3}}}, + }, + wantError: errArrayNoTable, + }, + "(error) map no string key": { + input: map[int]string{1: ""}, + wantError: errNonString, + }, + "(error) anonymous non-struct": { + input: struct{ NonStruct }{5}, + wantError: errAnonNonStruct, + }, + "(error) empty key name": { + input: map[string]int{"": 1}, + wantError: errAnything, + }, + "(error) empty map name": { + input: map[string]interface{}{ + "": map[string]int{"v": 1}, + }, + wantError: errAnything, }, } for label, test := range tests { - var buf bytes.Buffer - e := NewEncoder(&buf) - err := e.Encode(test.input) - if err != test.wantError { - if test.wantError != nil { - t.Errorf("%s: want Encode error %v, got %v", - label, test.wantError, err) - } else { - t.Errorf("%s: Encode failed: %s", label, err) + encodeExpected(t, label, test.input, test.wantOutput, test.wantError) + } +} + +func TestEncodeNestedTableArrays(t *testing.T) { + type song struct { + Name string `toml:"name"` + } + type album struct { + Name string `toml:"name"` + Songs []song `toml:"songs"` + } + type springsteen struct { + Albums []album `toml:"albums"` + } + value := springsteen{ + []album{ + {"Born to Run", + []song{{"Jungleland"}, {"Meeting Across the River"}}}, + {"Born in the USA", + []song{{"Glory Days"}, {"Dancing in the Dark"}}}, + }, + } + expected := `[[albums]] + name = "Born to Run" + + [[albums.songs]] + name = "Jungleland" + + [[albums.songs]] + name = "Meeting Across the River" + +[[albums]] + name = "Born in the USA" + + [[albums.songs]] + name = "Glory Days" + + [[albums.songs]] + name = "Dancing in the Dark" +` + encodeExpected(t, "nested table arrays", value, expected, nil) +} + +func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) { + type Alpha struct { + V int + } + type Beta struct { + V int + } + type Conf struct { + V int + A Alpha + B []Beta + } + + val := Conf{ + V: 1, + A: Alpha{2}, + B: []Beta{{3}}, + } + expected := "V = 1\n\n[A]\n V = 2\n\n[[B]]\n V = 3\n" + encodeExpected(t, "array hash with normal hash order", val, expected, nil) +} + +func encodeExpected( + t *testing.T, label string, val interface{}, wantStr string, wantErr error, +) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.Encode(val) + if err != wantErr { + if wantErr != nil { + if wantErr == errAnything && err != nil { + return } + t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err) + } else { + t.Errorf("%s: Encode failed: %s", label, err) } - if err != nil { - continue - } - if got := buf.String(); test.wantOutput != got { - t.Errorf("%s: want %q, got %q", label, test.wantOutput, got) - } + } + if err != nil { + return + } + if got := buf.String(); wantStr != got { + t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n", + label, wantStr, got) } } -type Embedded struct { - Int int `toml:"_int"` +func ExampleEncoder_Encode() { + date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC") + var config = map[string]interface{}{ + "date": date, + "counts": []int{1, 1, 2, 3, 5, 8}, + "hash": map[string]string{ + "key1": "val1", + "key2": "val2", + }, + } + buf := new(bytes.Buffer) + if err := NewEncoder(buf).Encode(config); err != nil { + log.Fatal(err) + } + fmt.Println(buf.String()) + + // Output: + // counts = [1, 1, 2, 3, 5, 8] + // date = 2010-03-14T18:00:00Z + // + // [hash] + // key1 = "val1" + // key2 = "val2" } diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go index 6500c9a9f..140c44c11 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types.go @@ -2,9 +2,18 @@ package toml +// In order to support Go 1.1, we define our own TextMarshaler and +// TextUnmarshaler types. For Go 1.2+, we just alias them with the +// standard library interfaces. + import ( "encoding" ) +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. type TextMarshaler encoding.TextMarshaler + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined here +// so that Go 1.1 can be supported. type TextUnmarshaler encoding.TextUnmarshaler diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go index 6211ceafd..fb285e7f5 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/encoding_types_1.1.go @@ -2,10 +2,17 @@ package toml +// These interfaces were introduced in Go 1.2, so we add them manually when +// compiling for Go 1.1. + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. type TextMarshaler interface { MarshalText() (text []byte, err error) } +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined here +// so that Go 1.1 can be supported. type TextUnmarshaler interface { UnmarshalText(text []byte) error } diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go index 01fcb491b..3821fa271 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/lex.go @@ -2,6 +2,7 @@ package toml import ( "fmt" + "strings" "unicode/utf8" ) @@ -112,6 +113,11 @@ func (lx *lexer) emit(typ itemType) { lx.start = lx.pos } +func (lx *lexer) emitTrim(typ itemType) { + lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} + lx.start = lx.pos +} + func (lx *lexer) next() (r rune) { if lx.pos >= len(lx.input) { lx.width = 0 @@ -251,7 +257,7 @@ func lexArrayTableEnd(lx *lexer) stateFn { func lexTableNameStart(lx *lexer) stateFn { switch lx.next() { - case tableEnd: + case tableEnd, eof: return lx.errorf("Unexpected end of table. (Tables cannot " + "be empty.)") case tableSep: @@ -265,6 +271,8 @@ func lexTableNameStart(lx *lexer) stateFn { // valid character for the table has already been read. func lexTableName(lx *lexer) stateFn { switch lx.peek() { + case eof: + return lx.errorf("Unexpected end of table name %q.", lx.current()) case tableStart: return lx.errorf("Table names cannot contain %q or %q.", tableStart, tableEnd) @@ -305,19 +313,24 @@ func lexKeyStart(lx *lexer) stateFn { func lexKey(lx *lexer) stateFn { r := lx.peek() + // Keys cannot contain a '#' character. + if r == commentStart { + return lx.errorf("Key cannot contain a '#' character.") + } + // XXX: Possible divergence from spec? // "Keys start with the first non-whitespace character and end with the // last non-whitespace character before the equals sign." // Note here that whitespace is either a tab or a space. // But we'll call it quits if we see a new line too. - if isWhitespace(r) || isNL(r) { - lx.emit(itemText) + if isNL(r) { + lx.emitTrim(itemText) return lexKeyEnd } // Let's also call it quits if we see an equals sign. if r == keySep { - lx.emit(itemText) + lx.emitTrim(itemText) return lexKeyEnd } @@ -326,14 +339,10 @@ func lexKey(lx *lexer) stateFn { } // lexKeyEnd consumes the end of a key (up to the key separator). -// Assumes that the first whitespace character after a key (or the '=' -// separator) has NOT been consumed. +// Assumes that any whitespace after a key has been consumed. func lexKeyEnd(lx *lexer) stateFn { r := lx.next() - switch { - case isWhitespace(r) || isNL(r): - return lexSkip(lx, lexKeyEnd) - case r == keySep: + if r == keySep { return lexSkip(lx, lexValue) } return lx.errorf("Expected key separator %q, but got %q instead.", diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/lex_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/lex_test.go deleted file mode 100644 index 6cfa21088..000000000 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/lex_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package toml - -import ( - "log" - "testing" -) - -func init() { - log.SetFlags(0) -} - -var testSmall = ` -# This is a TOML document. Boom. - -[owner] -[owner] # Whoa there. -andrew = "gallant # poopy" # weeeee -predicate = false -num = -5192 -f = -0.5192 -zulu = 1979-05-27T07:32:00Z -whoop = "poop" -arrs = [ - 1987-07-05T05:45:00Z, - 5, - "wat?", - "hehe \n\r kewl", - [6], [], - 5.0, - # sweetness -] # more comments -# hehe -` - -var testSmaller = ` -[a.b] # Do you ignore me? -andrew = "ga# ll\"ant" # what about me? -kait = "brady" -awesomeness = true -pi = 3.14 -dob = 1987-07-05T17:45:00Z -perfection = [ - [6, 28], - [496, 8128] -] -` - -func TestLexer(t *testing.T) { - lx := lex(testSmaller) - for { - item := lx.nextItem() - if item.typ == itemEOF { - break - } else if item.typ == itemError { - t.Fatal(item.val) - } - testf("%s\n", item) - } -} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/out_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/out_test.go deleted file mode 100644 index ab121e375..000000000 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/out_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package toml - -import ( - "flag" - "fmt" -) - -var flagOut = false - -func init() { - flag.BoolVar(&flagOut, "out", flagOut, "Print debug output.") - flag.Parse() -} - -func testf(format string, v ...interface{}) { - if flagOut { - fmt.Printf(format, v...) - } -} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/parse_test.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/parse_test.go deleted file mode 100644 index 09c5d6796..000000000 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/parse_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package toml - -import ( - "strings" - "testing" -) - -var testParseSmall = ` -# This is a TOML document. Boom. - -wat = "chipper" - -[owner.andrew.gallant] -hmm = "hi" - -[owner] # Whoa there. -andreW = "gallant # poopy" # weeeee -predicate = false -num = -5192 -f = -0.5192 -zulu = 1979-05-27T07:32:00Z -whoop = "poop" -tests = [ [1, 2, 3], ["abc", "xyz"] ] -arrs = [ # hmm - # more comments are awesome. - 1987-07-05T05:45:00Z, - # say wat? - 1987-07-05T05:45:00Z, - 1987-07-05T05:45:00Z, - # sweetness -] # more comments -# hehe -` - -var testParseSmall2 = ` -[a] -better = 43 - -[a.b.c] -answer = 42 -` - -func TestParse(t *testing.T) { - m, err := parse(testParseSmall) - if err != nil { - t.Fatal(err) - } - printMap(m.mapping, 0) -} - -func printMap(m map[string]interface{}, depth int) { - for k, v := range m { - testf("%s%s\n", strings.Repeat(" ", depth), k) - switch subm := v.(type) { - case map[string]interface{}: - printMap(subm, depth+1) - default: - testf("%s%v\n", strings.Repeat(" ", depth+1), v) - } - } -} - -var testParseSmall3 = ` -foo = [1, 2,3] -` - -func TestParseSmall3(t *testing.T) { - m, err := parse(testParseSmall3) - if err != nil { - t.Fatal(err) - } - printMap(m.mapping, 0) -} diff --git a/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go b/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go index 026ac6ae6..79dac6b19 100644 --- a/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go +++ b/Godeps/_workspace/src/github.com/BurntSushi/toml/type_check.go @@ -10,9 +10,16 @@ type tomlType interface { // typeEqual accepts any two types and returns true if they are equal. func typeEqual(t1, t2 tomlType) bool { + if t1 == nil || t2 == nil { + return false + } return t1.typeString() == t2.typeString() } +func typeIsHash(t tomlType) bool { + return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) +} + type tomlBaseType string func (btype tomlBaseType) typeString() string { diff --git a/Godeps/_workspace/src/github.com/armon/consul-kv/README.md b/Godeps/_workspace/src/github.com/armon/consul-kv/README.md index 0b6fd32a1..6992f6699 100644 --- a/Godeps/_workspace/src/github.com/armon/consul-kv/README.md +++ b/Godeps/_workspace/src/github.com/armon/consul-kv/README.md @@ -1,7 +1,11 @@ consul-kv ========= -This package provides the `consulkv` package which is a Key/Value client for Consul. It supports all the commands, and has a very simple API. +*DEPRECATED* Please use [consul-api](https://github.com/armon/consul-api) instead. + +This package provides the `consulkv` package which is a Key/Value +client for Consul. It supports all the commands as of Consul 0.1, +and has a very simple API. Documentation ============= diff --git a/Godeps/_workspace/src/github.com/armon/consul-kv/client.go b/Godeps/_workspace/src/github.com/armon/consul-kv/client.go index 6a1832d4f..ad4a3368c 100644 --- a/Godeps/_workspace/src/github.com/armon/consul-kv/client.go +++ b/Godeps/_workspace/src/github.com/armon/consul-kv/client.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "net/http" "net/url" - "path" "strconv" "strings" "time" @@ -222,7 +221,7 @@ func (c *Client) pathURL(key string) *url.URL { url := &url.URL{ Scheme: "http", Host: c.config.Address, - Path: path.Join("/v1/kv/", key), + Path: "/v1/kv/" + strings.TrimPrefix(key, "/"), } if c.config.Datacenter != "" { query := url.Query() diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go index f71507f57..f6ae54861 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go @@ -51,11 +51,11 @@ type Client struct { // If CheckRetry is nil, client will call the default one // `DefaultCheckRetry`. // Argument cluster is the etcd.Cluster object that these requests have been made on. - // Argument reqs is all of the http.Requests that have been made so far. - // Argument resps is all of the http.Responses from these requests. + // Argument numReqs is the number of http.Requests that have been made so far. + // Argument lastResp is the http.Responses from the last request. // Argument err is the reason of the failure. - CheckRetry func(cluster *Cluster, reqs []http.Request, - resps []http.Response, err error) error + CheckRetry func(cluster *Cluster, numReqs int, + lastResp http.Response, err error) error } // NewClient create a basic client that is configured to be used @@ -231,6 +231,11 @@ func (c *Client) SetConsistency(consistency string) error { return nil } +// Sets the DialTimeout value +func (c *Client) SetDialTimeout(d time.Duration) { + c.config.DialTimeout = d +} + // AddRootCA adds a root CA cert for the etcd client func (c *Client) AddRootCA(caCert string) error { if c.httpClient == nil { diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go index d170547eb..0f777886b 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go @@ -1,6 +1,7 @@ package etcd import ( + "fmt" "io/ioutil" "log" "strings" @@ -21,31 +22,31 @@ type etcdLogger struct { } func (p *etcdLogger) Debug(args ...interface{}) { - args[0] = "DEBUG: " + args[0].(string) - p.log.Println(args) + msg := "DEBUG: " + fmt.Sprint(args...) + p.log.Println(msg) } -func (p *etcdLogger) Debugf(fmt string, args ...interface{}) { - args[0] = "DEBUG: " + args[0].(string) +func (p *etcdLogger) Debugf(f string, args ...interface{}) { + msg := "DEBUG: " + fmt.Sprintf(f, args...) // Append newline if necessary - if !strings.HasSuffix(fmt, "\n") { - fmt = fmt + "\n" + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" } - p.log.Printf(fmt, args) + p.log.Print(msg) } func (p *etcdLogger) Warning(args ...interface{}) { - args[0] = "WARNING: " + args[0].(string) - p.log.Println(args) + msg := "WARNING: " + fmt.Sprint(args...) + p.log.Println(msg) } -func (p *etcdLogger) Warningf(fmt string, args ...interface{}) { +func (p *etcdLogger) Warningf(f string, args ...interface{}) { + msg := "WARNING: " + fmt.Sprintf(f, args...) // Append newline if necessary - if !strings.HasSuffix(fmt, "\n") { - fmt = fmt + "\n" + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" } - args[0] = "WARNING: " + args[0].(string) - p.log.Printf(fmt, args) + p.log.Print(msg) } func init() { diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go new file mode 100644 index 000000000..97f6d1110 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go @@ -0,0 +1,28 @@ +package etcd + +import ( + "testing" +) + +type Foo struct{} +type Bar struct { + one string + two int +} + +// Tests that logs don't panic with arbitrary interfaces +func TestDebug(t *testing.T) { + f := &Foo{} + b := &Bar{"asfd", 3} + for _, test := range []interface{}{ + 1234, + "asdf", + f, + b, + } { + logger.Debug(test) + logger.Debugf("something, %s", test) + logger.Warning(test) + logger.Warningf("something, %s", test) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go index 6335a0149..5d8b45a2d 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go @@ -136,8 +136,7 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { var err error var respBody []byte - reqs := make([]http.Request, 0) - resps := make([]http.Response, 0) + var numReqs = 1 checkRetry := c.CheckRetry if checkRetry == nil { @@ -176,15 +175,24 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { }() } - // if we connect to a follower, we will retry until we find a leader + // If we connect to a follower and consistency is required, retry until + // we connect to a leader + sleep := 25 * time.Millisecond + maxSleep := time.Second for attempt := 0; ; attempt++ { - select { - case <-cancelled: - return nil, ErrRequestCancelled - default: + if attempt > 0 { + select { + case <-cancelled: + return nil, ErrRequestCancelled + case <-time.After(sleep): + sleep = sleep * 2 + if sleep > maxSleep { + sleep = maxSleep + } + } } - logger.Debug("begin attempt", attempt, "for", rr.RelativePath) + logger.Debug("Connecting to etcd: attempt", attempt+1, "for", rr.RelativePath) if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY { // If it's a GET and consistency level is set to WEAK, @@ -223,6 +231,12 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { reqLock.Unlock() resp, err = c.httpClient.Do(req) + defer func() { + if resp != nil { + resp.Body.Close() + } + }() + // If the request was cancelled, return ErrRequestCancelled directly select { case <-cancelled: @@ -230,13 +244,13 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { default: } - reqs = append(reqs, *req) + numReqs++ // network error, change a machine! if err != nil { logger.Debug("network error:", err.Error()) - resps = append(resps, http.Response{}) - if checkErr := checkRetry(c.cluster, reqs, resps, err); checkErr != nil { + lastResp := http.Response{} + if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil { return nil, checkErr } @@ -245,8 +259,6 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { } // if there is no error, it should receive response - resps = append(resps, *resp) - defer resp.Body.Close() logger.Debug("recv.response.from", httpPath) if validHttpStatusCode[resp.StatusCode] { @@ -256,6 +268,12 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { logger.Debug("recv.success.", httpPath) break } + // ReadAll error may be caused due to cancel request + select { + case <-cancelled: + return nil, ErrRequestCancelled + default: + } } // if resp is TemporaryRedirect, set the new leader and retry @@ -270,13 +288,15 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { c.cluster.updateLeaderFromURL(u) logger.Debug("recv.response.relocate", u.String()) } + resp.Body.Close() continue } - if checkErr := checkRetry(c.cluster, reqs, resps, + if checkErr := checkRetry(c.cluster, numReqs, *resp, errors.New("Unexpected HTTP status code")); checkErr != nil { return nil, checkErr } + resp.Body.Close() } r := &RawResponse{ @@ -288,26 +308,18 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { return r, nil } -// DefaultCheckRetry checks retry cases -// If it has retried 2 * machine number, stop to retry it anymore -// If resp is nil, sleep for 200ms +// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests +// If we have retried 2 * machine number, stop retrying. // If status code is InternalServerError, sleep for 200ms. -func DefaultCheckRetry(cluster *Cluster, reqs []http.Request, - resps []http.Response, err error) error { +func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response, + err error) error { - if len(reqs) >= 2*len(cluster.Machines) { + if numReqs >= 2*len(cluster.Machines) { return newError(ErrCodeEtcdNotReachable, "Tried to connect to each peer twice and failed", 0) } - resp := &resps[len(resps)-1] - - if resp == nil { - time.Sleep(time.Millisecond * 200) - return nil - } - - code := resp.StatusCode + code := lastResp.StatusCode if code == http.StatusInternalServerError { time.Sleep(time.Millisecond * 200) From ab6431543610c766f1e8ffb10098b92fdf5a6d99 Mon Sep 17 00:00:00 2001 From: Chris Armstrong Date: Tue, 5 Aug 2014 11:51:04 -0700 Subject: [PATCH 5/8] Set DialTimeout in etcd client One second (the default) isn't long enough for many tasks. --- backends/etcd/etcdutil/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backends/etcd/etcdutil/client.go b/backends/etcd/etcdutil/client.go index 22ba308af..e494b810d 100644 --- a/backends/etcd/etcdutil/client.go +++ b/backends/etcd/etcdutil/client.go @@ -6,6 +6,7 @@ package etcdutil import ( "errors" "strings" + "time" "github.com/coreos/go-etcd/etcd" ) @@ -31,6 +32,8 @@ func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, } else { c = etcd.NewClient(machines) } + // Configure the DialTimeout, since 1 second is often too short + c.SetDialTimeout(time.Duration(3) * time.Second) success := c.SetCluster(machines) if !success { return &Client{c}, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ",")) From 0e563e50c738ba0d34be827287889ecba404aea2 Mon Sep 17 00:00:00 2001 From: Chris Armstrong Date: Fri, 29 Aug 2014 14:58:17 -0700 Subject: [PATCH 6/8] fix(resource.go): try writing to destination when rename fails --- resource/template/resource.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/resource/template/resource.go b/resource/template/resource.go index 59ae581cc..52c071511 100644 --- a/resource/template/resource.go +++ b/resource/template/resource.go @@ -160,8 +160,26 @@ func (t *TemplateResource) sync() error { } } log.Debug("Overwriting target config " + t.Dest) - if err := os.Rename(staged, t.Dest); err != nil { - return err + err := os.Rename(staged, t.Dest) + if err != nil { + if strings.Contains(err.Error(), "device or resource busy") { + log.Debug("Rename failed - target is likely a mount. Trying to write instead") + // try to open the file and write to it + var contents []byte + var rerr error + contents, rerr = ioutil.ReadFile(staged) + if rerr != nil { + return rerr + } + err := ioutil.WriteFile(t.Dest, contents, t.FileMode) + // make sure owner and group match the temp file, in case the file was created with WriteFile + os.Chown(t.Dest, t.Uid, t.Gid) + if err != nil { + return err + } + } else { + return err + } } if t.ReloadCmd != "" { if err := t.reload(); err != nil { From c47349513d0136ea6fb1f66cbad18689190f2edb Mon Sep 17 00:00:00 2001 From: Micah Hausler Date: Tue, 9 Sep 2014 13:12:25 -0400 Subject: [PATCH 7/8] Update installation to point to 0.5.0 --- docs/installation.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index b850b0600..e35af8933 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -6,24 +6,14 @@ Currently confd ships binaries for OS X and Linux 64bit systems. You can downloa > Note: You don't need Go installed to use confd unless you plan to build from source. -Download confd version 0.3.0 using wget from the command line: +Download confd version 0.5.0 using wget from the command line: * OSX ```Bash -wget https://github.com/kelseyhightower/confd/releases/download/v0.3.0/confd_0.3.0_darwin_amd64.zip +wget -O confd https://github.com/kelseyhightower/confd/releases/download/v0.5.0/confd-0.5.0-darwin-amd64 ``` * LINUX ```Bash -wget -O confd_0.3.0_linux_amd64.tar.gz https://github.com/kelseyhightower/confd/releases/download/v0.3.0/confd_0.3.0_linux_amd64.tar.gz -``` - -Unzip the confd package. -* OSX -```Bash -unzip confd_0.3.0_darwin_amd64.zip -``` -* LINUX -```Bash -tar -zxvf confd_0.3.0_linux_amd64.tar.gz +wget -O confd https://github.com/kelseyhightower/confd/releases/download/v0.5.0/confd-0.5.0-linux-amd64 ``` Copy the confd binary to a bin directory in your path. From dc4af0ada5ad66dd43bf703a21c920a98aedb754 Mon Sep 17 00:00:00 2001 From: Bearice Ren Date: Wed, 17 Sep 2014 17:04:35 +0800 Subject: [PATCH 8/8] fixes #119 "c, err := .... " will override variable "c" defined before. --- backends/etcd/etcdutil/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backends/etcd/etcdutil/client.go b/backends/etcd/etcdutil/client.go index e494b810d..5d385fd88 100644 --- a/backends/etcd/etcdutil/client.go +++ b/backends/etcd/etcdutil/client.go @@ -24,8 +24,9 @@ type EtcdClient interface { // It returns an error if a connection to the cluster cannot be made. func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, error) { var c *etcd.Client + var err error if cert != "" && key != "" { - c, err := etcd.NewTLSClient(machines, cert, key, caCert) + c, err = etcd.NewTLSClient(machines, cert, key, caCert) if err != nil { return &Client{c}, err }