diff options
| author | Dave Henderson <dhenderson@gmail.com> | 2017-08-05 13:21:05 -0400 |
|---|---|---|
| committer | Dave Henderson <dhenderson@gmail.com> | 2017-08-09 21:51:07 -0400 |
| commit | 51ddb6e800ab087fa3dff19686b0f1f39a1a4432 (patch) | |
| tree | 1892e841efa720c2cc387cd0de7f9c1b6d318c63 | |
| parent | dd5a7e412352f2e268973b428648cca6e549dc83 (diff) | |
Extracting data namespace, renaming typeconv to conv namespace
Signed-off-by: Dave Henderson <dhenderson@gmail.com>
| -rw-r--r-- | conv/conv.go | 110 | ||||
| -rw-r--r-- | conv/conv_test.go | 86 | ||||
| -rw-r--r-- | data/data.go (renamed from typeconv.go) | 115 | ||||
| -rw-r--r-- | data/data_test.go (renamed from typeconv_test.go) | 113 | ||||
| -rw-r--r-- | data/datasource.go (renamed from data.go) | 32 | ||||
| -rw-r--r-- | data/datasource_test.go (renamed from data_test.go) | 4 | ||||
| -rw-r--r-- | docs/content/functions/conv.md | 130 | ||||
| -rw-r--r-- | docs/content/functions/data.md (renamed from docs/content/functions/general.md) | 600 | ||||
| -rw-r--r-- | funcs.go | 33 | ||||
| -rw-r--r-- | funcs/conv.go | 64 | ||||
| -rw-r--r-- | funcs/data.go | 110 | ||||
| -rw-r--r-- | gomplate.go | 11 | ||||
| -rw-r--r-- | gomplate_test.go | 27 | ||||
| -rw-r--r-- | libkv/boltdb.go | 6 | ||||
| -rw-r--r-- | libkv/consul.go | 8 | ||||
| -rw-r--r-- | process_test.go | 9 | ||||
| -rw-r--r-- | typeconv/typeconv.go | 33 | ||||
| -rw-r--r-- | typeconv/typeconv_test.go | 47 | ||||
| -rw-r--r-- | vault/auth.go | 4 |
19 files changed, 864 insertions, 678 deletions
diff --git a/conv/conv.go b/conv/conv.go new file mode 100644 index 00000000..ec89e0fa --- /dev/null +++ b/conv/conv.go @@ -0,0 +1,110 @@ +package conv + +import ( + "fmt" + "log" + "reflect" + "strconv" + "strings" +) + +// Bool converts a string to a boolean value, using strconv.ParseBool under the covers. +// Possible true values are: 1, t, T, TRUE, true, True +// All other values are considered false. +func Bool(in string) bool { + if b, err := strconv.ParseBool(in); err == nil { + return b + } + return false +} + +// Slice creates a slice from a bunch of arguments +func Slice(args ...interface{}) []interface{} { + return args +} + +// Join concatenates the elements of a to create a single string. +// The separator string sep is placed between elements in the resulting string. +// +// This is functionally identical to strings.Join, except that each element is +// coerced to a string first +func Join(in interface{}, sep string) string { + s, ok := in.([]string) + if ok { + return strings.Join(s, sep) + } + + var a []interface{} + a, ok = in.([]interface{}) + if ok { + b := make([]string, len(a)) + for i := range a { + b[i] = toString(a[i]) + } + return strings.Join(b, sep) + } + + log.Fatal("Input to Join must be an array") + return "" +} + +// Has determines whether or not a given object has a property with the given key +func Has(in interface{}, key string) bool { + av := reflect.ValueOf(in) + kv := reflect.ValueOf(key) + + if av.Kind() == reflect.Map { + return av.MapIndex(kv).IsValid() + } + + return false +} + +func toString(in interface{}) string { + if s, ok := in.(string); ok { + return s + } + if s, ok := in.(fmt.Stringer); ok { + return s.String() + } + if i, ok := in.(int); ok { + return strconv.Itoa(i) + } + if u, ok := in.(uint64); ok { + return strconv.FormatUint(u, 10) + } + if f, ok := in.(float64); ok { + return strconv.FormatFloat(f, 'f', -1, 64) + } + if b, ok := in.(bool); ok { + return strconv.FormatBool(b) + } + if in == nil { + return "nil" + } + return fmt.Sprintf("%s", in) +} + +// MustParseInt - wrapper for strconv.ParseInt that returns 0 in the case of error +func MustParseInt(s string, base, bitSize int) int64 { + i, _ := strconv.ParseInt(s, base, bitSize) + return i +} + +// MustParseFloat - wrapper for strconv.ParseFloat that returns 0 in the case of error +func MustParseFloat(s string, bitSize int) float64 { + i, _ := strconv.ParseFloat(s, bitSize) + return i +} + +// MustParseUint - wrapper for strconv.ParseUint that returns 0 in the case of error +func MustParseUint(s string, base, bitSize int) uint64 { + i, _ := strconv.ParseUint(s, base, bitSize) + return i +} + +// MustAtoi - wrapper for strconv.Atoi that returns 0 in the case of error +func MustAtoi(s string) int { + i, _ := strconv.Atoi(s) + return i +} diff --git a/conv/conv_test.go b/conv/conv_test.go new file mode 100644 index 00000000..6b57f65a --- /dev/null +++ b/conv/conv_test.go @@ -0,0 +1,86 @@ +package conv + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBool(t *testing.T) { + assert.False(t, Bool("")) + assert.False(t, Bool("asdf")) + assert.False(t, Bool("1234")) + assert.False(t, Bool("False")) + assert.False(t, Bool("0")) + assert.False(t, Bool("false")) + assert.False(t, Bool("F")) + assert.False(t, Bool("f")) + assert.True(t, Bool("true")) + assert.True(t, Bool("True")) + assert.True(t, Bool("t")) + assert.True(t, Bool("T")) + assert.True(t, Bool("1")) +} + +func TestSlice(t *testing.T) { + expected := []string{"foo", "bar"} + actual := Slice("foo", "bar") + assert.Equal(t, expected[0], actual[0]) + assert.Equal(t, expected[1], actual[1]) +} + +func TestJoin(t *testing.T) { + + assert.Equal(t, "foo,bar", Join([]interface{}{"foo", "bar"}, ",")) + assert.Equal(t, "foo,\nbar", Join([]interface{}{"foo", "bar"}, ",\n")) + // Join handles all kinds of scalar types too... + assert.Equal(t, "42-18446744073709551615", Join([]interface{}{42, uint64(18446744073709551615)}, "-")) + assert.Equal(t, "1,,true,3.14,foo,nil", Join([]interface{}{1, "", true, 3.14, "foo", nil}, ",")) + // and best-effort with weird types + assert.Equal(t, "[foo],bar", Join([]interface{}{[]string{"foo"}, "bar"}, ",")) +} + +func TestHas(t *testing.T) { + + in := map[string]interface{}{ + "foo": "bar", + "baz": map[string]interface{}{ + "qux": "quux", + }, + } + + assert.True(t, Has(in, "foo")) + assert.False(t, Has(in, "bar")) + assert.True(t, Has(in["baz"], "qux")) +} + +func TestMustParseInt(t *testing.T) { + for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { + assert.Equal(t, 0, int(MustParseInt(i, 10, 64))) + } + assert.Equal(t, 1, int(MustParseInt("1", 10, 64))) + assert.Equal(t, -1, int(MustParseInt("-1", 10, 64))) +} + +func TestMustAtoi(t *testing.T) { + for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { + assert.Equal(t, 0, MustAtoi(i)) + } + assert.Equal(t, 1, MustAtoi("1")) + assert.Equal(t, -1, MustAtoi("-1")) +} + +func TestMustParseUint(t *testing.T) { + for _, i := range []string{"0", "-0", "-1", "foo", "", "*&^%"} { + assert.Equal(t, uint64(0), MustParseUint(i, 10, 64)) + } + assert.Equal(t, uint64(1), MustParseUint("1", 10, 64)) +} + +func TestMustParseFloat(t *testing.T) { + for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { + assert.Equal(t, 0.0, MustParseFloat(i, 64)) + } + assert.Equal(t, 1.0, MustParseFloat("1", 64)) + assert.Equal(t, -1.0, MustParseFloat("-1", 64)) +} diff --git a/typeconv.go b/data/data.go index 2ec433eb..10102eaa 100644 --- a/typeconv.go +++ b/data/data.go @@ -1,36 +1,18 @@ -package main +package data import ( "bytes" "encoding/csv" "encoding/json" - "fmt" "log" - "reflect" - "strconv" "strings" - yaml "gopkg.in/yaml.v2" - // XXX: replace once https://github.com/BurntSushi/toml/pull/179 is merged "github.com/hairyhenderson/toml" "github.com/ugorji/go/codec" + yaml "gopkg.in/yaml.v2" ) -// TypeConv - type conversion function -type TypeConv struct { -} - -// Bool converts a string to a boolean value, using strconv.ParseBool under the covers. -// Possible true values are: 1, t, T, TRUE, true, True -// All other values are considered false. -func (t *TypeConv) Bool(in string) bool { - if b, err := strconv.ParseBool(in); err == nil { - return b - } - return false -} - func unmarshalObj(obj map[string]interface{}, in string, f func([]byte, interface{}) error) map[string]interface{} { err := f([]byte(in), &obj) if err != nil { @@ -48,31 +30,31 @@ func unmarshalArray(obj []interface{}, in string, f func([]byte, interface{}) er } // JSON - Unmarshal a JSON Object -func (t *TypeConv) JSON(in string) map[string]interface{} { +func JSON(in string) map[string]interface{} { obj := make(map[string]interface{}) return unmarshalObj(obj, in, yaml.Unmarshal) } // JSONArray - Unmarshal a JSON Array -func (t *TypeConv) JSONArray(in string) []interface{} { +func JSONArray(in string) []interface{} { obj := make([]interface{}, 1) return unmarshalArray(obj, in, yaml.Unmarshal) } // YAML - Unmarshal a YAML Object -func (t *TypeConv) YAML(in string) map[string]interface{} { +func YAML(in string) map[string]interface{} { obj := make(map[string]interface{}) return unmarshalObj(obj, in, yaml.Unmarshal) } // YAMLArray - Unmarshal a YAML Array -func (t *TypeConv) YAMLArray(in string) []interface{} { +func YAMLArray(in string) []interface{} { obj := make([]interface{}, 1) return unmarshalArray(obj, in, yaml.Unmarshal) } // TOML - Unmarshal a TOML Object -func (t *TypeConv) TOML(in string) interface{} { +func TOML(in string) interface{} { obj := make(map[string]interface{}) return unmarshalObj(obj, in, toml.Unmarshal) } @@ -131,7 +113,7 @@ func autoIndex(i int) string { // in - the CSV-format string to parse // returns: // an array of rows, which are arrays of cells (strings) -func (t *TypeConv) CSV(args ...string) [][]string { +func CSV(args ...string) [][]string { records, hdr := parseCSV(args...) records = append(records, nil) copy(records[1:], records) @@ -148,7 +130,7 @@ func (t *TypeConv) CSV(args ...string) [][]string { // in - the CSV-format string to parse // returns: // an array of rows, indexed by the header name -func (t *TypeConv) CSVByRow(args ...string) (rows []map[string]string) { +func CSVByRow(args ...string) (rows []map[string]string) { records, hdr := parseCSV(args...) for _, record := range records { m := make(map[string]string) @@ -169,7 +151,7 @@ func (t *TypeConv) CSVByRow(args ...string) (rows []map[string]string) { // in - the CSV-format string to parse // returns: // a map of columns, indexed by the header name. values are arrays of strings -func (t *TypeConv) CSVByColumn(args ...string) (cols map[string][]string) { +func CSVByColumn(args ...string) (cols map[string][]string) { records, hdr := parseCSV(args...) cols = make(map[string][]string) for _, record := range records { @@ -181,7 +163,7 @@ func (t *TypeConv) CSVByColumn(args ...string) (cols map[string][]string) { } // ToCSV - -func (t *TypeConv) ToCSV(args ...interface{}) string { +func ToCSV(args ...interface{}) string { delim := "," var in [][]string if len(args) == 2 { @@ -236,12 +218,12 @@ func toJSONBytes(in interface{}) []byte { } // ToJSON - Stringify a struct as JSON -func (t *TypeConv) ToJSON(in interface{}) string { +func ToJSON(in interface{}) string { return string(toJSONBytes(in)) } // ToJSONPretty - Stringify a struct as JSON (indented) -func (t *TypeConv) toJSONPretty(indent string, in interface{}) string { +func ToJSONPretty(indent string, in interface{}) string { out := new(bytes.Buffer) b := toJSONBytes(in) err := json.Indent(out, b, "", indent) @@ -253,12 +235,12 @@ func (t *TypeConv) toJSONPretty(indent string, in interface{}) string { } // ToYAML - Stringify a struct as YAML -func (t *TypeConv) ToYAML(in interface{}) string { +func ToYAML(in interface{}) string { return marshalObj(in, yaml.Marshal) } // ToTOML - Stringify a struct as TOML -func (t *TypeConv) ToTOML(in interface{}) string { +func ToTOML(in interface{}) string { buf := new(bytes.Buffer) err := toml.NewEncoder(buf).Encode(in) if err != nil { @@ -266,70 +248,3 @@ func (t *TypeConv) ToTOML(in interface{}) string { } return string(buf.Bytes()) } - -// Slice creates a slice from a bunch of arguments -func (t *TypeConv) Slice(args ...interface{}) []interface{} { - return args -} - -// Join concatenates the elements of a to create a single string. -// The separator string sep is placed between elements in the resulting string. -// -// This is functionally identical to strings.Join, except that each element is -// coerced to a string first -func (t *TypeConv) Join(in interface{}, sep string) string { - s, ok := in.([]string) - if ok { - return strings.Join(s, sep) - } - - var a []interface{} - a, ok = in.([]interface{}) - if ok { - b := make([]string, len(a)) - for i := range a { - b[i] = toString(a[i]) - } - return strings.Join(b, sep) - } - - log.Fatal("Input to Join must be an array") - return "" -} - -// Has determines whether or not a given object has a property with the given key -func (t *TypeConv) Has(in interface{}, key string) bool { - av := reflect.ValueOf(in) - kv := reflect.ValueOf(key) - - if av.Kind() == reflect.Map { - return av.MapIndex(kv).IsValid() - } - - return false -} - -func toString(in interface{}) string { - if s, ok := in.(string); ok { - return s - } - if s, ok := in.(fmt.Stringer); ok { - return s.String() - } - if i, ok := in.(int); ok { - return strconv.Itoa(i) - } - if u, ok := in.(uint64); ok { - return strconv.FormatUint(u, 10) - } - if f, ok := in.(float64); ok { - return strconv.FormatFloat(f, 'f', -1, 64) - } - if b, ok := in.(bool); ok { - return strconv.FormatBool(b) - } - if in == nil { - return "nil" - } - return fmt.Sprintf("%s", in) -} diff --git a/typeconv_test.go b/data/data_test.go index d38b5352..eb284940 100644 --- a/typeconv_test.go +++ b/data/data_test.go @@ -1,4 +1,4 @@ -package main +package data import ( "testing" @@ -7,25 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBool(t *testing.T) { - ty := &TypeConv{} - assert.False(t, ty.Bool("")) - assert.False(t, ty.Bool("asdf")) - assert.False(t, ty.Bool("1234")) - assert.False(t, ty.Bool("False")) - assert.False(t, ty.Bool("0")) - assert.False(t, ty.Bool("false")) - assert.False(t, ty.Bool("F")) - assert.False(t, ty.Bool("f")) - assert.True(t, ty.Bool("true")) - assert.True(t, ty.Bool("True")) - assert.True(t, ty.Bool("t")) - assert.True(t, ty.Bool("T")) - assert.True(t, ty.Bool("1")) -} - func TestUnmarshalObj(t *testing.T) { - ty := new(TypeConv) expected := map[string]interface{}{ "foo": map[interface{}]interface{}{"bar": "baz"}, "one": 1.0, @@ -37,8 +19,8 @@ func TestUnmarshalObj(t *testing.T) { assert.Equal(t, expected["one"], actual["one"]) assert.Equal(t, expected["true"], actual["true"]) } - test(ty.JSON(`{"foo":{"bar":"baz"},"one":1.0,"true":true}`)) - test(ty.YAML(`foo: + test(JSON(`{"foo":{"bar":"baz"},"one":1.0,"true":true}`)) + test(YAML(`foo: bar: baz one: 1.0 true: true @@ -46,7 +28,6 @@ true: true } func TestUnmarshalArray(t *testing.T) { - ty := new(TypeConv) expected := []string{"foo", "bar"} @@ -54,15 +35,14 @@ func TestUnmarshalArray(t *testing.T) { assert.Equal(t, expected[0], actual[0]) assert.Equal(t, expected[1], actual[1]) } - test(ty.JSONArray(`["foo","bar"]`)) - test(ty.YAMLArray(` + test(JSONArray(`["foo","bar"]`)) + test(YAMLArray(` - foo - bar `)) } func TestToJSON(t *testing.T) { - ty := new(TypeConv) expected := `{"down":{"the":{"rabbit":{"hole":true}}},"foo":"bar","one":1,"true":true}` in := map[string]interface{}{ "foo": "bar", @@ -76,11 +56,10 @@ func TestToJSON(t *testing.T) { }, }, } - assert.Equal(t, expected, ty.ToJSON(in)) + assert.Equal(t, expected, ToJSON(in)) } func TestToJSONPretty(t *testing.T) { - ty := new(TypeConv) expected := `{ "down": { "the": { @@ -105,11 +84,10 @@ func TestToJSONPretty(t *testing.T) { }, }, } - assert.Equal(t, expected, ty.toJSONPretty(" ", in)) + assert.Equal(t, expected, ToJSONPretty(" ", in)) } func TestToYAML(t *testing.T) { - ty := new(TypeConv) expected := `d: 2006-01-02T15:04:05.999999999-07:00 foo: bar ? |- @@ -132,60 +110,23 @@ key`: map[string]interface{}{ }, "d": time.Date(2006, time.January, 2, 15, 4, 5, 999999999, mst), } - assert.Equal(t, expected, ty.ToYAML(in)) -} - -func TestSlice(t *testing.T) { - ty := new(TypeConv) - expected := []string{"foo", "bar"} - actual := ty.Slice("foo", "bar") - assert.Equal(t, expected[0], actual[0]) - assert.Equal(t, expected[1], actual[1]) -} - -func TestJoin(t *testing.T) { - ty := new(TypeConv) - - assert.Equal(t, "foo,bar", ty.Join([]interface{}{"foo", "bar"}, ",")) - assert.Equal(t, "foo,\nbar", ty.Join([]interface{}{"foo", "bar"}, ",\n")) - // Join handles all kinds of scalar types too... - assert.Equal(t, "42-18446744073709551615", ty.Join([]interface{}{42, uint64(18446744073709551615)}, "-")) - assert.Equal(t, "1,,true,3.14,foo,nil", ty.Join([]interface{}{1, "", true, 3.14, "foo", nil}, ",")) - // and best-effort with weird types - assert.Equal(t, "[foo],bar", ty.Join([]interface{}{[]string{"foo"}, "bar"}, ",")) -} - -func TestHas(t *testing.T) { - ty := new(TypeConv) - - in := map[string]interface{}{ - "foo": "bar", - "baz": map[string]interface{}{ - "qux": "quux", - }, - } - - assert.True(t, ty.Has(in, "foo")) - assert.False(t, ty.Has(in, "bar")) - assert.True(t, ty.Has(in["baz"], "qux")) + assert.Equal(t, expected, ToYAML(in)) } func TestCSV(t *testing.T) { - ty := new(TypeConv) in := "first,second,third\n1,2,3\n4,5,6" expected := [][]string{ {"first", "second", "third"}, {"1", "2", "3"}, {"4", "5", "6"}, } - assert.Equal(t, expected, ty.CSV(in)) + assert.Equal(t, expected, CSV(in)) in = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" - assert.Equal(t, expected, ty.CSV(";", in)) + assert.Equal(t, expected, CSV(";", in)) } func TestCSVByRow(t *testing.T) { - ty := new(TypeConv) in := "first,second,third\n1,2,3\n4,5,6" expected := []map[string]string{ { @@ -199,16 +140,16 @@ func TestCSVByRow(t *testing.T) { "third": "6", }, } - assert.Equal(t, expected, ty.CSVByRow(in)) + assert.Equal(t, expected, CSVByRow(in)) in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByRow("first,second,third", in)) + assert.Equal(t, expected, CSVByRow("first,second,third", in)) in = "1;2;3\n4;5;6" - assert.Equal(t, expected, ty.CSVByRow(";", "first;second;third", in)) + assert.Equal(t, expected, CSVByRow(";", "first;second;third", in)) in = "first;second;third\r\n1;2;3\r\n4;5;6" - assert.Equal(t, expected, ty.CSVByRow(";", in)) + assert.Equal(t, expected, CSVByRow(";", in)) expected = []map[string]string{ {"A": "1", "B": "2", "C": "3"}, @@ -216,34 +157,33 @@ func TestCSVByRow(t *testing.T) { } in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByRow("", in)) + assert.Equal(t, expected, CSVByRow("", in)) expected = []map[string]string{ {"A": "1", "B": "1", "C": "1", "D": "1", "E": "1", "F": "1", "G": "1", "H": "1", "I": "1", "J": "1", "K": "1", "L": "1", "M": "1", "N": "1", "O": "1", "P": "1", "Q": "1", "R": "1", "S": "1", "T": "1", "U": "1", "V": "1", "W": "1", "X": "1", "Y": "1", "Z": "1", "AA": "1", "BB": "1", "CC": "1", "DD": "1"}, } in = "1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1" - assert.Equal(t, expected, ty.CSVByRow("", in)) + assert.Equal(t, expected, CSVByRow("", in)) } func TestCSVByColumn(t *testing.T) { - ty := new(TypeConv) in := "first,second,third\n1,2,3\n4,5,6" expected := map[string][]string{ "first": {"1", "4"}, "second": {"2", "5"}, "third": {"3", "6"}, } - assert.Equal(t, expected, ty.CSVByColumn(in)) + assert.Equal(t, expected, CSVByColumn(in)) in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByColumn("first,second,third", in)) + assert.Equal(t, expected, CSVByColumn("first,second,third", in)) in = "1;2;3\n4;5;6" - assert.Equal(t, expected, ty.CSVByColumn(";", "first;second;third", in)) + assert.Equal(t, expected, CSVByColumn(";", "first;second;third", in)) in = "first;second;third\r\n1;2;3\r\n4;5;6" - assert.Equal(t, expected, ty.CSVByColumn(";", in)) + assert.Equal(t, expected, CSVByColumn(";", in)) expected = map[string][]string{ "A": {"1", "4"}, @@ -252,7 +192,7 @@ func TestCSVByColumn(t *testing.T) { } in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByColumn("", in)) + assert.Equal(t, expected, CSVByColumn("", in)) } func TestAutoIndex(t *testing.T) { @@ -266,7 +206,6 @@ func TestAutoIndex(t *testing.T) { } func TestToCSV(t *testing.T) { - ty := new(TypeConv) in := [][]string{ {"first", "second", "third"}, {"1", "2", "3"}, @@ -274,15 +213,14 @@ func TestToCSV(t *testing.T) { } expected := "first,second,third\r\n1,2,3\r\n4,5,6\r\n" - assert.Equal(t, expected, ty.ToCSV(in)) + assert.Equal(t, expected, ToCSV(in)) expected = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" - assert.Equal(t, expected, ty.ToCSV(";", in)) + assert.Equal(t, expected, ToCSV(";", in)) } func TestTOML(t *testing.T) { - ty := new(TypeConv) in := `# This is a TOML document. Boom. title = "TOML Example" @@ -352,11 +290,10 @@ hosts = [ }, } - assert.Equal(t, expected, ty.TOML(in)) + assert.Equal(t, expected, TOML(in)) } func TestToTOML(t *testing.T) { - ty := new(TypeConv) expected := `foo = "bar" one = 1 true = true @@ -378,5 +315,5 @@ true = true }, }, } - assert.Equal(t, expected, ty.ToTOML(in)) + assert.Equal(t, expected, ToTOML(in)) } diff --git a/data.go b/data/datasource.go index 7f41d2ec..5e626e0c 100644 --- a/data.go +++ b/data/datasource.go @@ -1,4 +1,4 @@ -package main +package data import ( "errors" @@ -63,6 +63,14 @@ type Data struct { cache map[string][]byte } +// Cleanup - clean up datasources before shutting the process down - things +// like Logging out happen here +func (d *Data) Cleanup() { + for _, s := range d.Sources { + s.cleanup() + } +} + // NewData - constructor for Data func NewData(datasourceArgs []string, headerArgs []string) *Data { sources := make(map[string]*Source) @@ -95,6 +103,15 @@ type Source struct { Header http.Header // used for http[s]: URLs, nil otherwise } +func (s *Source) cleanup() { + if s.VC != nil { + s.VC.Logout() + } + if s.KV != nil { + s.KV.Logout() + } +} + // NewSource - builds a &Source func NewSource(alias string, URL *url.URL) (s *Source) { ext := filepath.Ext(URL.Path) @@ -190,18 +207,17 @@ func (d *Data) Datasource(alias string, args ...string) interface{} { log.Fatalf("Couldn't read datasource '%s': %s", alias, err) } s := string(b) - ty := &TypeConv{} if source.Type == "application/json" { - return ty.JSON(s) + return JSON(s) } if source.Type == "application/yaml" { - return ty.YAML(s) + return YAML(s) } if source.Type == "text/csv" { - return ty.CSV(s) + return CSV(s) } if source.Type == "application/toml" { - return ty.TOML(s) + return TOML(s) } if source.Type == plaintext { return s @@ -211,7 +227,7 @@ func (d *Data) Datasource(alias string, args ...string) interface{} { } // Include - -func (d *Data) include(alias string, args ...string) interface{} { +func (d *Data) Include(alias string, args ...string) string { source, ok := d.Sources[alias] if !ok { log.Fatalf("Undefined datasource '%s'", alias) @@ -318,7 +334,6 @@ func readVault(source *Source, args ...string) ([]byte, error) { if source.VC == nil { source.VC = vault.New() source.VC.Login() - addCleanupHook(source.VC.Logout) } params := make(map[string]interface{}) @@ -364,7 +379,6 @@ func readConsul(source *Source, args ...string) ([]byte, error) { if source.KV == nil { source.KV = libkv.NewConsul(source.URL) err := source.KV.Login() - addCleanupHook(source.KV.Logout) if err != nil { return nil, err } diff --git a/data_test.go b/data/datasource_test.go index 1be57e3e..31b66eee 100644 --- a/data_test.go +++ b/data/datasource_test.go @@ -1,6 +1,6 @@ // +build !windows -package main +package data import ( "encoding/json" @@ -295,6 +295,6 @@ func TestInclude(t *testing.T) { data := &Data{ Sources: sources, } - actual := data.include("foo") + actual := data.Include("foo") assert.Equal(t, contents, actual) } diff --git a/docs/content/functions/conv.md b/docs/content/functions/conv.md new file mode 100644 index 00000000..177a37b7 --- /dev/null +++ b/docs/content/functions/conv.md @@ -0,0 +1,130 @@ +--- +title: conversion functions +menu: + main: + parent: functions +--- + +These are a collection of functions that mostly help converting from one type +to another - generally from a `string` to something else, and vice-versa. + +## `conv.Bool` + +**Alias:** `bool` + +Converts a true-ish string to a boolean. Can be used to simplify conditional statements based on environment variables or other text input. + +#### Example + +_`input.tmpl`:_ +``` +{{if bool (getenv "FOO")}}foo{{else}}bar{{end}} +``` + +```console +$ gomplate < input.tmpl +bar +$ FOO=true gomplate < input.tmpl +foo +``` + +## `conv.Slice` + +**Alias:** `slice` + +Creates a slice. Useful when needing to `range` over a bunch of variables. + +#### Example + +_`input.tmpl`:_ +``` +{{range slice "Bart" "Lisa" "Maggie"}} +Hello, {{.}} +{{- end}} +``` + +```console +$ gomplate < input.tmpl +Hello, Bart +Hello, Lisa +Hello, Maggie +``` + +## `conv.Has` + +**Alias:** `has` + +Has reports whether or not a given object has a property with the given key. Can be used with `if` to prevent the template from trying to access a non-existent property in an object. + +#### Example + +_Let's say we're using a Vault datasource..._ + +_`input.tmpl`:_ +``` +{{ $secret := datasource "vault" "mysecret" -}} +The secret is ' +{{- if (has $secret "value") }} +{{- $secret.value }} +{{- else }} +{{- $secret | toYAML }} +{{- end }}' +``` + +If the `secret/foo/mysecret` secret in Vault has a property named `value` set to `supersecret`: + +```console +$ gomplate -d vault:///secret/foo < input.tmpl +The secret is 'supersecret' +``` + +On the other hand, if there is no `value` property: + +```console +$ gomplate -d vault:///secret/foo < input.tmpl +The secret is 'foo: bar' +``` + +## `conv.Join` + +**Alias:** `join` + +Concatenates the elements of an array to create a string. The separator string sep is placed between elements in the resulting string. + +#### Example + +_`input.tmpl`_ +``` +{{ $a := `[1, 2, 3]` | jsonArray }} +{{ join $a "-" }} +``` + +```console +$ gomplate -f input.tmpl +1-2-3 +``` + + +## `conv.URL` + +**Alias:** `urlParse` + +Parses a string as a URL for later use. Equivalent to [url.Parse](https://golang.org/pkg/net/url/#Parse) + +#### Example + +_`input.tmpl`:_ +``` +{{ $u := conv.URL "https://example.com:443/foo/bar" }} +The scheme is {{ $u.Scheme }} +The host is {{ $u.Host }} +The path is {{ $u.Path }} +``` + +```console +$ gomplate < input.tmpl +The scheme is https +The host is example.com:443 +The path is /foo/bar +``` + diff --git a/docs/content/functions/general.md b/docs/content/functions/data.md index 7f33cfde..acdbf44f 100644 --- a/docs/content/functions/general.md +++ b/docs/content/functions/data.md @@ -1,120 +1,267 @@ --- -title: other functions +title: data functions menu: main: parent: functions --- -## `bool` +A collection of functions that retrieve, parse, and convert structured data. -Converts a true-ish string to a boolean. Can be used to simplify conditional statements based on environment variables or other text input. +## `datasource` -#### Example +Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument). + +Currently, `file://`, `http://`, `https://`, and `vault://` URLs are supported. + +Currently-supported formats are JSON, YAML, TOML, and CSV. + +### Basic usage + +_`person.json`:_ +```json +{ + "name": "Dave" +} +``` _`input.tmpl`:_ ``` -{{if bool (getenv "FOO")}}foo{{else}}bar{{end}} +Hello {{ (datasource "person").name }} ``` ```console -$ gomplate < input.tmpl +$ gomplate -d person.json < input.tmpl +Hello Dave +``` + +### Usage with HTTP data + +```console +$ echo 'Hello there, {{(datasource "foo").headers.Host}}...' | gomplate -d foo=https://httpbin.org/get +Hello there, httpbin.org... +``` + +Additional headers can be provided with the `--datasource-header`/`-H` option: + +```console +$ gomplate -d foo=https://httpbin.org/get -H 'foo=Foo: bar' -i '{{(datasource "foo").headers.Foo}}' bar -$ FOO=true gomplate < input.tmpl -foo ``` -## `slice` +### Usage with Consul data + +There are three supported URL schemes to retrieve data from [Consul](https://consul.io/). +The `consul://` (or `consul+http://`) scheme can optionally be used with a hostname and port to specify a server (e.g. `consul://localhost:8500`). +By default HTTP will be used, but the `consul+https://` form can be used to use HTTPS, alternatively `$CONSUL_HTTP_SSL` can be used. + +If the server address isn't part of the datasource URL, `$CONSUL_HTTP_ADDR` will be checked. + +The following optional environment variables can be set: + +| name | usage | +|------|-------| +| `CONSUL_HTTP_ADDR` | Hostname and optional port for connecting to Consul. Defaults to `http://localhost:8500` | +| `CONSUL_TIMEOUT` | Timeout (in seconds) when communicating to Consul. Defaults to 10 seconds. | +| `CONSUL_HTTP_TOKEN` | The Consul token to use when connecting to the server. | +| `CONSUL_HTTP_AUTH` | Should be specified as `<username>:<password>`. Used to authenticate to the server. | +| `CONSUL_HTTP_SSL` | Force HTTPS if set to `true` value. Disables if set to `false`. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | +| `CONSUL_TLS_SERVER_NAME` | The server name to use as the SNI host when connecting to Consul via TLS. | +| `CONSUL_CACERT` | Path to CA file for verifying Consul server using TLS. | +| `CONSUL_CAPATH` | Path to directory of CA files for verifying Consul server using TLS. | +| `CONSUL_CLIENT_CERT` | Client certificate file for certificate authentication. If this is set, `$CONSUL_CLIENT_KEY` must also be set. | +| `CONSUL_CLIENT_KEY` | Client key file for certificate authentication. If this is set, `$CONSUL_CLIENT_CERT` must also be set. | +| `CONSUL_HTTP_SSL_VERIFY` | Set to `false` to disable Consul TLS certificate checking. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. <br/> _Recommended only for testing and development scenarios!_ | +| `CONSUL_VAULT_ROLE` | Set to the name of the role to use for authenticating to Consul with [Vault's Consul secret backend](https://www.vaultproject.io/docs/secrets/consul/index.html). | +| `CONSUL_VAULT_MOUNT` | Used to override the mount-point when using Vault's Consul secret backend for authentication. Defaults to `consul`. | -Creates a slice. Useful when needing to `range` over a bunch of variables. +If a path is included it is used as a prefix for all uses of the datasource. #### Example -_`input.tmpl`:_ +```console +$ gomplate -d consul=consul:// -i '{{(datasource "consul" "foo")}}' +value for foo key ``` -{{range slice "Bart" "Lisa" "Maggie"}} -Hello, {{.}} -{{- end}} + +```console +$ gomplate -d consul=consul+https://my-consul-server.com:8533/foo -i '{{(datasource "consul" "bar")}}' +value for foo/bar key ``` ```console -$ gomplate < input.tmpl -Hello, Bart -Hello, Lisa -Hello, Maggie +$ gomplate -d consul=consul:///foo -i '{{(datasource "consul" "bar/baz")}}' +value for foo/bar/baz key ``` -## `urlParse` +Instead of using a non-authenticated Consul connection or connecting using the token set with the +`CONSUL_HTTP_TOKEN` environment variable, it is possible to authenticate using a dynamically generated +token fetched from Vault. This requires Vault to be configured to use the [Consul secret backend](https://www.vaultproject.io/docs/secrets/consul/index.html) and +is enabled by passing the name of the role to use in the `CONSUL_VAULT_ROLE` environment variable. -Parses a string as a URL for later use. Equivalent to [url.Parse](https://golang.org/pkg/net/url/#Parse) +### Usage with BoltDB data -#### Example +[BoltDB](https://github.com/boltdb/bolt) is a simple local key/value store used +by many Go tools. The `boltdb://` scheme can be used to access values stored in +a BoltDB database file. The full path is provided in the URL, and the bucket name +can be specified using a URL fragment (e.g. `boltdb:///tmp/database.db#bucket`). -_`input.tmpl`:_ -``` -{{ $u := urlParse "https://example.com:443/foo/bar" }} -The scheme is {{ $u.Scheme }} -The host is {{ $u.Host }} -The path is {{ $u.Path }} -``` +Access is implemented through [libkv](https://github.com/docker/libkv), and as +such, the first 8 bytes of all values are used as an incrementing last modified +index value. All values must therefore be at least 9 bytes long, with the first +8 being ignored. + +The following environment variables can be set: + +| name | usage | +|------|-------| +| `BOLTDB_TIMEOUT` | Timeout (in seconds) to wait for a lock on the database file when opening. | +| `BOLTDB_PERSIST` | If set keep the database open instead of closing after each read. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | + +### Example ```console -$ gomplate < input.tmpl -The scheme is https -The host is example.com:443 -The path is /foo/bar +$ gomplate -d config=boltdb:///tmp/config.db#Bucket1 -i '{{(datasource "config" "foo")}}' +bar ``` -## `has` +### Usage with Vault data -Has reports whether or not a given object has a property with the given key. Can be used with `if` to prevent the template from trying to access a non-existent property in an object. +The special `vault://` URL scheme can be used to retrieve data from [Hashicorp +Vault](https://vaultproject.io). To use this, you must put the Vault server's +URL in the `$VAULT_ADDR` environment variable. -#### Example +This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: -_Let's say we're using a Vault datasource..._ +| auth backend | configuration | +|-------------: |---------------| +| [`approle`](https://www.vaultproject.io/docs/auth/approle.html) | Environment variables `$VAULT_ROLE_ID` and `$VAULT_SECRET_ID` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_APPROLE_MOUNT`. | +| [`app-id`](https://www.vaultproject.io/docs/auth/app-id.html) | Environment variables `$VAULT_APP_ID` and `$VAULT_USER_ID` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_APP_ID_MOUNT`. | +| [`github`](https://www.vaultproject.io/docs/auth/github.html) | Environment variable `$VAULT_AUTH_GITHUB_TOKEN` must be set to an appropriate value.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_GITHUB_MOUNT`. | +| [`userpass`](https://www.vaultproject.io/docs/auth/userpass.html) | Environment variables `$VAULT_AUTH_USERNAME` and `$VAULT_AUTH_PASSWORD` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_USERPASS_MOUNT`. | +| [`token`](https://www.vaultproject.io/docs/auth/token.html) | Determined from either the `$VAULT_TOKEN` environment variable, or read from the file `~/.vault-token` | +| [`aws`](https://www.vaultproject.io/docs/auth/aws.html) | As a final option authentication will be attempted using the AWS auth backend. See below for more details. | -_`input.tmpl`:_ +_**Note:**_ The secret values listed in the above table can either be set in environment +variables or provided in files. This can increase security when using +[Docker Swarm Secrets](https://docs.docker.com/engine/swarm/secrets/), for example. +To use files, specify the filename by appending `_FILE` to the environment variable, +(i.e. `VAULT_USER_ID_FILE`). If the non-file variable is set, this will override +any `_FILE` variable and the secret file will be ignored. + +To use a Vault datasource with a single secret, just use a URL of +`vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left +empty. + +```console +$ echo 'My voice is my passport. {{(datasource "vault").value}}' \ + | gomplate -d vault=vault:///secret/sneakers +My voice is my passport. Verify me. ``` -{{ $secret := datasource "vault" "mysecret" -}} -The secret is ' -{{- if (has $secret "value") }} -{{- $secret.value }} -{{- else }} -{{- $secret | toYAML }} -{{- end }}' + +You can also specify the secret path in the template by using a URL of `vault://` +(or `vault:///`, or `vault:`): +```console +$ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \ + | gomplate -d vault=vault:// +My voice is my passport. Verify me. ``` -If the `secret/foo/mysecret` secret in Vault has a property named `value` set to `supersecret`: +And the two can be mixed to scope secrets to a specific namespace: ```console -$ gomplate -d vault:///secret/foo < input.tmpl -The secret is 'supersecret' +$ echo 'db_password={{(datasource "vault" "db/pass").value}}' \ + | gomplate -d vault=vault:///secret/production +db_password=prodsecret ``` -On the other hand, if there is no `value` property: +It is also possible to use dynamic secrets by using the write capability of the datasource. To use, +add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then +included within the JSON body of the request. ```console -$ gomplate -d vault:///secret/foo < input.tmpl -The secret is 'foo: bar' +$ echo 'otp={{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").key}}' \ + | gomplate -d vault=vault:/// +otp=604a4bd5-7afd-30a2-d2d8-80c4aebc6183 ``` -## `join` +#### Authentication using AWS details -Concatenates the elements of an array to create a string. The separator string sep is placed between elements in the resulting string. +If running on an EC2 instance authentication will be attempted using the AWS auth backend. The +optional `VAULT_AUTH_AWS_MOUNT` environment variable can be used to set the mount point to use if +it differs from the default of `aws`. Additionally `AWS_TIMEOUT` can be set (in seconds) to a value +to wait for AWS to respond before skipping the attempt. -#### Example +If set, the `VAULT_AUTH_AWS_ROLE` environment variable will be used to specify the role to authenticate +using. If not set the AMI ID of the EC2 instance will be used by Vault. + +## `datasourceExists` + +Tests whether or not a given datasource was defined on the commandline (with the +[`--datasource/-d`](#--datasource-d) argument). This is intended mainly to allow +a template to be rendered differently whether or not a given datasource was +defined. + +Note: this does _not_ verify if the datasource is reachable. + +Useful when used in an `if`/`else` block + +```console +$ echo '{{if (datasourceExists "test")}}{{datasource "test"}}{{else}}no worries{{end}}' | gomplate +no worries +``` + +## `ds` + +Alias to [`datasource`](#datasource) + +## `include` + +Includes the content of a given datasource (provided by the [`--datasource/-d`](../usage/#datasource-d) argument). + +This is similar to [`datasource`](#datasource), +except that the data is not parsed. -_`input.tmpl`_ +### Usage + +```go +include alias [subpath] +``` + +### Arguments + +| name | description | +|--------|-------| +| `alias` | the datasource alias, as provided by [`--datasource/-d`](../usage/#datasource-d) | +| `subpath` | _(optional)_ the subpath to use, if supported by the datasource | + +### Examples + +_`person.json`:_ +```json +{ "name": "Dave" } ``` -{{ $a := `[1, 2, 3]` | jsonArray }} -{{ join $a "-" }} + +_`input.tmpl`:_ +```go +{ + "people": [ + {{ include "person" }} + ] +} ``` ```console -$ gomplate -f input.tmpl -1-2-3 +$ gomplate -d person.json -f input.tmpl +{ + "people": [ + { "name": "Dave" } + ] +} ``` -## `json` +## `data.JSON` + +**Alias:** `json` Converts a JSON string into an object. Only works for JSON Objects (not Arrays or other valid JSON types). This can be used to access properties of JSON objects. @@ -131,7 +278,9 @@ $ gomplate < input.tmpl Hello world ``` -## `jsonArray` +## `data.JSONArray` + +**Alias:** `jsonArray` Converts a JSON string into a slice. Only works for JSON Arrays. @@ -148,7 +297,9 @@ $ gomplate < input.tmpl Hello world ``` -## `yaml` +## `data.YAML` + +**Alias:** `yaml` Converts a YAML string into an object. Only works for YAML Objects (not Arrays or other valid YAML types). This can be used to access properties of YAML objects. @@ -165,7 +316,9 @@ $ gomplate < input.tmpl Hello world ``` -## `yamlArray` +## `data.YAMLArray` + +**Alias:** `yamlArray` Converts a YAML string into a slice. Only works for YAML Arrays. @@ -182,7 +335,9 @@ $ gomplate < input.tmpl Hello world ``` -## `toml` +## `data.TOML` + +**Alias:** `toml` Converts a [TOML](https://github.com/toml-lang/toml) document into an object. This can be used to access properties of TOML documents. @@ -220,7 +375,9 @@ $ gomplate -f input.tmpl Hello world ``` -## `csv` +## `data.CSV` + +**Alias:** `csv` Converts a CSV-format string into a 2-dimensional string array. @@ -264,7 +421,9 @@ Go has 25 keywords. COBOL has 357 keywords. ``` -## `csvByRow` +## `data.CSVByRow` + +**Alias:** `csvByRow` Converts a CSV-format string into a slice of maps. @@ -315,7 +474,9 @@ Go has 25 keywords. COBOL has 357 keywords. ``` -## `csvByColumn` +## `data.CSVByColumn` + +**Alias:** `csvByColumn` Like [`csvByRow`](#csvByRow), except that the data is presented as a columnar (column-oriented) map. @@ -339,7 +500,9 @@ Go COBOL ``` -## `toJSON` +## `data.ToJSON` + +**Alias:** `toJSON` Converts an object to a JSON document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. @@ -357,9 +520,14 @@ $ gomplate < input.tmpl {"hello":"world"} ``` -## `toJSONPretty` +## `data.ToJSONPretty` -Converts an object to a pretty-printed (or _indented_) JSON document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. +**Alias:** `toJSONPretty` + +Converts an object to a pretty-printed (or _indented_) JSON document. +Input objects may be the result of functions like `data.JSON`, `data.YAML`, +`data.JSONArray`, or `data.YAMLArray` functions, or they could be provided +by a [`datasource`](../general/datasource). The indent string must be provided as an argument. @@ -367,7 +535,7 @@ The indent string must be provided as an argument. _`input.tmpl`:_ ``` -{{ `{"hello":"world"}` | json | toJSONPretty " " }} +{{ `{"hello":"world"}` | data.JSON | data.ToJSONPretty " " }} ``` ```console @@ -377,17 +545,21 @@ $ gomplate < input.tmpl } ``` -## `toYAML` +## `data.ToYAML` + +**Alias:** `toYAML` -Converts an object to a YAML document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. +Converts an object to a YAML document. Input objects may be the result of +`data.JSON`, `data.YAML`, `data.JSONArray`, or `data.YAMLArray` functions, +or they could be provided by a [`datasource`](../general/datasource). #### Example -_This is obviously contrived - `json` is used to create an object._ +_This is obviously contrived - `data.JSON` is used to create an object._ _`input.tmpl`:_ ``` -{{ (`{"foo":{"hello":"world"}}` | json).foo | toYAML }} +{{ (`{"foo":{"hello":"world"}}` | data.JSON).foo | data.ToYAML }} ``` ```console @@ -395,19 +567,21 @@ $ gomplate < input.tmpl hello: world ``` -## `toTOML` +## `data.ToTOML` + +**Alias:** `toTOML` Converts an object to a [TOML](https://github.com/toml-lang/toml) document. ### Usage ```go -toTOML obj +data.ToTOML obj ``` Can also be used in a pipeline: ```go -obj | toTOML +obj | data.ToTOML ``` ### Arguments @@ -419,31 +593,33 @@ obj | toTOML #### Example ```console -$ gomplate -i '{{ `{"foo":"bar"}` | json | toTOML }}' +$ gomplate -i '{{ `{"foo":"bar"}` | data.JSON | data.ToTOML }}' foo = "bar" ``` -## `toCSV` +## `data.ToCSV` + +**Alias:** `toCSV` Converts an object to a CSV document. The input object must be a 2-dimensional -array of strings (a `[][]string`). Objects produced by [`csvByRow`](#csvByRow) -and [`csvByColumn`](#csvByColumn) cannot yet be converted back to CSV documents. +array of strings (a `[][]string`). Objects produced by [`data.CSVByRow`](#conv-csvbyrow) +and [`data.CSVByColumn`](#conv-csvbycolumn) cannot yet be converted back to CSV documents. -**Note:** With the exception that a custom delimiter can be used, `toCSV` +**Note:** With the exception that a custom delimiter can be used, `data.ToCSV` outputs according to the [RFC 4180](https://tools.ietf.org/html/rfc4180) format, which means that line terminators are `CRLF` (Windows format, or `\r\n`). If you require `LF` (UNIX format, or `\n`), the output can be piped through -[`replaceAll`](#replaceAll) to replace `"\r\n"` with `"\n"`. +[`strings.ReplaceAll`](../strings/#strings-replaceall) to replace `"\r\n"` with `"\n"`. ### Usage ```go -toCSV [delim] input +data.ToCSV [delim] input ``` Can also be used in a pipeline: ```go -input | toCSV [delim] +input | data.ToCSV [delim] ``` ### Arguments @@ -458,7 +634,7 @@ input | toCSV [delim] _`input.tmpl`:_ ```go {{ $rows := (jsonArray `[["first","second"],["1","2"],["3","4"]]`) -}} -{{ toCSV ";" $rows }} +{{ data.ToCSV ";" $rows }} ``` ```console @@ -467,255 +643,3 @@ first,second 1,2 3,4 ``` - -## `datasource` - -Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument). - -Currently, `file://`, `http://`, `https://`, and `vault://` URLs are supported. - -Currently-supported formats are JSON, YAML, TOML, and CSV. - -### Basic usage - -_`person.json`:_ -```json -{ - "name": "Dave" -} -``` - -_`input.tmpl`:_ -``` -Hello {{ (datasource "person").name }} -``` - -```console -$ gomplate -d person.json < input.tmpl -Hello Dave -``` - -### Usage with HTTP data - -```console -$ echo 'Hello there, {{(datasource "foo").headers.Host}}...' | gomplate -d foo=https://httpbin.org/get -Hello there, httpbin.org... -``` - -Additional headers can be provided with the `--datasource-header`/`-H` option: - -```console -$ gomplate -d foo=https://httpbin.org/get -H 'foo=Foo: bar' -i '{{(datasource "foo").headers.Foo}}' -bar -``` - -### Usage with Consul data - -There are three supported URL schemes to retrieve data from [Consul](https://consul.io/). -The `consul://` (or `consul+http://`) scheme can optionally be used with a hostname and port to specify a server (e.g. `consul://localhost:8500`). -By default HTTP will be used, but the `consul+https://` form can be used to use HTTPS, alternatively `$CONSUL_HTTP_SSL` can be used. - -If the server address isn't part of the datasource URL, `$CONSUL_HTTP_ADDR` will be checked. - -The following optional environment variables can be set: - -| name | usage | -|------|-------| -| `CONSUL_HTTP_ADDR` | Hostname and optional port for connecting to Consul. Defaults to `http://localhost:8500` | -| `CONSUL_TIMEOUT` | Timeout (in seconds) when communicating to Consul. Defaults to 10 seconds. | -| `CONSUL_HTTP_TOKEN` | The Consul token to use when connecting to the server. | -| `CONSUL_HTTP_AUTH` | Should be specified as `<username>:<password>`. Used to authenticate to the server. | -| `CONSUL_HTTP_SSL` | Force HTTPS if set to `true` value. Disables if set to `false`. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | -| `CONSUL_TLS_SERVER_NAME` | The server name to use as the SNI host when connecting to Consul via TLS. | -| `CONSUL_CACERT` | Path to CA file for verifying Consul server using TLS. | -| `CONSUL_CAPATH` | Path to directory of CA files for verifying Consul server using TLS. | -| `CONSUL_CLIENT_CERT` | Client certificate file for certificate authentication. If this is set, `$CONSUL_CLIENT_KEY` must also be set. | -| `CONSUL_CLIENT_KEY` | Client key file for certificate authentication. If this is set, `$CONSUL_CLIENT_CERT` must also be set. | -| `CONSUL_HTTP_SSL_VERIFY` | Set to `false` to disable Consul TLS certificate checking. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. <br/> _Recommended only for testing and development scenarios!_ | -| `CONSUL_VAULT_ROLE` | If set will fetch the Consul authentication from Vault using the Consul dynamic secret backend. Value should be the name of the role to use. | -| `CONSUL_VAULT_MOUNT` | If using Vault for authentication set the name of the Consul dynamic secret backend. Defaults to `consul`. | - -If a path is included it is used as a prefix for all uses of the datasource. - -#### Example - -```console -$ gomplate -d consul=consul:// -i '{{(datasource "consul" "foo")}}' -value for foo key -``` - -```console -$ gomplate -d consul=consul+https://my-consul-server.com:8533/foo -i '{{(datasource "consul" "bar")}}' -value for foo/bar key -``` - -```console -$ gomplate -d consul=consul:///foo -i '{{(datasource "consul" "bar/baz")}}' -value for foo/bar/baz key -``` - -Instead of using a non-authenticated Consul connection or connecting using the token set with the -`CONSUL_HTTP_TOKEN` environment variable, it is possible to authenticate using a dynamically generated -token fetched from Vault. This requires Vault to be configured to use the Consul secret backend and -is enabled by passing the name of the role to use in the `CONSUL_VAULT_ROLE` environment variable. - -### Usage with BoltDB data - -[BoltDB](https://github.com/boltdb/bolt) is a simple local key/value store used -by many Go tools. The `boltdb://` scheme can be used to access values stored in -a BoltDB database file. The full path is provided in the URL, and the bucket name -can be specified using a URL fragment (e.g. `boltdb:///tmp/database.db#bucket`). - -Access is implemented through [libkv](https://github.com/docker/libkv), and as -such, the first 8 bytes of all values are used as an incrementing last modified -index value. All values must therefore be at least 9 bytes long, with the first -8 being ignored. - -The following environment variables can be set: - -| name | usage | -|------|-------| -| `BOLTDB_TIMEOUT` | Timeout (in seconds) to wait for a lock on the database file when opening. | -| `BOLTDB_PERSIST` | If set keep the database open instead of closing after each read. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | - -### Example - -```console -$ gomplate -d config=boltdb:///tmp/config.db#Bucket1 -i '{{(datasource "config" "foo")}}' -bar -``` - -### Usage with Vault data - -The special `vault://` URL scheme can be used to retrieve data from [Hashicorp -Vault](https://vaultproject.io). To use this, you must put the Vault server's -URL in the `$VAULT_ADDR` environment variable. - -This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: - -| auth backend | configuration | -|-------------: |---------------| -| [`approle`](https://www.vaultproject.io/docs/auth/approle.html) | Environment variables `$VAULT_ROLE_ID` and `$VAULT_SECRET_ID` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_APPROLE_MOUNT`. | -| [`app-id`](https://www.vaultproject.io/docs/auth/app-id.html) | Environment variables `$VAULT_APP_ID` and `$VAULT_USER_ID` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_APP_ID_MOUNT`. | -| [`github`](https://www.vaultproject.io/docs/auth/github.html) | Environment variable `$VAULT_AUTH_GITHUB_TOKEN` must be set to an appropriate value.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_GITHUB_MOUNT`. | -| [`userpass`](https://www.vaultproject.io/docs/auth/userpass.html) | Environment variables `$VAULT_AUTH_USERNAME` and `$VAULT_AUTH_PASSWORD` must be set to the appropriate values.<br/> If the backend is mounted to a different location, set `$VAULT_AUTH_USERPASS_MOUNT`. | -| [`token`](https://www.vaultproject.io/docs/auth/token.html) | Determined from either the `$VAULT_TOKEN` environment variable, or read from the file `~/.vault-token` | -| [`aws`](https://www.vaultproject.io/docs/auth/aws.html) | As a final option authentication will be attempted using the AWS auth backend. See below for more details. | - -_**Note:**_ The secret values listed in the above table can either be set in environment -variables or provided in files. This can increase security when using -[Docker Swarm Secrets](https://docs.docker.com/engine/swarm/secrets/), for example. -To use files, specify the filename by appending `_FILE` to the environment variable, -(i.e. `VAULT_USER_ID_FILE`). If the non-file variable is set, this will override -any `_FILE` variable and the secret file will be ignored. - -To use a Vault datasource with a single secret, just use a URL of -`vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left -empty. - -```console -$ echo 'My voice is my passport. {{(datasource "vault").value}}' \ - | gomplate -d vault=vault:///secret/sneakers -My voice is my passport. Verify me. -``` - -You can also specify the secret path in the template by using a URL of `vault://` -(or `vault:///`, or `vault:`): -```console -$ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \ - | gomplate -d vault=vault:// -My voice is my passport. Verify me. -``` - -And the two can be mixed to scope secrets to a specific namespace: - -```console -$ echo 'db_password={{(datasource "vault" "db/pass").value}}' \ - | gomplate -d vault=vault:///secret/production -db_password=prodsecret -``` - -It is also possible to use dynamic secrets by using the write capability of the datasource. To use, -add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then -included within the JSON body of the request. - -```console -$ echo 'otp={{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").key}}' \ - | gomplate -d vault=vault:/// -otp=604a4bd5-7afd-30a2-d2d8-80c4aebc6183 -``` - -#### Authentication using AWS details - -If running on an EC2 instance authentication will be attempted using the AWS auth backend. The -optional `VAULT_AUTH_AWS_MOUNT` environment variable can be used to set the mount point to use if -it differs from the default of `aws`. Additionally `AWS_TIMEOUT` can be set (in seconds) to a value -to wait for AWS to respond before skipping the attempt. - -If set, the `VAULT_AUTH_AWS_ROLE` environment variable will be used to specify the role to authenticate -using. If not set the AMI ID of the EC2 instance will be used by Vault. - -## `datasourceExists` - -Tests whether or not a given datasource was defined on the commandline (with the -[`--datasource/-d`](#--datasource-d) argument). This is intended mainly to allow -a template to be rendered differently whether or not a given datasource was -defined. - -Note: this does _not_ verify if the datasource is reachable. - -Useful when used in an `if`/`else` block - -```console -$ echo '{{if (datasourceExists "test")}}{{datasource "test"}}{{else}}no worries{{end}}' | gomplate -no worries -``` - -## `ds` - -Alias to [`datasource`](#datasource) - -## `include` - -Includes the content of a given datasource (provided by the [`--datasource/-d`](../usage/#datasource-d) argument). - -This is similar to [`datasource`](#datasource), -except that the data is not parsed. - -### Usage - -```go -include alias [subpath] -``` - -### Arguments - -| name | description | -|--------|-------| -| `alias` | the datasource alias, as provided by [`--datasource/-d`](../usage/#datasource-d) | -| `subpath` | _(optional)_ the subpath to use, if supported by the datasource | - -### Examples - -_`person.json`:_ -```json -{ "name": "Dave" } -``` - -_`input.tmpl`:_ -```go -{ - "people": [ - {{ include "person" }} - ] -} -``` - -```console -$ gomplate -d person.json -f input.tmpl -{ - "people": [ - { "name": "Dave" } - ] -} -``` @@ -1,45 +1,22 @@ package main import ( - "net/url" "text/template" + "github.com/hairyhenderson/gomplate/data" "github.com/hairyhenderson/gomplate/funcs" ) // initFuncs - The function mappings are defined here! -func initFuncs(data *Data) template.FuncMap { - typeconv := &TypeConv{} - - f := template.FuncMap{ - "bool": typeconv.Bool, - "has": typeconv.Has, - "json": typeconv.JSON, - "jsonArray": typeconv.JSONArray, - "yaml": typeconv.YAML, - "yamlArray": typeconv.YAMLArray, - "toml": typeconv.TOML, - "csv": typeconv.CSV, - "csvByRow": typeconv.CSVByRow, - "csvByColumn": typeconv.CSVByColumn, - "slice": typeconv.Slice, - "join": typeconv.Join, - "toJSON": typeconv.ToJSON, - "toJSONPretty": typeconv.toJSONPretty, - "toYAML": typeconv.ToYAML, - "toTOML": typeconv.ToTOML, - "toCSV": typeconv.ToCSV, - "urlParse": url.Parse, - "datasource": data.Datasource, - "ds": data.Datasource, - "datasourceExists": data.DatasourceExists, - "include": data.include, - } +func initFuncs(d *data.Data) template.FuncMap { + f := template.FuncMap{} + funcs.AddDataFuncs(f, d) funcs.AWSFuncs(f) funcs.AddBase64Funcs(f) funcs.AddNetFuncs(f) funcs.AddReFuncs(f) funcs.AddStringFuncs(f) funcs.AddEnvFuncs(f) + funcs.AddConvFuncs(f) return f } diff --git a/funcs/conv.go b/funcs/conv.go new file mode 100644 index 00000000..2e5327b1 --- /dev/null +++ b/funcs/conv.go @@ -0,0 +1,64 @@ +package funcs + +import ( + "net/url" + "sync" + + "github.com/hairyhenderson/gomplate/conv" +) + +var ( + convNS *ConvFuncs + convNSInit sync.Once +) + +// ConvNS - +func ConvNS() *ConvFuncs { + convNSInit.Do(func() { convNS = &ConvFuncs{} }) + return convNS +} + +// AddConvFuncs - +func AddConvFuncs(f map[string]interface{}) { + f["conv"] = ConvNS + + f["urlParse"] = ConvNS().URL + f["bool"] = ConvNS().Bool + f["has"] = ConvNS().Has + f["slice"] = ConvNS().Slice + f["join"] = ConvNS().Join +} + +// ConvFuncs - +type ConvFuncs struct{} + +func (f *ConvFuncs) Bool(s string) bool { + return conv.Bool(s) +} + +func (f *ConvFuncs) Slice(args ...interface{}) []interface{} { + return conv.Slice(args...) +} +func (f *ConvFuncs) Join(in interface{}, sep string) string { + return conv.Join(in, sep) +} +func (f *ConvFuncs) Has(in interface{}, key string) bool { + return conv.Has(in, key) +} + +func (f *ConvFuncs) ParseInt(s string, base, bitSize int) int64 { + return conv.MustParseInt(s, base, bitSize) +} +func (f *ConvFuncs) ParseFloat(s string, bitSize int) float64 { + return conv.MustParseFloat(s, bitSize) +} +func (f *ConvFuncs) ParseUint(s string, base, bitSize int) uint64 { + return conv.MustParseUint(s, base, bitSize) +} +func (f *ConvFuncs) Atoi(s string) int { + return conv.MustAtoi(s) +} + +func (f *ConvFuncs) URL(s string) (*url.URL, error) { + return url.Parse(s) +} diff --git a/funcs/data.go b/funcs/data.go new file mode 100644 index 00000000..0a432c1e --- /dev/null +++ b/funcs/data.go @@ -0,0 +1,110 @@ +package funcs + +import ( + "sync" + + "github.com/hairyhenderson/gomplate/data" +) + +var ( + dataNS *DataFuncs + dataNSInit sync.Once +) + +// DataNS - +func DataNS() *DataFuncs { + dataNSInit.Do(func() { dataNS = &DataFuncs{} }) + return dataNS +} + +// AddDataFuncs - +func AddDataFuncs(f map[string]interface{}, d *data.Data) { + f["datasource"] = d.Datasource + f["ds"] = d.Datasource + f["datasourceExists"] = d.DatasourceExists + f["include"] = d.Include + + f["data"] = DataNS + + f["json"] = DataNS().JSON + f["jsonArray"] = DataNS().JSONArray + f["yaml"] = DataNS().YAML + f["yamlArray"] = DataNS().YAMLArray + f["toml"] = DataNS().TOML + f["csv"] = DataNS().CSV + f["csvByRow"] = DataNS().CSVByRow + f["csvByColumn"] = DataNS().CSVByColumn + f["toJSON"] = DataNS().ToJSON + f["toJSONPretty"] = DataNS().ToJSONPretty + f["toYAML"] = DataNS().ToYAML + f["toTOML"] = DataNS().ToTOML + f["toCSV"] = DataNS().ToCSV +} + +// DataFuncs - +type DataFuncs struct{} + +// JSON - +func (f *DataFuncs) JSON(in string) map[string]interface{} { + return data.JSON(in) +} + +// JSONArray - +func (f *DataFuncs) JSONArray(in string) []interface{} { + return data.JSONArray(in) +} + +// YAML - +func (f *DataFuncs) YAML(in string) map[string]interface{} { + return data.YAML(in) +} + +// YAMLArray - +func (f *DataFuncs) YAMLArray(in string) []interface{} { + return data.YAMLArray(in) +} + +// TOML - +func (f *DataFuncs) TOML(in string) interface{} { + return data.TOML(in) +} + +// CSV - +func (f *DataFuncs) CSV(args ...string) [][]string { + return data.CSV(args...) +} + +// CSVByRow - +func (f *DataFuncs) CSVByRow(args ...string) (rows []map[string]string) { + return data.CSVByRow(args...) +} + +// CSVByColumn - +func (f *DataFuncs) CSVByColumn(args ...string) (cols map[string][]string) { + return data.CSVByColumn(args...) +} + +// ToCSV - +func (f *DataFuncs) ToCSV(args ...interface{}) string { + return data.ToCSV(args...) +} + +// ToJSON - +func (f *DataFuncs) ToJSON(in interface{}) string { + return data.ToJSON(in) +} + +// ToJSONPretty - +func (f *DataFuncs) ToJSONPretty(indent string, in interface{}) string { + return data.ToJSONPretty(indent, in) +} + +// ToYAML - +func (f *DataFuncs) ToYAML(in interface{}) string { + return data.ToYAML(in) +} + +// ToTOML - +func (f *DataFuncs) ToTOML(in interface{}) string { + return data.ToTOML(in) +} diff --git a/gomplate.go b/gomplate.go index 71530368..25e2a317 100644 --- a/gomplate.go +++ b/gomplate.go @@ -4,6 +4,8 @@ import ( "io" "log" "text/template" + + "github.com/hairyhenderson/gomplate/data" ) func (g *Gomplate) createTemplate() *template.Template { @@ -31,19 +33,20 @@ func (g *Gomplate) RunTemplate(text string, out io.Writer) { } // NewGomplate - -func NewGomplate(data *Data, leftDelim, rightDelim string) *Gomplate { +func NewGomplate(d *data.Data, leftDelim, rightDelim string) *Gomplate { return &Gomplate{ leftDelim: leftDelim, rightDelim: rightDelim, - funcMap: initFuncs(data), + funcMap: initFuncs(d), } } func runTemplate(o *GomplateOpts) error { defer runCleanupHooks() - data := NewData(o.dataSources, o.dataSourceHeaders) + d := data.NewData(o.dataSources, o.dataSourceHeaders) + addCleanupHook(d.Cleanup) - g := NewGomplate(data, o.lDelim, o.rDelim) + g := NewGomplate(d, o.lDelim, o.rDelim) if o.inputDir != "" { return processInputDir(o.inputDir, o.outputDir, g) diff --git a/gomplate_test.go b/gomplate_test.go index cd940d30..112f6ae3 100644 --- a/gomplate_test.go +++ b/gomplate_test.go @@ -9,6 +9,8 @@ import ( "text/template" "github.com/hairyhenderson/gomplate/aws" + "github.com/hairyhenderson/gomplate/conv" + "github.com/hairyhenderson/gomplate/data" "github.com/hairyhenderson/gomplate/env" "github.com/stretchr/testify/assert" ) @@ -20,11 +22,10 @@ func testTemplate(g *Gomplate, template string) string { } func TestGetenvTemplates(t *testing.T) { - typeconv := &TypeConv{} g := &Gomplate{ funcMap: template.FuncMap{ "getenv": env.Getenv, - "bool": typeconv.Bool, + "bool": conv.Bool, }, } assert.Empty(t, testTemplate(g, `{{getenv "BLAHBLAHBLAH"}}`)) @@ -33,10 +34,9 @@ func TestGetenvTemplates(t *testing.T) { } func TestBoolTemplates(t *testing.T) { - typeconv := &TypeConv{} g := &Gomplate{ funcMap: template.FuncMap{ - "bool": typeconv.Bool, + "bool": conv.Bool, }, } assert.Equal(t, "true", testTemplate(g, `{{bool "true"}}`)) @@ -66,12 +66,11 @@ func TestEc2MetaTemplates(t *testing.T) { func TestEc2MetaTemplates_WithJSON(t *testing.T) { server, ec2meta := aws.MockServer(200, `{"foo":"bar"}`) defer server.Close() - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ "ec2meta": ec2meta.Meta, "ec2dynamic": ec2meta.Dynamic, - "json": ty.JSON, + "json": data.JSON, }, } @@ -80,10 +79,9 @@ func TestEc2MetaTemplates_WithJSON(t *testing.T) { } func TestJSONArrayTemplates(t *testing.T) { - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ - "jsonArray": ty.JSONArray, + "jsonArray": data.JSONArray, }, } @@ -92,11 +90,10 @@ func TestJSONArrayTemplates(t *testing.T) { } func TestYAMLTemplates(t *testing.T) { - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ - "yaml": ty.YAML, - "yamlArray": ty.YAMLArray, + "yaml": data.YAML, + "yamlArray": data.YAMLArray, }, } @@ -106,10 +103,9 @@ func TestYAMLTemplates(t *testing.T) { } func TestSliceTemplates(t *testing.T) { - typeconv := &TypeConv{} g := &Gomplate{ funcMap: template.FuncMap{ - "slice": typeconv.Slice, + "slice": conv.Slice, }, } assert.Equal(t, "foo", testTemplate(g, `{{index (slice "foo") 0}}`)) @@ -118,11 +114,10 @@ func TestSliceTemplates(t *testing.T) { } func TestHasTemplate(t *testing.T) { - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ - "yaml": ty.YAML, - "has": ty.Has, + "yaml": data.YAML, + "has": conv.Has, }, } assert.Equal(t, "true", testTemplate(g, `{{has ("foo:\n bar: true" | yaml) "foo"}}`)) diff --git a/libkv/boltdb.go b/libkv/boltdb.go index d3efe6db..024e21dd 100644 --- a/libkv/boltdb.go +++ b/libkv/boltdb.go @@ -7,8 +7,8 @@ import ( "github.com/docker/libkv" "github.com/docker/libkv/store" "github.com/docker/libkv/store/boltdb" + "github.com/hairyhenderson/gomplate/conv" "github.com/hairyhenderson/gomplate/env" - "github.com/hairyhenderson/gomplate/typeconv" ) // NewBoltDB - initialize a new BoltDB datasource handler @@ -28,10 +28,10 @@ func setupBoltDB(bucket string) *store.Config { logFatal("missing bucket - must specify BoltDB bucket in URL fragment") } - t := typeconv.MustParseInt(env.Getenv("BOLTDB_TIMEOUT"), 10, 16) + t := conv.MustParseInt(env.Getenv("BOLTDB_TIMEOUT"), 10, 16) return &store.Config{ Bucket: bucket, ConnectionTimeout: time.Duration(t) * time.Second, - PersistConnection: typeconv.MustParseBool(env.Getenv("BOLTDB_PERSIST")), + PersistConnection: conv.Bool(env.Getenv("BOLTDB_PERSIST")), } } diff --git a/libkv/consul.go b/libkv/consul.go index 3cf441f7..d4fb53c4 100644 --- a/libkv/consul.go +++ b/libkv/consul.go @@ -11,8 +11,8 @@ import ( "github.com/docker/libkv" "github.com/docker/libkv/store" "github.com/docker/libkv/store/consul" + "github.com/hairyhenderson/gomplate/conv" "github.com/hairyhenderson/gomplate/env" - "github.com/hairyhenderson/gomplate/typeconv" "github.com/hairyhenderson/gomplate/vault" consulapi "github.com/hashicorp/consul/api" ) @@ -66,7 +66,7 @@ func consulURL(u *url.URL) *url.URL { case "consul+https", "https": c.Scheme = "https" case "consul": - if typeconv.MustParseBool(env.Getenv("CONSUL_HTTP_SSL")) { + if conv.Bool(env.Getenv("CONSUL_HTTP_SSL")) { c.Scheme = "https" } else { c.Scheme = "http" @@ -83,7 +83,7 @@ func consulURL(u *url.URL) *url.URL { } func consulConfig(useTLS bool) *store.Config { - t := typeconv.MustAtoi(env.Getenv("CONSUL_TIMEOUT")) + t := conv.MustAtoi(env.Getenv("CONSUL_TIMEOUT")) config := &store.Config{ ConnectionTimeout: time.Duration(t) * time.Second, } @@ -107,7 +107,7 @@ func setupTLS(prefix string) *consulapi.TLSConfig { KeyFile: env.Getenv(prefix + "_CLIENT_KEY"), } if v := env.Getenv(prefix + "_HTTP_SSL_VERIFY"); v != "" { - verify := typeconv.MustParseBool(v) + verify := conv.Bool(v) tlsConfig.InsecureSkipVerify = !verify } return tlsConfig diff --git a/process_test.go b/process_test.go index 23a4e1a7..b63ad6eb 100644 --- a/process_test.go +++ b/process_test.go @@ -11,6 +11,7 @@ import ( "log" + "github.com/hairyhenderson/gomplate/data" "github.com/stretchr/testify/assert" ) @@ -40,13 +41,13 @@ func TestInputDir(t *testing.T) { } })() - src, err := ParseSource("config=test/files/input-dir/config.yml") + src, err := data.ParseSource("config=test/files/input-dir/config.yml") assert.Nil(t, err) - data := &Data{ - Sources: map[string]*Source{"config": src}, + d := &data.Data{ + Sources: map[string]*data.Source{"config": src}, } - gomplate := NewGomplate(data, "{{", "}}") + gomplate := NewGomplate(d, "{{", "}}") err = processInputDir(filepath.Join("test", "files", "input-dir", "in"), outDir, gomplate) assert.Nil(t, err) diff --git a/typeconv/typeconv.go b/typeconv/typeconv.go deleted file mode 100644 index 60e5cf9c..00000000 --- a/typeconv/typeconv.go +++ /dev/null @@ -1,33 +0,0 @@ -package typeconv - -import "strconv" - -// MustParseBool - wrapper for strconv.ParseBool that returns false in the case of error -func MustParseBool(s string) bool { - b, _ := strconv.ParseBool(s) - return b -} - -// MustParseInt - wrapper for strconv.ParseInt that returns 0 in the case of error -func MustParseInt(s string, base, bitSize int) int64 { - i, _ := strconv.ParseInt(s, base, bitSize) - return i -} - -// MustParseFloat - wrapper for strconv.ParseFloat that returns 0 in the case of error -func MustParseFloat(s string, bitSize int) float64 { - i, _ := strconv.ParseFloat(s, bitSize) - return i -} - -// MustParseUint - wrapper for strconv.ParseUint that returns 0 in the case of error -func MustParseUint(s string, base, bitSize int) uint64 { - i, _ := strconv.ParseUint(s, base, bitSize) - return i -} - -// MustAtoi - wrapper for strconv.Atoi that returns 0 in the case of error -func MustAtoi(s string) int { - i, _ := strconv.Atoi(s) - return i -} diff --git a/typeconv/typeconv_test.go b/typeconv/typeconv_test.go deleted file mode 100644 index e07eaa28..00000000 --- a/typeconv/typeconv_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package typeconv - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMustParseBool(t *testing.T) { - for _, b := range []string{"1", "t", "T", "true", "TRUE", "True"} { - assert.True(t, MustParseBool(b)) - } - for _, b := range []string{"0", "f", "F", "false", "FALSE", "False", "", "gibberish", "12345"} { - assert.False(t, MustParseBool(b)) - } -} - -func TestMustParseInt(t *testing.T) { - for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { - assert.Equal(t, 0, int(MustParseInt(i, 10, 64))) - } - assert.Equal(t, 1, int(MustParseInt("1", 10, 64))) - assert.Equal(t, -1, int(MustParseInt("-1", 10, 64))) -} - -func TestMustAtoi(t *testing.T) { - for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { - assert.Equal(t, 0, MustAtoi(i)) - } - assert.Equal(t, 1, MustAtoi("1")) - assert.Equal(t, -1, MustAtoi("-1")) -} - -func TestMustParseUint(t *testing.T) { - for _, i := range []string{"0", "-0", "-1", "foo", "", "*&^%"} { - assert.Equal(t, uint64(0), MustParseUint(i, 10, 64)) - } - assert.Equal(t, uint64(1), MustParseUint("1", 10, 64)) -} - -func TestMustParseFloat(t *testing.T) { - for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { - assert.Equal(t, 0.0, MustParseFloat(i, 64)) - } - assert.Equal(t, 1.0, MustParseFloat("1", 64)) - assert.Equal(t, -1.0, MustParseFloat("-1", 64)) -} diff --git a/vault/auth.go b/vault/auth.go index 960e61ef..50d23df3 100644 --- a/vault/auth.go +++ b/vault/auth.go @@ -10,8 +10,8 @@ import ( "github.com/blang/vfs" "github.com/hairyhenderson/gomplate/aws" + "github.com/hairyhenderson/gomplate/conv" "github.com/hairyhenderson/gomplate/env" - "github.com/hairyhenderson/gomplate/typeconv" ) // GetToken - @@ -167,7 +167,7 @@ func (v *Vault) EC2Login() string { } opts := aws.ClientOptions{ - Timeout: time.Duration(typeconv.MustAtoi(os.Getenv("AWS_TIMEOUT"))) * time.Millisecond, + Timeout: time.Duration(conv.MustAtoi(os.Getenv("AWS_TIMEOUT"))) * time.Millisecond, } meta := aws.NewEc2Meta(opts) |
