From 51ddb6e800ab087fa3dff19686b0f1f39a1a4432 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sat, 5 Aug 2017 13:21:05 -0400 Subject: Extracting data namespace, renaming typeconv to conv namespace Signed-off-by: Dave Henderson --- conv/conv.go | 110 ++++++ conv/conv_test.go | 86 +++++ data.go | 438 ----------------------- data/data.go | 250 +++++++++++++ data/data_test.go | 319 +++++++++++++++++ data/datasource.go | 452 ++++++++++++++++++++++++ data/datasource_test.go | 300 ++++++++++++++++ data_test.go | 300 ---------------- docs/content/functions/conv.md | 130 +++++++ docs/content/functions/data.md | 645 ++++++++++++++++++++++++++++++++++ docs/content/functions/general.md | 721 -------------------------------------- funcs.go | 33 +- funcs/conv.go | 64 ++++ funcs/data.go | 110 ++++++ gomplate.go | 11 +- gomplate_test.go | 27 +- libkv/boltdb.go | 6 +- libkv/consul.go | 8 +- process_test.go | 9 +- typeconv.go | 335 ------------------ typeconv/typeconv.go | 33 -- typeconv/typeconv_test.go | 47 --- typeconv_test.go | 382 -------------------- vault/auth.go | 4 +- 24 files changed, 2503 insertions(+), 2317 deletions(-) create mode 100644 conv/conv.go create mode 100644 conv/conv_test.go delete mode 100644 data.go create mode 100644 data/data.go create mode 100644 data/data_test.go create mode 100644 data/datasource.go create mode 100644 data/datasource_test.go delete mode 100644 data_test.go create mode 100644 docs/content/functions/conv.md create mode 100644 docs/content/functions/data.md delete mode 100644 docs/content/functions/general.md create mode 100644 funcs/conv.go create mode 100644 funcs/data.go delete mode 100644 typeconv.go delete mode 100644 typeconv/typeconv.go delete mode 100644 typeconv/typeconv_test.go delete mode 100644 typeconv_test.go diff --git a/conv/conv.go b/conv/conv.go new file mode 100644 index 00000000..ec89e0fa --- /dev/null +++ b/conv/conv.go @@ -0,0 +1,110 @@ +package conv + +import ( + "fmt" + "log" + "reflect" + "strconv" + "strings" +) + +// Bool converts a string to a boolean value, using strconv.ParseBool under the covers. +// Possible true values are: 1, t, T, TRUE, true, True +// All other values are considered false. +func Bool(in string) bool { + if b, err := strconv.ParseBool(in); err == nil { + return b + } + return false +} + +// Slice creates a slice from a bunch of arguments +func Slice(args ...interface{}) []interface{} { + return args +} + +// Join concatenates the elements of a to create a single string. +// The separator string sep is placed between elements in the resulting string. +// +// This is functionally identical to strings.Join, except that each element is +// coerced to a string first +func Join(in interface{}, sep string) string { + s, ok := in.([]string) + if ok { + return strings.Join(s, sep) + } + + var a []interface{} + a, ok = in.([]interface{}) + if ok { + b := make([]string, len(a)) + for i := range a { + b[i] = toString(a[i]) + } + return strings.Join(b, sep) + } + + log.Fatal("Input to Join must be an array") + return "" +} + +// Has determines whether or not a given object has a property with the given key +func Has(in interface{}, key string) bool { + av := reflect.ValueOf(in) + kv := reflect.ValueOf(key) + + if av.Kind() == reflect.Map { + return av.MapIndex(kv).IsValid() + } + + return false +} + +func toString(in interface{}) string { + if s, ok := in.(string); ok { + return s + } + if s, ok := in.(fmt.Stringer); ok { + return s.String() + } + if i, ok := in.(int); ok { + return strconv.Itoa(i) + } + if u, ok := in.(uint64); ok { + return strconv.FormatUint(u, 10) + } + if f, ok := in.(float64); ok { + return strconv.FormatFloat(f, 'f', -1, 64) + } + if b, ok := in.(bool); ok { + return strconv.FormatBool(b) + } + if in == nil { + return "nil" + } + return fmt.Sprintf("%s", in) +} + +// MustParseInt - wrapper for strconv.ParseInt that returns 0 in the case of error +func MustParseInt(s string, base, bitSize int) int64 { + i, _ := strconv.ParseInt(s, base, bitSize) + return i +} + +// MustParseFloat - wrapper for strconv.ParseFloat that returns 0 in the case of error +func MustParseFloat(s string, bitSize int) float64 { + i, _ := strconv.ParseFloat(s, bitSize) + return i +} + +// MustParseUint - wrapper for strconv.ParseUint that returns 0 in the case of error +func MustParseUint(s string, base, bitSize int) uint64 { + i, _ := strconv.ParseUint(s, base, bitSize) + return i +} + +// MustAtoi - wrapper for strconv.Atoi that returns 0 in the case of error +func MustAtoi(s string) int { + i, _ := strconv.Atoi(s) + return i +} diff --git a/conv/conv_test.go b/conv/conv_test.go new file mode 100644 index 00000000..6b57f65a --- /dev/null +++ b/conv/conv_test.go @@ -0,0 +1,86 @@ +package conv + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBool(t *testing.T) { + assert.False(t, Bool("")) + assert.False(t, Bool("asdf")) + assert.False(t, Bool("1234")) + assert.False(t, Bool("False")) + assert.False(t, Bool("0")) + assert.False(t, Bool("false")) + assert.False(t, Bool("F")) + assert.False(t, Bool("f")) + assert.True(t, Bool("true")) + assert.True(t, Bool("True")) + assert.True(t, Bool("t")) + assert.True(t, Bool("T")) + assert.True(t, Bool("1")) +} + +func TestSlice(t *testing.T) { + expected := []string{"foo", "bar"} + actual := Slice("foo", "bar") + assert.Equal(t, expected[0], actual[0]) + assert.Equal(t, expected[1], actual[1]) +} + +func TestJoin(t *testing.T) { + + assert.Equal(t, "foo,bar", Join([]interface{}{"foo", "bar"}, ",")) + assert.Equal(t, "foo,\nbar", Join([]interface{}{"foo", "bar"}, ",\n")) + // Join handles all kinds of scalar types too... + assert.Equal(t, "42-18446744073709551615", Join([]interface{}{42, uint64(18446744073709551615)}, "-")) + assert.Equal(t, "1,,true,3.14,foo,nil", Join([]interface{}{1, "", true, 3.14, "foo", nil}, ",")) + // and best-effort with weird types + assert.Equal(t, "[foo],bar", Join([]interface{}{[]string{"foo"}, "bar"}, ",")) +} + +func TestHas(t *testing.T) { + + in := map[string]interface{}{ + "foo": "bar", + "baz": map[string]interface{}{ + "qux": "quux", + }, + } + + assert.True(t, Has(in, "foo")) + assert.False(t, Has(in, "bar")) + assert.True(t, Has(in["baz"], "qux")) +} + +func TestMustParseInt(t *testing.T) { + for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { + assert.Equal(t, 0, int(MustParseInt(i, 10, 64))) + } + assert.Equal(t, 1, int(MustParseInt("1", 10, 64))) + assert.Equal(t, -1, int(MustParseInt("-1", 10, 64))) +} + +func TestMustAtoi(t *testing.T) { + for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { + assert.Equal(t, 0, MustAtoi(i)) + } + assert.Equal(t, 1, MustAtoi("1")) + assert.Equal(t, -1, MustAtoi("-1")) +} + +func TestMustParseUint(t *testing.T) { + for _, i := range []string{"0", "-0", "-1", "foo", "", "*&^%"} { + assert.Equal(t, uint64(0), MustParseUint(i, 10, 64)) + } + assert.Equal(t, uint64(1), MustParseUint("1", 10, 64)) +} + +func TestMustParseFloat(t *testing.T) { + for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { + assert.Equal(t, 0.0, MustParseFloat(i, 64)) + } + assert.Equal(t, 1.0, MustParseFloat("1", 64)) + assert.Equal(t, -1.0, MustParseFloat("-1", 64)) +} diff --git a/data.go b/data.go deleted file mode 100644 index 7f41d2ec..00000000 --- a/data.go +++ /dev/null @@ -1,438 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "io/ioutil" - "log" - "mime" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "strings" - "time" - - "github.com/blang/vfs" - "github.com/hairyhenderson/gomplate/libkv" - "github.com/hairyhenderson/gomplate/vault" -) - -// logFatal is defined so log.Fatal calls can be overridden for testing -var logFatalf = log.Fatalf - -func regExtension(ext, typ string) { - err := mime.AddExtensionType(ext, typ) - if err != nil { - log.Fatal(err) - } -} - -func init() { - // Add some types we want to be able to handle which can be missing by default - regExtension(".json", "application/json") - regExtension(".yml", "application/yaml") - regExtension(".yaml", "application/yaml") - regExtension(".csv", "text/csv") - regExtension(".toml", "application/toml") - - sourceReaders = make(map[string]func(*Source, ...string) ([]byte, error)) - - // Register our source-reader functions - addSourceReader("http", readHTTP) - addSourceReader("https", readHTTP) - addSourceReader("file", readFile) - addSourceReader("vault", readVault) - addSourceReader("consul", readConsul) - addSourceReader("consul+http", readConsul) - addSourceReader("consul+https", readConsul) - addSourceReader("boltdb", readBoltDB) -} - -var sourceReaders map[string]func(*Source, ...string) ([]byte, error) - -// addSourceReader - -func addSourceReader(scheme string, readFunc func(*Source, ...string) ([]byte, error)) { - sourceReaders[scheme] = readFunc -} - -// Data - -type Data struct { - Sources map[string]*Source - cache map[string][]byte -} - -// NewData - constructor for Data -func NewData(datasourceArgs []string, headerArgs []string) *Data { - sources := make(map[string]*Source) - headers := parseHeaderArgs(headerArgs) - for _, v := range datasourceArgs { - s, err := ParseSource(v) - if err != nil { - log.Fatalf("error parsing datasource %v", err) - return nil - } - s.Header = headers[s.Alias] - sources[s.Alias] = s - } - return &Data{ - Sources: sources, - } -} - -// Source - a data source -type Source struct { - Alias string - URL *url.URL - Ext string - Type string - Params map[string]string - FS vfs.Filesystem // used for file: URLs, nil otherwise - HC *http.Client // used for http[s]: URLs, nil otherwise - VC *vault.Vault // used for vault: URLs, nil otherwise - KV *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise - Header http.Header // used for http[s]: URLs, nil otherwise -} - -// NewSource - builds a &Source -func NewSource(alias string, URL *url.URL) (s *Source) { - ext := filepath.Ext(URL.Path) - - s = &Source{ - Alias: alias, - URL: URL, - Ext: ext, - } - - if ext != "" && URL.Scheme != "boltdb" { - mediatype := mime.TypeByExtension(ext) - t, params, err := mime.ParseMediaType(mediatype) - if err != nil { - log.Fatal(err) - } - s.Type = t - s.Params = params - } - return -} - -// String is the method to format the flag's value, part of the flag.Value interface. -// The String method's output will be used in diagnostics. -func (s *Source) String() string { - return fmt.Sprintf("%s=%s (%s)", s.Alias, s.URL.String(), s.Type) -} - -// ParseSource - -func ParseSource(value string) (*Source, error) { - var ( - alias string - srcURL *url.URL - ) - parts := strings.SplitN(value, "=", 2) - if len(parts) == 1 { - f := parts[0] - alias = strings.SplitN(value, ".", 2)[0] - if path.Base(f) != f { - err := fmt.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory", value) - return nil, err - } - srcURL = absURL(f) - } else if len(parts) == 2 { - alias = parts[0] - var err error - srcURL, err = url.Parse(parts[1]) - if err != nil { - return nil, err - } - - if !srcURL.IsAbs() { - srcURL = absURL(parts[1]) - } - } - - s := NewSource(alias, srcURL) - return s, nil -} - -func absURL(value string) *url.URL { - cwd, err := os.Getwd() - if err != nil { - log.Fatalf("Can't get working directory: %s", err) - } - urlCwd := strings.Replace(cwd, string(os.PathSeparator), "/", -1) - baseURL := &url.URL{ - Scheme: "file", - Path: urlCwd + "/", - } - relURL := &url.URL{ - Path: value, - } - return baseURL.ResolveReference(relURL) -} - -// DatasourceExists - -func (d *Data) DatasourceExists(alias string) bool { - _, ok := d.Sources[alias] - return ok -} - -const plaintext = "text/plain" - -// Datasource - -func (d *Data) Datasource(alias string, args ...string) interface{} { - source, ok := d.Sources[alias] - if !ok { - log.Fatalf("Undefined datasource '%s'", alias) - } - b, err := d.ReadSource(source, args...) - if err != nil { - log.Fatalf("Couldn't read datasource '%s': %s", alias, err) - } - s := string(b) - ty := &TypeConv{} - if source.Type == "application/json" { - return ty.JSON(s) - } - if source.Type == "application/yaml" { - return ty.YAML(s) - } - if source.Type == "text/csv" { - return ty.CSV(s) - } - if source.Type == "application/toml" { - return ty.TOML(s) - } - if source.Type == plaintext { - return s - } - log.Fatalf("Datasources of type %s not yet supported", source.Type) - return nil -} - -// Include - -func (d *Data) include(alias string, args ...string) interface{} { - source, ok := d.Sources[alias] - if !ok { - log.Fatalf("Undefined datasource '%s'", alias) - } - b, err := d.ReadSource(source, args...) - if err != nil { - log.Fatalf("Couldn't read datasource '%s': %s", alias, err) - } - return string(b) -} - -// ReadSource - -func (d *Data) ReadSource(source *Source, args ...string) ([]byte, error) { - if d.cache == nil { - d.cache = make(map[string][]byte) - } - cacheKey := source.Alias - for _, v := range args { - cacheKey += v - } - cached, ok := d.cache[cacheKey] - if ok { - return cached, nil - } - if r, ok := sourceReaders[source.URL.Scheme]; ok { - data, err := r(source, args...) - if err != nil { - return nil, err - } - d.cache[cacheKey] = data - return data, nil - } - - log.Fatalf("Datasources with scheme %s not yet supported", source.URL.Scheme) - return nil, nil -} - -func readFile(source *Source, args ...string) ([]byte, error) { - if source.FS == nil { - source.FS = vfs.OS() - } - - p := filepath.FromSlash(source.URL.Path) - - // make sure we can access the file - _, err := source.FS.Stat(p) - if err != nil { - log.Fatalf("Can't stat %s: %#v", p, err) - return nil, err - } - - f, err := source.FS.OpenFile(p, os.O_RDONLY, 0) - if err != nil { - log.Fatalf("Can't open %s: %#v", p, err) - return nil, err - } - - b, err := ioutil.ReadAll(f) - if err != nil { - log.Fatalf("Can't read %s: %#v", p, err) - return nil, err - } - return b, nil -} - -func readHTTP(source *Source, args ...string) ([]byte, error) { - if source.HC == nil { - source.HC = &http.Client{Timeout: time.Second * 5} - } - req, err := http.NewRequest("GET", source.URL.String(), nil) - if err != nil { - return nil, err - } - req.Header = source.Header - res, err := source.HC.Do(req) - if err != nil { - return nil, err - } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - err = res.Body.Close() - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - err := fmt.Errorf("Unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body)) - return nil, err - } - ctypeHdr := res.Header.Get("Content-Type") - if ctypeHdr != "" { - mediatype, params, e := mime.ParseMediaType(ctypeHdr) - if e != nil { - return nil, e - } - source.Type = mediatype - source.Params = params - } - return body, nil -} - -func readVault(source *Source, args ...string) ([]byte, error) { - if source.VC == nil { - source.VC = vault.New() - source.VC.Login() - addCleanupHook(source.VC.Logout) - } - - params := make(map[string]interface{}) - - p := source.URL.Path - - for key, val := range source.URL.Query() { - params[key] = strings.Join(val, " ") - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, err - } - - if parsed.Path != "" { - p = p + "/" + parsed.Path - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - - var data []byte - var err error - - if len(params) > 0 { - data, err = source.VC.Write(p, params) - } else { - data, err = source.VC.Read(p) - } - if err != nil { - return nil, err - } - source.Type = "application/json" - - return data, nil -} - -func readConsul(source *Source, args ...string) ([]byte, error) { - if source.KV == nil { - source.KV = libkv.NewConsul(source.URL) - err := source.KV.Login() - addCleanupHook(source.KV.Logout) - if err != nil { - return nil, err - } - } - - p := source.URL.Path - if len(args) == 1 { - p = p + "/" + args[0] - } - - data, err := source.KV.Read(p) - if err != nil { - return nil, err - } - source.Type = plaintext - - return data, nil -} - -func readBoltDB(source *Source, args ...string) ([]byte, error) { - if source.KV == nil { - source.KV = libkv.NewBoltDB(source.URL) - } - - if len(args) != 1 { - return nil, errors.New("missing key") - } - p := args[0] - - data, err := source.KV.Read(p) - if err != nil { - return nil, err - } - source.Type = plaintext - - return data, nil -} - -func parseHeaderArgs(headerArgs []string) map[string]http.Header { - headers := make(map[string]http.Header) - for _, v := range headerArgs { - ds, name, value := splitHeaderArg(v) - if _, ok := headers[ds]; !ok { - headers[ds] = make(http.Header) - } - headers[ds][name] = append(headers[ds][name], strings.TrimSpace(value)) - } - return headers -} - -func splitHeaderArg(arg string) (datasourceAlias, name, value string) { - parts := strings.SplitN(arg, "=", 2) - if len(parts) != 2 { - logFatalf("Invalid datasource-header option '%s'", arg) - return "", "", "" - } - datasourceAlias = parts[0] - name, value = splitHeader(parts[1]) - return datasourceAlias, name, value -} - -func splitHeader(header string) (name, value string) { - parts := strings.SplitN(header, ":", 2) - if len(parts) != 2 { - logFatalf("Invalid HTTP Header format '%s'", header) - return "", "" - } - name = http.CanonicalHeaderKey(parts[0]) - value = parts[1] - return name, value -} diff --git a/data/data.go b/data/data.go new file mode 100644 index 00000000..10102eaa --- /dev/null +++ b/data/data.go @@ -0,0 +1,250 @@ +package data + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "log" + "strings" + + // XXX: replace once https://github.com/BurntSushi/toml/pull/179 is merged + "github.com/hairyhenderson/toml" + "github.com/ugorji/go/codec" + yaml "gopkg.in/yaml.v2" +) + +func unmarshalObj(obj map[string]interface{}, in string, f func([]byte, interface{}) error) map[string]interface{} { + err := f([]byte(in), &obj) + if err != nil { + log.Fatalf("Unable to unmarshal object %s: %v", in, err) + } + return obj +} + +func unmarshalArray(obj []interface{}, in string, f func([]byte, interface{}) error) []interface{} { + err := f([]byte(in), &obj) + if err != nil { + log.Fatalf("Unable to unmarshal array %s: %v", in, err) + } + return obj +} + +// JSON - Unmarshal a JSON Object +func JSON(in string) map[string]interface{} { + obj := make(map[string]interface{}) + return unmarshalObj(obj, in, yaml.Unmarshal) +} + +// JSONArray - Unmarshal a JSON Array +func JSONArray(in string) []interface{} { + obj := make([]interface{}, 1) + return unmarshalArray(obj, in, yaml.Unmarshal) +} + +// YAML - Unmarshal a YAML Object +func YAML(in string) map[string]interface{} { + obj := make(map[string]interface{}) + return unmarshalObj(obj, in, yaml.Unmarshal) +} + +// YAMLArray - Unmarshal a YAML Array +func YAMLArray(in string) []interface{} { + obj := make([]interface{}, 1) + return unmarshalArray(obj, in, yaml.Unmarshal) +} + +// TOML - Unmarshal a TOML Object +func TOML(in string) interface{} { + obj := make(map[string]interface{}) + return unmarshalObj(obj, in, toml.Unmarshal) +} + +func parseCSV(args ...string) (records [][]string, hdr []string) { + delim := "," + var in string + if len(args) == 1 { + in = args[0] + } + if len(args) == 2 { + in = args[1] + if len(args[0]) == 1 { + delim = args[0] + } else if len(args[0]) == 0 { + hdr = []string{} + } else { + hdr = strings.Split(args[0], delim) + } + } + if len(args) == 3 { + delim = args[0] + hdr = strings.Split(args[1], delim) + in = args[2] + } + c := csv.NewReader(strings.NewReader(in)) + c.Comma = rune(delim[0]) + records, err := c.ReadAll() + if err != nil { + log.Fatal(err) + } + if hdr == nil { + hdr = records[0] + records = records[1:] + } else if len(hdr) == 0 { + hdr = make([]string, len(records[0])) + for i := range hdr { + hdr[i] = autoIndex(i) + } + } + return records, hdr +} + +// autoIndex - calculates a default string column name given a numeric value +func autoIndex(i int) string { + s := "" + for n := 0; n <= i/26; n++ { + s += string('A' + i%26) + } + return s +} + +// CSV - Unmarshal CSV +// parameters: +// delim - (optional) the (single-character!) field delimiter, defaults to "," +// in - the CSV-format string to parse +// returns: +// an array of rows, which are arrays of cells (strings) +func CSV(args ...string) [][]string { + records, hdr := parseCSV(args...) + records = append(records, nil) + copy(records[1:], records) + records[0] = hdr + return records +} + +// CSVByRow - Unmarshal CSV in a row-oriented form +// parameters: +// delim - (optional) the (single-character!) field delimiter, defaults to "," +// hdr - (optional) comma-separated list of column names, +// set to "" to get auto-named columns (A-Z), omit +// to use the first line +// in - the CSV-format string to parse +// returns: +// an array of rows, indexed by the header name +func CSVByRow(args ...string) (rows []map[string]string) { + records, hdr := parseCSV(args...) + for _, record := range records { + m := make(map[string]string) + for i, v := range record { + m[hdr[i]] = v + } + rows = append(rows, m) + } + return rows +} + +// CSVByColumn - Unmarshal CSV in a Columnar form +// parameters: +// delim - (optional) the (single-character!) field delimiter, defaults to "," +// hdr - (optional) comma-separated list of column names, +// set to "" to get auto-named columns (A-Z), omit +// to use the first line +// in - the CSV-format string to parse +// returns: +// a map of columns, indexed by the header name. values are arrays of strings +func CSVByColumn(args ...string) (cols map[string][]string) { + records, hdr := parseCSV(args...) + cols = make(map[string][]string) + for _, record := range records { + for i, v := range record { + cols[hdr[i]] = append(cols[hdr[i]], v) + } + } + return cols +} + +// ToCSV - +func ToCSV(args ...interface{}) string { + delim := "," + var in [][]string + if len(args) == 2 { + d, ok := args[0].(string) + if ok { + delim = d + } else { + log.Fatalf("Can't parse ToCSV delimiter (%v) - must be string (is a %T)", args[0], args[0]) + } + in, ok = args[1].([][]string) + if !ok { + log.Fatal("Can't parse ToCSV input - must be of type [][]string") + } + } + if len(args) == 1 { + var ok bool + in, ok = args[0].([][]string) + if !ok { + log.Fatal("Can't parse ToCSV input - must be of type [][]string") + } + } + b := &bytes.Buffer{} + c := csv.NewWriter(b) + c.Comma = rune(delim[0]) + // We output RFC4180 CSV, so force this to CRLF + c.UseCRLF = true + err := c.WriteAll(in) + if err != nil { + log.Fatal(err) + } + return string(b.Bytes()) +} + +func marshalObj(obj interface{}, f func(interface{}) ([]byte, error)) string { + b, err := f(obj) + if err != nil { + log.Fatalf("Unable to marshal object %s: %v", obj, err) + } + + return string(b) +} + +func toJSONBytes(in interface{}) []byte { + h := &codec.JsonHandle{} + h.Canonical = true + buf := new(bytes.Buffer) + err := codec.NewEncoder(buf, h).Encode(in) + if err != nil { + log.Fatalf("Unable to marshal %s: %v", in, err) + } + return buf.Bytes() +} + +// ToJSON - Stringify a struct as JSON +func ToJSON(in interface{}) string { + return string(toJSONBytes(in)) +} + +// ToJSONPretty - Stringify a struct as JSON (indented) +func ToJSONPretty(indent string, in interface{}) string { + out := new(bytes.Buffer) + b := toJSONBytes(in) + err := json.Indent(out, b, "", indent) + if err != nil { + log.Fatalf("Unable to indent JSON %s: %v", b, err) + } + + return string(out.Bytes()) +} + +// ToYAML - Stringify a struct as YAML +func ToYAML(in interface{}) string { + return marshalObj(in, yaml.Marshal) +} + +// ToTOML - Stringify a struct as TOML +func ToTOML(in interface{}) string { + buf := new(bytes.Buffer) + err := toml.NewEncoder(buf).Encode(in) + if err != nil { + log.Fatalf("Unable to marshal %s: %v", in, err) + } + return string(buf.Bytes()) +} diff --git a/data/data_test.go b/data/data_test.go new file mode 100644 index 00000000..eb284940 --- /dev/null +++ b/data/data_test.go @@ -0,0 +1,319 @@ +package data + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestUnmarshalObj(t *testing.T) { + expected := map[string]interface{}{ + "foo": map[interface{}]interface{}{"bar": "baz"}, + "one": 1.0, + "true": true, + } + + test := func(actual map[string]interface{}) { + assert.Equal(t, expected["foo"], actual["foo"]) + assert.Equal(t, expected["one"], actual["one"]) + assert.Equal(t, expected["true"], actual["true"]) + } + test(JSON(`{"foo":{"bar":"baz"},"one":1.0,"true":true}`)) + test(YAML(`foo: + bar: baz +one: 1.0 +true: true +`)) +} + +func TestUnmarshalArray(t *testing.T) { + + expected := []string{"foo", "bar"} + + test := func(actual []interface{}) { + assert.Equal(t, expected[0], actual[0]) + assert.Equal(t, expected[1], actual[1]) + } + test(JSONArray(`["foo","bar"]`)) + test(YAMLArray(` +- foo +- bar +`)) +} + +func TestToJSON(t *testing.T) { + expected := `{"down":{"the":{"rabbit":{"hole":true}}},"foo":"bar","one":1,"true":true}` + in := map[string]interface{}{ + "foo": "bar", + "one": 1, + "true": true, + "down": map[interface{}]interface{}{ + "the": map[interface{}]interface{}{ + "rabbit": map[interface{}]interface{}{ + "hole": true, + }, + }, + }, + } + assert.Equal(t, expected, ToJSON(in)) +} + +func TestToJSONPretty(t *testing.T) { + expected := `{ + "down": { + "the": { + "rabbit": { + "hole": true + } + } + }, + "foo": "bar", + "one": 1, + "true": true +}` + in := map[string]interface{}{ + "foo": "bar", + "one": 1, + "true": true, + "down": map[string]interface{}{ + "the": map[string]interface{}{ + "rabbit": map[string]interface{}{ + "hole": true, + }, + }, + }, + } + assert.Equal(t, expected, ToJSONPretty(" ", in)) +} + +func TestToYAML(t *testing.T) { + expected := `d: 2006-01-02T15:04:05.999999999-07:00 +foo: bar +? |- + multi + line + key +: hello: world +one: 1 +"true": true +` + mst, _ := time.LoadLocation("MST") + in := map[string]interface{}{ + "foo": "bar", + "one": 1, + "true": true, + `multi +line +key`: map[string]interface{}{ + "hello": "world", + }, + "d": time.Date(2006, time.January, 2, 15, 4, 5, 999999999, mst), + } + assert.Equal(t, expected, ToYAML(in)) +} + +func TestCSV(t *testing.T) { + in := "first,second,third\n1,2,3\n4,5,6" + expected := [][]string{ + {"first", "second", "third"}, + {"1", "2", "3"}, + {"4", "5", "6"}, + } + assert.Equal(t, expected, CSV(in)) + + in = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" + assert.Equal(t, expected, CSV(";", in)) +} + +func TestCSVByRow(t *testing.T) { + in := "first,second,third\n1,2,3\n4,5,6" + expected := []map[string]string{ + { + "first": "1", + "second": "2", + "third": "3", + }, + { + "first": "4", + "second": "5", + "third": "6", + }, + } + assert.Equal(t, expected, CSVByRow(in)) + + in = "1,2,3\n4,5,6" + assert.Equal(t, expected, CSVByRow("first,second,third", in)) + + in = "1;2;3\n4;5;6" + assert.Equal(t, expected, CSVByRow(";", "first;second;third", in)) + + in = "first;second;third\r\n1;2;3\r\n4;5;6" + assert.Equal(t, expected, CSVByRow(";", in)) + + expected = []map[string]string{ + {"A": "1", "B": "2", "C": "3"}, + {"A": "4", "B": "5", "C": "6"}, + } + + in = "1,2,3\n4,5,6" + assert.Equal(t, expected, CSVByRow("", in)) + + expected = []map[string]string{ + {"A": "1", "B": "1", "C": "1", "D": "1", "E": "1", "F": "1", "G": "1", "H": "1", "I": "1", "J": "1", "K": "1", "L": "1", "M": "1", "N": "1", "O": "1", "P": "1", "Q": "1", "R": "1", "S": "1", "T": "1", "U": "1", "V": "1", "W": "1", "X": "1", "Y": "1", "Z": "1", "AA": "1", "BB": "1", "CC": "1", "DD": "1"}, + } + + in = "1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1" + assert.Equal(t, expected, CSVByRow("", in)) +} + +func TestCSVByColumn(t *testing.T) { + in := "first,second,third\n1,2,3\n4,5,6" + expected := map[string][]string{ + "first": {"1", "4"}, + "second": {"2", "5"}, + "third": {"3", "6"}, + } + assert.Equal(t, expected, CSVByColumn(in)) + + in = "1,2,3\n4,5,6" + assert.Equal(t, expected, CSVByColumn("first,second,third", in)) + + in = "1;2;3\n4;5;6" + assert.Equal(t, expected, CSVByColumn(";", "first;second;third", in)) + + in = "first;second;third\r\n1;2;3\r\n4;5;6" + assert.Equal(t, expected, CSVByColumn(";", in)) + + expected = map[string][]string{ + "A": {"1", "4"}, + "B": {"2", "5"}, + "C": {"3", "6"}, + } + + in = "1,2,3\n4,5,6" + assert.Equal(t, expected, CSVByColumn("", in)) +} + +func TestAutoIndex(t *testing.T) { + assert.Equal(t, "A", autoIndex(0)) + assert.Equal(t, "B", autoIndex(1)) + assert.Equal(t, "Z", autoIndex(25)) + assert.Equal(t, "AA", autoIndex(26)) + assert.Equal(t, "ZZ", autoIndex(51)) + assert.Equal(t, "AAA", autoIndex(52)) + assert.Equal(t, "YYYYY", autoIndex(128)) +} + +func TestToCSV(t *testing.T) { + in := [][]string{ + {"first", "second", "third"}, + {"1", "2", "3"}, + {"4", "5", "6"}, + } + expected := "first,second,third\r\n1,2,3\r\n4,5,6\r\n" + + assert.Equal(t, expected, ToCSV(in)) + + expected = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" + + assert.Equal(t, expected, ToCSV(";", in)) +} + +func TestTOML(t *testing.T) { + in := `# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] +` + expected := map[string]interface{}{ + "title": "TOML Example", + "owner": map[string]interface{}{ + "name": "Tom Preston-Werner", + "organization": "GitHub", + "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", + "dob": time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), + }, + "database": map[string]interface{}{ + "server": "192.168.1.1", + "ports": []interface{}{int64(8001), int64(8001), int64(8002)}, + "connection_max": int64(5000), + "enabled": true, + }, + "servers": map[string]interface{}{ + "alpha": map[string]interface{}{ + "ip": "10.0.0.1", + "dc": "eqdc10", + }, + "beta": map[string]interface{}{ + "ip": "10.0.0.2", + "dc": "eqdc10", + }, + }, + "clients": map[string]interface{}{ + "data": []interface{}{ + []interface{}{"gamma", "delta"}, + []interface{}{int64(1), int64(2)}, + }, + "hosts": []interface{}{"alpha", "omega"}, + }, + } + + assert.Equal(t, expected, TOML(in)) +} + +func TestToTOML(t *testing.T) { + expected := `foo = "bar" +one = 1 +true = true + +[down] + [down.the] + [down.the.rabbit] + hole = true +` + in := map[string]interface{}{ + "foo": "bar", + "one": 1, + "true": true, + "down": map[interface{}]interface{}{ + "the": map[interface{}]interface{}{ + "rabbit": map[interface{}]interface{}{ + "hole": true, + }, + }, + }, + } + assert.Equal(t, expected, ToTOML(in)) +} diff --git a/data/datasource.go b/data/datasource.go new file mode 100644 index 00000000..5e626e0c --- /dev/null +++ b/data/datasource.go @@ -0,0 +1,452 @@ +package data + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "mime" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/blang/vfs" + "github.com/hairyhenderson/gomplate/libkv" + "github.com/hairyhenderson/gomplate/vault" +) + +// logFatal is defined so log.Fatal calls can be overridden for testing +var logFatalf = log.Fatalf + +func regExtension(ext, typ string) { + err := mime.AddExtensionType(ext, typ) + if err != nil { + log.Fatal(err) + } +} + +func init() { + // Add some types we want to be able to handle which can be missing by default + regExtension(".json", "application/json") + regExtension(".yml", "application/yaml") + regExtension(".yaml", "application/yaml") + regExtension(".csv", "text/csv") + regExtension(".toml", "application/toml") + + sourceReaders = make(map[string]func(*Source, ...string) ([]byte, error)) + + // Register our source-reader functions + addSourceReader("http", readHTTP) + addSourceReader("https", readHTTP) + addSourceReader("file", readFile) + addSourceReader("vault", readVault) + addSourceReader("consul", readConsul) + addSourceReader("consul+http", readConsul) + addSourceReader("consul+https", readConsul) + addSourceReader("boltdb", readBoltDB) +} + +var sourceReaders map[string]func(*Source, ...string) ([]byte, error) + +// addSourceReader - +func addSourceReader(scheme string, readFunc func(*Source, ...string) ([]byte, error)) { + sourceReaders[scheme] = readFunc +} + +// Data - +type Data struct { + Sources map[string]*Source + cache map[string][]byte +} + +// Cleanup - clean up datasources before shutting the process down - things +// like Logging out happen here +func (d *Data) Cleanup() { + for _, s := range d.Sources { + s.cleanup() + } +} + +// NewData - constructor for Data +func NewData(datasourceArgs []string, headerArgs []string) *Data { + sources := make(map[string]*Source) + headers := parseHeaderArgs(headerArgs) + for _, v := range datasourceArgs { + s, err := ParseSource(v) + if err != nil { + log.Fatalf("error parsing datasource %v", err) + return nil + } + s.Header = headers[s.Alias] + sources[s.Alias] = s + } + return &Data{ + Sources: sources, + } +} + +// Source - a data source +type Source struct { + Alias string + URL *url.URL + Ext string + Type string + Params map[string]string + FS vfs.Filesystem // used for file: URLs, nil otherwise + HC *http.Client // used for http[s]: URLs, nil otherwise + VC *vault.Vault // used for vault: URLs, nil otherwise + KV *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise + Header http.Header // used for http[s]: URLs, nil otherwise +} + +func (s *Source) cleanup() { + if s.VC != nil { + s.VC.Logout() + } + if s.KV != nil { + s.KV.Logout() + } +} + +// NewSource - builds a &Source +func NewSource(alias string, URL *url.URL) (s *Source) { + ext := filepath.Ext(URL.Path) + + s = &Source{ + Alias: alias, + URL: URL, + Ext: ext, + } + + if ext != "" && URL.Scheme != "boltdb" { + mediatype := mime.TypeByExtension(ext) + t, params, err := mime.ParseMediaType(mediatype) + if err != nil { + log.Fatal(err) + } + s.Type = t + s.Params = params + } + return +} + +// String is the method to format the flag's value, part of the flag.Value interface. +// The String method's output will be used in diagnostics. +func (s *Source) String() string { + return fmt.Sprintf("%s=%s (%s)", s.Alias, s.URL.String(), s.Type) +} + +// ParseSource - +func ParseSource(value string) (*Source, error) { + var ( + alias string + srcURL *url.URL + ) + parts := strings.SplitN(value, "=", 2) + if len(parts) == 1 { + f := parts[0] + alias = strings.SplitN(value, ".", 2)[0] + if path.Base(f) != f { + err := fmt.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory", value) + return nil, err + } + srcURL = absURL(f) + } else if len(parts) == 2 { + alias = parts[0] + var err error + srcURL, err = url.Parse(parts[1]) + if err != nil { + return nil, err + } + + if !srcURL.IsAbs() { + srcURL = absURL(parts[1]) + } + } + + s := NewSource(alias, srcURL) + return s, nil +} + +func absURL(value string) *url.URL { + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("Can't get working directory: %s", err) + } + urlCwd := strings.Replace(cwd, string(os.PathSeparator), "/", -1) + baseURL := &url.URL{ + Scheme: "file", + Path: urlCwd + "/", + } + relURL := &url.URL{ + Path: value, + } + return baseURL.ResolveReference(relURL) +} + +// DatasourceExists - +func (d *Data) DatasourceExists(alias string) bool { + _, ok := d.Sources[alias] + return ok +} + +const plaintext = "text/plain" + +// Datasource - +func (d *Data) Datasource(alias string, args ...string) interface{} { + source, ok := d.Sources[alias] + if !ok { + log.Fatalf("Undefined datasource '%s'", alias) + } + b, err := d.ReadSource(source, args...) + if err != nil { + log.Fatalf("Couldn't read datasource '%s': %s", alias, err) + } + s := string(b) + if source.Type == "application/json" { + return JSON(s) + } + if source.Type == "application/yaml" { + return YAML(s) + } + if source.Type == "text/csv" { + return CSV(s) + } + if source.Type == "application/toml" { + return TOML(s) + } + if source.Type == plaintext { + return s + } + log.Fatalf("Datasources of type %s not yet supported", source.Type) + return nil +} + +// Include - +func (d *Data) Include(alias string, args ...string) string { + source, ok := d.Sources[alias] + if !ok { + log.Fatalf("Undefined datasource '%s'", alias) + } + b, err := d.ReadSource(source, args...) + if err != nil { + log.Fatalf("Couldn't read datasource '%s': %s", alias, err) + } + return string(b) +} + +// ReadSource - +func (d *Data) ReadSource(source *Source, args ...string) ([]byte, error) { + if d.cache == nil { + d.cache = make(map[string][]byte) + } + cacheKey := source.Alias + for _, v := range args { + cacheKey += v + } + cached, ok := d.cache[cacheKey] + if ok { + return cached, nil + } + if r, ok := sourceReaders[source.URL.Scheme]; ok { + data, err := r(source, args...) + if err != nil { + return nil, err + } + d.cache[cacheKey] = data + return data, nil + } + + log.Fatalf("Datasources with scheme %s not yet supported", source.URL.Scheme) + return nil, nil +} + +func readFile(source *Source, args ...string) ([]byte, error) { + if source.FS == nil { + source.FS = vfs.OS() + } + + p := filepath.FromSlash(source.URL.Path) + + // make sure we can access the file + _, err := source.FS.Stat(p) + if err != nil { + log.Fatalf("Can't stat %s: %#v", p, err) + return nil, err + } + + f, err := source.FS.OpenFile(p, os.O_RDONLY, 0) + if err != nil { + log.Fatalf("Can't open %s: %#v", p, err) + return nil, err + } + + b, err := ioutil.ReadAll(f) + if err != nil { + log.Fatalf("Can't read %s: %#v", p, err) + return nil, err + } + return b, nil +} + +func readHTTP(source *Source, args ...string) ([]byte, error) { + if source.HC == nil { + source.HC = &http.Client{Timeout: time.Second * 5} + } + req, err := http.NewRequest("GET", source.URL.String(), nil) + if err != nil { + return nil, err + } + req.Header = source.Header + res, err := source.HC.Do(req) + if err != nil { + return nil, err + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + err = res.Body.Close() + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + err := fmt.Errorf("Unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body)) + return nil, err + } + ctypeHdr := res.Header.Get("Content-Type") + if ctypeHdr != "" { + mediatype, params, e := mime.ParseMediaType(ctypeHdr) + if e != nil { + return nil, e + } + source.Type = mediatype + source.Params = params + } + return body, nil +} + +func readVault(source *Source, args ...string) ([]byte, error) { + if source.VC == nil { + source.VC = vault.New() + source.VC.Login() + } + + params := make(map[string]interface{}) + + p := source.URL.Path + + for key, val := range source.URL.Query() { + params[key] = strings.Join(val, " ") + } + + if len(args) == 1 { + parsed, err := url.Parse(args[0]) + if err != nil { + return nil, err + } + + if parsed.Path != "" { + p = p + "/" + parsed.Path + } + + for key, val := range parsed.Query() { + params[key] = strings.Join(val, " ") + } + } + + var data []byte + var err error + + if len(params) > 0 { + data, err = source.VC.Write(p, params) + } else { + data, err = source.VC.Read(p) + } + if err != nil { + return nil, err + } + source.Type = "application/json" + + return data, nil +} + +func readConsul(source *Source, args ...string) ([]byte, error) { + if source.KV == nil { + source.KV = libkv.NewConsul(source.URL) + err := source.KV.Login() + if err != nil { + return nil, err + } + } + + p := source.URL.Path + if len(args) == 1 { + p = p + "/" + args[0] + } + + data, err := source.KV.Read(p) + if err != nil { + return nil, err + } + source.Type = plaintext + + return data, nil +} + +func readBoltDB(source *Source, args ...string) ([]byte, error) { + if source.KV == nil { + source.KV = libkv.NewBoltDB(source.URL) + } + + if len(args) != 1 { + return nil, errors.New("missing key") + } + p := args[0] + + data, err := source.KV.Read(p) + if err != nil { + return nil, err + } + source.Type = plaintext + + return data, nil +} + +func parseHeaderArgs(headerArgs []string) map[string]http.Header { + headers := make(map[string]http.Header) + for _, v := range headerArgs { + ds, name, value := splitHeaderArg(v) + if _, ok := headers[ds]; !ok { + headers[ds] = make(http.Header) + } + headers[ds][name] = append(headers[ds][name], strings.TrimSpace(value)) + } + return headers +} + +func splitHeaderArg(arg string) (datasourceAlias, name, value string) { + parts := strings.SplitN(arg, "=", 2) + if len(parts) != 2 { + logFatalf("Invalid datasource-header option '%s'", arg) + return "", "", "" + } + datasourceAlias = parts[0] + name, value = splitHeader(parts[1]) + return datasourceAlias, name, value +} + +func splitHeader(header string) (name, value string) { + parts := strings.SplitN(header, ":", 2) + if len(parts) != 2 { + logFatalf("Invalid HTTP Header format '%s'", header) + return "", "" + } + name = http.CanonicalHeaderKey(parts[0]) + value = parts[1] + return name, value +} diff --git a/data/datasource_test.go b/data/datasource_test.go new file mode 100644 index 00000000..31b66eee --- /dev/null +++ b/data/datasource_test.go @@ -0,0 +1,300 @@ +// +build !windows + +package data + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/blang/vfs" + "github.com/blang/vfs/memfs" + "github.com/stretchr/testify/assert" +) + +var spyLogFatalfMsg string + +func restoreLogFatalf() { + logFatalf = log.Fatalf +} + +func mockLogFatalf(msg string, args ...interface{}) { + spyLogFatalfMsg = msg + panic(spyLogFatalfMsg) +} + +func setupMockLogFatalf() { + logFatalf = mockLogFatalf + spyLogFatalfMsg = "" +} + +func TestNewSource(t *testing.T) { + s := NewSource("foo", &url.URL{ + Scheme: "file", + Path: "/foo.json", + }) + assert.Equal(t, "application/json", s.Type) + assert.Equal(t, ".json", s.Ext) + + s = NewSource("foo", &url.URL{ + Scheme: "http", + Host: "example.com", + Path: "/foo.json", + }) + assert.Equal(t, "application/json", s.Type) + assert.Equal(t, ".json", s.Ext) + + s = NewSource("foo", &url.URL{ + Scheme: "ftp", + Host: "example.com", + Path: "/foo.json", + }) + assert.Equal(t, "application/json", s.Type) + assert.Equal(t, ".json", s.Ext) +} + +func TestNewData(t *testing.T) { + d := NewData(nil, nil) + assert.Len(t, d.Sources, 0) + + d = NewData([]string{"foo=http:///foo.json"}, nil) + assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) + + d = NewData([]string{"foo=http:///foo.json"}, []string{}) + assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) + assert.Empty(t, d.Sources["foo"].Header) + + d = NewData([]string{"foo=http:///foo.json"}, []string{"bar=Accept: blah"}) + assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) + assert.Empty(t, d.Sources["foo"].Header) + + d = NewData([]string{"foo=http:///foo.json"}, []string{"foo=Accept: blah"}) + assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) + assert.Equal(t, "blah", d.Sources["foo"].Header["Accept"][0]) +} + +func TestParseSourceNoAlias(t *testing.T) { + s, err := ParseSource("foo.json") + assert.NoError(t, err) + assert.Equal(t, "foo", s.Alias) + + _, err = ParseSource("../foo.json") + assert.Error(t, err) + + _, err = ParseSource("ftp://example.com/foo.yml") + assert.Error(t, err) +} + +func TestParseSourceWithAlias(t *testing.T) { + s, err := ParseSource("data=foo.json") + assert.NoError(t, err) + assert.Equal(t, "data", s.Alias) + assert.Equal(t, "file", s.URL.Scheme) + assert.Equal(t, "application/json", s.Type) + assert.True(t, s.URL.IsAbs()) + + s, err = ParseSource("data=/otherdir/foo.json") + assert.NoError(t, err) + assert.Equal(t, "data", s.Alias) + assert.Equal(t, "file", s.URL.Scheme) + assert.True(t, s.URL.IsAbs()) + assert.Equal(t, "/otherdir/foo.json", s.URL.Path) + + s, err = ParseSource("data=sftp://example.com/blahblah/foo.json") + assert.NoError(t, err) + assert.Equal(t, "data", s.Alias) + assert.Equal(t, "sftp", s.URL.Scheme) + assert.True(t, s.URL.IsAbs()) + assert.Equal(t, "/blahblah/foo.json", s.URL.Path) +} + +func TestDatasource(t *testing.T) { + test := func(ext, mime, contents string) { + fname := "foo." + ext + fs := memfs.Create() + _ = fs.Mkdir("/tmp", 0777) + f, _ := vfs.Create(fs, "/tmp/"+fname) + _, _ = f.Write([]byte(contents)) + + sources := map[string]*Source{ + "foo": { + Alias: "foo", + URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, + Ext: ext, + Type: mime, + FS: fs, + }, + } + data := &Data{ + Sources: sources, + } + expected := map[string]interface{}{"hello": map[interface{}]interface{}{"cruel": "world"}} + actual := data.Datasource("foo") + assert.Equal(t, expected, actual) + } + + test("json", "application/json", `{"hello":{"cruel":"world"}}`) + test("yml", "application/yaml", "hello:\n cruel: world\n") +} + +func TestDatasourceExists(t *testing.T) { + sources := map[string]*Source{ + "foo": {Alias: "foo"}, + } + data := &Data{Sources: sources} + assert.True(t, data.DatasourceExists("foo")) + assert.False(t, data.DatasourceExists("bar")) +} + +func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + w.Header().Set("Content-Type", mimetype) + w.WriteHeader(code) + if body == "" { + // mirror back the headers + fmt.Fprintln(w, marshalObj(r.Header, json.Marshal)) + } else { + fmt.Fprintln(w, body) + } + })) + + client := &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(server.URL) + }, + }, + } + + return server, client +} + +func TestHTTPFile(t *testing.T) { + server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`) + defer server.Close() + + sources := make(map[string]*Source) + sources["foo"] = &Source{ + Alias: "foo", + URL: &url.URL{ + Scheme: "http", + Host: "example.com", + Path: "/foo", + }, + HC: client, + } + data := &Data{ + Sources: sources, + } + expected := make(map[string]interface{}) + expected["hello"] = "world" + actual := data.Datasource("foo").(map[string]interface{}) + assert.Equal(t, expected["hello"], actual["hello"]) +} + +func TestHTTPFileWithHeaders(t *testing.T) { + server, client := setupHTTP(200, "application/json", "") + defer server.Close() + + sources := make(map[string]*Source) + sources["foo"] = &Source{ + Alias: "foo", + URL: &url.URL{ + Scheme: "http", + Host: "example.com", + Path: "/foo", + }, + HC: client, + Header: http.Header{ + "Foo": {"bar"}, + "foo": {"baz"}, + "User-Agent": {}, + "Accept-Encoding": {"test"}, + }, + } + data := &Data{ + Sources: sources, + } + expected := http.Header{ + "Accept-Encoding": {"test"}, + "Foo": {"bar", "baz"}, + } + actual := data.Datasource("foo") + assert.Equal(t, marshalObj(expected, json.Marshal), marshalObj(actual, json.Marshal)) +} + +func TestParseHeaderArgs(t *testing.T) { + args := []string{ + "foo=Accept: application/json", + "bar=Authorization: Bearer supersecret", + } + expected := map[string]http.Header{ + "foo": { + "Accept": {"application/json"}, + }, + "bar": { + "Authorization": {"Bearer supersecret"}, + }, + } + assert.Equal(t, expected, parseHeaderArgs(args)) + + defer restoreLogFatalf() + setupMockLogFatalf() + assert.Panics(t, func() { + parseHeaderArgs([]string{"foo"}) + }) + + defer restoreLogFatalf() + setupMockLogFatalf() + assert.Panics(t, func() { + parseHeaderArgs([]string{"foo=bar"}) + }) + + args = []string{ + "foo=Accept: application/json", + "foo=Foo: bar", + "foo=foo: baz", + "foo=fOO: qux", + "bar=Authorization: Bearer supersecret", + } + expected = map[string]http.Header{ + "foo": { + "Accept": {"application/json"}, + "Foo": {"bar", "baz", "qux"}, + }, + "bar": { + "Authorization": {"Bearer supersecret"}, + }, + } + assert.Equal(t, expected, parseHeaderArgs(args)) +} + +func TestInclude(t *testing.T) { + ext := "txt" + contents := "hello world" + fname := "foo." + ext + fs := memfs.Create() + _ = fs.Mkdir("/tmp", 0777) + f, _ := vfs.Create(fs, "/tmp/"+fname) + _, _ = f.Write([]byte(contents)) + + sources := map[string]*Source{ + "foo": { + Alias: "foo", + URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, + Ext: ext, + Type: "text/plain", + FS: fs, + }, + } + data := &Data{ + Sources: sources, + } + actual := data.Include("foo") + assert.Equal(t, contents, actual) +} diff --git a/data_test.go b/data_test.go deleted file mode 100644 index 1be57e3e..00000000 --- a/data_test.go +++ /dev/null @@ -1,300 +0,0 @@ -// +build !windows - -package main - -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/blang/vfs" - "github.com/blang/vfs/memfs" - "github.com/stretchr/testify/assert" -) - -var spyLogFatalfMsg string - -func restoreLogFatalf() { - logFatalf = log.Fatalf -} - -func mockLogFatalf(msg string, args ...interface{}) { - spyLogFatalfMsg = msg - panic(spyLogFatalfMsg) -} - -func setupMockLogFatalf() { - logFatalf = mockLogFatalf - spyLogFatalfMsg = "" -} - -func TestNewSource(t *testing.T) { - s := NewSource("foo", &url.URL{ - Scheme: "file", - Path: "/foo.json", - }) - assert.Equal(t, "application/json", s.Type) - assert.Equal(t, ".json", s.Ext) - - s = NewSource("foo", &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo.json", - }) - assert.Equal(t, "application/json", s.Type) - assert.Equal(t, ".json", s.Ext) - - s = NewSource("foo", &url.URL{ - Scheme: "ftp", - Host: "example.com", - Path: "/foo.json", - }) - assert.Equal(t, "application/json", s.Type) - assert.Equal(t, ".json", s.Ext) -} - -func TestNewData(t *testing.T) { - d := NewData(nil, nil) - assert.Len(t, d.Sources, 0) - - d = NewData([]string{"foo=http:///foo.json"}, nil) - assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - - d = NewData([]string{"foo=http:///foo.json"}, []string{}) - assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - assert.Empty(t, d.Sources["foo"].Header) - - d = NewData([]string{"foo=http:///foo.json"}, []string{"bar=Accept: blah"}) - assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - assert.Empty(t, d.Sources["foo"].Header) - - d = NewData([]string{"foo=http:///foo.json"}, []string{"foo=Accept: blah"}) - assert.Equal(t, "/foo.json", d.Sources["foo"].URL.Path) - assert.Equal(t, "blah", d.Sources["foo"].Header["Accept"][0]) -} - -func TestParseSourceNoAlias(t *testing.T) { - s, err := ParseSource("foo.json") - assert.NoError(t, err) - assert.Equal(t, "foo", s.Alias) - - _, err = ParseSource("../foo.json") - assert.Error(t, err) - - _, err = ParseSource("ftp://example.com/foo.yml") - assert.Error(t, err) -} - -func TestParseSourceWithAlias(t *testing.T) { - s, err := ParseSource("data=foo.json") - assert.NoError(t, err) - assert.Equal(t, "data", s.Alias) - assert.Equal(t, "file", s.URL.Scheme) - assert.Equal(t, "application/json", s.Type) - assert.True(t, s.URL.IsAbs()) - - s, err = ParseSource("data=/otherdir/foo.json") - assert.NoError(t, err) - assert.Equal(t, "data", s.Alias) - assert.Equal(t, "file", s.URL.Scheme) - assert.True(t, s.URL.IsAbs()) - assert.Equal(t, "/otherdir/foo.json", s.URL.Path) - - s, err = ParseSource("data=sftp://example.com/blahblah/foo.json") - assert.NoError(t, err) - assert.Equal(t, "data", s.Alias) - assert.Equal(t, "sftp", s.URL.Scheme) - assert.True(t, s.URL.IsAbs()) - assert.Equal(t, "/blahblah/foo.json", s.URL.Path) -} - -func TestDatasource(t *testing.T) { - test := func(ext, mime, contents string) { - fname := "foo." + ext - fs := memfs.Create() - _ = fs.Mkdir("/tmp", 0777) - f, _ := vfs.Create(fs, "/tmp/"+fname) - _, _ = f.Write([]byte(contents)) - - sources := map[string]*Source{ - "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, - Ext: ext, - Type: mime, - FS: fs, - }, - } - data := &Data{ - Sources: sources, - } - expected := map[string]interface{}{"hello": map[interface{}]interface{}{"cruel": "world"}} - actual := data.Datasource("foo") - assert.Equal(t, expected, actual) - } - - test("json", "application/json", `{"hello":{"cruel":"world"}}`) - test("yml", "application/yaml", "hello:\n cruel: world\n") -} - -func TestDatasourceExists(t *testing.T) { - sources := map[string]*Source{ - "foo": {Alias: "foo"}, - } - data := &Data{Sources: sources} - assert.True(t, data.DatasourceExists("foo")) - assert.False(t, data.DatasourceExists("bar")) -} - -func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - w.Header().Set("Content-Type", mimetype) - w.WriteHeader(code) - if body == "" { - // mirror back the headers - fmt.Fprintln(w, marshalObj(r.Header, json.Marshal)) - } else { - fmt.Fprintln(w, body) - } - })) - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - }, - } - - return server, client -} - -func TestHTTPFile(t *testing.T) { - server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`) - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - HC: client, - } - data := &Data{ - Sources: sources, - } - expected := make(map[string]interface{}) - expected["hello"] = "world" - actual := data.Datasource("foo").(map[string]interface{}) - assert.Equal(t, expected["hello"], actual["hello"]) -} - -func TestHTTPFileWithHeaders(t *testing.T) { - server, client := setupHTTP(200, "application/json", "") - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - HC: client, - Header: http.Header{ - "Foo": {"bar"}, - "foo": {"baz"}, - "User-Agent": {}, - "Accept-Encoding": {"test"}, - }, - } - data := &Data{ - Sources: sources, - } - expected := http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - } - actual := data.Datasource("foo") - assert.Equal(t, marshalObj(expected, json.Marshal), marshalObj(actual, json.Marshal)) -} - -func TestParseHeaderArgs(t *testing.T) { - args := []string{ - "foo=Accept: application/json", - "bar=Authorization: Bearer supersecret", - } - expected := map[string]http.Header{ - "foo": { - "Accept": {"application/json"}, - }, - "bar": { - "Authorization": {"Bearer supersecret"}, - }, - } - assert.Equal(t, expected, parseHeaderArgs(args)) - - defer restoreLogFatalf() - setupMockLogFatalf() - assert.Panics(t, func() { - parseHeaderArgs([]string{"foo"}) - }) - - defer restoreLogFatalf() - setupMockLogFatalf() - assert.Panics(t, func() { - parseHeaderArgs([]string{"foo=bar"}) - }) - - args = []string{ - "foo=Accept: application/json", - "foo=Foo: bar", - "foo=foo: baz", - "foo=fOO: qux", - "bar=Authorization: Bearer supersecret", - } - expected = map[string]http.Header{ - "foo": { - "Accept": {"application/json"}, - "Foo": {"bar", "baz", "qux"}, - }, - "bar": { - "Authorization": {"Bearer supersecret"}, - }, - } - assert.Equal(t, expected, parseHeaderArgs(args)) -} - -func TestInclude(t *testing.T) { - ext := "txt" - contents := "hello world" - fname := "foo." + ext - fs := memfs.Create() - _ = fs.Mkdir("/tmp", 0777) - f, _ := vfs.Create(fs, "/tmp/"+fname) - _, _ = f.Write([]byte(contents)) - - sources := map[string]*Source{ - "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: "/tmp/" + fname}, - Ext: ext, - Type: "text/plain", - FS: fs, - }, - } - data := &Data{ - Sources: sources, - } - actual := data.include("foo") - assert.Equal(t, contents, actual) -} diff --git a/docs/content/functions/conv.md b/docs/content/functions/conv.md new file mode 100644 index 00000000..177a37b7 --- /dev/null +++ b/docs/content/functions/conv.md @@ -0,0 +1,130 @@ +--- +title: conversion functions +menu: + main: + parent: functions +--- + +These are a collection of functions that mostly help converting from one type +to another - generally from a `string` to something else, and vice-versa. + +## `conv.Bool` + +**Alias:** `bool` + +Converts a true-ish string to a boolean. Can be used to simplify conditional statements based on environment variables or other text input. + +#### Example + +_`input.tmpl`:_ +``` +{{if bool (getenv "FOO")}}foo{{else}}bar{{end}} +``` + +```console +$ gomplate < input.tmpl +bar +$ FOO=true gomplate < input.tmpl +foo +``` + +## `conv.Slice` + +**Alias:** `slice` + +Creates a slice. Useful when needing to `range` over a bunch of variables. + +#### Example + +_`input.tmpl`:_ +``` +{{range slice "Bart" "Lisa" "Maggie"}} +Hello, {{.}} +{{- end}} +``` + +```console +$ gomplate < input.tmpl +Hello, Bart +Hello, Lisa +Hello, Maggie +``` + +## `conv.Has` + +**Alias:** `has` + +Has reports whether or not a given object has a property with the given key. Can be used with `if` to prevent the template from trying to access a non-existent property in an object. + +#### Example + +_Let's say we're using a Vault datasource..._ + +_`input.tmpl`:_ +``` +{{ $secret := datasource "vault" "mysecret" -}} +The secret is ' +{{- if (has $secret "value") }} +{{- $secret.value }} +{{- else }} +{{- $secret | toYAML }} +{{- end }}' +``` + +If the `secret/foo/mysecret` secret in Vault has a property named `value` set to `supersecret`: + +```console +$ gomplate -d vault:///secret/foo < input.tmpl +The secret is 'supersecret' +``` + +On the other hand, if there is no `value` property: + +```console +$ gomplate -d vault:///secret/foo < input.tmpl +The secret is 'foo: bar' +``` + +## `conv.Join` + +**Alias:** `join` + +Concatenates the elements of an array to create a string. The separator string sep is placed between elements in the resulting string. + +#### Example + +_`input.tmpl`_ +``` +{{ $a := `[1, 2, 3]` | jsonArray }} +{{ join $a "-" }} +``` + +```console +$ gomplate -f input.tmpl +1-2-3 +``` + + +## `conv.URL` + +**Alias:** `urlParse` + +Parses a string as a URL for later use. Equivalent to [url.Parse](https://golang.org/pkg/net/url/#Parse) + +#### Example + +_`input.tmpl`:_ +``` +{{ $u := conv.URL "https://example.com:443/foo/bar" }} +The scheme is {{ $u.Scheme }} +The host is {{ $u.Host }} +The path is {{ $u.Path }} +``` + +```console +$ gomplate < input.tmpl +The scheme is https +The host is example.com:443 +The path is /foo/bar +``` + diff --git a/docs/content/functions/data.md b/docs/content/functions/data.md new file mode 100644 index 00000000..acdbf44f --- /dev/null +++ b/docs/content/functions/data.md @@ -0,0 +1,645 @@ +--- +title: data functions +menu: + main: + parent: functions +--- + +A collection of functions that retrieve, parse, and convert structured data. + +## `datasource` + +Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument). + +Currently, `file://`, `http://`, `https://`, and `vault://` URLs are supported. + +Currently-supported formats are JSON, YAML, TOML, and CSV. + +### Basic usage + +_`person.json`:_ +```json +{ + "name": "Dave" +} +``` + +_`input.tmpl`:_ +``` +Hello {{ (datasource "person").name }} +``` + +```console +$ gomplate -d person.json < input.tmpl +Hello Dave +``` + +### Usage with HTTP data + +```console +$ echo 'Hello there, {{(datasource "foo").headers.Host}}...' | gomplate -d foo=https://httpbin.org/get +Hello there, httpbin.org... +``` + +Additional headers can be provided with the `--datasource-header`/`-H` option: + +```console +$ gomplate -d foo=https://httpbin.org/get -H 'foo=Foo: bar' -i '{{(datasource "foo").headers.Foo}}' +bar +``` + +### Usage with Consul data + +There are three supported URL schemes to retrieve data from [Consul](https://consul.io/). +The `consul://` (or `consul+http://`) scheme can optionally be used with a hostname and port to specify a server (e.g. `consul://localhost:8500`). +By default HTTP will be used, but the `consul+https://` form can be used to use HTTPS, alternatively `$CONSUL_HTTP_SSL` can be used. + +If the server address isn't part of the datasource URL, `$CONSUL_HTTP_ADDR` will be checked. + +The following optional environment variables can be set: + +| name | usage | +|------|-------| +| `CONSUL_HTTP_ADDR` | Hostname and optional port for connecting to Consul. Defaults to `http://localhost:8500` | +| `CONSUL_TIMEOUT` | Timeout (in seconds) when communicating to Consul. Defaults to 10 seconds. | +| `CONSUL_HTTP_TOKEN` | The Consul token to use when connecting to the server. | +| `CONSUL_HTTP_AUTH` | Should be specified as `:`. Used to authenticate to the server. | +| `CONSUL_HTTP_SSL` | Force HTTPS if set to `true` value. Disables if set to `false`. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | +| `CONSUL_TLS_SERVER_NAME` | The server name to use as the SNI host when connecting to Consul via TLS. | +| `CONSUL_CACERT` | Path to CA file for verifying Consul server using TLS. | +| `CONSUL_CAPATH` | Path to directory of CA files for verifying Consul server using TLS. | +| `CONSUL_CLIENT_CERT` | Client certificate file for certificate authentication. If this is set, `$CONSUL_CLIENT_KEY` must also be set. | +| `CONSUL_CLIENT_KEY` | Client key file for certificate authentication. If this is set, `$CONSUL_CLIENT_CERT` must also be set. | +| `CONSUL_HTTP_SSL_VERIFY` | Set to `false` to disable Consul TLS certificate checking. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided.
_Recommended only for testing and development scenarios!_ | +| `CONSUL_VAULT_ROLE` | Set to the name of the role to use for authenticating to Consul with [Vault's Consul secret backend](https://www.vaultproject.io/docs/secrets/consul/index.html). | +| `CONSUL_VAULT_MOUNT` | Used to override the mount-point when using Vault's Consul secret backend for authentication. Defaults to `consul`. | + +If a path is included it is used as a prefix for all uses of the datasource. + +#### Example + +```console +$ gomplate -d consul=consul:// -i '{{(datasource "consul" "foo")}}' +value for foo key +``` + +```console +$ gomplate -d consul=consul+https://my-consul-server.com:8533/foo -i '{{(datasource "consul" "bar")}}' +value for foo/bar key +``` + +```console +$ gomplate -d consul=consul:///foo -i '{{(datasource "consul" "bar/baz")}}' +value for foo/bar/baz key +``` + +Instead of using a non-authenticated Consul connection or connecting using the token set with the +`CONSUL_HTTP_TOKEN` environment variable, it is possible to authenticate using a dynamically generated +token fetched from Vault. This requires Vault to be configured to use the [Consul secret backend](https://www.vaultproject.io/docs/secrets/consul/index.html) and +is enabled by passing the name of the role to use in the `CONSUL_VAULT_ROLE` environment variable. + +### Usage with BoltDB data + +[BoltDB](https://github.com/boltdb/bolt) is a simple local key/value store used +by many Go tools. The `boltdb://` scheme can be used to access values stored in +a BoltDB database file. The full path is provided in the URL, and the bucket name +can be specified using a URL fragment (e.g. `boltdb:///tmp/database.db#bucket`). + +Access is implemented through [libkv](https://github.com/docker/libkv), and as +such, the first 8 bytes of all values are used as an incrementing last modified +index value. All values must therefore be at least 9 bytes long, with the first +8 being ignored. + +The following environment variables can be set: + +| name | usage | +|------|-------| +| `BOLTDB_TIMEOUT` | Timeout (in seconds) to wait for a lock on the database file when opening. | +| `BOLTDB_PERSIST` | If set keep the database open instead of closing after each read. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | + +### Example + +```console +$ gomplate -d config=boltdb:///tmp/config.db#Bucket1 -i '{{(datasource "config" "foo")}}' +bar +``` + +### Usage with Vault data + +The special `vault://` URL scheme can be used to retrieve data from [Hashicorp +Vault](https://vaultproject.io). To use this, you must put the Vault server's +URL in the `$VAULT_ADDR` environment variable. + +This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: + +| auth backend | configuration | +|-------------: |---------------| +| [`approle`](https://www.vaultproject.io/docs/auth/approle.html) | Environment variables `$VAULT_ROLE_ID` and `$VAULT_SECRET_ID` must be set to the appropriate values.
If the backend is mounted to a different location, set `$VAULT_AUTH_APPROLE_MOUNT`. | +| [`app-id`](https://www.vaultproject.io/docs/auth/app-id.html) | Environment variables `$VAULT_APP_ID` and `$VAULT_USER_ID` must be set to the appropriate values.
If the backend is mounted to a different location, set `$VAULT_AUTH_APP_ID_MOUNT`. | +| [`github`](https://www.vaultproject.io/docs/auth/github.html) | Environment variable `$VAULT_AUTH_GITHUB_TOKEN` must be set to an appropriate value.
If the backend is mounted to a different location, set `$VAULT_AUTH_GITHUB_MOUNT`. | +| [`userpass`](https://www.vaultproject.io/docs/auth/userpass.html) | Environment variables `$VAULT_AUTH_USERNAME` and `$VAULT_AUTH_PASSWORD` must be set to the appropriate values.
If the backend is mounted to a different location, set `$VAULT_AUTH_USERPASS_MOUNT`. | +| [`token`](https://www.vaultproject.io/docs/auth/token.html) | Determined from either the `$VAULT_TOKEN` environment variable, or read from the file `~/.vault-token` | +| [`aws`](https://www.vaultproject.io/docs/auth/aws.html) | As a final option authentication will be attempted using the AWS auth backend. See below for more details. | + +_**Note:**_ The secret values listed in the above table can either be set in environment +variables or provided in files. This can increase security when using +[Docker Swarm Secrets](https://docs.docker.com/engine/swarm/secrets/), for example. +To use files, specify the filename by appending `_FILE` to the environment variable, +(i.e. `VAULT_USER_ID_FILE`). If the non-file variable is set, this will override +any `_FILE` variable and the secret file will be ignored. + +To use a Vault datasource with a single secret, just use a URL of +`vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left +empty. + +```console +$ echo 'My voice is my passport. {{(datasource "vault").value}}' \ + | gomplate -d vault=vault:///secret/sneakers +My voice is my passport. Verify me. +``` + +You can also specify the secret path in the template by using a URL of `vault://` +(or `vault:///`, or `vault:`): +```console +$ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \ + | gomplate -d vault=vault:// +My voice is my passport. Verify me. +``` + +And the two can be mixed to scope secrets to a specific namespace: + +```console +$ echo 'db_password={{(datasource "vault" "db/pass").value}}' \ + | gomplate -d vault=vault:///secret/production +db_password=prodsecret +``` + +It is also possible to use dynamic secrets by using the write capability of the datasource. To use, +add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then +included within the JSON body of the request. + +```console +$ echo 'otp={{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").key}}' \ + | gomplate -d vault=vault:/// +otp=604a4bd5-7afd-30a2-d2d8-80c4aebc6183 +``` + +#### Authentication using AWS details + +If running on an EC2 instance authentication will be attempted using the AWS auth backend. The +optional `VAULT_AUTH_AWS_MOUNT` environment variable can be used to set the mount point to use if +it differs from the default of `aws`. Additionally `AWS_TIMEOUT` can be set (in seconds) to a value +to wait for AWS to respond before skipping the attempt. + +If set, the `VAULT_AUTH_AWS_ROLE` environment variable will be used to specify the role to authenticate +using. If not set the AMI ID of the EC2 instance will be used by Vault. + +## `datasourceExists` + +Tests whether or not a given datasource was defined on the commandline (with the +[`--datasource/-d`](#--datasource-d) argument). This is intended mainly to allow +a template to be rendered differently whether or not a given datasource was +defined. + +Note: this does _not_ verify if the datasource is reachable. + +Useful when used in an `if`/`else` block + +```console +$ echo '{{if (datasourceExists "test")}}{{datasource "test"}}{{else}}no worries{{end}}' | gomplate +no worries +``` + +## `ds` + +Alias to [`datasource`](#datasource) + +## `include` + +Includes the content of a given datasource (provided by the [`--datasource/-d`](../usage/#datasource-d) argument). + +This is similar to [`datasource`](#datasource), +except that the data is not parsed. + +### Usage + +```go +include alias [subpath] +``` + +### Arguments + +| name | description | +|--------|-------| +| `alias` | the datasource alias, as provided by [`--datasource/-d`](../usage/#datasource-d) | +| `subpath` | _(optional)_ the subpath to use, if supported by the datasource | + +### Examples + +_`person.json`:_ +```json +{ "name": "Dave" } +``` + +_`input.tmpl`:_ +```go +{ + "people": [ + {{ include "person" }} + ] +} +``` + +```console +$ gomplate -d person.json -f input.tmpl +{ + "people": [ + { "name": "Dave" } + ] +} +``` + +## `data.JSON` + +**Alias:** `json` + +Converts a JSON string into an object. Only works for JSON Objects (not Arrays or other valid JSON types). This can be used to access properties of JSON objects. + +#### Example + +_`input.tmpl`:_ +``` +Hello {{ (getenv "FOO" | json).hello }} +``` + +```console +$ export FOO='{"hello":"world"}' +$ gomplate < input.tmpl +Hello world +``` + +## `data.JSONArray` + +**Alias:** `jsonArray` + +Converts a JSON string into a slice. Only works for JSON Arrays. + +#### Example + +_`input.tmpl`:_ +``` +Hello {{ index (getenv "FOO" | jsonArray) 1 }} +``` + +```console +$ export FOO='[ "you", "world" ]' +$ gomplate < input.tmpl +Hello world +``` + +## `data.YAML` + +**Alias:** `yaml` + +Converts a YAML string into an object. Only works for YAML Objects (not Arrays or other valid YAML types). This can be used to access properties of YAML objects. + +#### Example + +_`input.tmpl`:_ +``` +Hello {{ (getenv "FOO" | yaml).hello }} +``` + +```console +$ export FOO='hello: world' +$ gomplate < input.tmpl +Hello world +``` + +## `data.YAMLArray` + +**Alias:** `yamlArray` + +Converts a YAML string into a slice. Only works for YAML Arrays. + +#### Example + +_`input.tmpl`:_ +``` +Hello {{ index (getenv "FOO" | yamlArray) 1 }} +``` + +```console +$ export FOO='[ "you", "world" ]' +$ gomplate < input.tmpl +Hello world +``` + +## `data.TOML` + +**Alias:** `toml` + +Converts a [TOML](https://github.com/toml-lang/toml) document into an object. +This can be used to access properties of TOML documents. + +Compatible with [TOML v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md). + +### Usage + +```go +toml input +``` + +Can also be used in a pipeline: +```go +input | toml +``` + +### Arguments + +| name | description | +|--------|-------| +| `input` | the TOML document to parse | + +#### Example + +_`input.tmpl`:_ +``` +{{ $t := `[data] +hello = "world"` -}} +Hello {{ (toml $t).hello }} +``` + +```console +$ gomplate -f input.tmpl +Hello world +``` + +## `data.CSV` + +**Alias:** `csv` + +Converts a CSV-format string into a 2-dimensional string array. + +By default, the [RFC 4180](https://tools.ietf.org/html/rfc4180) format is +supported, but any single-character delimiter can be specified. + +### Usage + +```go +csv [delim] input +``` + +Can also be used in a pipeline: +```go +input | csv [delim] +``` + +### Arguments + +| name | description | +|--------|-------| +| `delim` | _(optional)_ the (single-character!) field delimiter, defaults to `","` | +| `input` | the CSV-format string to parse | + +### Example + +_`input.tmpl`:_ +``` +{{ $c := `C,32 +Go,25 +COBOL,357` -}} +{{ range ($c | csv) -}} +{{ index . 0 }} has {{ index . 1 }} keywords. +{{ end }} +``` + +```console +$ gomplate < input.tmpl +C has 32 keywords. +Go has 25 keywords. +COBOL has 357 keywords. +``` + +## `data.CSVByRow` + +**Alias:** `csvByRow` + +Converts a CSV-format string into a slice of maps. + +By default, the [RFC 4180](https://tools.ietf.org/html/rfc4180) format is +supported, but any single-character delimiter can be specified. + +Also by default, the first line of the string will be assumed to be the header, +but this can be overridden by providing an explicit header, or auto-indexing +can be used. + + +### Usage + +```go +csv [delim] [header] input +``` + +Can also be used in a pipeline: +```go +input | csv [delim] [header] +``` + +### Arguments + +| name | description | +|--------|-------| +| `delim` | _(optional)_ the (single-character!) field delimiter, defaults to `","` | +| `header`| _(optional)_ comma-separated list of column names, set to `""` to get auto-named columns (A-Z), defaults to using the first line of `input` | +| `input` | the CSV-format string to parse | + +### Example + +_`input.tmpl`:_ +``` +{{ $c := `lang,keywords +C,32 +Go,25 +COBOL,357` -}} +{{ range ($c | csvByRow) -}} +{{ .lang }} has {{ .keywords }} keywords. +{{ end }} +``` + +```console +$ gomplate < input.tmpl +C has 32 keywords. +Go has 25 keywords. +COBOL has 357 keywords. +``` + +## `data.CSVByColumn` + +**Alias:** `csvByColumn` + +Like [`csvByRow`](#csvByRow), except that the data is presented as a columnar +(column-oriented) map. + +### Example + +_`input.tmpl`:_ +``` +{{ $c := `C;32 +Go;25 +COBOL;357` -}} +{{ $langs := ($c | csvByColumn ";" "lang,keywords").lang -}} +{{ range $langs }}{{ . }} +{{ end -}} +``` + +```console +$ gomplate < input.tmpl +C +Go +COBOL +``` + +## `data.ToJSON` + +**Alias:** `toJSON` + +Converts an object to a JSON document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. + +#### Example + +_This is obviously contrived - `json` is used to create an object._ + +_`input.tmpl`:_ +``` +{{ (`{"foo":{"hello":"world"}}` | json).foo | toJSON }} +``` + +```console +$ gomplate < input.tmpl +{"hello":"world"} +``` + +## `data.ToJSONPretty` + +**Alias:** `toJSONPretty` + +Converts an object to a pretty-printed (or _indented_) JSON document. +Input objects may be the result of functions like `data.JSON`, `data.YAML`, +`data.JSONArray`, or `data.YAMLArray` functions, or they could be provided +by a [`datasource`](../general/datasource). + +The indent string must be provided as an argument. + +#### Example + +_`input.tmpl`:_ +``` +{{ `{"hello":"world"}` | data.JSON | data.ToJSONPretty " " }} +``` + +```console +$ gomplate < input.tmpl +{ + "hello": "world" +} +``` + +## `data.ToYAML` + +**Alias:** `toYAML` + +Converts an object to a YAML document. Input objects may be the result of +`data.JSON`, `data.YAML`, `data.JSONArray`, or `data.YAMLArray` functions, +or they could be provided by a [`datasource`](../general/datasource). + +#### Example + +_This is obviously contrived - `data.JSON` is used to create an object._ + +_`input.tmpl`:_ +``` +{{ (`{"foo":{"hello":"world"}}` | data.JSON).foo | data.ToYAML }} +``` + +```console +$ gomplate < input.tmpl +hello: world +``` + +## `data.ToTOML` + +**Alias:** `toTOML` + +Converts an object to a [TOML](https://github.com/toml-lang/toml) document. + +### Usage + +```go +data.ToTOML obj +``` + +Can also be used in a pipeline: +```go +obj | data.ToTOML +``` + +### Arguments + +| name | description | +|--------|-------| +| `obj` | the object to marshal as a TOML document | + +#### Example + +```console +$ gomplate -i '{{ `{"foo":"bar"}` | data.JSON | data.ToTOML }}' +foo = "bar" +``` + +## `data.ToCSV` + +**Alias:** `toCSV` + +Converts an object to a CSV document. The input object must be a 2-dimensional +array of strings (a `[][]string`). Objects produced by [`data.CSVByRow`](#conv-csvbyrow) +and [`data.CSVByColumn`](#conv-csvbycolumn) cannot yet be converted back to CSV documents. + +**Note:** With the exception that a custom delimiter can be used, `data.ToCSV` +outputs according to the [RFC 4180](https://tools.ietf.org/html/rfc4180) format, +which means that line terminators are `CRLF` (Windows format, or `\r\n`). If +you require `LF` (UNIX format, or `\n`), the output can be piped through +[`strings.ReplaceAll`](../strings/#strings-replaceall) to replace `"\r\n"` with `"\n"`. + +### Usage + +```go +data.ToCSV [delim] input +``` + +Can also be used in a pipeline: +```go +input | data.ToCSV [delim] +``` + +### Arguments + +| name | description | +|--------|-------| +| `delim` | _(optional)_ the (single-character!) field delimiter, defaults to `","` | +| `input` | the object to convert to a CSV | + +### Examples + +_`input.tmpl`:_ +```go +{{ $rows := (jsonArray `[["first","second"],["1","2"],["3","4"]]`) -}} +{{ data.ToCSV ";" $rows }} +``` + +```console +$ gomplate -f input.tmpl +first,second +1,2 +3,4 +``` diff --git a/docs/content/functions/general.md b/docs/content/functions/general.md deleted file mode 100644 index 7f33cfde..00000000 --- a/docs/content/functions/general.md +++ /dev/null @@ -1,721 +0,0 @@ ---- -title: other functions -menu: - main: - parent: functions ---- - -## `bool` - -Converts a true-ish string to a boolean. Can be used to simplify conditional statements based on environment variables or other text input. - -#### Example - -_`input.tmpl`:_ -``` -{{if bool (getenv "FOO")}}foo{{else}}bar{{end}} -``` - -```console -$ gomplate < input.tmpl -bar -$ FOO=true gomplate < input.tmpl -foo -``` - -## `slice` - -Creates a slice. Useful when needing to `range` over a bunch of variables. - -#### Example - -_`input.tmpl`:_ -``` -{{range slice "Bart" "Lisa" "Maggie"}} -Hello, {{.}} -{{- end}} -``` - -```console -$ gomplate < input.tmpl -Hello, Bart -Hello, Lisa -Hello, Maggie -``` - -## `urlParse` - -Parses a string as a URL for later use. Equivalent to [url.Parse](https://golang.org/pkg/net/url/#Parse) - -#### Example - -_`input.tmpl`:_ -``` -{{ $u := urlParse "https://example.com:443/foo/bar" }} -The scheme is {{ $u.Scheme }} -The host is {{ $u.Host }} -The path is {{ $u.Path }} -``` - -```console -$ gomplate < input.tmpl -The scheme is https -The host is example.com:443 -The path is /foo/bar -``` - -## `has` - -Has reports whether or not a given object has a property with the given key. Can be used with `if` to prevent the template from trying to access a non-existent property in an object. - -#### Example - -_Let's say we're using a Vault datasource..._ - -_`input.tmpl`:_ -``` -{{ $secret := datasource "vault" "mysecret" -}} -The secret is ' -{{- if (has $secret "value") }} -{{- $secret.value }} -{{- else }} -{{- $secret | toYAML }} -{{- end }}' -``` - -If the `secret/foo/mysecret` secret in Vault has a property named `value` set to `supersecret`: - -```console -$ gomplate -d vault:///secret/foo < input.tmpl -The secret is 'supersecret' -``` - -On the other hand, if there is no `value` property: - -```console -$ gomplate -d vault:///secret/foo < input.tmpl -The secret is 'foo: bar' -``` - -## `join` - -Concatenates the elements of an array to create a string. The separator string sep is placed between elements in the resulting string. - -#### Example - -_`input.tmpl`_ -``` -{{ $a := `[1, 2, 3]` | jsonArray }} -{{ join $a "-" }} -``` - -```console -$ gomplate -f input.tmpl -1-2-3 -``` - -## `json` - -Converts a JSON string into an object. Only works for JSON Objects (not Arrays or other valid JSON types). This can be used to access properties of JSON objects. - -#### Example - -_`input.tmpl`:_ -``` -Hello {{ (getenv "FOO" | json).hello }} -``` - -```console -$ export FOO='{"hello":"world"}' -$ gomplate < input.tmpl -Hello world -``` - -## `jsonArray` - -Converts a JSON string into a slice. Only works for JSON Arrays. - -#### Example - -_`input.tmpl`:_ -``` -Hello {{ index (getenv "FOO" | jsonArray) 1 }} -``` - -```console -$ export FOO='[ "you", "world" ]' -$ gomplate < input.tmpl -Hello world -``` - -## `yaml` - -Converts a YAML string into an object. Only works for YAML Objects (not Arrays or other valid YAML types). This can be used to access properties of YAML objects. - -#### Example - -_`input.tmpl`:_ -``` -Hello {{ (getenv "FOO" | yaml).hello }} -``` - -```console -$ export FOO='hello: world' -$ gomplate < input.tmpl -Hello world -``` - -## `yamlArray` - -Converts a YAML string into a slice. Only works for YAML Arrays. - -#### Example - -_`input.tmpl`:_ -``` -Hello {{ index (getenv "FOO" | yamlArray) 1 }} -``` - -```console -$ export FOO='[ "you", "world" ]' -$ gomplate < input.tmpl -Hello world -``` - -## `toml` - -Converts a [TOML](https://github.com/toml-lang/toml) document into an object. -This can be used to access properties of TOML documents. - -Compatible with [TOML v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md). - -### Usage - -```go -toml input -``` - -Can also be used in a pipeline: -```go -input | toml -``` - -### Arguments - -| name | description | -|--------|-------| -| `input` | the TOML document to parse | - -#### Example - -_`input.tmpl`:_ -``` -{{ $t := `[data] -hello = "world"` -}} -Hello {{ (toml $t).hello }} -``` - -```console -$ gomplate -f input.tmpl -Hello world -``` - -## `csv` - -Converts a CSV-format string into a 2-dimensional string array. - -By default, the [RFC 4180](https://tools.ietf.org/html/rfc4180) format is -supported, but any single-character delimiter can be specified. - -### Usage - -```go -csv [delim] input -``` - -Can also be used in a pipeline: -```go -input | csv [delim] -``` - -### Arguments - -| name | description | -|--------|-------| -| `delim` | _(optional)_ the (single-character!) field delimiter, defaults to `","` | -| `input` | the CSV-format string to parse | - -### Example - -_`input.tmpl`:_ -``` -{{ $c := `C,32 -Go,25 -COBOL,357` -}} -{{ range ($c | csv) -}} -{{ index . 0 }} has {{ index . 1 }} keywords. -{{ end }} -``` - -```console -$ gomplate < input.tmpl -C has 32 keywords. -Go has 25 keywords. -COBOL has 357 keywords. -``` - -## `csvByRow` - -Converts a CSV-format string into a slice of maps. - -By default, the [RFC 4180](https://tools.ietf.org/html/rfc4180) format is -supported, but any single-character delimiter can be specified. - -Also by default, the first line of the string will be assumed to be the header, -but this can be overridden by providing an explicit header, or auto-indexing -can be used. - - -### Usage - -```go -csv [delim] [header] input -``` - -Can also be used in a pipeline: -```go -input | csv [delim] [header] -``` - -### Arguments - -| name | description | -|--------|-------| -| `delim` | _(optional)_ the (single-character!) field delimiter, defaults to `","` | -| `header`| _(optional)_ comma-separated list of column names, set to `""` to get auto-named columns (A-Z), defaults to using the first line of `input` | -| `input` | the CSV-format string to parse | - -### Example - -_`input.tmpl`:_ -``` -{{ $c := `lang,keywords -C,32 -Go,25 -COBOL,357` -}} -{{ range ($c | csvByRow) -}} -{{ .lang }} has {{ .keywords }} keywords. -{{ end }} -``` - -```console -$ gomplate < input.tmpl -C has 32 keywords. -Go has 25 keywords. -COBOL has 357 keywords. -``` - -## `csvByColumn` - -Like [`csvByRow`](#csvByRow), except that the data is presented as a columnar -(column-oriented) map. - -### Example - -_`input.tmpl`:_ -``` -{{ $c := `C;32 -Go;25 -COBOL;357` -}} -{{ $langs := ($c | csvByColumn ";" "lang,keywords").lang -}} -{{ range $langs }}{{ . }} -{{ end -}} -``` - -```console -$ gomplate < input.tmpl -C -Go -COBOL -``` - -## `toJSON` - -Converts an object to a JSON document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. - -#### Example - -_This is obviously contrived - `json` is used to create an object._ - -_`input.tmpl`:_ -``` -{{ (`{"foo":{"hello":"world"}}` | json).foo | toJSON }} -``` - -```console -$ gomplate < input.tmpl -{"hello":"world"} -``` - -## `toJSONPretty` - -Converts an object to a pretty-printed (or _indented_) JSON document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. - -The indent string must be provided as an argument. - -#### Example - -_`input.tmpl`:_ -``` -{{ `{"hello":"world"}` | json | toJSONPretty " " }} -``` - -```console -$ gomplate < input.tmpl -{ - "hello": "world" -} -``` - -## `toYAML` - -Converts an object to a YAML document. Input objects may be the result of `json`, `yaml`, `jsonArray`, or `yamlArray` functions, or they could be provided by a `datasource`. - -#### Example - -_This is obviously contrived - `json` is used to create an object._ - -_`input.tmpl`:_ -``` -{{ (`{"foo":{"hello":"world"}}` | json).foo | toYAML }} -``` - -```console -$ gomplate < input.tmpl -hello: world -``` - -## `toTOML` - -Converts an object to a [TOML](https://github.com/toml-lang/toml) document. - -### Usage - -```go -toTOML obj -``` - -Can also be used in a pipeline: -```go -obj | toTOML -``` - -### Arguments - -| name | description | -|--------|-------| -| `obj` | the object to marshal as a TOML document | - -#### Example - -```console -$ gomplate -i '{{ `{"foo":"bar"}` | json | toTOML }}' -foo = "bar" -``` - -## `toCSV` - -Converts an object to a CSV document. The input object must be a 2-dimensional -array of strings (a `[][]string`). Objects produced by [`csvByRow`](#csvByRow) -and [`csvByColumn`](#csvByColumn) cannot yet be converted back to CSV documents. - -**Note:** With the exception that a custom delimiter can be used, `toCSV` -outputs according to the [RFC 4180](https://tools.ietf.org/html/rfc4180) format, -which means that line terminators are `CRLF` (Windows format, or `\r\n`). If -you require `LF` (UNIX format, or `\n`), the output can be piped through -[`replaceAll`](#replaceAll) to replace `"\r\n"` with `"\n"`. - -### Usage - -```go -toCSV [delim] input -``` - -Can also be used in a pipeline: -```go -input | toCSV [delim] -``` - -### Arguments - -| name | description | -|--------|-------| -| `delim` | _(optional)_ the (single-character!) field delimiter, defaults to `","` | -| `input` | the object to convert to a CSV | - -### Examples - -_`input.tmpl`:_ -```go -{{ $rows := (jsonArray `[["first","second"],["1","2"],["3","4"]]`) -}} -{{ toCSV ";" $rows }} -``` - -```console -$ gomplate -f input.tmpl -first,second -1,2 -3,4 -``` - -## `datasource` - -Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument). - -Currently, `file://`, `http://`, `https://`, and `vault://` URLs are supported. - -Currently-supported formats are JSON, YAML, TOML, and CSV. - -### Basic usage - -_`person.json`:_ -```json -{ - "name": "Dave" -} -``` - -_`input.tmpl`:_ -``` -Hello {{ (datasource "person").name }} -``` - -```console -$ gomplate -d person.json < input.tmpl -Hello Dave -``` - -### Usage with HTTP data - -```console -$ echo 'Hello there, {{(datasource "foo").headers.Host}}...' | gomplate -d foo=https://httpbin.org/get -Hello there, httpbin.org... -``` - -Additional headers can be provided with the `--datasource-header`/`-H` option: - -```console -$ gomplate -d foo=https://httpbin.org/get -H 'foo=Foo: bar' -i '{{(datasource "foo").headers.Foo}}' -bar -``` - -### Usage with Consul data - -There are three supported URL schemes to retrieve data from [Consul](https://consul.io/). -The `consul://` (or `consul+http://`) scheme can optionally be used with a hostname and port to specify a server (e.g. `consul://localhost:8500`). -By default HTTP will be used, but the `consul+https://` form can be used to use HTTPS, alternatively `$CONSUL_HTTP_SSL` can be used. - -If the server address isn't part of the datasource URL, `$CONSUL_HTTP_ADDR` will be checked. - -The following optional environment variables can be set: - -| name | usage | -|------|-------| -| `CONSUL_HTTP_ADDR` | Hostname and optional port for connecting to Consul. Defaults to `http://localhost:8500` | -| `CONSUL_TIMEOUT` | Timeout (in seconds) when communicating to Consul. Defaults to 10 seconds. | -| `CONSUL_HTTP_TOKEN` | The Consul token to use when connecting to the server. | -| `CONSUL_HTTP_AUTH` | Should be specified as `:`. Used to authenticate to the server. | -| `CONSUL_HTTP_SSL` | Force HTTPS if set to `true` value. Disables if set to `false`. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | -| `CONSUL_TLS_SERVER_NAME` | The server name to use as the SNI host when connecting to Consul via TLS. | -| `CONSUL_CACERT` | Path to CA file for verifying Consul server using TLS. | -| `CONSUL_CAPATH` | Path to directory of CA files for verifying Consul server using TLS. | -| `CONSUL_CLIENT_CERT` | Client certificate file for certificate authentication. If this is set, `$CONSUL_CLIENT_KEY` must also be set. | -| `CONSUL_CLIENT_KEY` | Client key file for certificate authentication. If this is set, `$CONSUL_CLIENT_CERT` must also be set. | -| `CONSUL_HTTP_SSL_VERIFY` | Set to `false` to disable Consul TLS certificate checking. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided.
_Recommended only for testing and development scenarios!_ | -| `CONSUL_VAULT_ROLE` | If set will fetch the Consul authentication from Vault using the Consul dynamic secret backend. Value should be the name of the role to use. | -| `CONSUL_VAULT_MOUNT` | If using Vault for authentication set the name of the Consul dynamic secret backend. Defaults to `consul`. | - -If a path is included it is used as a prefix for all uses of the datasource. - -#### Example - -```console -$ gomplate -d consul=consul:// -i '{{(datasource "consul" "foo")}}' -value for foo key -``` - -```console -$ gomplate -d consul=consul+https://my-consul-server.com:8533/foo -i '{{(datasource "consul" "bar")}}' -value for foo/bar key -``` - -```console -$ gomplate -d consul=consul:///foo -i '{{(datasource "consul" "bar/baz")}}' -value for foo/bar/baz key -``` - -Instead of using a non-authenticated Consul connection or connecting using the token set with the -`CONSUL_HTTP_TOKEN` environment variable, it is possible to authenticate using a dynamically generated -token fetched from Vault. This requires Vault to be configured to use the Consul secret backend and -is enabled by passing the name of the role to use in the `CONSUL_VAULT_ROLE` environment variable. - -### Usage with BoltDB data - -[BoltDB](https://github.com/boltdb/bolt) is a simple local key/value store used -by many Go tools. The `boltdb://` scheme can be used to access values stored in -a BoltDB database file. The full path is provided in the URL, and the bucket name -can be specified using a URL fragment (e.g. `boltdb:///tmp/database.db#bucket`). - -Access is implemented through [libkv](https://github.com/docker/libkv), and as -such, the first 8 bytes of all values are used as an incrementing last modified -index value. All values must therefore be at least 9 bytes long, with the first -8 being ignored. - -The following environment variables can be set: - -| name | usage | -|------|-------| -| `BOLTDB_TIMEOUT` | Timeout (in seconds) to wait for a lock on the database file when opening. | -| `BOLTDB_PERSIST` | If set keep the database open instead of closing after each read. Any value acceptable to [`strconv.ParseBool`](https://golang.org/pkg/strconv/#ParseBool) can be provided. | - -### Example - -```console -$ gomplate -d config=boltdb:///tmp/config.db#Bucket1 -i '{{(datasource "config" "foo")}}' -bar -``` - -### Usage with Vault data - -The special `vault://` URL scheme can be used to retrieve data from [Hashicorp -Vault](https://vaultproject.io). To use this, you must put the Vault server's -URL in the `$VAULT_ADDR` environment variable. - -This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: - -| auth backend | configuration | -|-------------: |---------------| -| [`approle`](https://www.vaultproject.io/docs/auth/approle.html) | Environment variables `$VAULT_ROLE_ID` and `$VAULT_SECRET_ID` must be set to the appropriate values.
If the backend is mounted to a different location, set `$VAULT_AUTH_APPROLE_MOUNT`. | -| [`app-id`](https://www.vaultproject.io/docs/auth/app-id.html) | Environment variables `$VAULT_APP_ID` and `$VAULT_USER_ID` must be set to the appropriate values.
If the backend is mounted to a different location, set `$VAULT_AUTH_APP_ID_MOUNT`. | -| [`github`](https://www.vaultproject.io/docs/auth/github.html) | Environment variable `$VAULT_AUTH_GITHUB_TOKEN` must be set to an appropriate value.
If the backend is mounted to a different location, set `$VAULT_AUTH_GITHUB_MOUNT`. | -| [`userpass`](https://www.vaultproject.io/docs/auth/userpass.html) | Environment variables `$VAULT_AUTH_USERNAME` and `$VAULT_AUTH_PASSWORD` must be set to the appropriate values.
If the backend is mounted to a different location, set `$VAULT_AUTH_USERPASS_MOUNT`. | -| [`token`](https://www.vaultproject.io/docs/auth/token.html) | Determined from either the `$VAULT_TOKEN` environment variable, or read from the file `~/.vault-token` | -| [`aws`](https://www.vaultproject.io/docs/auth/aws.html) | As a final option authentication will be attempted using the AWS auth backend. See below for more details. | - -_**Note:**_ The secret values listed in the above table can either be set in environment -variables or provided in files. This can increase security when using -[Docker Swarm Secrets](https://docs.docker.com/engine/swarm/secrets/), for example. -To use files, specify the filename by appending `_FILE` to the environment variable, -(i.e. `VAULT_USER_ID_FILE`). If the non-file variable is set, this will override -any `_FILE` variable and the secret file will be ignored. - -To use a Vault datasource with a single secret, just use a URL of -`vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left -empty. - -```console -$ echo 'My voice is my passport. {{(datasource "vault").value}}' \ - | gomplate -d vault=vault:///secret/sneakers -My voice is my passport. Verify me. -``` - -You can also specify the secret path in the template by using a URL of `vault://` -(or `vault:///`, or `vault:`): -```console -$ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \ - | gomplate -d vault=vault:// -My voice is my passport. Verify me. -``` - -And the two can be mixed to scope secrets to a specific namespace: - -```console -$ echo 'db_password={{(datasource "vault" "db/pass").value}}' \ - | gomplate -d vault=vault:///secret/production -db_password=prodsecret -``` - -It is also possible to use dynamic secrets by using the write capability of the datasource. To use, -add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then -included within the JSON body of the request. - -```console -$ echo 'otp={{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").key}}' \ - | gomplate -d vault=vault:/// -otp=604a4bd5-7afd-30a2-d2d8-80c4aebc6183 -``` - -#### Authentication using AWS details - -If running on an EC2 instance authentication will be attempted using the AWS auth backend. The -optional `VAULT_AUTH_AWS_MOUNT` environment variable can be used to set the mount point to use if -it differs from the default of `aws`. Additionally `AWS_TIMEOUT` can be set (in seconds) to a value -to wait for AWS to respond before skipping the attempt. - -If set, the `VAULT_AUTH_AWS_ROLE` environment variable will be used to specify the role to authenticate -using. If not set the AMI ID of the EC2 instance will be used by Vault. - -## `datasourceExists` - -Tests whether or not a given datasource was defined on the commandline (with the -[`--datasource/-d`](#--datasource-d) argument). This is intended mainly to allow -a template to be rendered differently whether or not a given datasource was -defined. - -Note: this does _not_ verify if the datasource is reachable. - -Useful when used in an `if`/`else` block - -```console -$ echo '{{if (datasourceExists "test")}}{{datasource "test"}}{{else}}no worries{{end}}' | gomplate -no worries -``` - -## `ds` - -Alias to [`datasource`](#datasource) - -## `include` - -Includes the content of a given datasource (provided by the [`--datasource/-d`](../usage/#datasource-d) argument). - -This is similar to [`datasource`](#datasource), -except that the data is not parsed. - -### Usage - -```go -include alias [subpath] -``` - -### Arguments - -| name | description | -|--------|-------| -| `alias` | the datasource alias, as provided by [`--datasource/-d`](../usage/#datasource-d) | -| `subpath` | _(optional)_ the subpath to use, if supported by the datasource | - -### Examples - -_`person.json`:_ -```json -{ "name": "Dave" } -``` - -_`input.tmpl`:_ -```go -{ - "people": [ - {{ include "person" }} - ] -} -``` - -```console -$ gomplate -d person.json -f input.tmpl -{ - "people": [ - { "name": "Dave" } - ] -} -``` diff --git a/funcs.go b/funcs.go index aa98f029..f4a5d6fc 100644 --- a/funcs.go +++ b/funcs.go @@ -1,45 +1,22 @@ package main import ( - "net/url" "text/template" + "github.com/hairyhenderson/gomplate/data" "github.com/hairyhenderson/gomplate/funcs" ) // initFuncs - The function mappings are defined here! -func initFuncs(data *Data) template.FuncMap { - typeconv := &TypeConv{} - - f := template.FuncMap{ - "bool": typeconv.Bool, - "has": typeconv.Has, - "json": typeconv.JSON, - "jsonArray": typeconv.JSONArray, - "yaml": typeconv.YAML, - "yamlArray": typeconv.YAMLArray, - "toml": typeconv.TOML, - "csv": typeconv.CSV, - "csvByRow": typeconv.CSVByRow, - "csvByColumn": typeconv.CSVByColumn, - "slice": typeconv.Slice, - "join": typeconv.Join, - "toJSON": typeconv.ToJSON, - "toJSONPretty": typeconv.toJSONPretty, - "toYAML": typeconv.ToYAML, - "toTOML": typeconv.ToTOML, - "toCSV": typeconv.ToCSV, - "urlParse": url.Parse, - "datasource": data.Datasource, - "ds": data.Datasource, - "datasourceExists": data.DatasourceExists, - "include": data.include, - } +func initFuncs(d *data.Data) template.FuncMap { + f := template.FuncMap{} + funcs.AddDataFuncs(f, d) funcs.AWSFuncs(f) funcs.AddBase64Funcs(f) funcs.AddNetFuncs(f) funcs.AddReFuncs(f) funcs.AddStringFuncs(f) funcs.AddEnvFuncs(f) + funcs.AddConvFuncs(f) return f } diff --git a/funcs/conv.go b/funcs/conv.go new file mode 100644 index 00000000..2e5327b1 --- /dev/null +++ b/funcs/conv.go @@ -0,0 +1,64 @@ +package funcs + +import ( + "net/url" + "sync" + + "github.com/hairyhenderson/gomplate/conv" +) + +var ( + convNS *ConvFuncs + convNSInit sync.Once +) + +// ConvNS - +func ConvNS() *ConvFuncs { + convNSInit.Do(func() { convNS = &ConvFuncs{} }) + return convNS +} + +// AddConvFuncs - +func AddConvFuncs(f map[string]interface{}) { + f["conv"] = ConvNS + + f["urlParse"] = ConvNS().URL + f["bool"] = ConvNS().Bool + f["has"] = ConvNS().Has + f["slice"] = ConvNS().Slice + f["join"] = ConvNS().Join +} + +// ConvFuncs - +type ConvFuncs struct{} + +func (f *ConvFuncs) Bool(s string) bool { + return conv.Bool(s) +} + +func (f *ConvFuncs) Slice(args ...interface{}) []interface{} { + return conv.Slice(args...) +} +func (f *ConvFuncs) Join(in interface{}, sep string) string { + return conv.Join(in, sep) +} +func (f *ConvFuncs) Has(in interface{}, key string) bool { + return conv.Has(in, key) +} + +func (f *ConvFuncs) ParseInt(s string, base, bitSize int) int64 { + return conv.MustParseInt(s, base, bitSize) +} +func (f *ConvFuncs) ParseFloat(s string, bitSize int) float64 { + return conv.MustParseFloat(s, bitSize) +} +func (f *ConvFuncs) ParseUint(s string, base, bitSize int) uint64 { + return conv.MustParseUint(s, base, bitSize) +} +func (f *ConvFuncs) Atoi(s string) int { + return conv.MustAtoi(s) +} + +func (f *ConvFuncs) URL(s string) (*url.URL, error) { + return url.Parse(s) +} diff --git a/funcs/data.go b/funcs/data.go new file mode 100644 index 00000000..0a432c1e --- /dev/null +++ b/funcs/data.go @@ -0,0 +1,110 @@ +package funcs + +import ( + "sync" + + "github.com/hairyhenderson/gomplate/data" +) + +var ( + dataNS *DataFuncs + dataNSInit sync.Once +) + +// DataNS - +func DataNS() *DataFuncs { + dataNSInit.Do(func() { dataNS = &DataFuncs{} }) + return dataNS +} + +// AddDataFuncs - +func AddDataFuncs(f map[string]interface{}, d *data.Data) { + f["datasource"] = d.Datasource + f["ds"] = d.Datasource + f["datasourceExists"] = d.DatasourceExists + f["include"] = d.Include + + f["data"] = DataNS + + f["json"] = DataNS().JSON + f["jsonArray"] = DataNS().JSONArray + f["yaml"] = DataNS().YAML + f["yamlArray"] = DataNS().YAMLArray + f["toml"] = DataNS().TOML + f["csv"] = DataNS().CSV + f["csvByRow"] = DataNS().CSVByRow + f["csvByColumn"] = DataNS().CSVByColumn + f["toJSON"] = DataNS().ToJSON + f["toJSONPretty"] = DataNS().ToJSONPretty + f["toYAML"] = DataNS().ToYAML + f["toTOML"] = DataNS().ToTOML + f["toCSV"] = DataNS().ToCSV +} + +// DataFuncs - +type DataFuncs struct{} + +// JSON - +func (f *DataFuncs) JSON(in string) map[string]interface{} { + return data.JSON(in) +} + +// JSONArray - +func (f *DataFuncs) JSONArray(in string) []interface{} { + return data.JSONArray(in) +} + +// YAML - +func (f *DataFuncs) YAML(in string) map[string]interface{} { + return data.YAML(in) +} + +// YAMLArray - +func (f *DataFuncs) YAMLArray(in string) []interface{} { + return data.YAMLArray(in) +} + +// TOML - +func (f *DataFuncs) TOML(in string) interface{} { + return data.TOML(in) +} + +// CSV - +func (f *DataFuncs) CSV(args ...string) [][]string { + return data.CSV(args...) +} + +// CSVByRow - +func (f *DataFuncs) CSVByRow(args ...string) (rows []map[string]string) { + return data.CSVByRow(args...) +} + +// CSVByColumn - +func (f *DataFuncs) CSVByColumn(args ...string) (cols map[string][]string) { + return data.CSVByColumn(args...) +} + +// ToCSV - +func (f *DataFuncs) ToCSV(args ...interface{}) string { + return data.ToCSV(args...) +} + +// ToJSON - +func (f *DataFuncs) ToJSON(in interface{}) string { + return data.ToJSON(in) +} + +// ToJSONPretty - +func (f *DataFuncs) ToJSONPretty(indent string, in interface{}) string { + return data.ToJSONPretty(indent, in) +} + +// ToYAML - +func (f *DataFuncs) ToYAML(in interface{}) string { + return data.ToYAML(in) +} + +// ToTOML - +func (f *DataFuncs) ToTOML(in interface{}) string { + return data.ToTOML(in) +} diff --git a/gomplate.go b/gomplate.go index 71530368..25e2a317 100644 --- a/gomplate.go +++ b/gomplate.go @@ -4,6 +4,8 @@ import ( "io" "log" "text/template" + + "github.com/hairyhenderson/gomplate/data" ) func (g *Gomplate) createTemplate() *template.Template { @@ -31,19 +33,20 @@ func (g *Gomplate) RunTemplate(text string, out io.Writer) { } // NewGomplate - -func NewGomplate(data *Data, leftDelim, rightDelim string) *Gomplate { +func NewGomplate(d *data.Data, leftDelim, rightDelim string) *Gomplate { return &Gomplate{ leftDelim: leftDelim, rightDelim: rightDelim, - funcMap: initFuncs(data), + funcMap: initFuncs(d), } } func runTemplate(o *GomplateOpts) error { defer runCleanupHooks() - data := NewData(o.dataSources, o.dataSourceHeaders) + d := data.NewData(o.dataSources, o.dataSourceHeaders) + addCleanupHook(d.Cleanup) - g := NewGomplate(data, o.lDelim, o.rDelim) + g := NewGomplate(d, o.lDelim, o.rDelim) if o.inputDir != "" { return processInputDir(o.inputDir, o.outputDir, g) diff --git a/gomplate_test.go b/gomplate_test.go index cd940d30..112f6ae3 100644 --- a/gomplate_test.go +++ b/gomplate_test.go @@ -9,6 +9,8 @@ import ( "text/template" "github.com/hairyhenderson/gomplate/aws" + "github.com/hairyhenderson/gomplate/conv" + "github.com/hairyhenderson/gomplate/data" "github.com/hairyhenderson/gomplate/env" "github.com/stretchr/testify/assert" ) @@ -20,11 +22,10 @@ func testTemplate(g *Gomplate, template string) string { } func TestGetenvTemplates(t *testing.T) { - typeconv := &TypeConv{} g := &Gomplate{ funcMap: template.FuncMap{ "getenv": env.Getenv, - "bool": typeconv.Bool, + "bool": conv.Bool, }, } assert.Empty(t, testTemplate(g, `{{getenv "BLAHBLAHBLAH"}}`)) @@ -33,10 +34,9 @@ func TestGetenvTemplates(t *testing.T) { } func TestBoolTemplates(t *testing.T) { - typeconv := &TypeConv{} g := &Gomplate{ funcMap: template.FuncMap{ - "bool": typeconv.Bool, + "bool": conv.Bool, }, } assert.Equal(t, "true", testTemplate(g, `{{bool "true"}}`)) @@ -66,12 +66,11 @@ func TestEc2MetaTemplates(t *testing.T) { func TestEc2MetaTemplates_WithJSON(t *testing.T) { server, ec2meta := aws.MockServer(200, `{"foo":"bar"}`) defer server.Close() - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ "ec2meta": ec2meta.Meta, "ec2dynamic": ec2meta.Dynamic, - "json": ty.JSON, + "json": data.JSON, }, } @@ -80,10 +79,9 @@ func TestEc2MetaTemplates_WithJSON(t *testing.T) { } func TestJSONArrayTemplates(t *testing.T) { - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ - "jsonArray": ty.JSONArray, + "jsonArray": data.JSONArray, }, } @@ -92,11 +90,10 @@ func TestJSONArrayTemplates(t *testing.T) { } func TestYAMLTemplates(t *testing.T) { - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ - "yaml": ty.YAML, - "yamlArray": ty.YAMLArray, + "yaml": data.YAML, + "yamlArray": data.YAMLArray, }, } @@ -106,10 +103,9 @@ func TestYAMLTemplates(t *testing.T) { } func TestSliceTemplates(t *testing.T) { - typeconv := &TypeConv{} g := &Gomplate{ funcMap: template.FuncMap{ - "slice": typeconv.Slice, + "slice": conv.Slice, }, } assert.Equal(t, "foo", testTemplate(g, `{{index (slice "foo") 0}}`)) @@ -118,11 +114,10 @@ func TestSliceTemplates(t *testing.T) { } func TestHasTemplate(t *testing.T) { - ty := new(TypeConv) g := &Gomplate{ funcMap: template.FuncMap{ - "yaml": ty.YAML, - "has": ty.Has, + "yaml": data.YAML, + "has": conv.Has, }, } assert.Equal(t, "true", testTemplate(g, `{{has ("foo:\n bar: true" | yaml) "foo"}}`)) diff --git a/libkv/boltdb.go b/libkv/boltdb.go index d3efe6db..024e21dd 100644 --- a/libkv/boltdb.go +++ b/libkv/boltdb.go @@ -7,8 +7,8 @@ import ( "github.com/docker/libkv" "github.com/docker/libkv/store" "github.com/docker/libkv/store/boltdb" + "github.com/hairyhenderson/gomplate/conv" "github.com/hairyhenderson/gomplate/env" - "github.com/hairyhenderson/gomplate/typeconv" ) // NewBoltDB - initialize a new BoltDB datasource handler @@ -28,10 +28,10 @@ func setupBoltDB(bucket string) *store.Config { logFatal("missing bucket - must specify BoltDB bucket in URL fragment") } - t := typeconv.MustParseInt(env.Getenv("BOLTDB_TIMEOUT"), 10, 16) + t := conv.MustParseInt(env.Getenv("BOLTDB_TIMEOUT"), 10, 16) return &store.Config{ Bucket: bucket, ConnectionTimeout: time.Duration(t) * time.Second, - PersistConnection: typeconv.MustParseBool(env.Getenv("BOLTDB_PERSIST")), + PersistConnection: conv.Bool(env.Getenv("BOLTDB_PERSIST")), } } diff --git a/libkv/consul.go b/libkv/consul.go index 3cf441f7..d4fb53c4 100644 --- a/libkv/consul.go +++ b/libkv/consul.go @@ -11,8 +11,8 @@ import ( "github.com/docker/libkv" "github.com/docker/libkv/store" "github.com/docker/libkv/store/consul" + "github.com/hairyhenderson/gomplate/conv" "github.com/hairyhenderson/gomplate/env" - "github.com/hairyhenderson/gomplate/typeconv" "github.com/hairyhenderson/gomplate/vault" consulapi "github.com/hashicorp/consul/api" ) @@ -66,7 +66,7 @@ func consulURL(u *url.URL) *url.URL { case "consul+https", "https": c.Scheme = "https" case "consul": - if typeconv.MustParseBool(env.Getenv("CONSUL_HTTP_SSL")) { + if conv.Bool(env.Getenv("CONSUL_HTTP_SSL")) { c.Scheme = "https" } else { c.Scheme = "http" @@ -83,7 +83,7 @@ func consulURL(u *url.URL) *url.URL { } func consulConfig(useTLS bool) *store.Config { - t := typeconv.MustAtoi(env.Getenv("CONSUL_TIMEOUT")) + t := conv.MustAtoi(env.Getenv("CONSUL_TIMEOUT")) config := &store.Config{ ConnectionTimeout: time.Duration(t) * time.Second, } @@ -107,7 +107,7 @@ func setupTLS(prefix string) *consulapi.TLSConfig { KeyFile: env.Getenv(prefix + "_CLIENT_KEY"), } if v := env.Getenv(prefix + "_HTTP_SSL_VERIFY"); v != "" { - verify := typeconv.MustParseBool(v) + verify := conv.Bool(v) tlsConfig.InsecureSkipVerify = !verify } return tlsConfig diff --git a/process_test.go b/process_test.go index 23a4e1a7..b63ad6eb 100644 --- a/process_test.go +++ b/process_test.go @@ -11,6 +11,7 @@ import ( "log" + "github.com/hairyhenderson/gomplate/data" "github.com/stretchr/testify/assert" ) @@ -40,13 +41,13 @@ func TestInputDir(t *testing.T) { } })() - src, err := ParseSource("config=test/files/input-dir/config.yml") + src, err := data.ParseSource("config=test/files/input-dir/config.yml") assert.Nil(t, err) - data := &Data{ - Sources: map[string]*Source{"config": src}, + d := &data.Data{ + Sources: map[string]*data.Source{"config": src}, } - gomplate := NewGomplate(data, "{{", "}}") + gomplate := NewGomplate(d, "{{", "}}") err = processInputDir(filepath.Join("test", "files", "input-dir", "in"), outDir, gomplate) assert.Nil(t, err) diff --git a/typeconv.go b/typeconv.go deleted file mode 100644 index 2ec433eb..00000000 --- a/typeconv.go +++ /dev/null @@ -1,335 +0,0 @@ -package main - -import ( - "bytes" - "encoding/csv" - "encoding/json" - "fmt" - "log" - "reflect" - "strconv" - "strings" - - yaml "gopkg.in/yaml.v2" - - // XXX: replace once https://github.com/BurntSushi/toml/pull/179 is merged - "github.com/hairyhenderson/toml" - "github.com/ugorji/go/codec" -) - -// TypeConv - type conversion function -type TypeConv struct { -} - -// Bool converts a string to a boolean value, using strconv.ParseBool under the covers. -// Possible true values are: 1, t, T, TRUE, true, True -// All other values are considered false. -func (t *TypeConv) Bool(in string) bool { - if b, err := strconv.ParseBool(in); err == nil { - return b - } - return false -} - -func unmarshalObj(obj map[string]interface{}, in string, f func([]byte, interface{}) error) map[string]interface{} { - err := f([]byte(in), &obj) - if err != nil { - log.Fatalf("Unable to unmarshal object %s: %v", in, err) - } - return obj -} - -func unmarshalArray(obj []interface{}, in string, f func([]byte, interface{}) error) []interface{} { - err := f([]byte(in), &obj) - if err != nil { - log.Fatalf("Unable to unmarshal array %s: %v", in, err) - } - return obj -} - -// JSON - Unmarshal a JSON Object -func (t *TypeConv) JSON(in string) map[string]interface{} { - obj := make(map[string]interface{}) - return unmarshalObj(obj, in, yaml.Unmarshal) -} - -// JSONArray - Unmarshal a JSON Array -func (t *TypeConv) JSONArray(in string) []interface{} { - obj := make([]interface{}, 1) - return unmarshalArray(obj, in, yaml.Unmarshal) -} - -// YAML - Unmarshal a YAML Object -func (t *TypeConv) YAML(in string) map[string]interface{} { - obj := make(map[string]interface{}) - return unmarshalObj(obj, in, yaml.Unmarshal) -} - -// YAMLArray - Unmarshal a YAML Array -func (t *TypeConv) YAMLArray(in string) []interface{} { - obj := make([]interface{}, 1) - return unmarshalArray(obj, in, yaml.Unmarshal) -} - -// TOML - Unmarshal a TOML Object -func (t *TypeConv) TOML(in string) interface{} { - obj := make(map[string]interface{}) - return unmarshalObj(obj, in, toml.Unmarshal) -} - -func parseCSV(args ...string) (records [][]string, hdr []string) { - delim := "," - var in string - if len(args) == 1 { - in = args[0] - } - if len(args) == 2 { - in = args[1] - if len(args[0]) == 1 { - delim = args[0] - } else if len(args[0]) == 0 { - hdr = []string{} - } else { - hdr = strings.Split(args[0], delim) - } - } - if len(args) == 3 { - delim = args[0] - hdr = strings.Split(args[1], delim) - in = args[2] - } - c := csv.NewReader(strings.NewReader(in)) - c.Comma = rune(delim[0]) - records, err := c.ReadAll() - if err != nil { - log.Fatal(err) - } - if hdr == nil { - hdr = records[0] - records = records[1:] - } else if len(hdr) == 0 { - hdr = make([]string, len(records[0])) - for i := range hdr { - hdr[i] = autoIndex(i) - } - } - return records, hdr -} - -// autoIndex - calculates a default string column name given a numeric value -func autoIndex(i int) string { - s := "" - for n := 0; n <= i/26; n++ { - s += string('A' + i%26) - } - return s -} - -// CSV - Unmarshal CSV -// parameters: -// delim - (optional) the (single-character!) field delimiter, defaults to "," -// in - the CSV-format string to parse -// returns: -// an array of rows, which are arrays of cells (strings) -func (t *TypeConv) CSV(args ...string) [][]string { - records, hdr := parseCSV(args...) - records = append(records, nil) - copy(records[1:], records) - records[0] = hdr - return records -} - -// CSVByRow - Unmarshal CSV in a row-oriented form -// parameters: -// delim - (optional) the (single-character!) field delimiter, defaults to "," -// hdr - (optional) comma-separated list of column names, -// set to "" to get auto-named columns (A-Z), omit -// to use the first line -// in - the CSV-format string to parse -// returns: -// an array of rows, indexed by the header name -func (t *TypeConv) CSVByRow(args ...string) (rows []map[string]string) { - records, hdr := parseCSV(args...) - for _, record := range records { - m := make(map[string]string) - for i, v := range record { - m[hdr[i]] = v - } - rows = append(rows, m) - } - return rows -} - -// CSVByColumn - Unmarshal CSV in a Columnar form -// parameters: -// delim - (optional) the (single-character!) field delimiter, defaults to "," -// hdr - (optional) comma-separated list of column names, -// set to "" to get auto-named columns (A-Z), omit -// to use the first line -// in - the CSV-format string to parse -// returns: -// a map of columns, indexed by the header name. values are arrays of strings -func (t *TypeConv) CSVByColumn(args ...string) (cols map[string][]string) { - records, hdr := parseCSV(args...) - cols = make(map[string][]string) - for _, record := range records { - for i, v := range record { - cols[hdr[i]] = append(cols[hdr[i]], v) - } - } - return cols -} - -// ToCSV - -func (t *TypeConv) ToCSV(args ...interface{}) string { - delim := "," - var in [][]string - if len(args) == 2 { - d, ok := args[0].(string) - if ok { - delim = d - } else { - log.Fatalf("Can't parse ToCSV delimiter (%v) - must be string (is a %T)", args[0], args[0]) - } - in, ok = args[1].([][]string) - if !ok { - log.Fatal("Can't parse ToCSV input - must be of type [][]string") - } - } - if len(args) == 1 { - var ok bool - in, ok = args[0].([][]string) - if !ok { - log.Fatal("Can't parse ToCSV input - must be of type [][]string") - } - } - b := &bytes.Buffer{} - c := csv.NewWriter(b) - c.Comma = rune(delim[0]) - // We output RFC4180 CSV, so force this to CRLF - c.UseCRLF = true - err := c.WriteAll(in) - if err != nil { - log.Fatal(err) - } - return string(b.Bytes()) -} - -func marshalObj(obj interface{}, f func(interface{}) ([]byte, error)) string { - b, err := f(obj) - if err != nil { - log.Fatalf("Unable to marshal object %s: %v", obj, err) - } - - return string(b) -} - -func toJSONBytes(in interface{}) []byte { - h := &codec.JsonHandle{} - h.Canonical = true - buf := new(bytes.Buffer) - err := codec.NewEncoder(buf, h).Encode(in) - if err != nil { - log.Fatalf("Unable to marshal %s: %v", in, err) - } - return buf.Bytes() -} - -// ToJSON - Stringify a struct as JSON -func (t *TypeConv) ToJSON(in interface{}) string { - return string(toJSONBytes(in)) -} - -// ToJSONPretty - Stringify a struct as JSON (indented) -func (t *TypeConv) toJSONPretty(indent string, in interface{}) string { - out := new(bytes.Buffer) - b := toJSONBytes(in) - err := json.Indent(out, b, "", indent) - if err != nil { - log.Fatalf("Unable to indent JSON %s: %v", b, err) - } - - return string(out.Bytes()) -} - -// ToYAML - Stringify a struct as YAML -func (t *TypeConv) ToYAML(in interface{}) string { - return marshalObj(in, yaml.Marshal) -} - -// ToTOML - Stringify a struct as TOML -func (t *TypeConv) ToTOML(in interface{}) string { - buf := new(bytes.Buffer) - err := toml.NewEncoder(buf).Encode(in) - if err != nil { - log.Fatalf("Unable to marshal %s: %v", in, err) - } - return string(buf.Bytes()) -} - -// Slice creates a slice from a bunch of arguments -func (t *TypeConv) Slice(args ...interface{}) []interface{} { - return args -} - -// Join concatenates the elements of a to create a single string. -// The separator string sep is placed between elements in the resulting string. -// -// This is functionally identical to strings.Join, except that each element is -// coerced to a string first -func (t *TypeConv) Join(in interface{}, sep string) string { - s, ok := in.([]string) - if ok { - return strings.Join(s, sep) - } - - var a []interface{} - a, ok = in.([]interface{}) - if ok { - b := make([]string, len(a)) - for i := range a { - b[i] = toString(a[i]) - } - return strings.Join(b, sep) - } - - log.Fatal("Input to Join must be an array") - return "" -} - -// Has determines whether or not a given object has a property with the given key -func (t *TypeConv) Has(in interface{}, key string) bool { - av := reflect.ValueOf(in) - kv := reflect.ValueOf(key) - - if av.Kind() == reflect.Map { - return av.MapIndex(kv).IsValid() - } - - return false -} - -func toString(in interface{}) string { - if s, ok := in.(string); ok { - return s - } - if s, ok := in.(fmt.Stringer); ok { - return s.String() - } - if i, ok := in.(int); ok { - return strconv.Itoa(i) - } - if u, ok := in.(uint64); ok { - return strconv.FormatUint(u, 10) - } - if f, ok := in.(float64); ok { - return strconv.FormatFloat(f, 'f', -1, 64) - } - if b, ok := in.(bool); ok { - return strconv.FormatBool(b) - } - if in == nil { - return "nil" - } - return fmt.Sprintf("%s", in) -} diff --git a/typeconv/typeconv.go b/typeconv/typeconv.go deleted file mode 100644 index 60e5cf9c..00000000 --- a/typeconv/typeconv.go +++ /dev/null @@ -1,33 +0,0 @@ -package typeconv - -import "strconv" - -// MustParseBool - wrapper for strconv.ParseBool that returns false in the case of error -func MustParseBool(s string) bool { - b, _ := strconv.ParseBool(s) - return b -} - -// MustParseInt - wrapper for strconv.ParseInt that returns 0 in the case of error -func MustParseInt(s string, base, bitSize int) int64 { - i, _ := strconv.ParseInt(s, base, bitSize) - return i -} - -// MustParseFloat - wrapper for strconv.ParseFloat that returns 0 in the case of error -func MustParseFloat(s string, bitSize int) float64 { - i, _ := strconv.ParseFloat(s, bitSize) - return i -} - -// MustParseUint - wrapper for strconv.ParseUint that returns 0 in the case of error -func MustParseUint(s string, base, bitSize int) uint64 { - i, _ := strconv.ParseUint(s, base, bitSize) - return i -} - -// MustAtoi - wrapper for strconv.Atoi that returns 0 in the case of error -func MustAtoi(s string) int { - i, _ := strconv.Atoi(s) - return i -} diff --git a/typeconv/typeconv_test.go b/typeconv/typeconv_test.go deleted file mode 100644 index e07eaa28..00000000 --- a/typeconv/typeconv_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package typeconv - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMustParseBool(t *testing.T) { - for _, b := range []string{"1", "t", "T", "true", "TRUE", "True"} { - assert.True(t, MustParseBool(b)) - } - for _, b := range []string{"0", "f", "F", "false", "FALSE", "False", "", "gibberish", "12345"} { - assert.False(t, MustParseBool(b)) - } -} - -func TestMustParseInt(t *testing.T) { - for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { - assert.Equal(t, 0, int(MustParseInt(i, 10, 64))) - } - assert.Equal(t, 1, int(MustParseInt("1", 10, 64))) - assert.Equal(t, -1, int(MustParseInt("-1", 10, 64))) -} - -func TestMustAtoi(t *testing.T) { - for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { - assert.Equal(t, 0, MustAtoi(i)) - } - assert.Equal(t, 1, MustAtoi("1")) - assert.Equal(t, -1, MustAtoi("-1")) -} - -func TestMustParseUint(t *testing.T) { - for _, i := range []string{"0", "-0", "-1", "foo", "", "*&^%"} { - assert.Equal(t, uint64(0), MustParseUint(i, 10, 64)) - } - assert.Equal(t, uint64(1), MustParseUint("1", 10, 64)) -} - -func TestMustParseFloat(t *testing.T) { - for _, i := range []string{"0", "-0", "foo", "", "*&^%"} { - assert.Equal(t, 0.0, MustParseFloat(i, 64)) - } - assert.Equal(t, 1.0, MustParseFloat("1", 64)) - assert.Equal(t, -1.0, MustParseFloat("-1", 64)) -} diff --git a/typeconv_test.go b/typeconv_test.go deleted file mode 100644 index d38b5352..00000000 --- a/typeconv_test.go +++ /dev/null @@ -1,382 +0,0 @@ -package main - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestBool(t *testing.T) { - ty := &TypeConv{} - assert.False(t, ty.Bool("")) - assert.False(t, ty.Bool("asdf")) - assert.False(t, ty.Bool("1234")) - assert.False(t, ty.Bool("False")) - assert.False(t, ty.Bool("0")) - assert.False(t, ty.Bool("false")) - assert.False(t, ty.Bool("F")) - assert.False(t, ty.Bool("f")) - assert.True(t, ty.Bool("true")) - assert.True(t, ty.Bool("True")) - assert.True(t, ty.Bool("t")) - assert.True(t, ty.Bool("T")) - assert.True(t, ty.Bool("1")) -} - -func TestUnmarshalObj(t *testing.T) { - ty := new(TypeConv) - expected := map[string]interface{}{ - "foo": map[interface{}]interface{}{"bar": "baz"}, - "one": 1.0, - "true": true, - } - - test := func(actual map[string]interface{}) { - assert.Equal(t, expected["foo"], actual["foo"]) - assert.Equal(t, expected["one"], actual["one"]) - assert.Equal(t, expected["true"], actual["true"]) - } - test(ty.JSON(`{"foo":{"bar":"baz"},"one":1.0,"true":true}`)) - test(ty.YAML(`foo: - bar: baz -one: 1.0 -true: true -`)) -} - -func TestUnmarshalArray(t *testing.T) { - ty := new(TypeConv) - - expected := []string{"foo", "bar"} - - test := func(actual []interface{}) { - assert.Equal(t, expected[0], actual[0]) - assert.Equal(t, expected[1], actual[1]) - } - test(ty.JSONArray(`["foo","bar"]`)) - test(ty.YAMLArray(` -- foo -- bar -`)) -} - -func TestToJSON(t *testing.T) { - ty := new(TypeConv) - expected := `{"down":{"the":{"rabbit":{"hole":true}}},"foo":"bar","one":1,"true":true}` - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - "down": map[interface{}]interface{}{ - "the": map[interface{}]interface{}{ - "rabbit": map[interface{}]interface{}{ - "hole": true, - }, - }, - }, - } - assert.Equal(t, expected, ty.ToJSON(in)) -} - -func TestToJSONPretty(t *testing.T) { - ty := new(TypeConv) - expected := `{ - "down": { - "the": { - "rabbit": { - "hole": true - } - } - }, - "foo": "bar", - "one": 1, - "true": true -}` - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - "down": map[string]interface{}{ - "the": map[string]interface{}{ - "rabbit": map[string]interface{}{ - "hole": true, - }, - }, - }, - } - assert.Equal(t, expected, ty.toJSONPretty(" ", in)) -} - -func TestToYAML(t *testing.T) { - ty := new(TypeConv) - expected := `d: 2006-01-02T15:04:05.999999999-07:00 -foo: bar -? |- - multi - line - key -: hello: world -one: 1 -"true": true -` - mst, _ := time.LoadLocation("MST") - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - `multi -line -key`: map[string]interface{}{ - "hello": "world", - }, - "d": time.Date(2006, time.January, 2, 15, 4, 5, 999999999, mst), - } - assert.Equal(t, expected, ty.ToYAML(in)) -} - -func TestSlice(t *testing.T) { - ty := new(TypeConv) - expected := []string{"foo", "bar"} - actual := ty.Slice("foo", "bar") - assert.Equal(t, expected[0], actual[0]) - assert.Equal(t, expected[1], actual[1]) -} - -func TestJoin(t *testing.T) { - ty := new(TypeConv) - - assert.Equal(t, "foo,bar", ty.Join([]interface{}{"foo", "bar"}, ",")) - assert.Equal(t, "foo,\nbar", ty.Join([]interface{}{"foo", "bar"}, ",\n")) - // Join handles all kinds of scalar types too... - assert.Equal(t, "42-18446744073709551615", ty.Join([]interface{}{42, uint64(18446744073709551615)}, "-")) - assert.Equal(t, "1,,true,3.14,foo,nil", ty.Join([]interface{}{1, "", true, 3.14, "foo", nil}, ",")) - // and best-effort with weird types - assert.Equal(t, "[foo],bar", ty.Join([]interface{}{[]string{"foo"}, "bar"}, ",")) -} - -func TestHas(t *testing.T) { - ty := new(TypeConv) - - in := map[string]interface{}{ - "foo": "bar", - "baz": map[string]interface{}{ - "qux": "quux", - }, - } - - assert.True(t, ty.Has(in, "foo")) - assert.False(t, ty.Has(in, "bar")) - assert.True(t, ty.Has(in["baz"], "qux")) -} - -func TestCSV(t *testing.T) { - ty := new(TypeConv) - in := "first,second,third\n1,2,3\n4,5,6" - expected := [][]string{ - {"first", "second", "third"}, - {"1", "2", "3"}, - {"4", "5", "6"}, - } - assert.Equal(t, expected, ty.CSV(in)) - - in = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" - assert.Equal(t, expected, ty.CSV(";", in)) -} - -func TestCSVByRow(t *testing.T) { - ty := new(TypeConv) - in := "first,second,third\n1,2,3\n4,5,6" - expected := []map[string]string{ - { - "first": "1", - "second": "2", - "third": "3", - }, - { - "first": "4", - "second": "5", - "third": "6", - }, - } - assert.Equal(t, expected, ty.CSVByRow(in)) - - in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByRow("first,second,third", in)) - - in = "1;2;3\n4;5;6" - assert.Equal(t, expected, ty.CSVByRow(";", "first;second;third", in)) - - in = "first;second;third\r\n1;2;3\r\n4;5;6" - assert.Equal(t, expected, ty.CSVByRow(";", in)) - - expected = []map[string]string{ - {"A": "1", "B": "2", "C": "3"}, - {"A": "4", "B": "5", "C": "6"}, - } - - in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByRow("", in)) - - expected = []map[string]string{ - {"A": "1", "B": "1", "C": "1", "D": "1", "E": "1", "F": "1", "G": "1", "H": "1", "I": "1", "J": "1", "K": "1", "L": "1", "M": "1", "N": "1", "O": "1", "P": "1", "Q": "1", "R": "1", "S": "1", "T": "1", "U": "1", "V": "1", "W": "1", "X": "1", "Y": "1", "Z": "1", "AA": "1", "BB": "1", "CC": "1", "DD": "1"}, - } - - in = "1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1" - assert.Equal(t, expected, ty.CSVByRow("", in)) -} - -func TestCSVByColumn(t *testing.T) { - ty := new(TypeConv) - in := "first,second,third\n1,2,3\n4,5,6" - expected := map[string][]string{ - "first": {"1", "4"}, - "second": {"2", "5"}, - "third": {"3", "6"}, - } - assert.Equal(t, expected, ty.CSVByColumn(in)) - - in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByColumn("first,second,third", in)) - - in = "1;2;3\n4;5;6" - assert.Equal(t, expected, ty.CSVByColumn(";", "first;second;third", in)) - - in = "first;second;third\r\n1;2;3\r\n4;5;6" - assert.Equal(t, expected, ty.CSVByColumn(";", in)) - - expected = map[string][]string{ - "A": {"1", "4"}, - "B": {"2", "5"}, - "C": {"3", "6"}, - } - - in = "1,2,3\n4,5,6" - assert.Equal(t, expected, ty.CSVByColumn("", in)) -} - -func TestAutoIndex(t *testing.T) { - assert.Equal(t, "A", autoIndex(0)) - assert.Equal(t, "B", autoIndex(1)) - assert.Equal(t, "Z", autoIndex(25)) - assert.Equal(t, "AA", autoIndex(26)) - assert.Equal(t, "ZZ", autoIndex(51)) - assert.Equal(t, "AAA", autoIndex(52)) - assert.Equal(t, "YYYYY", autoIndex(128)) -} - -func TestToCSV(t *testing.T) { - ty := new(TypeConv) - in := [][]string{ - {"first", "second", "third"}, - {"1", "2", "3"}, - {"4", "5", "6"}, - } - expected := "first,second,third\r\n1,2,3\r\n4,5,6\r\n" - - assert.Equal(t, expected, ty.ToCSV(in)) - - expected = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" - - assert.Equal(t, expected, ty.ToCSV(";", in)) -} - -func TestTOML(t *testing.T) { - ty := new(TypeConv) - in := `# This is a TOML document. Boom. - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -organization = "GitHub" -bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." -dob = 1979-05-27T07:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -connection_max = 5000 -enabled = true - -[servers] - - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] -` - expected := map[string]interface{}{ - "title": "TOML Example", - "owner": map[string]interface{}{ - "name": "Tom Preston-Werner", - "organization": "GitHub", - "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", - "dob": time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), - }, - "database": map[string]interface{}{ - "server": "192.168.1.1", - "ports": []interface{}{int64(8001), int64(8001), int64(8002)}, - "connection_max": int64(5000), - "enabled": true, - }, - "servers": map[string]interface{}{ - "alpha": map[string]interface{}{ - "ip": "10.0.0.1", - "dc": "eqdc10", - }, - "beta": map[string]interface{}{ - "ip": "10.0.0.2", - "dc": "eqdc10", - }, - }, - "clients": map[string]interface{}{ - "data": []interface{}{ - []interface{}{"gamma", "delta"}, - []interface{}{int64(1), int64(2)}, - }, - "hosts": []interface{}{"alpha", "omega"}, - }, - } - - assert.Equal(t, expected, ty.TOML(in)) -} - -func TestToTOML(t *testing.T) { - ty := new(TypeConv) - expected := `foo = "bar" -one = 1 -true = true - -[down] - [down.the] - [down.the.rabbit] - hole = true -` - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - "down": map[interface{}]interface{}{ - "the": map[interface{}]interface{}{ - "rabbit": map[interface{}]interface{}{ - "hole": true, - }, - }, - }, - } - assert.Equal(t, expected, ty.ToTOML(in)) -} diff --git a/vault/auth.go b/vault/auth.go index 960e61ef..50d23df3 100644 --- a/vault/auth.go +++ b/vault/auth.go @@ -10,8 +10,8 @@ import ( "github.com/blang/vfs" "github.com/hairyhenderson/gomplate/aws" + "github.com/hairyhenderson/gomplate/conv" "github.com/hairyhenderson/gomplate/env" - "github.com/hairyhenderson/gomplate/typeconv" ) // GetToken - @@ -167,7 +167,7 @@ func (v *Vault) EC2Login() string { } opts := aws.ClientOptions{ - Timeout: time.Duration(typeconv.MustAtoi(os.Getenv("AWS_TIMEOUT"))) * time.Millisecond, + Timeout: time.Duration(conv.MustAtoi(os.Getenv("AWS_TIMEOUT"))) * time.Millisecond, } meta := aws.NewEc2Meta(opts) -- cgit v1.2.3