diff options
Diffstat (limited to 'data')
28 files changed, 408 insertions, 4201 deletions
diff --git a/data/data.go b/data/data.go deleted file mode 100644 index d2fc32fb..00000000 --- a/data/data.go +++ /dev/null @@ -1,507 +0,0 @@ -// Package data contains functions that parse and produce data structures in -// different formats. -// -// Supported formats are: JSON, YAML, TOML, and CSV. -package data - -import ( - "bytes" - "encoding/csv" - "encoding/json" - "fmt" - "io" - "strings" - - "cuelang.org/go/cue" - "cuelang.org/go/cue/cuecontext" - "cuelang.org/go/cue/format" - "github.com/Shopify/ejson" - ejsonJson "github.com/Shopify/ejson/json" - "github.com/hairyhenderson/gomplate/v4/conv" - "github.com/hairyhenderson/gomplate/v4/env" - "github.com/joho/godotenv" - - // XXX: replace once https://github.com/BurntSushi/toml/pull/179 is merged - "github.com/hairyhenderson/toml" - "github.com/ugorji/go/codec" - - "github.com/hairyhenderson/yaml" -) - -func unmarshalObj(obj map[string]interface{}, in string, f func([]byte, interface{}) error) (map[string]interface{}, error) { - err := f([]byte(in), &obj) - if err != nil { - return nil, fmt.Errorf("unable to unmarshal object %s: %w", in, err) - } - return obj, nil -} - -func unmarshalArray(obj []interface{}, in string, f func([]byte, interface{}) error) ([]interface{}, error) { - err := f([]byte(in), &obj) - if err != nil { - return nil, fmt.Errorf("unable to unmarshal array %s: %w", in, err) - } - return obj, nil -} - -// JSON - Unmarshal a JSON Object. Can be ejson-encrypted. -func JSON(in string) (map[string]interface{}, error) { - obj := make(map[string]interface{}) - out, err := unmarshalObj(obj, in, yaml.Unmarshal) - if err != nil { - return out, err - } - - _, ok := out[ejsonJson.PublicKeyField] - if ok { - out, err = decryptEJSON(in) - } - return out, err -} - -// decryptEJSON - decrypts an ejson input, and unmarshals it, stripping the _public_key field. -func decryptEJSON(in string) (map[string]interface{}, error) { - keyDir := env.Getenv("EJSON_KEYDIR", "/opt/ejson/keys") - key := env.Getenv("EJSON_KEY") - - rIn := bytes.NewBufferString(in) - rOut := &bytes.Buffer{} - err := ejson.Decrypt(rIn, rOut, keyDir, key) - if err != nil { - return nil, err - } - obj := make(map[string]interface{}) - out, err := unmarshalObj(obj, rOut.String(), yaml.Unmarshal) - if err != nil { - return nil, err - } - delete(out, ejsonJson.PublicKeyField) - return out, nil -} - -// JSONArray - Unmarshal a JSON Array -func JSONArray(in string) ([]interface{}, error) { - obj := make([]interface{}, 1) - return unmarshalArray(obj, in, yaml.Unmarshal) -} - -// YAML - Unmarshal a YAML Object -func YAML(in string) (map[string]interface{}, error) { - obj := make(map[string]interface{}) - s := strings.NewReader(in) - d := yaml.NewDecoder(s) - for { - err := d.Decode(&obj) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if obj != nil { - break - } - } - - err := stringifyYAMLMapMapKeys(obj) - return obj, err -} - -// YAMLArray - Unmarshal a YAML Array -func YAMLArray(in string) ([]interface{}, error) { - obj := make([]interface{}, 1) - s := strings.NewReader(in) - d := yaml.NewDecoder(s) - for { - err := d.Decode(&obj) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if obj != nil { - break - } - } - err := stringifyYAMLArrayMapKeys(obj) - return obj, err -} - -// stringifyYAMLArrayMapKeys recurses into the input array and changes all -// non-string map keys to string map keys. Modifies the input array. -func stringifyYAMLArrayMapKeys(in []interface{}) error { - if _, changed := stringifyMapKeys(in); changed { - return fmt.Errorf("stringifyYAMLArrayMapKeys: output type did not match input type, this should be impossible") - } - return nil -} - -// stringifyYAMLMapMapKeys recurses into the input map and changes all -// non-string map keys to string map keys. Modifies the input map. -func stringifyYAMLMapMapKeys(in map[string]interface{}) error { - if _, changed := stringifyMapKeys(in); changed { - return fmt.Errorf("stringifyYAMLMapMapKeys: output type did not match input type, this should be impossible") - } - return nil -} - -// stringifyMapKeys recurses into in and changes all instances of -// map[interface{}]interface{} to map[string]interface{}. This is useful to -// work around the impedance mismatch between JSON and YAML unmarshaling that's -// described here: https://github.com/go-yaml/yaml/issues/139 -// -// Taken and modified from https://github.com/gohugoio/hugo/blob/cdfd1c99baa22d69e865294dfcd783811f96c880/parser/metadecoders/decoder.go#L257, Apache License 2.0 -// Originally inspired by https://github.com/stripe/stripe-mock/blob/24a2bb46a49b2a416cfea4150ab95781f69ee145/mapstr.go#L13, MIT License -func stringifyMapKeys(in interface{}) (interface{}, bool) { - switch in := in.(type) { - case []interface{}: - for i, v := range in { - if vv, replaced := stringifyMapKeys(v); replaced { - in[i] = vv - } - } - case map[string]interface{}: - for k, v := range in { - if vv, changed := stringifyMapKeys(v); changed { - in[k] = vv - } - } - case map[interface{}]interface{}: - res := make(map[string]interface{}) - - for k, v := range in { - ks := conv.ToString(k) - if vv, replaced := stringifyMapKeys(v); replaced { - res[ks] = vv - } else { - res[ks] = v - } - } - return res, true - } - - return nil, false -} - -// TOML - Unmarshal a TOML Object -func TOML(in string) (interface{}, error) { - obj := make(map[string]interface{}) - return unmarshalObj(obj, in, toml.Unmarshal) -} - -// dotEnv - Unmarshal a dotenv file -func dotEnv(in string) (interface{}, error) { - env, err := godotenv.Unmarshal(in) - if err != nil { - return nil, err - } - out := make(map[string]interface{}) - for k, v := range env { - out[k] = v - } - return out, nil -} - -func parseCSV(args ...string) ([][]string, []string, error) { - in, delim, hdr := csvParseArgs(args...) - c := csv.NewReader(strings.NewReader(in)) - c.Comma = rune(delim[0]) - records, err := c.ReadAll() - if err != nil { - return nil, nil, err - } - if len(records) > 0 { - if hdr == nil { - hdr = records[0] - records = records[1:] - } else if len(hdr) == 0 { - hdr = make([]string, len(records[0])) - for i := range hdr { - hdr[i] = autoIndex(i) - } - } - } - return records, hdr, nil -} - -func csvParseArgs(args ...string) (in, delim string, hdr []string) { - delim = "," - switch len(args) { - case 1: - in = args[0] - case 2: - in = args[1] - switch len(args[0]) { - case 1: - delim = args[0] - case 0: - hdr = []string{} - default: - hdr = strings.Split(args[0], delim) - } - case 3: - delim = args[0] - hdr = strings.Split(args[1], delim) - in = args[2] - } - return in, delim, hdr -} - -// autoIndex - calculates a default string column name given a numeric value -func autoIndex(i int) string { - s := &strings.Builder{} - for n := 0; n <= i/26; n++ { - s.WriteRune('A' + rune(i%26)) - } - return s.String() -} - -// CSV - Unmarshal CSV -// parameters: -// -// delim - (optional) the (single-character!) field delimiter, defaults to "," -// in - the CSV-format string to parse -// -// returns: -// -// an array of rows, which are arrays of cells (strings) -func CSV(args ...string) ([][]string, error) { - records, hdr, err := parseCSV(args...) - if err != nil { - return nil, err - } - records = append(records, nil) - copy(records[1:], records) - records[0] = hdr - return records, nil -} - -// CSVByRow - Unmarshal CSV in a row-oriented form -// parameters: -// -// delim - (optional) the (single-character!) field delimiter, defaults to "," -// hdr - (optional) list of column names separated by `delim`, -// set to "" to get auto-named columns (A-Z), omit -// to use the first line -// in - the CSV-format string to parse -// -// returns: -// -// an array of rows, indexed by the header name -func CSVByRow(args ...string) (rows []map[string]string, err error) { - records, hdr, err := parseCSV(args...) - if err != nil { - return nil, err - } - for _, record := range records { - m := make(map[string]string) - for i, v := range record { - m[hdr[i]] = v - } - rows = append(rows, m) - } - return rows, nil -} - -// CSVByColumn - Unmarshal CSV in a Columnar form -// parameters: -// -// delim - (optional) the (single-character!) field delimiter, defaults to "," -// hdr - (optional) list of column names separated by `delim`, -// set to "" to get auto-named columns (A-Z), omit -// to use the first line -// in - the CSV-format string to parse -// -// returns: -// -// a map of columns, indexed by the header name. values are arrays of strings -func CSVByColumn(args ...string) (cols map[string][]string, err error) { - records, hdr, err := parseCSV(args...) - if err != nil { - return nil, err - } - cols = make(map[string][]string) - for _, record := range records { - for i, v := range record { - cols[hdr[i]] = append(cols[hdr[i]], v) - } - } - return cols, nil -} - -// ToCSV - -func ToCSV(args ...interface{}) (string, error) { - delim := "," - var in [][]string - if len(args) == 2 { - var ok bool - delim, ok = args[0].(string) - if !ok { - return "", fmt.Errorf("can't parse ToCSV delimiter (%v) - must be string (is a %T)", args[0], args[0]) - } - args = args[1:] - } - if len(args) == 1 { - switch a := args[0].(type) { - case [][]string: - in = a - case [][]interface{}: - in = make([][]string, len(a)) - for i, v := range a { - in[i] = conv.ToStrings(v...) - } - case []interface{}: - in = make([][]string, len(a)) - for i, v := range a { - ar, ok := v.([]interface{}) - if !ok { - return "", fmt.Errorf("can't parse ToCSV input - must be a two-dimensional array (like [][]string or [][]interface{}) (was %T)", args[0]) - } - in[i] = conv.ToStrings(ar...) - } - default: - return "", fmt.Errorf("can't parse ToCSV input - must be a two-dimensional array (like [][]string or [][]interface{}) (was %T)", args[0]) - } - } - b := &bytes.Buffer{} - c := csv.NewWriter(b) - c.Comma = rune(delim[0]) - // We output RFC4180 CSV, so force this to CRLF - c.UseCRLF = true - err := c.WriteAll(in) - if err != nil { - return "", err - } - return b.String(), nil -} - -func marshalObj(obj interface{}, f func(interface{}) ([]byte, error)) (string, error) { - b, err := f(obj) - if err != nil { - return "", fmt.Errorf("unable to marshal object %s: %w", obj, err) - } - - return string(b), nil -} - -func toJSONBytes(in interface{}) ([]byte, error) { - h := &codec.JsonHandle{} - h.Canonical = true - buf := new(bytes.Buffer) - err := codec.NewEncoder(buf, h).Encode(in) - if err != nil { - return nil, fmt.Errorf("unable to marshal %s: %w", in, err) - } - return buf.Bytes(), nil -} - -// ToJSON - Stringify a struct as JSON -func ToJSON(in interface{}) (string, error) { - s, err := toJSONBytes(in) - if err != nil { - return "", err - } - return string(s), nil -} - -// ToJSONPretty - Stringify a struct as JSON (indented) -func ToJSONPretty(indent string, in interface{}) (string, error) { - out := new(bytes.Buffer) - b, err := toJSONBytes(in) - if err != nil { - return "", err - } - err = json.Indent(out, b, "", indent) - if err != nil { - return "", fmt.Errorf("unable to indent JSON %s: %w", b, err) - } - - return out.String(), nil -} - -// ToYAML - Stringify a struct as YAML -func ToYAML(in interface{}) (string, error) { - // I'd use yaml.Marshal, but between v2 and v3 the indent has changed from - // 2 to 4. This explicitly sets it back to 2. - marshal := func(in interface{}) (out []byte, err error) { - buf := &bytes.Buffer{} - e := yaml.NewEncoder(buf) - e.SetIndent(2) - defer e.Close() - err = e.Encode(in) - return buf.Bytes(), err - } - - return marshalObj(in, marshal) -} - -// ToTOML - Stringify a struct as TOML -func ToTOML(in interface{}) (string, error) { - buf := new(bytes.Buffer) - err := toml.NewEncoder(buf).Encode(in) - if err != nil { - return "", fmt.Errorf("unable to marshal %s: %w", in, err) - } - return buf.String(), nil -} - -// CUE - Unmarshal a CUE expression into the appropriate type -func CUE(in string) (interface{}, error) { - cuectx := cuecontext.New() - val := cuectx.CompileString(in) - - if val.Err() != nil { - return nil, fmt.Errorf("unable to process CUE: %w", val.Err()) - } - - switch val.Kind() { - case cue.StructKind: - out := map[string]interface{}{} - err := val.Decode(&out) - return out, err - case cue.ListKind: - out := []interface{}{} - err := val.Decode(&out) - return out, err - case cue.BytesKind: - out := []byte{} - err := val.Decode(&out) - return out, err - case cue.StringKind: - out := "" - err := val.Decode(&out) - return out, err - case cue.IntKind: - out := 0 - err := val.Decode(&out) - return out, err - case cue.NumberKind, cue.FloatKind: - out := 0.0 - err := val.Decode(&out) - return out, err - case cue.BoolKind: - out := false - err := val.Decode(&out) - return out, err - case cue.NullKind: - return nil, nil - default: - return nil, fmt.Errorf("unsupported CUE type %q", val.Kind()) - } -} - -func ToCUE(in interface{}) (string, error) { - cuectx := cuecontext.New() - v := cuectx.Encode(in) - if v.Err() != nil { - return "", v.Err() - } - - bs, err := format.Node(v.Syntax()) - if err != nil { - return "", err - } - - return string(bs), nil -} diff --git a/data/data_test.go b/data/data_test.go deleted file mode 100644 index e4f0f555..00000000 --- a/data/data_test.go +++ /dev/null @@ -1,777 +0,0 @@ -package data - -import ( - "fmt" - "os" - "testing" - "time" - - "github.com/ugorji/go/codec" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "gotest.tools/v3/fs" -) - -func TestUnmarshalObj(t *testing.T) { - expected := map[string]interface{}{ - "foo": map[string]interface{}{"bar": "baz"}, - "one": 1.0, - "true": true, - "escaped": "\"/\\\b\f\n\r\t∞", - } - - test := func(actual map[string]interface{}, err error) { - t.Helper() - require.NoError(t, err) - assert.Equal(t, expected["foo"], actual["foo"], "foo") - assert.Equal(t, expected["one"], actual["one"], "one") - assert.Equal(t, expected["true"], actual["true"], "true") - assert.Equal(t, expected["escaped"], actual["escaped"], "escaped") - } - test(JSON(`{"foo":{"bar":"baz"},"one":1.0,"true":true,"escaped":"\"\/\\\b\f\n\r\t\u221e"}`)) - test(YAML(`foo: - bar: baz -one: 1.0 -'true': true -escaped: "\"\/\\\b\f\n\r\t\u221e" -`)) - test(YAML(`anchor: &anchor - bar: baz -foo: - <<: *anchor -one: 1.0 -'true': true -escaped: "\"\/\\\b\f\n\r\t\u221e" -`)) - test(YAML(`# this comment marks an empty (nil!) document ---- -# this one too, for good measure ---- -foo: - bar: baz -one: 1.0 -'true': true -escaped: "\"\/\\\b\f\n\r\t\u221e" -`)) - - obj := make(map[string]interface{}) - _, err := unmarshalObj(obj, "SOMETHING", func(in []byte, out interface{}) error { - return fmt.Errorf("fail") - }) - assert.EqualError(t, err, "unable to unmarshal object SOMETHING: fail") -} - -func TestUnmarshalArray(t *testing.T) { - expected := []interface{}{ - "foo", "bar", - map[string]interface{}{ - "baz": map[string]interface{}{"qux": true}, - "quux": map[string]interface{}{"42": 18}, - "corge": map[string]interface{}{"false": "blah"}, - }, - } - - test := func(actual []interface{}, err error) { - require.NoError(t, err) - assert.EqualValues(t, expected, actual) - } - test(JSONArray(`["foo","bar",{"baz":{"qux": true},"quux":{"42":18},"corge":{"false":"blah"}}]`)) - test(YAMLArray(` -- foo -- bar -- baz: - qux: true - quux: - "42": 18 - corge: - "false": blah -`)) - test(YAMLArray(`--- -# blah blah blah ignore this! ---- -- foo -- bar -- baz: - qux: true - quux: - "42": 18 - corge: - "false": blah ---- -this shouldn't be reached -`)) - - actual, err := YAMLArray(`--- -- foo: &foo - bar: baz -- qux: - <<: *foo - quux: corge -- baz: - qux: true - 42: 18 - false: blah -`) - require.NoError(t, err) - assert.EqualValues(t, - []interface{}{ - map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": "baz", - }, - }, - map[string]interface{}{ - "qux": map[string]interface{}{ - "bar": "baz", - "quux": "corge", - }, - }, - map[string]interface{}{ - "baz": map[string]interface{}{ - "qux": true, - "42": 18, - "false": "blah", - }, - }, - }, - actual) - - obj := make([]interface{}, 1) - _, err = unmarshalArray(obj, "SOMETHING", func(in []byte, out interface{}) error { - return fmt.Errorf("fail") - }) - assert.EqualError(t, err, "unable to unmarshal array SOMETHING: fail") -} - -func TestMarshalObj(t *testing.T) { - expected := "foo" - actual, err := marshalObj(nil, func(in interface{}) ([]byte, error) { - return []byte("foo"), nil - }) - require.NoError(t, err) - assert.Equal(t, expected, actual) - _, err = marshalObj(nil, func(in interface{}) ([]byte, error) { - return nil, fmt.Errorf("fail") - }) - assert.Error(t, err) -} - -func TestToJSONBytes(t *testing.T) { - expected := []byte("null") - actual, err := toJSONBytes(nil) - require.NoError(t, err) - assert.Equal(t, expected, actual) - - _, err = toJSONBytes(&badObject{}) - assert.Error(t, err) -} - -type badObject struct{} - -func (b *badObject) CodecEncodeSelf(_ *codec.Encoder) { - panic("boom") -} - -func (b *badObject) CodecDecodeSelf(_ *codec.Decoder) { -} - -func TestToJSON(t *testing.T) { - expected := `{"down":{"the":{"rabbit":{"hole":true}}},"foo":"bar","one":1,"true":true}` - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - "down": map[interface{}]interface{}{ - "the": map[interface{}]interface{}{ - "rabbit": map[interface{}]interface{}{ - "hole": true, - }, - }, - }, - } - out, err := ToJSON(in) - require.NoError(t, err) - assert.Equal(t, expected, out) - - _, err = ToJSON(&badObject{}) - assert.Error(t, err) -} - -func TestToJSONPretty(t *testing.T) { - expected := `{ - "down": { - "the": { - "rabbit": { - "hole": true - } - } - }, - "foo": "bar", - "one": 1, - "true": true -}` - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - "down": map[string]interface{}{ - "the": map[string]interface{}{ - "rabbit": map[string]interface{}{ - "hole": true, - }, - }, - }, - } - out, err := ToJSONPretty(" ", in) - require.NoError(t, err) - assert.Equal(t, expected, out) - - _, err = ToJSONPretty(" ", &badObject{}) - assert.Error(t, err) -} - -func TestToYAML(t *testing.T) { - expected := `d: 2006-01-02T15:04:05.999999999-07:00 -foo: bar -? |- - multi - line - key -: hello: world -one: 1 -"true": true -` - mst, _ := time.LoadLocation("MST") - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - `multi -line -key`: map[string]interface{}{ - "hello": "world", - }, - "d": time.Date(2006, time.January, 2, 15, 4, 5, 999999999, mst), - } - out, err := ToYAML(in) - require.NoError(t, err) - assert.Equal(t, expected, out) -} - -func TestCSV(t *testing.T) { - expected := [][]string{ - {"first", "second", "third"}, - {"1", "2", "3"}, - {"4", "5", "6"}, - } - testdata := []struct { - args []string - out [][]string - }{ - {[]string{"first,second,third\n1,2,3\n4,5,6"}, expected}, - {[]string{";", "first;second;third\r\n1;2;3\r\n4;5;6\r\n"}, expected}, - - {[]string{""}, [][]string{nil}}, - {[]string{"\n"}, [][]string{nil}}, - {[]string{"foo"}, [][]string{{"foo"}}}, - } - for _, d := range testdata { - out, err := CSV(d.args...) - require.NoError(t, err) - assert.Equal(t, d.out, out) - } -} - -func TestCSVByRow(t *testing.T) { - in := "first,second,third\n1,2,3\n4,5,6" - expected := []map[string]string{ - { - "first": "1", - "second": "2", - "third": "3", - }, - { - "first": "4", - "second": "5", - "third": "6", - }, - } - testdata := []struct { - args []string - out []map[string]string - }{ - {[]string{in}, expected}, - {[]string{"first,second,third", "1,2,3\n4,5,6"}, expected}, - {[]string{";", "first;second;third", "1;2;3\n4;5;6"}, expected}, - {[]string{";", "first;second;third\r\n1;2;3\r\n4;5;6"}, expected}, - {[]string{"", "1,2,3\n4,5,6"}, []map[string]string{ - {"A": "1", "B": "2", "C": "3"}, - {"A": "4", "B": "5", "C": "6"}, - }}, - {[]string{"", "1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1"}, []map[string]string{ - {"A": "1", "B": "1", "C": "1", "D": "1", "E": "1", "F": "1", "G": "1", "H": "1", "I": "1", "J": "1", "K": "1", "L": "1", "M": "1", "N": "1", "O": "1", "P": "1", "Q": "1", "R": "1", "S": "1", "T": "1", "U": "1", "V": "1", "W": "1", "X": "1", "Y": "1", "Z": "1", "AA": "1", "BB": "1", "CC": "1", "DD": "1"}, - }}, - } - for _, d := range testdata { - out, err := CSVByRow(d.args...) - require.NoError(t, err) - assert.Equal(t, d.out, out) - } -} - -func TestCSVByColumn(t *testing.T) { - expected := map[string][]string{ - "first": {"1", "4"}, - "second": {"2", "5"}, - "third": {"3", "6"}, - } - - testdata := []struct { - out map[string][]string - args []string - }{ - {expected, []string{"first,second,third\n1,2,3\n4,5,6"}}, - {expected, []string{"first,second,third", "1,2,3\n4,5,6"}}, - {expected, []string{";", "first;second;third", "1;2;3\n4;5;6"}}, - {expected, []string{";", "first;second;third\r\n1;2;3\r\n4;5;6"}}, - {map[string][]string{ - "A": {"1", "4"}, - "B": {"2", "5"}, - "C": {"3", "6"}, - }, []string{"", "1,2,3\n4,5,6"}}, - } - for _, d := range testdata { - out, err := CSVByColumn(d.args...) - require.NoError(t, err) - assert.Equal(t, d.out, out) - } -} - -func TestAutoIndex(t *testing.T) { - assert.Equal(t, "A", autoIndex(0)) - assert.Equal(t, "B", autoIndex(1)) - assert.Equal(t, "Z", autoIndex(25)) - assert.Equal(t, "AA", autoIndex(26)) - assert.Equal(t, "ZZ", autoIndex(51)) - assert.Equal(t, "AAA", autoIndex(52)) - assert.Equal(t, "YYYYY", autoIndex(128)) -} - -func TestToCSV(t *testing.T) { - in := [][]string{ - {"first", "second", "third"}, - {"1", "2", "3"}, - {"4", "5", "6"}, - } - expected := "first,second,third\r\n1,2,3\r\n4,5,6\r\n" - - out, err := ToCSV(in) - require.NoError(t, err) - assert.Equal(t, expected, out) - - expected = "first;second;third\r\n1;2;3\r\n4;5;6\r\n" - - out, err = ToCSV(";", in) - require.NoError(t, err) - assert.Equal(t, expected, out) - - _, err = ToCSV(42, [][]int{{1, 2}}) - assert.Error(t, err) - - _, err = ToCSV([][]int{{1, 2}}) - assert.Error(t, err) - - expected = "first,second,third\r\n1,2,3\r\n4,5,6\r\n" - out, err = ToCSV([][]interface{}{ - {"first", "second", "third"}, - {"1", "2", "3"}, - {"4", "5", "6"}, - }) - require.NoError(t, err) - assert.Equal(t, expected, out) - - expected = "first|second|third\r\n1|2|3\r\n4|5|6\r\n" - out, err = ToCSV("|", []interface{}{ - []interface{}{"first", "second", "third"}, - []interface{}{1, "2", 3}, - []interface{}{"4", 5, "6"}, - }) - require.NoError(t, err) - assert.Equal(t, expected, out) -} - -func TestTOML(t *testing.T) { - in := `# This is a TOML document. Boom. - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -organization = "GitHub" -bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." -dob = 1979-05-27T07:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -connection_max = 5000 -enabled = true - -[servers] - - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] -` - expected := map[string]interface{}{ - "title": "TOML Example", - "owner": map[string]interface{}{ - "name": "Tom Preston-Werner", - "organization": "GitHub", - "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", - "dob": time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC), - }, - "database": map[string]interface{}{ - "server": "192.168.1.1", - "ports": []interface{}{int64(8001), int64(8001), int64(8002)}, - "connection_max": int64(5000), - "enabled": true, - }, - "servers": map[string]interface{}{ - "alpha": map[string]interface{}{ - "ip": "10.0.0.1", - "dc": "eqdc10", - }, - "beta": map[string]interface{}{ - "ip": "10.0.0.2", - "dc": "eqdc10", - }, - }, - "clients": map[string]interface{}{ - "data": []interface{}{ - []interface{}{"gamma", "delta"}, - []interface{}{int64(1), int64(2)}, - }, - "hosts": []interface{}{"alpha", "omega"}, - }, - } - - out, err := TOML(in) - require.NoError(t, err) - assert.Equal(t, expected, out) -} - -func TestToTOML(t *testing.T) { - expected := `foo = "bar" -one = 1 -true = true - -[down] - [down.the] - [down.the.rabbit] - hole = true -` - in := map[string]interface{}{ - "foo": "bar", - "one": 1, - "true": true, - "down": map[interface{}]interface{}{ - "the": map[interface{}]interface{}{ - "rabbit": map[interface{}]interface{}{ - "hole": true, - }, - }, - }, - } - out, err := ToTOML(in) - require.NoError(t, err) - assert.Equal(t, expected, out) -} - -func TestDecryptEJSON(t *testing.T) { - privateKey := "e282d979654f88267f7e6c2d8268f1f4314b8673579205ed0029b76de9c8223f" - publicKey := "6e05ec625bcdca34864181cc43e6fcc20a57732a453bc2f4a2e117ffdf1a6762" - expected := map[string]interface{}{ - "password": "supersecret", - "_unencrypted": "notsosecret", - } - in := `{ - "_public_key": "` + publicKey + `", - "password": "EJ[1:yJ7n4UorqxkJZMoKevIA1dJeDvaQhkbgENIVZW18jig=:0591iW+paVSh4APOytKBVW/ZcxHO/5wO:TssnpVtkiXmpDIxPlXSiYdgnWyd44stGcwG1]", - "_unencrypted": "notsosecret" - }` - - t.Setenv("EJSON_KEY", privateKey) - actual, err := decryptEJSON(in) - require.NoError(t, err) - assert.EqualValues(t, expected, actual) - - actual, err = JSON(in) - require.NoError(t, err) - assert.EqualValues(t, expected, actual) - - tmpDir := fs.NewDir(t, "gomplate-ejsontest", - fs.WithFile(publicKey, privateKey), - ) - t.Cleanup(tmpDir.Remove) - - os.Unsetenv("EJSON_KEY") - t.Setenv("EJSON_KEY_FILE", tmpDir.Join(publicKey)) - actual, err = decryptEJSON(in) - require.NoError(t, err) - assert.EqualValues(t, expected, actual) - - os.Unsetenv("EJSON_KEY") - os.Unsetenv("EJSON_KEY_FILE") - t.Setenv("EJSON_KEYDIR", tmpDir.Path()) - actual, err = decryptEJSON(in) - require.NoError(t, err) - assert.EqualValues(t, expected, actual) -} - -func TestDotEnv(t *testing.T) { - in := `FOO=a regular unquoted value -export BAR=another value, exports are ignored - -# comments are totally ignored, as are blank lines -FOO.BAR = "values can be double-quoted, and shell\nescapes are supported" - -BAZ = "variable expansion: ${FOO}" -QUX='single quotes ignore $variables' -` - expected := map[string]interface{}{ - "FOO": "a regular unquoted value", - "BAR": "another value, exports are ignored", - "FOO.BAR": "values can be double-quoted, and shell\nescapes are supported", - "BAZ": "variable expansion: a regular unquoted value", - "QUX": "single quotes ignore $variables", - } - out, err := dotEnv(in) - require.NoError(t, err) - assert.EqualValues(t, expected, out) -} - -func TestStringifyYAMLArrayMapKeys(t *testing.T) { - cases := []struct { - input []interface{} - want []interface{} - replaced bool - }{ - { - []interface{}{map[interface{}]interface{}{"a": 1, "b": 2}}, - []interface{}{map[string]interface{}{"a": 1, "b": 2}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}}}, - []interface{}{map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{true: 1, "b": false}}, - []interface{}{map[string]interface{}{"true": 1, "b": false}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}, - []interface{}{map[string]interface{}{"1": "a", "2": "b"}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}}}, - []interface{}{map[string]interface{}{"a": map[string]interface{}{"b": 1}}}, - false, - }, - { - []interface{}{map[string]interface{}{"a": map[string]interface{}{"b": 1}}}, - []interface{}{map[string]interface{}{"a": map[string]interface{}{"b": 1}}}, - false, - }, - { - []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}, - []interface{}{map[string]interface{}{"1": "a", "2": "b"}}, - false, - }, - } - - for _, c := range cases { - err := stringifyYAMLArrayMapKeys(c.input) - require.NoError(t, err) - assert.EqualValues(t, c.want, c.input) - } -} - -func TestStringifyYAMLMapMapKeys(t *testing.T) { - cases := []struct { - input map[string]interface{} - want map[string]interface{} - }{ - { - map[string]interface{}{"root": map[interface{}]interface{}{"a": 1, "b": 2}}, - map[string]interface{}{"root": map[string]interface{}{"a": 1, "b": 2}}, - }, - { - map[string]interface{}{"root": map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}}}, - map[string]interface{}{"root": map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}}}, - }, - { - map[string]interface{}{"root": map[interface{}]interface{}{true: 1, "b": false}}, - map[string]interface{}{"root": map[string]interface{}{"true": 1, "b": false}}, - }, - { - map[string]interface{}{"root": map[interface{}]interface{}{1: "a", 2: "b"}}, - map[string]interface{}{"root": map[string]interface{}{"1": "a", "2": "b"}}, - }, - { - map[string]interface{}{"root": map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}}}, - map[string]interface{}{"root": map[string]interface{}{"a": map[string]interface{}{"b": 1}}}, - }, - { - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - map[string]interface{}{"a": map[string]interface{}{"b": 1}}, - }, - { - map[string]interface{}{"root": []interface{}{map[interface{}]interface{}{1: "a", 2: "b"}}}, - map[string]interface{}{"root": []interface{}{map[string]interface{}{"1": "a", "2": "b"}}}, - }, - } - - for _, c := range cases { - err := stringifyYAMLMapMapKeys(c.input) - require.NoError(t, err) - assert.EqualValues(t, c.want, c.input) - } -} - -func TestCUE(t *testing.T) { - in := `package foo -import "regexp" -matches: regexp.FindSubmatch(#"^([^:]*):(\d+)$"#, "localhost:443") -one: 1 -two: 2 -// A field using quotes. -"two-and-a-half": 2.5 -list: [ 1, 2, 3 ] -` - - expected := map[string]interface{}{ - "matches": []interface{}{ - "localhost:443", - "localhost", - "443", - }, - "one": 1, - "two": 2, - "two-and-a-half": 2.5, - "list": []interface{}{1, 2, 3}, - } - - out, err := CUE(in) - require.NoError(t, err) - assert.EqualValues(t, expected, out) - - out, err = CUE(`[1,2,3]`) - require.NoError(t, err) - assert.EqualValues(t, []interface{}{1, 2, 3}, out) - - out, err = CUE(`"hello world"`) - require.NoError(t, err) - assert.EqualValues(t, "hello world", out) - - out, err = CUE(`true`) - require.NoError(t, err) - assert.EqualValues(t, true, out) - - out, err = CUE(`'\x00\x01\x02\x03\x04'`) - require.NoError(t, err) - assert.EqualValues(t, []byte{0, 1, 2, 3, 4}, out) - - out, err = CUE(`42`) - require.NoError(t, err) - assert.EqualValues(t, 42, out) - - out, err = CUE(`42.0`) - require.NoError(t, err) - assert.EqualValues(t, 42.0, out) - - out, err = CUE(`null`) - require.NoError(t, err) - assert.EqualValues(t, nil, out) - - _, err = CUE(`>=0 & <=7 & >=3 & <=10`) - require.Error(t, err) -} - -func TestToCUE(t *testing.T) { - in := map[string]interface{}{ - "matches": []interface{}{ - "localhost:443", - "localhost", - "443", - }, - "one": 1, - "two": 2, - "two-and-a-half": 2.5, - "list": []interface{}{1, 2, 3}, - } - - expected := `{ - "two-and-a-half": 2.5 - list: [1, 2, 3] - two: 2 - one: 1 - matches: ["localhost:443", "localhost", "443"] -}` - - out, err := ToCUE(in) - require.NoError(t, err) - assert.EqualValues(t, expected, out) - - out, err = ToCUE([]interface{}{1, 2, 3}) - require.NoError(t, err) - assert.EqualValues(t, `[1, 2, 3]`, out) - - out, err = ToCUE("hello world") - require.NoError(t, err) - assert.EqualValues(t, `"hello world"`, out) - - out, err = ToCUE(true) - require.NoError(t, err) - assert.EqualValues(t, `true`, out) - - out, err = ToCUE([]byte{0, 1, 2, 3, 4}) - require.NoError(t, err) - assert.EqualValues(t, `'\x00\x01\x02\x03\x04'`, out) - - out, err = ToCUE(42) - require.NoError(t, err) - assert.EqualValues(t, `42`, out) - - out, err = ToCUE(42.0) - require.NoError(t, err) - assert.EqualValues(t, `42.0`, out) - - out, err = ToCUE(nil) - require.NoError(t, err) - assert.EqualValues(t, `null`, out) - - out, err = ToCUE(struct{}{}) - require.NoError(t, err) - assert.EqualValues(t, `{}`, out) -} diff --git a/data/datafuncs.go b/data/datafuncs.go new file mode 100644 index 00000000..ccd767e1 --- /dev/null +++ b/data/datafuncs.go @@ -0,0 +1,98 @@ +package data + +import ( + "github.com/hairyhenderson/gomplate/v4/internal/parsers" +) + +// temporary aliases for parser functions while I figure out if they need to be +// exported from the internal parsers package + +// JSON - Unmarshal a JSON Object. Can be ejson-encrypted. +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var JSON = parsers.JSON + +// JSONArray - Unmarshal a JSON Array +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var JSONArray = parsers.JSONArray + +// YAML - Unmarshal a YAML Object +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var YAML = parsers.YAML + +// YAMLArray - Unmarshal a YAML Array +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var YAMLArray = parsers.YAMLArray + +// TOML - Unmarshal a TOML Object +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var TOML = parsers.TOML + +// CSV - Unmarshal CSV +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CSV = parsers.CSV + +// CSVByRow - Unmarshal CSV in a row-oriented form +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CSVByRow = parsers.CSVByRow + +// CSVByColumn - Unmarshal CSV in a Columnar form +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CSVByColumn = parsers.CSVByColumn + +// ToCSV - +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToCSV = parsers.ToCSV + +// ToJSON - Stringify a struct as JSON +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToJSON = parsers.ToJSON + +// ToJSONPretty - Stringify a struct as JSON (indented) +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToJSONPretty = parsers.ToJSONPretty + +// ToYAML - Stringify a struct as YAML +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToYAML = parsers.ToYAML + +// ToTOML - Stringify a struct as TOML +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToTOML = parsers.ToTOML + +// CUE - Unmarshal a CUE expression into the appropriate type +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CUE = parsers.CUE + +// ToCUE - Stringify a struct as CUE +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToCUE = parsers.ToCUE diff --git a/data/datasource.go b/data/datasource.go index fe3f0877..27260ac1 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -2,98 +2,43 @@ package data import ( "context" + "encoding/json" "fmt" + "io" "io/fs" - "mime" "net/http" "net/url" - "path/filepath" + "runtime" "sort" "strings" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v4/internal/config" "github.com/hairyhenderson/gomplate/v4/internal/datafs" - "github.com/hairyhenderson/gomplate/v4/libkv" - "github.com/hairyhenderson/gomplate/v4/vault" + "github.com/hairyhenderson/gomplate/v4/internal/parsers" + "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" ) -func regExtension(ext, typ string) { - err := mime.AddExtensionType(ext, typ) - if err != nil { - panic(err) - } -} - -func init() { - // Add some types we want to be able to handle which can be missing by default - regExtension(".json", jsonMimetype) - regExtension(".yml", yamlMimetype) - regExtension(".yaml", yamlMimetype) - regExtension(".csv", csvMimetype) - regExtension(".toml", tomlMimetype) - regExtension(".env", envMimetype) - regExtension(".cue", cueMimetype) -} - -// registerReaders registers the source-reader functions -func (d *Data) registerReaders() { - d.sourceReaders = make(map[string]func(context.Context, *Source, ...string) ([]byte, error)) - - d.sourceReaders["aws+smp"] = readAWSSMP - d.sourceReaders["aws+sm"] = readAWSSecretsManager - d.sourceReaders["consul"] = readConsul - d.sourceReaders["consul+http"] = readConsul - d.sourceReaders["consul+https"] = readConsul - d.sourceReaders["env"] = readEnv - d.sourceReaders["file"] = readFile - d.sourceReaders["http"] = readHTTP - d.sourceReaders["https"] = readHTTP - d.sourceReaders["merge"] = d.readMerge - d.sourceReaders["stdin"] = readStdin - d.sourceReaders["vault"] = readVault - d.sourceReaders["vault+http"] = readVault - d.sourceReaders["vault+https"] = readVault - d.sourceReaders["s3"] = readBlob - d.sourceReaders["gs"] = readBlob - d.sourceReaders["git"] = readGit - d.sourceReaders["git+file"] = readGit - d.sourceReaders["git+http"] = readGit - d.sourceReaders["git+https"] = readGit - d.sourceReaders["git+ssh"] = readGit -} - -// lookupReader - return the reader function for the given scheme. Empty scheme -// will return the file reader. -func (d *Data) lookupReader(scheme string) (func(context.Context, *Source, ...string) ([]byte, error), error) { - if d.sourceReaders == nil { - d.registerReaders() - } - if scheme == "" { - scheme = "file" - } - - r, ok := d.sourceReaders[scheme] - if !ok { - return nil, fmt.Errorf("scheme %s not registered", scheme) - } - return r, nil -} - // Data - // // Deprecated: will be replaced in future type Data struct { Ctx context.Context + // TODO: remove this before 4.0 Sources map[string]*Source - sourceReaders map[string]func(context.Context, *Source, ...string) ([]byte, error) - cache map[string][]byte + cache map[string]*fileContent // headers from the --datasource-header/-H option that don't reference datasources from the commandline ExtraHeaders map[string]http.Header } +type fileContent struct { + contentType string + b []byte +} + // Cleanup - clean up datasources before shutting the process down - things // like Logging out happen here func (d *Data) Cleanup() { @@ -119,7 +64,7 @@ func NewData(datasourceArgs, headerArgs []string) (*Data, error) { func FromConfig(ctx context.Context, cfg *config.Config) *Data { // XXX: This is temporary, and will be replaced with something a bit cleaner // when datasources are refactored - ctx = ContextWithStdin(ctx, cfg.Stdin) + ctx = datafs.ContextWithStdin(ctx, cfg.Stdin) sources := map[string]*Source{} for alias, d := range cfg.DataSources { @@ -147,89 +92,17 @@ func FromConfig(ctx context.Context, cfg *config.Config) *Data { // // Deprecated: will be replaced in future type Source struct { - Alias string - URL *url.URL - Header http.Header // used for http[s]: URLs, nil otherwise - fs fs.FS // used for file: URLs, nil otherwise - hc *http.Client // used for http[s]: URLs, nil otherwise - vc *vault.Vault // used for vault: URLs, nil otherwise - kv *libkv.LibKV // used for consul:, etcd:, zookeeper: URLs, nil otherwise - asmpg awssmpGetter // used for aws+smp:, nil otherwise - awsSecretsManager awsSecretsManagerGetter // used for aws+sm, nil otherwise - mediaType string -} - -func (s *Source) inherit(parent *Source) { - s.fs = parent.fs - s.hc = parent.hc - s.vc = parent.vc - s.kv = parent.kv - s.asmpg = parent.asmpg + Alias string + URL *url.URL + Header http.Header // used for http[s]: URLs, nil otherwise + mediaType string } +// Deprecated: no-op func (s *Source) cleanup() { - if s.vc != nil { - s.vc.Logout() - } - if s.kv != nil { - s.kv.Logout() - } -} - -// mimeType returns the MIME type to use as a hint for parsing the datasource. -// It's expected that the datasource will have already been read before -// this function is called, and so the Source's Type property may be already set. -// -// The MIME type is determined by these rules: -// 1. the 'type' URL query parameter is used if present -// 2. otherwise, the Type property on the Source is used, if present -// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered -// 4. otherwise, the default type of 'text/plain' is used -func (s *Source) mimeType(arg string) (mimeType string, err error) { - if len(arg) > 0 { - if strings.HasPrefix(arg, "//") { - arg = arg[1:] - } - if !strings.HasPrefix(arg, "/") { - arg = "/" + arg - } - } - argURL, err := url.Parse(arg) - if err != nil { - return "", fmt.Errorf("mimeType: couldn't parse arg %q: %w", arg, err) - } - mediatype := argURL.Query().Get("type") - if mediatype == "" { - mediatype = s.URL.Query().Get("type") - } - - if mediatype == "" { - mediatype = s.mediaType - } - - // make it so + doesn't need to be escaped - mediatype = strings.ReplaceAll(mediatype, " ", "+") - - if mediatype == "" { - ext := filepath.Ext(argURL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype == "" { - ext := filepath.Ext(s.URL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype != "" { - t, _, err := mime.ParseMediaType(mediatype) - if err != nil { - return "", fmt.Errorf("MIME type was %q: %w", mediatype, err) - } - mediatype = t - return mediatype, nil - } - - return textMimetype, nil + // if s.kv != nil { + // s.kv.Logout() + // } } // String is the method to format the flag's value, part of the flag.Value interface. @@ -246,7 +119,7 @@ func (d *Data) DefineDatasource(alias, value string) (string, error) { if d.DatasourceExists(alias) { return "", nil } - srcURL, err := datafs.ParseSourceURL(value) + srcURL, err := urlhelpers.ParseSourceURL(value) if err != nil { return "", err } @@ -288,73 +161,37 @@ func (d *Data) lookupSource(alias string) (*Source, error) { return source, nil } -func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (data, mimeType string, err error) { +func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (*fileContent, error) { source, err := d.lookupSource(alias) if err != nil { - return "", "", err + return nil, err } - b, err := d.readSource(ctx, source, args...) + fc, err := d.readSource(ctx, source, args...) if err != nil { - return "", "", fmt.Errorf("couldn't read datasource '%s': %w", alias, err) + return nil, fmt.Errorf("couldn't read datasource '%s': %w", alias, err) } - subpath := "" - if len(args) > 0 { - subpath = args[0] - } - mimeType, err = source.mimeType(subpath) - if err != nil { - return "", "", err - } - return string(b), mimeType, nil + return fc, nil } // Include - func (d *Data) Include(alias string, args ...string) (string, error) { - data, _, err := d.readDataSource(d.Ctx, alias, args...) - return data, err + fc, err := d.readDataSource(d.Ctx, alias, args...) + if err != nil { + return "", err + } + + return string(fc.b), err } // Datasource - func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { - data, mimeType, err := d.readDataSource(d.Ctx, alias, args...) + fc, err := d.readDataSource(d.Ctx, alias, args...) if err != nil { return nil, err } - return parseData(mimeType, data) -} - -func parseData(mimeType, s string) (out interface{}, err error) { - switch mimeAlias(mimeType) { - case jsonMimetype: - out, err = JSON(s) - if err != nil { - // maybe it's a JSON array - out, err = JSONArray(s) - } - case jsonArrayMimetype: - out, err = JSONArray(s) - case yamlMimetype: - out, err = YAML(s) - if err != nil { - // maybe it's a YAML array - out, err = YAMLArray(s) - } - case csvMimetype: - out, err = CSV(s) - case tomlMimetype: - out, err = TOML(s) - case envMimetype: - out, err = dotEnv(s) - case textMimetype: - out = s - case cueMimetype: - out, err = CUE(s) - default: - return nil, fmt.Errorf("datasources of type %s not yet supported", mimeType) - } - return out, err + return parsers.ParseData(fc.contentType, string(fc.b)) } // DatasourceReachable - Determines if the named datasource is reachable with @@ -370,9 +207,9 @@ func (d *Data) DatasourceReachable(alias string, args ...string) bool { // readSource returns the (possibly cached) data from the given source, // as referenced by the given args -func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ([]byte, error) { +func (d *Data) readSource(ctx context.Context, source *Source, args ...string) (*fileContent, error) { if d.cache == nil { - d.cache = make(map[string][]byte) + d.cache = make(map[string]*fileContent) } cacheKey := source.Alias for _, v := range args { @@ -382,16 +219,107 @@ func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ( if ok { return cached, nil } - r, err := d.lookupReader(source.URL.Scheme) - if err != nil { - return nil, fmt.Errorf("Datasource not yet supported") + + arg := "" + if len(args) > 0 { + arg = args[0] } - data, err := r(ctx, source, args...) + u, err := resolveURL(source.URL, arg) if err != nil { return nil, err } - d.cache[cacheKey] = data - return data, nil + + fc, err := d.readFileContent(ctx, u, source.Header) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", u, err) + } + d.cache[cacheKey] = fc + return fc, nil +} + +// readFileContent returns content from the given URL +func (d Data) readFileContent(ctx context.Context, u *url.URL, hdr http.Header) (*fileContent, error) { + fsys, err := datafs.FSysForPath(ctx, u.String()) + if err != nil { + return nil, fmt.Errorf("fsys for path %v: %w", u, err) + } + + u, fname := datafs.SplitFSMuxURL(u) + + // need to support absolute paths on local filesystem too + // TODO: this is a hack, probably fix this? + if u.Scheme == "file" && runtime.GOOS != "windows" { + fname = u.Path + fname + } + + fsys = fsimpl.WithContextFS(ctx, fsys) + fsys = fsimpl.WithHeaderFS(hdr, fsys) + + // convert d.Sources to a map[string]config.DataSources + // TODO: remove this when d.Sources is removed + ds := make(map[string]config.DataSource) + for k, v := range d.Sources { + ds[k] = config.DataSource{ + URL: v.URL, + Header: v.Header, + } + } + + fsys = datafs.WithDataSourcesFS(ds, fsys) + + f, err := fsys.Open(fname) + if err != nil { + return nil, fmt.Errorf("open (url: %q, name: %q): %w", u, fname, err) + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("stat (url: %q, name: %q): %w", u, fname, err) + } + + // possible type hint in the type query param. Contrary to spec, we allow + // unescaped '+' characters to make it simpler to provide types like + // "application/array+json" + mimeType := u.Query().Get("type") + mimeType = strings.ReplaceAll(mimeType, " ", "+") + + if mimeType == "" { + mimeType = fsimpl.ContentType(fi) + } + + var data []byte + + if fi.IsDir() { + var dirents []fs.DirEntry + dirents, err = fs.ReadDir(fsys, fname) + if err != nil { + return nil, fmt.Errorf("readDir (url: %q, name: %s): %w", u, fname, err) + } + + entries := make([]string, len(dirents)) + for i, e := range dirents { + entries[i] = e.Name() + } + data, err = json.Marshal(entries) + if err != nil { + return nil, fmt.Errorf("json.Marshal: %w", err) + } + + mimeType = jsonArrayMimetype + } else { + data, err = io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("read (url: %q, name: %s): %w", u, fname, err) + } + } + + if mimeType == "" { + // default to text/plain + mimeType = textMimetype + } + + return &fileContent{contentType: mimeType, b: data}, nil } // Show all datasources - @@ -403,3 +331,62 @@ func (d *Data) ListDatasources() []string { sort.Strings(datasources) return datasources } + +// resolveURL parses the relative URL rel against base, and returns the +// resolved URL. Differs from url.ResolveReference in that query parameters are +// added. In case of duplicates, params from rel are used. +func resolveURL(base *url.URL, rel string) (*url.URL, error) { + // if there's an opaque part, there's no resolving to do - just return the + // base URL + if base.Opaque != "" { + return base, nil + } + + // git URLs are special - they have double-slashes that separate a repo + // from a path in the repo. A missing double-slash means the path is the + // root. + switch base.Scheme { + case "git", "git+file", "git+http", "git+https", "git+ssh": + if strings.Contains(base.Path, "//") && strings.Contains(rel, "//") { + return nil, fmt.Errorf("both base URL and subpath contain '//', which is not allowed in git URLs") + } + + // If there's a subpath, the base path must end with '/'. This behaviour + // is unique to git URLs - other schemes would instead drop the last + // path element and replace with the subpath. + if rel != "" && !strings.HasSuffix(base.Path, "/") { + base.Path += "/" + } + + // If subpath starts with '//', make it relative by prefixing a '.', + // otherwise it'll be treated as a schemeless URI and the first part + // will be interpreted as a hostname. + if strings.HasPrefix(rel, "//") { + rel = "." + rel + } + } + + relURL, err := url.Parse(rel) + if err != nil { + return nil, err + } + + // URL.ResolveReference requires (or assumes, at least) that the base is + // absolute. We want to support relative URLs too though, so we need to + // correct for that. + out := base.ResolveReference(relURL) + if out.Scheme == "" && out.Path[0] == '/' { + out.Path = out.Path[1:] + } + + if base.RawQuery != "" { + bq := base.Query() + rq := relURL.Query() + for k := range rq { + bq.Set(k, rq.Get(k)) + } + out.RawQuery = bq.Encode() + } + + return out, nil +} diff --git a/data/datasource_aws_sm.go b/data/datasource_aws_sm.go deleted file mode 100644 index aa42fdf9..00000000 --- a/data/datasource_aws_sm.go +++ /dev/null @@ -1,87 +0,0 @@ -package data - -import ( - "context" - "fmt" - "net/url" - "path" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - - gaws "github.com/hairyhenderson/gomplate/v4/aws" -) - -// awsSecretsManagerGetter - A subset of Secrets Manager API for use in unit testing -type awsSecretsManagerGetter interface { - GetSecretValueWithContext(ctx context.Context, input *secretsmanager.GetSecretValueInput, opts ...request.Option) (*secretsmanager.GetSecretValueOutput, error) -} - -func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { - if len(args) >= 2 { - err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", - sourceURL.Scheme, len(args)) - return nil, "", err - } - - p = sourceURL.Path - params = make(map[string]interface{}) - for key, val := range sourceURL.Query() { - params[key] = strings.Join(val, " ") - } - - if p == "" && sourceURL.Opaque != "" { - p = sourceURL.Opaque - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, "", err - } - - if parsed.Path != "" { - p = path.Join(p, parsed.Path) - if strings.HasSuffix(parsed.Path, "/") { - p += "/" - } - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - return params, p, nil -} - -func readAWSSecretsManager(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.awsSecretsManager == nil { - source.awsSecretsManager = secretsmanager.New(gaws.SDKSession()) - } - - _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - return readAWSSecretsManagerParam(ctx, source, paramPath) -} - -func readAWSSecretsManagerParam(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(paramPath), - } - - response, err := source.awsSecretsManager.GetSecretValueWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("reading aws+sm source %q: %w", source.Alias, err) - } - - if response.SecretString != nil { - return []byte(*response.SecretString), nil - } - - return response.SecretBinary, nil -} diff --git a/data/datasource_aws_sm_test.go b/data/datasource_aws_sm_test.go deleted file mode 100644 index e6e2cef3..00000000 --- a/data/datasource_aws_sm_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// DummyAWSSecretsManagerSecretGetter - test double -type DummyAWSSecretsManagerSecretGetter struct { - t *testing.T - secretValut *secretsmanager.GetSecretValueOutput - err awserr.Error - mockGetSecretValue func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) -} - -func (d DummyAWSSecretsManagerSecretGetter) GetSecretValueWithContext(_ context.Context, input *secretsmanager.GetSecretValueInput, _ ...request.Option) (*secretsmanager.GetSecretValueOutput, error) { - if d.mockGetSecretValue != nil { - output, err := d.mockGetSecretValue(input) - return output, err - } - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.secretValut, "Must provide a param if no error!") - return d.secretValut, nil -} - -func simpleAWSSecretsManagerSourceHelper(dummyGetter awsSecretsManagerGetter) *Source { - return &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "aws+sm", - Path: "/foo", - }, - awsSecretsManager: dummyGetter, - } -} - -func TestAWSSecretsManager_ParseAWSSecretsManagerArgs(t *testing.T) { - _, _, err := parseDatasourceURLArgs(mustParseURL("base"), "extra", "too many!") - assert.Error(t, err) - - tplain := map[string]interface{}{"type": "text/plain"} - - data := []struct { - eParams map[string]interface{} - u string - ePath string - args string - }{ - {u: "noddy", ePath: "noddy"}, - {u: "base", ePath: "base/extra", args: "extra"}, - {u: "/foo/", ePath: "/foo/extra", args: "/extra"}, - {u: "aws+sm:///foo", ePath: "/foo/bar", args: "bar"}, - {u: "aws+sm:foo", ePath: "foo"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar"}, - {u: "aws+sm:foo", ePath: "foo/baz", args: "baz"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar/baz", args: "baz"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar/baz", args: "baz"}, - {u: "aws+sm:///foo", ePath: "/foo/dir/", args: "dir/"}, - {u: "aws+sm:///foo/", ePath: "/foo/"}, - {u: "aws+sm:///foo/", ePath: "/foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo?type=text/plain", ePath: "foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo/bar?type=text/plain", ePath: "foo/bar/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:/foo/bar?type=text/plain", ePath: "/foo/bar/baz", args: "baz"}, - { - eParams: map[string]interface{}{ - "type": "application/json", - "param": "quux", - }, - u: "aws+sm:/foo/bar?type=text/plain", - ePath: "/foo/bar/baz/qux", - args: "baz/qux?type=application/json¶m=quux", - }, - } - - for _, d := range data { - args := []string{d.args} - if d.args == "" { - args = nil - } - params, p, err := parseDatasourceURLArgs(mustParseURL(d.u), args...) - require.NoError(t, err) - if d.eParams == nil { - assert.Empty(t, params) - } else { - assert.EqualValues(t, d.eParams, params) - } - assert.Equal(t, d.ePath, p) - } -} - -func TestAWSSecretsManager_GetParameterSetup(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - assert.Nil(t, err) -} - -func TestAWSSecretsManager_GetParameterSetupWrongArgs(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar", "/foo", "/bla") - assert.False(t, calledOk) - assert.Error(t, err) -} - -func TestAWSSecretsManager_GetParameterMissing(t *testing.T) { - expectedErr := awserr.New("ParameterNotFound", "Test of error message", nil) - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - err: expectedErr, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "") - assert.Error(t, err, "Test of error message") -} - -func TestAWSSecretsManager_ReadSecret(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - output, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - require.NoError(t, err) - assert.Equal(t, []byte("blub"), output) -} - -func TestAWSSecretsManager_ReadSecretBinary(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretBinary: []byte("supersecret")}, nil - }, - }) - - output, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - require.NoError(t, err) - assert.Equal(t, []byte("supersecret"), output) -} diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go deleted file mode 100644 index 74979129..00000000 --- a/data/datasource_awssmp.go +++ /dev/null @@ -1,77 +0,0 @@ -package data - -import ( - "context" - "fmt" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/ssm" - - gaws "github.com/hairyhenderson/gomplate/v4/aws" -) - -// awssmpGetter - A subset of SSM API for use in unit testing -type awssmpGetter interface { - GetParameterWithContext(ctx context.Context, input *ssm.GetParameterInput, opts ...request.Option) (*ssm.GetParameterOutput, error) - GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error) -} - -func readAWSSMP(ctx context.Context, source *Source, args ...string) (data []byte, err error) { - if source.asmpg == nil { - source.asmpg = ssm.New(gaws.SDKSession()) - } - - _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - source.mediaType = jsonMimetype - switch { - case strings.HasSuffix(paramPath, "/"): - source.mediaType = jsonArrayMimetype - data, err = listAWSSMPParams(ctx, source, paramPath) - default: - data, err = readAWSSMPParam(ctx, source, paramPath) - } - return data, err -} - -func readAWSSMPParam(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &ssm.GetParameterInput{ - Name: aws.String(paramPath), - WithDecryption: aws.Bool(true), - } - - response, err := source.asmpg.GetParameterWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("error reading aws+smp from AWS using GetParameter with input %v: %w", input, err) - } - - result := *response.Parameter - - output, err := ToJSON(result) - return []byte(output), err -} - -// listAWSSMPParams - supports directory semantics, returns array -func listAWSSMPParams(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &ssm.GetParametersByPathInput{ - Path: aws.String(paramPath), - } - - response, err := source.asmpg.GetParametersByPathWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("error reading aws+smp from AWS using GetParameter with input %v: %w", input, err) - } - - listing := make([]string, len(response.Parameters)) - for i, p := range response.Parameters { - listing[i] = (*p.Name)[len(paramPath):] - } - - output, err := ToJSON(listing) - return []byte(output), err -} diff --git a/data/datasource_awssmp_test.go b/data/datasource_awssmp_test.go deleted file mode 100644 index 5c5e02db..00000000 --- a/data/datasource_awssmp_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package data - -import ( - "context" - "encoding/json" - "net/url" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// DummyParamGetter - test double -type DummyParamGetter struct { - err awserr.Error - t *testing.T - param *ssm.Parameter - mockGetParameter func(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) - params []*ssm.Parameter -} - -func (d DummyParamGetter) GetParameterWithContext(_ context.Context, input *ssm.GetParameterInput, _ ...request.Option) (*ssm.GetParameterOutput, error) { - if d.mockGetParameter != nil { - output, err := d.mockGetParameter(input) - return output, err - } - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.param, "Must provide a param if no error!") - return &ssm.GetParameterOutput{ - Parameter: d.param, - }, nil -} - -func (d DummyParamGetter) GetParametersByPathWithContext(_ context.Context, _ *ssm.GetParametersByPathInput, _ ...request.Option) (*ssm.GetParametersByPathOutput, error) { - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.params, "Must provide a param if no error!") - return &ssm.GetParametersByPathOutput{ - Parameters: d.params, - }, nil -} - -func simpleAWSSourceHelper(dummy awssmpGetter) *Source { - return &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "aws+smp", - Path: "/foo", - }, - asmpg: dummy, - } -} - -func TestAWSSMP_GetParameterSetup(t *testing.T) { - calledOk := false - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - mockGetParameter: func(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { - assert.Equal(t, "/foo/bar", *input.Name) - assert.True(t, *input.WithDecryption) - calledOk = true - return &ssm.GetParameterOutput{ - Parameter: &ssm.Parameter{}, - }, nil - }, - }) - - _, err := readAWSSMP(context.Background(), s, "/bar") - assert.True(t, calledOk) - assert.Nil(t, err) -} - -func TestAWSSMP_GetParameterValidOutput(t *testing.T) { - expected := &ssm.Parameter{ - Name: aws.String("/foo"), - Type: aws.String("String"), - Value: aws.String("val"), - Version: aws.Int64(1), - } - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - param: expected, - }) - - output, err := readAWSSMP(context.Background(), s, "") - assert.Nil(t, err) - actual := &ssm.Parameter{} - err = json.Unmarshal(output, &actual) - assert.Nil(t, err) - assert.Equal(t, expected, actual) - assert.Equal(t, jsonMimetype, s.mediaType) -} - -func TestAWSSMP_GetParameterMissing(t *testing.T) { - expectedErr := awserr.New("ParameterNotFound", "Test of error message", nil) - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - err: expectedErr, - }) - - _, err := readAWSSMP(context.Background(), s, "") - assert.Error(t, err, "Test of error message") -} - -func TestAWSSMP_listAWSSMPParams(t *testing.T) { - ctx := context.Background() - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - err: awserr.New("ParameterNotFound", "foo", nil), - }) - _, err := listAWSSMPParams(ctx, s, "") - assert.Error(t, err) - - s = simpleAWSSourceHelper(DummyParamGetter{ - t: t, - params: []*ssm.Parameter{ - {Name: aws.String("/a")}, - {Name: aws.String("/b")}, - {Name: aws.String("/c")}, - }, - }) - data, err := listAWSSMPParams(ctx, s, "/") - require.NoError(t, err) - assert.Equal(t, []byte(`["a","b","c"]`), data) - - s = simpleAWSSourceHelper(DummyParamGetter{ - t: t, - params: []*ssm.Parameter{ - {Name: aws.String("/a/a")}, - {Name: aws.String("/a/b")}, - {Name: aws.String("/a/c")}, - }, - }) - data, err = listAWSSMPParams(ctx, s, "/a/") - require.NoError(t, err) - assert.Equal(t, []byte(`["a","b","c"]`), data) -} diff --git a/data/datasource_blob.go b/data/datasource_blob.go deleted file mode 100644 index 1dac584a..00000000 --- a/data/datasource_blob.go +++ /dev/null @@ -1,173 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "mime" - "net/url" - "path" - "strings" - - gaws "github.com/hairyhenderson/gomplate/v4/aws" - "github.com/hairyhenderson/gomplate/v4/env" - - "gocloud.dev/blob" - "gocloud.dev/blob/gcsblob" - "gocloud.dev/blob/s3blob" - "gocloud.dev/gcp" -) - -func readBlob(ctx context.Context, source *Source, args ...string) (output []byte, err error) { - if len(args) >= 2 { - return nil, fmt.Errorf("maximum two arguments to blob datasource: alias, extraPath") - } - - key := source.URL.Path - if len(args) == 1 { - key = path.Join(key, args[0]) - } - - opener, err := newOpener(ctx, source.URL) - if err != nil { - return nil, err - } - - mux := blob.URLMux{} - mux.RegisterBucket(source.URL.Scheme, opener) - - u := blobURL(source.URL) - bucket, err := mux.OpenBucket(ctx, u) - if err != nil { - return nil, err - } - defer bucket.Close() - - var r func(context.Context, *blob.Bucket, string) (string, []byte, error) - if strings.HasSuffix(key, "/") { - r = listBucket - } else { - r = getBlob - } - - mediaType, data, err := r(ctx, bucket, key) - if mediaType != "" { - source.mediaType = mediaType - } - return data, err -} - -// create the correct kind of blob.BucketURLOpener for the given URL -func newOpener(ctx context.Context, u *url.URL) (opener blob.BucketURLOpener, err error) { - switch u.Scheme { - case "s3": - // set up a "regular" gomplate AWS SDK session - sess := gaws.SDKSession() - // see https://gocloud.dev/concepts/urls/#muxes - opener = &s3blob.URLOpener{ConfigProvider: sess} - case "gs": - creds, err := gcp.DefaultCredentials(ctx) - if err != nil { - return nil, fmt.Errorf("failed to retrieve GCP credentials: %w", err) - } - - client, err := gcp.NewHTTPClient( - gcp.DefaultTransport(), - gcp.CredentialsTokenSource(creds)) - if err != nil { - return nil, fmt.Errorf("failed to create GCP HTTP client: %w", err) - } - opener = &gcsblob.URLOpener{ - Client: client, - } - } - return opener, nil -} - -func getBlob(ctx context.Context, bucket *blob.Bucket, key string) (mediaType string, data []byte, err error) { - key = strings.TrimPrefix(key, "/") - attr, err := bucket.Attributes(ctx, key) - if err != nil { - return "", nil, fmt.Errorf("failed to retrieve attributes for %s: %w", key, err) - } - if attr.ContentType != "" { - mt, _, e := mime.ParseMediaType(attr.ContentType) - if e != nil { - return "", nil, e - } - mediaType = mt - } - data, err = bucket.ReadAll(ctx, key) - if err != nil { - return "", nil, fmt.Errorf("failed to read %s: %w", key, err) - } - return mediaType, data, nil -} - -// calls the bucket listing API, returning a JSON Array -func listBucket(ctx context.Context, bucket *blob.Bucket, path string) (mediaType string, data []byte, err error) { - path = strings.TrimPrefix(path, "/") - opts := &blob.ListOptions{ - Prefix: path, - Delimiter: "/", - } - li := bucket.List(opts) - keys := []string{} - for { - obj, err := li.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return "", nil, err - } - keys = append(keys, strings.TrimPrefix(obj.Key, path)) - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(keys); err != nil { - return "", nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - data = b[:len(b)-1] - return jsonArrayMimetype, data, nil -} - -// copy/sanitize the URL for the Go CDK - it doesn't like params it can't parse -func blobURL(u *url.URL) string { - out := cloneURL(u) - q := out.Query() - - for param := range q { - switch u.Scheme { - case "s3": - switch param { - case "region", "endpoint", "disableSSL", "s3ForcePathStyle": - default: - q.Del(param) - } - case "gs": - switch param { - case "access_id", "private_key_path": - default: - q.Del(param) - } - } - } - - if u.Scheme == "s3" { - // handle AWS_S3_ENDPOINT env var - endpoint := env.Getenv("AWS_S3_ENDPOINT") - if endpoint != "" { - q.Set("endpoint", endpoint) - } - } - - out.RawQuery = q.Encode() - - return out.String() -} diff --git a/data/datasource_blob_test.go b/data/datasource_blob_test.go deleted file mode 100644 index 6be8ea00..00000000 --- a/data/datasource_blob_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package data - -import ( - "bytes" - "context" - "net/http/httptest" - "net/url" - "testing" - - "github.com/johannesboyne/gofakes3" - "github.com/johannesboyne/gofakes3/backend/s3mem" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func setupTestBucket(t *testing.T) (*httptest.Server, *url.URL) { - backend := s3mem.New() - faker := gofakes3.New(backend) - ts := httptest.NewServer(faker.Server()) - - err := backend.CreateBucket("mybucket") - require.NoError(t, err) - c := "hello" - err = putFile(backend, "file1", "text/plain", c) - require.NoError(t, err) - - c = `{"value": "goodbye world"}` - err = putFile(backend, "file2", "application/json", c) - require.NoError(t, err) - - c = `value: what a world` - err = putFile(backend, "file3", "application/yaml", c) - require.NoError(t, err) - - c = `value: out of this world` - err = putFile(backend, "dir1/file1", "application/yaml", c) - require.NoError(t, err) - - c = `value: foo` - err = putFile(backend, "dir1/file2", "application/yaml", c) - require.NoError(t, err) - - u, _ := url.Parse(ts.URL) - return ts, u -} - -func putFile(backend gofakes3.Backend, file, mime, content string) error { - _, err := backend.PutObject( - "mybucket", - file, - map[string]string{"Content-Type": mime}, - bytes.NewBufferString(content), - int64(len(content)), - ) - return err -} - -func TestReadBlob(t *testing.T) { - _, err := readBlob(context.Background(), nil, "foo", "bar") - assert.Error(t, err) - - ts, u := setupTestBucket(t) - defer ts.Close() - - t.Run("no authentication", func(t *testing.T) { - t.Setenv("AWS_ANON", "true") - - d, err := NewData([]string{"-d", "data=s3://mybucket/file1?region=us-east-1&disableSSL=true&s3ForcePathStyle=true&type=text/plain&endpoint=" + u.Host}, nil) - require.NoError(t, err) - - expected := "hello" - out, err := d.Datasource("data") - require.NoError(t, err) - assert.Equal(t, expected, out) - }) - - t.Run("with authentication", func(t *testing.T) { - t.Setenv("AWS_ACCESS_KEY_ID", "fake") - t.Setenv("AWS_SECRET_ACCESS_KEY", "fake") - t.Setenv("AWS_S3_ENDPOINT", u.Host) - - d, err := NewData([]string{"-d", "data=s3://mybucket/file2?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - require.NoError(t, err) - - var expected interface{} - expected = map[string]interface{}{"value": "goodbye world"} - out, err := d.Datasource("data") - require.NoError(t, err) - assert.Equal(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - require.NoError(t, err) - - expected = []interface{}{"dir1/", "file1", "file2", "file3"} - out, err = d.Datasource("data") - require.NoError(t, err) - assert.EqualValues(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/dir1/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - require.NoError(t, err) - - expected = []interface{}{"file1", "file2"} - out, err = d.Datasource("data") - require.NoError(t, err) - assert.EqualValues(t, expected, out) - }) -} - -func TestBlobURL(t *testing.T) { - data := []struct { - in string - expected string - }{ - {"s3://foo/bar/baz", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?type=hello/world", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?region=us-east-1", "s3://foo/bar/baz?region=us-east-1"}, - {"s3://foo/bar/baz?disableSSL=true&type=text/csv", "s3://foo/bar/baz?disableSSL=true"}, - {"s3://foo/bar/baz?type=text/csv&s3ForcePathStyle=true&endpoint=1.2.3.4", "s3://foo/bar/baz?endpoint=1.2.3.4&s3ForcePathStyle=true"}, - {"gs://foo/bar/baz", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?type=foo/bar", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?access_id=123", "gs://foo/bar/baz?access_id=123"}, - {"gs://foo/bar/baz?private_key_path=/foo/bar", "gs://foo/bar/baz?private_key_path=%2Ffoo%2Fbar"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar", "gs://foo/bar/baz?private_key_path=key.json"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar&access_id=abcd", "gs://foo/bar/baz?access_id=abcd&private_key_path=key.json"}, - } - - for _, d := range data { - u, _ := url.Parse(d.in) - out := blobURL(u) - assert.Equal(t, d.expected, out) - } -} diff --git a/data/datasource_consul.go b/data/datasource_consul.go deleted file mode 100644 index ecc7e516..00000000 --- a/data/datasource_consul.go +++ /dev/null @@ -1,39 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/hairyhenderson/gomplate/v4/libkv" -) - -func readConsul(_ context.Context, source *Source, args ...string) (data []byte, err error) { - if source.kv == nil { - source.kv, err = libkv.NewConsul(source.URL) - if err != nil { - return nil, err - } - err = source.kv.Login() - if err != nil { - return nil, err - } - } - - p := source.URL.Path - if len(args) == 1 { - p = strings.TrimRight(p, "/") + "/" + args[0] - } - - if strings.HasSuffix(p, "/") { - source.mediaType = jsonArrayMimetype - data, err = source.kv.List(p) - } else { - data, err = source.kv.Read(p) - } - - if err != nil { - return nil, err - } - - return data, nil -} diff --git a/data/datasource_env.go b/data/datasource_env.go deleted file mode 100644 index e5bf180f..00000000 --- a/data/datasource_env.go +++ /dev/null @@ -1,20 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/hairyhenderson/gomplate/v4/env" -) - -//nolint:unparam -func readEnv(_ context.Context, source *Source, _ ...string) (b []byte, err error) { - n := source.URL.Path - n = strings.TrimPrefix(n, "/") - if n == "" { - n = source.URL.Opaque - } - - b = []byte(env.Getenv(n)) - return b, nil -} diff --git a/data/datasource_env_test.go b/data/datasource_env_test.go deleted file mode 100644 index 6512578c..00000000 --- a/data/datasource_env_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func mustParseURL(in string) *url.URL { - u, _ := url.Parse(in) - return u -} - -func TestReadEnv(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - t.Setenv("HELLO_WORLD", "hello world") - t.Setenv("HELLO_UNIVERSE", "hello universe") - - source := &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD")} - - actual, err := readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:/HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) -} diff --git a/data/datasource_file.go b/data/datasource_file.go deleted file mode 100644 index f5c764fe..00000000 --- a/data/datasource_file.go +++ /dev/null @@ -1,87 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/fs" - "net/url" - "path/filepath" - "strings" - - "github.com/hairyhenderson/gomplate/v4/internal/datafs" -) - -func readFile(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.fs == nil { - fsp := datafs.FSProviderFromContext(ctx) - fsys, err := fsp.New(source.URL) - if err != nil { - return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", source.URL, err) - } - source.fs = fsys - } - - p := filepath.FromSlash(source.URL.Path) - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, err - } - - if parsed.Path != "" { - p = filepath.Join(p, parsed.Path) - } - - // reset the media type - it may have been set by a parent dir read - source.mediaType = "" - } - - isDir := strings.HasSuffix(p, string(filepath.Separator)) - if strings.HasSuffix(p, string(filepath.Separator)) { - p = p[:len(p)-1] - } - - // make sure we can access the file - i, err := fs.Stat(source.fs, p) - if err != nil { - return nil, fmt.Errorf("stat %s: %w", p, err) - } - - if isDir { - source.mediaType = jsonArrayMimetype - if i.IsDir() { - return readFileDir(source, p) - } - return nil, fmt.Errorf("%s is not a directory", p) - } - - b, err := fs.ReadFile(source.fs, p) - if err != nil { - return nil, fmt.Errorf("readFile %s: %w", p, err) - } - return b, nil -} - -func readFileDir(source *Source, p string) ([]byte, error) { - names, err := fs.ReadDir(source.fs, p) - if err != nil { - return nil, err - } - - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} diff --git a/data/datasource_file_test.go b/data/datasource_file_test.go deleted file mode 100644 index 7ad1c2a0..00000000 --- a/data/datasource_file_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package data - -import ( - "context" - "io/fs" - "testing" - "testing/fstest" - - "github.com/hairyhenderson/gomplate/v4/internal/datafs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadFile(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - - fsys := datafs.WrapWdFS(fstest.MapFS{ - "tmp": {Mode: fs.ModeDir | 0o777}, - "tmp/foo": {Data: content}, - "tmp/partial": {Mode: fs.ModeDir | 0o777}, - "tmp/partial/foo.txt": {Data: content}, - "tmp/partial/bar.txt": {}, - "tmp/partial/baz.txt": {}, - }) - - source := &Source{Alias: "foo", URL: mustParseURL("file:///tmp/foo")} - source.fs = fsys - - actual, err := readFile(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "bogus", URL: mustParseURL("file:///bogus")} - source.fs = fsys - _, err = readFile(ctx, source) - assert.Error(t, err) - - source = &Source{Alias: "partial", URL: mustParseURL("file:///tmp/partial")} - source.fs = fsys - actual, err = readFile(ctx, source, "foo.txt") - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/")} - source.fs = fsys - actual, err = readFile(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fsys - actual, err = readFile(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - mime, err := source.mimeType("") - require.NoError(t, err) - assert.Equal(t, "application/json", mime) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fsys - actual, err = readFile(ctx, source, "foo.txt") - require.NoError(t, err) - assert.Equal(t, content, actual) - mime, err = source.mimeType("") - require.NoError(t, err) - assert.Equal(t, "application/json", mime) -} diff --git a/data/datasource_git.go b/data/datasource_git.go deleted file mode 100644 index 3859d344..00000000 --- a/data/datasource_git.go +++ /dev/null @@ -1,328 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/url" - "os" - "path" - "path/filepath" - "strings" - - "github.com/hairyhenderson/gomplate/v4/base64" - "github.com/hairyhenderson/gomplate/v4/env" - "github.com/rs/zerolog" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/memory" -) - -func readGit(ctx context.Context, source *Source, args ...string) ([]byte, error) { - g := gitsource{} - - u := source.URL - repoURL, path, err := g.parseGitPath(u, args...) - if err != nil { - return nil, err - } - - depth := 1 - if u.Scheme == "git+file" { - // we can't do shallow clones for filesystem repos apparently - depth = 0 - } - - fs, _, err := g.clone(ctx, repoURL, depth) - if err != nil { - return nil, err - } - - mimeType, out, err := g.read(fs, path) - if mimeType != "" { - source.mediaType = mimeType - } - return out, err -} - -type gitsource struct{} - -func (g gitsource) parseArgURL(arg string) (u *url.URL, err error) { - if strings.HasPrefix(arg, "//") { - u, err = url.Parse(arg[1:]) - u.Path = "/" + u.Path - } else { - u, err = url.Parse(arg) - } - - if err != nil { - return nil, fmt.Errorf("failed to parse arg %s: %w", arg, err) - } - return u, err -} - -func (g gitsource) parseQuery(orig, arg *url.URL) string { - q := orig.Query() - pq := arg.Query() - for k, vs := range pq { - for _, v := range vs { - q.Add(k, v) - } - } - return q.Encode() -} - -func (g gitsource) parseArgPath(u *url.URL, arg string) (repo, p string) { - // if the source URL already specified a repo and subpath, the whole - // arg is interpreted as subpath - if strings.Contains(u.Path, "//") || strings.HasPrefix(arg, "//") { - return "", arg - } - - parts := strings.SplitN(arg, "//", 2) - repo = parts[0] - if len(parts) == 2 { - p = "/" + parts[1] - } - return repo, p -} - -// Massage the URL and args together to produce the repo to clone, -// and the path to read. -// The path is delimited from the repo by '//' -func (g gitsource) parseGitPath(u *url.URL, args ...string) (out *url.URL, p string, err error) { - if u == nil { - return nil, "", fmt.Errorf("parseGitPath: no url provided (%v)", u) - } - // copy the input url so we can modify it - out = cloneURL(u) - - parts := strings.SplitN(out.Path, "//", 2) - switch len(parts) { - case 1: - p = "/" - case 2: - p = "/" + parts[1] - - i := strings.LastIndex(out.Path, p) - out.Path = out.Path[:i-1] - } - - if len(args) > 0 { - argURL, uerr := g.parseArgURL(args[0]) - if uerr != nil { - return nil, "", uerr - } - repo, argpath := g.parseArgPath(u, argURL.Path) - out.Path = path.Join(out.Path, repo) - p = path.Join(p, argpath) - - out.RawQuery = g.parseQuery(u, argURL) - - if argURL.Fragment != "" { - out.Fragment = argURL.Fragment - } - } - return out, p, err -} - -//nolint:interfacer -func cloneURL(u *url.URL) *url.URL { - out, _ := url.Parse(u.String()) - return out -} - -// refFromURL - extract the ref from the URL fragment if present -func (g gitsource) refFromURL(u *url.URL) plumbing.ReferenceName { - switch { - case strings.HasPrefix(u.Fragment, "refs/"): - return plumbing.ReferenceName(u.Fragment) - case u.Fragment != "": - return plumbing.NewBranchReferenceName(u.Fragment) - default: - return plumbing.ReferenceName("") - } -} - -// refFromRemoteHead - extract the ref from the remote HEAD, to work around -// hard-coded 'master' default branch in go-git. -// Should be unnecessary once https://github.com/go-git/go-git/issues/249 is -// fixed. -func (g gitsource) refFromRemoteHead(ctx context.Context, u *url.URL, auth transport.AuthMethod) (plumbing.ReferenceName, error) { - e, err := transport.NewEndpoint(u.String()) - if err != nil { - return "", err - } - - cli, err := client.NewClient(e) - if err != nil { - return "", err - } - - s, err := cli.NewUploadPackSession(e, auth) - if err != nil { - return "", err - } - - info, err := s.AdvertisedReferencesContext(ctx) - if err != nil { - return "", err - } - - refs, err := info.AllReferences() - if err != nil { - return "", err - } - - headRef, ok := refs["HEAD"] - if !ok { - return "", fmt.Errorf("no HEAD ref found") - } - - return headRef.Target(), nil -} - -// clone a repo for later reading through http(s), git, or ssh. u must be the URL to the repo -// itself, and must have any file path stripped -func (g gitsource) clone(ctx context.Context, repoURL *url.URL, depth int) (billy.Filesystem, *git.Repository, error) { - fs := memfs.New() - storer := memory.NewStorage() - - // preserve repoURL by cloning it - u := cloneURL(repoURL) - - auth, err := g.auth(u) - if err != nil { - return nil, nil, err - } - - if strings.HasPrefix(u.Scheme, "git+") { - scheme := u.Scheme[len("git+"):] - u.Scheme = scheme - } - - ref := g.refFromURL(u) - u.Fragment = "" - u.RawQuery = "" - - // attempt to get the ref from the remote so we don't default to master - if ref == "" { - ref, err = g.refFromRemoteHead(ctx, u, auth) - if err != nil { - zerolog.Ctx(ctx).Warn(). - Stringer("repoURL", u). - Err(err). - Msg("failed to get ref from remote, using default") - } - } - - opts := &git.CloneOptions{ - URL: u.String(), - Auth: auth, - Depth: depth, - ReferenceName: ref, - SingleBranch: true, - Tags: git.NoTags, - } - repo, err := git.CloneContext(ctx, storer, fs, opts) - if u.Scheme == "file" && err == transport.ErrRepositoryNotFound && !strings.HasSuffix(u.Path, ".git") { - // maybe this has a `.git` subdirectory... - u = cloneURL(repoURL) - u.Path = path.Join(u.Path, ".git") - return g.clone(ctx, u, depth) - } - if err != nil { - return nil, nil, fmt.Errorf("git clone %s: %w", u, err) - } - return fs, repo, nil -} - -// read - reads the provided path out of a git repo -func (g gitsource) read(fsys billy.Filesystem, path string) (string, []byte, error) { - fi, err := fsys.Stat(path) - if err != nil { - return "", nil, fmt.Errorf("can't stat %s: %w", path, err) - } - if fi.IsDir() || strings.HasSuffix(path, string(filepath.Separator)) { - out, rerr := g.readDir(fsys, path) - return jsonArrayMimetype, out, rerr - } - - f, err := fsys.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return "", nil, fmt.Errorf("can't open %s: %w", path, err) - } - - b, err := io.ReadAll(f) - if err != nil { - return "", nil, fmt.Errorf("can't read %s: %w", path, err) - } - - return "", b, nil -} - -func (g gitsource) readDir(fs billy.Filesystem, path string) ([]byte, error) { - names, err := fs.ReadDir(path) - if err != nil { - return nil, fmt.Errorf("couldn't read dir %s: %w", path, err) - } - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} - -/* -auth methods: -- ssh named key (no password support) - - GIT_SSH_KEY (base64-encoded) or GIT_SSH_KEY_FILE (base64-encoded, or not) - -- ssh agent auth (preferred) -- http basic auth (for github, gitlab, bitbucket tokens) -- http token auth (bearer token, somewhat unusual) -*/ -func (g gitsource) auth(u *url.URL) (auth transport.AuthMethod, err error) { - user := u.User.Username() - switch u.Scheme { - case "git+http", "git+https": - if pass, ok := u.User.Password(); ok { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if pass := env.Getenv("GIT_HTTP_PASSWORD"); pass != "" { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if tok := env.Getenv("GIT_HTTP_TOKEN"); tok != "" { - // note docs on TokenAuth - this is rarely to be used - auth = &http.TokenAuth{Token: tok} - } - case "git+ssh": - k := env.Getenv("GIT_SSH_KEY") - if k != "" { - var key []byte - key, err = base64.Decode(k) - if err != nil { - key = []byte(k) - } - auth, err = ssh.NewPublicKeys(user, key, "") - } else { - auth, err = ssh.NewSSHAgentAuth(user) - } - } - return auth, err -} diff --git a/data/datasource_git_test.go b/data/datasource_git_test.go deleted file mode 100644 index 3b187ecc..00000000 --- a/data/datasource_git_test.go +++ /dev/null @@ -1,551 +0,0 @@ -package data - -import ( - "context" - "encoding/base64" - "fmt" - "io" - "net/url" - "os" - "strings" - "testing" - "time" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-billy/v5/osfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/server" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/filesystem" - - "golang.org/x/crypto/ssh/testdata" - - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -func TestParseArgPath(t *testing.T) { - t.Parallel() - g := gitsource{} - - data := []struct { - url string - arg string - repo, path string - }{ - { - "git+file:///foo//foo", - "/bar", - "", "/bar", - }, - { - "git+file:///foo//bar", - "/baz//qux", - "", "/baz//qux", - }, - { - "git+https://example.com/foo", - "/bar", - "/bar", "", - }, - { - "git+https://example.com/foo", - "//bar", - "", "//bar", - }, - { - "git+https://example.com/foo//bar", - "//baz", - "", "//baz", - }, - { - "git+https://example.com/foo", - "/bar//baz", - "/bar", "/baz", - }, - { - "git+https://example.com/foo?type=t", - "/bar//baz", - "/bar", "/baz", - }, - { - "git+https://example.com/foo#master", - "/bar//baz", - "/bar", "/baz", - }, - { - "git+https://example.com/foo", - "//bar", - "", "//bar", - }, - { - "git+https://example.com/foo?type=t", - "//baz", - "", "//baz", - }, - { - "git+https://example.com/foo?type=t#v1", - "//bar", - "", "//bar", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.arg, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - repo, path := g.parseArgPath(u, d.arg) - assert.Equal(t, d.repo, repo) - assert.Equal(t, d.path, path) - }) - } -} - -func TestParseGitPath(t *testing.T) { - t.Parallel() - g := gitsource{} - _, _, err := g.parseGitPath(nil) - assert.ErrorContains(t, err, "") - - u := mustParseURL("http://example.com//foo") - assert.Equal(t, "//foo", u.Path) - parts := strings.SplitN(u.Path, "//", 2) - assert.Equal(t, 2, len(parts)) - assert.DeepEqual(t, []string{"", "foo"}, parts) - - data := []struct { - url string - args string - repo, path string - }{ - { - "git+https://github.com/hairyhenderson/gomplate//docs-src/content/functions/aws.yml", - "", - "git+https://github.com/hairyhenderson/gomplate", - "/docs-src/content/functions/aws.yml", - }, - { - "git+ssh://github.com/hairyhenderson/gomplate.git", - "", - "git+ssh://github.com/hairyhenderson/gomplate.git", - "/", - }, - { - "https://github.com", - "", - "https://github.com", - "/", - }, - { - "git://example.com/foo//file.txt#someref", - "", - "git://example.com/foo#someref", "/file.txt", - }, - { - "git+file:///home/foo/repo//file.txt#someref", - "", - "git+file:///home/foo/repo#someref", "/file.txt", - }, - { - "git+file:///repo", - "", - "git+file:///repo", "/", - }, - { - "git+file:///foo//foo", - "", - "git+file:///foo", "/foo", - }, - { - "git+file:///foo//foo", - "/bar", - "git+file:///foo", "/foo/bar", - }, - { - "git+file:///foo//bar", - // in this case the // is meaningless - "/baz//qux", - "git+file:///foo", "/bar/baz/qux", - }, - { - "git+https://example.com/foo", - "/bar", - "git+https://example.com/foo/bar", "/", - }, - { - "git+https://example.com/foo", - "//bar", - "git+https://example.com/foo", "/bar", - }, - { - "git+https://example.com//foo", - "/bar", - "git+https://example.com", "/foo/bar", - }, - { - "git+https://example.com/foo//bar", - "//baz", - "git+https://example.com/foo", "/bar/baz", - }, - { - "git+https://example.com/foo", - "/bar//baz", - "git+https://example.com/foo/bar", "/baz", - }, - { - "git+https://example.com/foo?type=t", - "/bar//baz", - "git+https://example.com/foo/bar?type=t", "/baz", - }, - { - "git+https://example.com/foo#master", - "/bar//baz", - "git+https://example.com/foo/bar#master", "/baz", - }, - { - "git+https://example.com/foo", - "/bar//baz?type=t", - "git+https://example.com/foo/bar?type=t", "/baz", - }, - { - "git+https://example.com/foo", - "/bar//baz#master", - "git+https://example.com/foo/bar#master", "/baz", - }, - { - "git+https://example.com/foo", - "//bar?type=t", - "git+https://example.com/foo?type=t", "/bar", - }, - { - "git+https://example.com/foo", - "//bar#master", - "git+https://example.com/foo#master", "/bar", - }, - { - "git+https://example.com/foo?type=t", - "//bar#master", - "git+https://example.com/foo?type=t#master", "/bar", - }, - { - "git+https://example.com/foo?type=t#v1", - "//bar?type=j#v2", - "git+https://example.com/foo?type=t&type=j#v2", "/bar", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.args, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - args := []string{d.args} - if d.args == "" { - args = nil - } - repo, path, err := g.parseGitPath(u, args...) - assert.NilError(t, err) - assert.Equal(t, d.repo, repo.String()) - assert.Equal(t, d.path, path) - }) - } -} - -func TestReadGitRepo(t *testing.T) { - g := gitsource{} - fs := setupGitRepo(t) - fs, err := fs.Chroot("/repo") - assert.NilError(t, err) - - _, _, err = g.read(fs, "/bogus") - assert.ErrorContains(t, err, "can't stat /bogus") - - mtype, out, err := g.read(fs, "/foo") - assert.NilError(t, err) - assert.Equal(t, `["bar"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar") - assert.NilError(t, err) - assert.Equal(t, `["hi.txt"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar/hi.txt") - assert.NilError(t, err) - assert.Equal(t, `hello world`, string(out)) - assert.Equal(t, "", mtype) -} - -var testHashes = map[string]string{} - -func setupGitRepo(t *testing.T) billy.Filesystem { - fs := memfs.New() - fs.MkdirAll("/repo/.git", os.ModeDir) - repo, _ := fs.Chroot("/repo") - dot, _ := repo.Chroot("/.git") - s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - - r, err := git.Init(s, repo) - assert.NilError(t, err) - - // default to main - h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main")) - err = s.SetReference(h) - assert.NilError(t, err) - - // config needs to be created after setting up a "normal" fs repo - // this is possibly a bug in git-go? - c, err := r.Config() - assert.NilError(t, err) - - c.Init.DefaultBranch = "main" - - s.SetConfig(c) - assert.NilError(t, err) - - w, err := r.Worktree() - assert.NilError(t, err) - - repo.MkdirAll("/foo/bar", os.ModeDir) - f, err := repo.Create("/foo/bar/hi.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("hello world")) - assert.NilError(t, err) - _, err = w.Add(f.Name()) - assert.NilError(t, err) - hash, err := w.Commit("initial commit", &git.CommitOptions{Author: &object.Signature{}}) - assert.NilError(t, err) - - ref, err := r.CreateTag("v1", hash, nil) - assert.NilError(t, err) - testHashes["v1"] = hash.String() - - branchName := plumbing.NewBranchReferenceName("mybranch") - err = w.Checkout(&git.CheckoutOptions{ - Branch: branchName, - Hash: ref.Hash(), - Create: true, - }) - assert.NilError(t, err) - - f, err = repo.Create("/secondfile.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("another file\n")) - assert.NilError(t, err) - n := f.Name() - _, err = w.Add(n) - assert.NilError(t, err) - hash, err = w.Commit("second commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - }, - }) - ref = plumbing.NewHashReference(branchName, hash) - assert.NilError(t, err) - testHashes["mybranch"] = ref.Hash().String() - - // make the repo dirty - _, err = f.Write([]byte("dirty file")) - assert.NilError(t, err) - - // set up a bare repo - fs.MkdirAll("/bare.git", os.ModeDir) - fs.MkdirAll("/barewt", os.ModeDir) - repo, _ = fs.Chroot("/barewt") - dot, _ = fs.Chroot("/bare.git") - s = filesystem.NewStorage(dot, nil) - - r, err = git.Init(s, repo) - assert.NilError(t, err) - - w, err = r.Worktree() - assert.NilError(t, err) - - f, err = repo.Create("/hello.txt") - assert.NilError(t, err) - f.Write([]byte("hello world")) - w.Add(f.Name()) - _, err = w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - Email: "john@doe.org", - When: time.Now(), - }, - }) - assert.NilError(t, err) - - return fs -} - -func overrideFSLoader(fs billy.Filesystem) { - l := server.NewFilesystemLoader(fs) - client.InstallProtocol("file", server.NewClient(l)) -} - -func TestOpenFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fsys, _, err := g.clone(ctx, mustParseURL("git+file:///repo"), 0) - assert.NilError(t, err) - - f, err := fsys.Open("/foo/bar/hi.txt") - assert.NilError(t, err) - b, _ := io.ReadAll(f) - assert.Equal(t, "hello world", string(b)) - - _, repo, err := g.clone(ctx, mustParseURL("git+file:///repo#main"), 0) - assert.NilError(t, err) - - ref, err := repo.Reference(plumbing.NewBranchReferenceName("main"), true) - assert.NilError(t, err) - assert.Equal(t, "refs/heads/main", ref.Name().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo#refs/tags/v1"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, testHashes["v1"], ref.Hash().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo/#mybranch"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, "refs/heads/mybranch", ref.Name().String()) - assert.Equal(t, testHashes["mybranch"], ref.Hash().String()) -} - -func TestOpenBareFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fsys, _, err := g.clone(ctx, mustParseURL("git+file:///bare.git"), 0) - assert.NilError(t, err) - - f, err := fsys.Open("/hello.txt") - assert.NilError(t, err) - b, _ := io.ReadAll(f) - assert.Equal(t, "hello world", string(b)) -} - -func TestReadGit(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - s := &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git//hello.txt"), - } - b, err := readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "hello world", string(b)) - - s = &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git"), - } - b, err = readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "application/array+json", s.mediaType) - assert.Equal(t, `["hello.txt"]`, string(b)) -} - -func TestGitAuth(t *testing.T) { - g := gitsource{} - a, err := g.auth(mustParseURL("git+file:///bare.git")) - assert.NilError(t, err) - assert.Equal(t, nil, a) - - a, err = g.auth(mustParseURL("git+https://example.com/foo")) - assert.NilError(t, err) - assert.Assert(t, is.Nil(a)) - - a, err = g.auth(mustParseURL("git+https://user:swordfish@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - - t.Setenv("GIT_HTTP_PASSWORD", "swordfish") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - os.Unsetenv("GIT_HTTP_PASSWORD") - - t.Setenv("GIT_HTTP_TOKEN", "mytoken") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.TokenAuth{Token: "mytoken"}, a) - os.Unsetenv("GIT_HTTP_TOKEN") - - if os.Getenv("SSH_AUTH_SOCK") == "" { - t.Log("no SSH_AUTH_SOCK - skipping ssh agent test") - } else { - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - sa, ok := a.(*ssh.PublicKeysCallback) - assert.Equal(t, true, ok) - assert.Equal(t, "git", sa.User) - } - - t.Run("plain string ed25519", func(t *testing.T) { - key := string(testdata.PEMBytes["ed25519"]) - t.Setenv("GIT_SSH_KEY", key) - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok := a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - }) - - t.Run("base64 ed25519", func(t *testing.T) { - key := base64.StdEncoding.EncodeToString(testdata.PEMBytes["ed25519"]) - t.Setenv("GIT_SSH_KEY", key) - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok := a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - os.Unsetenv("GIT_SSH_KEY") - }) -} - -func TestRefFromURL(t *testing.T) { - t.Parallel() - g := gitsource{} - data := []struct { - url, expected string - }{ - {"git://localhost:1234/foo/bar.git//baz", ""}, - {"git+http://localhost:1234/foo/bar.git//baz", ""}, - {"git+ssh://localhost:1234/foo/bar.git//baz", ""}, - {"git+file:///foo/bar.git//baz", ""}, - {"git://localhost:1234/foo/bar.git//baz#master", "refs/heads/master"}, - {"git+http://localhost:1234/foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - {"git+ssh://localhost:1234/foo/bar.git//baz#refs/tags/foo", "refs/tags/foo"}, - {"git+file:///foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - } - - for _, d := range data { - out := g.refFromURL(mustParseURL(d.url)) - assert.Equal(t, plumbing.ReferenceName(d.expected), out) - } -} diff --git a/data/datasource_http.go b/data/datasource_http.go deleted file mode 100644 index 23c7dc36..00000000 --- a/data/datasource_http.go +++ /dev/null @@ -1,62 +0,0 @@ -package data - -import ( - "context" - "fmt" - "io" - "mime" - "net/http" - "net/url" - "time" -) - -func buildURL(base *url.URL, args ...string) (*url.URL, error) { - if len(args) == 0 { - return base, nil - } - p, err := url.Parse(args[0]) - if err != nil { - return nil, fmt.Errorf("bad sub-path %s: %w", args[0], err) - } - return base.ResolveReference(p), nil -} - -func readHTTP(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.hc == nil { - source.hc = &http.Client{Timeout: time.Second * 5} - } - u, err := buildURL(source.URL, args...) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - req.Header = source.Header - res, err := source.hc.Do(req) - if err != nil { - return nil, err - } - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - err = res.Body.Close() - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - err := fmt.Errorf("unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body)) - return nil, err - } - ctypeHdr := res.Header.Get("Content-Type") - if ctypeHdr != "" { - mediatype, _, e := mime.ParseMediaType(ctypeHdr) - if e != nil { - return nil, e - } - source.mediaType = mediatype - } - return body, nil -} diff --git a/data/datasource_http_test.go b/data/datasource_http_test.go deleted file mode 100644 index 90c4a7f9..00000000 --- a/data/datasource_http_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package data - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func must(r interface{}, err error) interface{} { - if err != nil { - panic(err) - } - return r -} - -func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", mimetype) - w.WriteHeader(code) - if body == "" { - // mirror back the headers - fmt.Fprintln(w, must(marshalObj(r.Header, json.Marshal))) - } else { - fmt.Fprintln(w, body) - } - })) - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - }, - } - - return server, client -} - -func TestHTTPFile(t *testing.T) { - server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`) - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - } - data := &Data{ - Ctx: context.Background(), - Sources: sources, - } - - expected := map[string]interface{}{ - "hello": "world", - } - - actual, err := data.Datasource("foo") - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - actual, err = data.Datasource(server.URL) - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestHTTPFileWithHeaders(t *testing.T) { - server, client := setupHTTP(200, jsonMimetype, "") - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - Header: http.Header{ - "Foo": {"bar"}, - "foo": {"baz"}, - "User-Agent": {}, - "Accept-Encoding": {"test"}, - }, - } - data := &Data{ - Ctx: context.Background(), - Sources: sources, - } - expected := http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - } - actual, err := data.Datasource("foo") - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - expected = http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - "User-Agent": {"Go-http-client/1.1"}, - } - data = &Data{ - Ctx: context.Background(), - Sources: sources, - ExtraHeaders: map[string]http.Header{server.URL: expected}, - } - actual, err = data.Datasource(server.URL) - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestBuildURL(t *testing.T) { - expected := "https://example.com/index.html" - base := mustParseURL(expected) - u, err := buildURL(base) - require.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/index.html" - base = mustParseURL("https://example.com") - u, err = buildURL(base, "index.html") - require.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/a/b/c/index.html" - base = mustParseURL("https://example.com/a/") - u, err = buildURL(base, "b/c/index.html") - require.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/bar/baz/index.html" - base = mustParseURL("https://example.com/foo") - u, err = buildURL(base, "bar/baz/index.html") - require.NoError(t, err) - assert.Equal(t, expected, u.String()) -} diff --git a/data/datasource_merge.go b/data/datasource_merge.go deleted file mode 100644 index b0d8b6bd..00000000 --- a/data/datasource_merge.go +++ /dev/null @@ -1,100 +0,0 @@ -package data - -import ( - "context" - "fmt" - "strings" - - "github.com/hairyhenderson/gomplate/v4/coll" - "github.com/hairyhenderson/gomplate/v4/internal/datafs" -) - -// readMerge demultiplexes a `merge:` datasource. The 'args' parameter currently -// has no meaning for this source. -// -// URI format is 'merge:<source 1>|<source 2>[|<source n>...]' where `<source #>` -// is a supported URI or a pre-defined alias name. -// -// Query strings and fragments are interpreted relative to the merged data, not -// the source data. To merge datasources with query strings or fragments, define -// separate sources first and specify the alias names. HTTP headers are also not -// supported directly. -func (d *Data) readMerge(ctx context.Context, source *Source, _ ...string) ([]byte, error) { - opaque := source.URL.Opaque - parts := strings.Split(opaque, "|") - if len(parts) < 2 { - return nil, fmt.Errorf("need at least 2 datasources to merge") - } - data := make([]map[string]interface{}, len(parts)) - for i, part := range parts { - // supports either URIs or aliases - subSource, err := d.lookupSource(part) - if err != nil { - // maybe it's a relative filename? - u, uerr := datafs.ParseSourceURL(part) - if uerr != nil { - return nil, uerr - } - subSource = &Source{ - Alias: part, - URL: u, - } - } - subSource.inherit(source) - - b, err := d.readSource(ctx, subSource) - if err != nil { - return nil, fmt.Errorf("couldn't read datasource '%s': %w", part, err) - } - - mimeType, err := subSource.mimeType("") - if err != nil { - return nil, fmt.Errorf("failed to read datasource %s: %w", subSource.URL, err) - } - - data[i], err = parseMap(mimeType, string(b)) - if err != nil { - return nil, err - } - } - - // Merge the data together - b, err := mergeData(data) - if err != nil { - return nil, err - } - - source.mediaType = yamlMimetype - return b, nil -} - -func mergeData(data []map[string]interface{}) (out []byte, err error) { - dst := data[0] - data = data[1:] - - dst, err = coll.Merge(dst, data...) - if err != nil { - return nil, err - } - - s, err := ToYAML(dst) - if err != nil { - return nil, err - } - return []byte(s), nil -} - -func parseMap(mimeType, data string) (map[string]interface{}, error) { - datum, err := parseData(mimeType, data) - if err != nil { - return nil, err - } - var m map[string]interface{} - switch datum := datum.(type) { - case map[string]interface{}: - m = datum - default: - return nil, fmt.Errorf("unexpected data type '%T' for datasource (type %s); merge: can only merge maps", datum, mimeType) - } - return m, nil -} diff --git a/data/datasource_merge_test.go b/data/datasource_merge_test.go deleted file mode 100644 index 48d1f85e..00000000 --- a/data/datasource_merge_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package data - -import ( - "context" - "io/fs" - "net/url" - "os" - "path" - "path/filepath" - "testing" - "testing/fstest" - - "github.com/hairyhenderson/gomplate/v4/internal/datafs" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadMerge(t *testing.T) { - ctx := context.Background() - - jsonContent := `{"hello": "world"}` - yamlContent := "hello: earth\ngoodnight: moon\n" - arrayContent := `["hello", "world"]` - - mergedContent := "goodnight: moon\nhello: world\n" - - wd, _ := os.Getwd() - - // MapFS doesn't support windows path separators, so we use / exclusively - // in this test - vol := filepath.VolumeName(wd) - if vol != "" && wd != vol { - wd = wd[len(vol)+1:] - } else if wd[0] == '/' { - wd = wd[1:] - } - wd = filepath.ToSlash(wd) - - fsys := datafs.WrapWdFS(fstest.MapFS{ - "tmp": {Mode: fs.ModeDir | 0o777}, - "tmp/jsonfile.json": {Data: []byte(jsonContent)}, - "tmp/array.json": {Data: []byte(arrayContent)}, - "tmp/yamlfile.yaml": {Data: []byte(yamlContent)}, - "tmp/textfile.txt": {Data: []byte(`plain text...`)}, - path.Join(wd, "jsonfile.json"): {Data: []byte(jsonContent)}, - path.Join(wd, "array.json"): {Data: []byte(arrayContent)}, - path.Join(wd, "yamlfile.yaml"): {Data: []byte(yamlContent)}, - path.Join(wd, "textfile.txt"): {Data: []byte(`plain text...`)}, - }) - - source := &Source{Alias: "foo", URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml")} - source.fs = fsys - d := &Data{ - Sources: map[string]*Source{ - "foo": source, - "bar": {Alias: "bar", URL: mustParseURL("file:///tmp/jsonfile.json")}, - "baz": {Alias: "baz", URL: mustParseURL("file:///tmp/yamlfile.yaml")}, - "text": {Alias: "text", URL: mustParseURL("file:///tmp/textfile.txt")}, - "badscheme": {Alias: "badscheme", URL: mustParseURL("bad:///scheme.json")}, - "badtype": {Alias: "badtype", URL: mustParseURL("file:///tmp/textfile.txt?type=foo/bar")}, - "array": {Alias: "array", URL: mustParseURL("file:///tmp/array.json?type=" + url.QueryEscape(jsonArrayMimetype))}, - }, - } - - actual, err := d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:bar|baz") - actual, err = d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:jsonfile.json|baz") - actual, err = d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:./jsonfile.json|baz") - actual, err = d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:bogusalias|file:///tmp/jsonfile.json") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|badscheme") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|badtype") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|array") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) -} - -func TestMergeData(t *testing.T) { - def := map[string]interface{}{ - "f": true, - "t": false, - "z": "def", - } - out, err := mergeData([]map[string]interface{}{def}) - require.NoError(t, err) - assert.Equal(t, "f: true\nt: false\nz: def\n", string(out)) - - over := map[string]interface{}{ - "f": false, - "t": true, - "z": "over", - } - out, err = mergeData([]map[string]interface{}{over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nt: true\nz: over\n", string(out)) - - over = map[string]interface{}{ - "f": false, - "t": true, - "z": "over", - "m": map[string]interface{}{ - "a": "aaa", - }, - } - out, err = mergeData([]map[string]interface{}{over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm:\n a: aaa\nt: true\nz: over\n", string(out)) - - uber := map[string]interface{}{ - "z": "über", - } - out, err = mergeData([]map[string]interface{}{uber, over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm:\n a: aaa\nt: true\nz: über\n", string(out)) - - uber = map[string]interface{}{ - "m": "notamap", - "z": map[string]interface{}{ - "b": "bbb", - }, - } - out, err = mergeData([]map[string]interface{}{uber, over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm: notamap\nt: true\nz:\n b: bbb\n", string(out)) - - uber = map[string]interface{}{ - "m": map[string]interface{}{ - "b": "bbb", - }, - } - out, err = mergeData([]map[string]interface{}{uber, over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm:\n a: aaa\n b: bbb\nt: true\nz: over\n", string(out)) -} diff --git a/data/datasource_stdin.go b/data/datasource_stdin.go deleted file mode 100644 index 13bb5fa4..00000000 --- a/data/datasource_stdin.go +++ /dev/null @@ -1,32 +0,0 @@ -package data - -import ( - "context" - "fmt" - "io" - "os" -) - -func readStdin(ctx context.Context, _ *Source, _ ...string) ([]byte, error) { - stdin := stdinFromContext(ctx) - - b, err := io.ReadAll(stdin) - if err != nil { - return nil, fmt.Errorf("can't read %s: %w", stdin, err) - } - return b, nil -} - -type stdinCtxKey struct{} - -func ContextWithStdin(ctx context.Context, r io.Reader) context.Context { - return context.WithValue(ctx, stdinCtxKey{}, r) -} - -func stdinFromContext(ctx context.Context) io.Reader { - if r, ok := ctx.Value(stdinCtxKey{}).(io.Reader); ok { - return r - } - - return os.Stdin -} diff --git a/data/datasource_stdin_test.go b/data/datasource_stdin_test.go deleted file mode 100644 index 8cef6827..00000000 --- a/data/datasource_stdin_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package data - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadStdin(t *testing.T) { - ctx := context.Background() - - ctx = ContextWithStdin(ctx, strings.NewReader("foo")) - out, err := readStdin(ctx, nil) - require.NoError(t, err) - assert.Equal(t, []byte("foo"), out) - - ctx = ContextWithStdin(ctx, errorReader{}) - _, err = readStdin(ctx, nil) - assert.Error(t, err) -} diff --git a/data/datasource_test.go b/data/datasource_test.go index a77f2645..da3d3094 100644 --- a/data/datasource_test.go +++ b/data/datasource_test.go @@ -2,13 +2,16 @@ package data import ( "context" - "fmt" "net/http" + "net/http/httptest" "net/url" + "os" "runtime" "testing" "testing/fstest" + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/httpfs" "github.com/hairyhenderson/gomplate/v4/internal/config" "github.com/hairyhenderson/gomplate/v4/internal/datafs" @@ -18,6 +21,11 @@ import ( const osWindows = "windows" +func mustParseURL(in string) *url.URL { + u, _ := url.Parse(in) + return u +} + func TestNewData(t *testing.T) { d, err := NewData(nil, nil) require.NoError(t, err) @@ -56,21 +64,22 @@ func TestDatasource(t *testing.T) { fsys := datafs.WrapWdFS(fstest.MapFS{ "tmp/" + fname: &fstest.MapFile{Data: contents}, }) + ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "file", "")) sources := map[string]*Source{ "foo": { Alias: "foo", URL: &url.URL{Scheme: "file", Path: uPath}, mediaType: mime, - fs: fsys, }, } - return &Data{Sources: sources} + return &Data{Sources: sources, Ctx: ctx} } + test := func(ext, mime string, contents []byte, expected interface{}) { data := setup(ext, mime, contents) - actual, err := data.Datasource("foo") + actual, err := data.Datasource("foo", "?type="+mime) require.NoError(t, err) assert.Equal(t, expected, actual) } @@ -110,21 +119,20 @@ func TestDatasourceReachable(t *testing.T) { fsys := datafs.WrapWdFS(fstest.MapFS{ "tmp/" + fname: &fstest.MapFile{Data: []byte("{}")}, }) + ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "file", "")) sources := map[string]*Source{ "foo": { Alias: "foo", URL: &url.URL{Scheme: "file", Path: uPath}, mediaType: jsonMimetype, - fs: fsys, }, "bar": { Alias: "bar", URL: &url.URL{Scheme: "file", Path: "/bogus"}, - fs: fsys, }, } - data := &Data{Sources: sources} + data := &Data{Sources: sources, Ctx: ctx} assert.True(t, data.DatasourceReachable("foo")) assert.False(t, data.DatasourceReachable("bar")) @@ -154,29 +162,21 @@ func TestInclude(t *testing.T) { fsys := datafs.WrapWdFS(fstest.MapFS{ "tmp/" + fname: &fstest.MapFile{Data: []byte(contents)}, }) + ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "file", "")) sources := map[string]*Source{ "foo": { Alias: "foo", URL: &url.URL{Scheme: "file", Path: uPath}, mediaType: textMimetype, - fs: fsys, }, } - data := &Data{ - Sources: sources, - } + data := &Data{Sources: sources, Ctx: ctx} actual, err := data.Include("foo") require.NoError(t, err) assert.Equal(t, contents, actual) } -type errorReader struct{} - -func (e errorReader) Read(_ []byte) (n int, err error) { - return 0, fmt.Errorf("error") -} - func TestDefineDatasource(t *testing.T) { d := &Data{} _, err := d.DefineDatasource("", "foo.json") @@ -231,134 +231,6 @@ func TestDefineDatasource(t *testing.T) { s = d.Sources["data"] require.NoError(t, err) assert.Equal(t, "data", s.Alias) - m, err := s.mimeType("") - require.NoError(t, err) - assert.Equal(t, "application/x-env", m) -} - -func TestMimeType(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com/list?type=a/b/c")} - _, err := s.mimeType("") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - expected string - }{ - { - "http://example.com/foo.json", - "", - jsonMimetype, - }, - { - "http://example.com/foo.json", - "text/foo", - "text/foo", - }, - { - "http://example.com/foo.json?type=application/yaml", - "text/foo", - "application/yaml", - }, - { - "http://example.com/list?type=application/array%2Bjson", - "text/foo", - "application/array+json", - }, - { - "http://example.com/list?type=application/array+json", - "", - "application/array+json", - }, - { - "http://example.com/unknown", - "", - "text/plain", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q==%q", i, d.url, d.mediaType, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType("") - require.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } -} - -func TestMimeTypeWithArg(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com")} - _, err := s.mimeType("h\nttp://foo") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - arg string - expected string - }{ - { - "http://example.com/unknown", - "", - "/foo.json", - "application/json", - }, - { - "http://example.com/unknown", - "", - "foo.json", - "application/json", - }, - { - "http://example.com/", - "text/foo", - "/foo.json", - "text/foo", - }, - { - "git+https://example.com/myrepo", - "", - "//foo.yaml", - "application/yaml", - }, - { - "http://example.com/foo.json", - "", - "/foo.yaml", - "application/yaml", - }, - { - "http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml", - "application/array+yaml", - }, - { - "http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml?type=application/yaml", - "application/yaml", - }, - { - "http://example.com/foo.json?type=application/array+yaml", - "text/plain", - "/foo.yaml?type=application/yaml", - "application/yaml", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q,%q==%q", i, d.url, d.mediaType, d.arg, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType(d.arg) - require.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } } func TestFromConfig(t *testing.T) { @@ -445,3 +317,94 @@ func TestListDatasources(t *testing.T) { assert.Equal(t, []string{"bar", "foo"}, data.ListDatasources()) } + +func TestResolveURL(t *testing.T) { + out, err := resolveURL(mustParseURL("http://example.com/foo.json"), "bar.json") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/bar.json", out.String()) + + out, err = resolveURL(mustParseURL("http://example.com/a/b/?n=2"), "bar.json?q=1") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/a/b/bar.json?n=2&q=1", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo"), "//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/myrepo//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/foo/bar/"), "//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/foo/bar//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo/"), ".//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/myrepo//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/repo//foo.txt"), "") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/repo//foo.txt", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo"), ".//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/myrepo//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo//foo/?type=application/json"), "bar/myfile") + assert.NoError(t, err) + // note that the '/' in the query string is encoded to %2F - that's OK + assert.Equal(t, "git+file:///tmp/myrepo//foo/bar/myfile?type=application%2Fjson", out.String()) + + // both base and relative may not contain "//" + _, err = resolveURL(mustParseURL("git+ssh://git@example.com/foo//bar"), ".//myfile") + assert.Error(t, err) + + _, err = resolveURL(mustParseURL("git+ssh://git@example.com/foo//bar"), "baz//myfile") + assert.Error(t, err) + + // relative urls must remain relative + out, err = resolveURL(mustParseURL("tmp/foo.json"), "") + require.NoError(t, err) + assert.Equal(t, "tmp/foo.json", out.String()) +} + +func TestReadFileContent(t *testing.T) { + wd, _ := os.Getwd() + t.Cleanup(func() { + _ = os.Chdir(wd) + }) + _ = os.Chdir("/") + + mux := http.NewServeMux() + mux.HandleFunc("/foo.json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", jsonMimetype) + w.Write([]byte(`{"foo": "bar"}`)) + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + fsys := datafs.WrapWdFS(fstest.MapFS{ + "foo.json": &fstest.MapFile{Data: []byte(`{"foo": "bar"}`)}, + "dir/1.yaml": &fstest.MapFile{Data: []byte(`foo: bar`)}, + "dir/2.yaml": &fstest.MapFile{Data: []byte(`baz: qux`)}, + "dir/sub/sub1.yaml": &fstest.MapFile{Data: []byte(`quux: corge`)}, + }) + + fsp := fsimpl.NewMux() + fsp.Add(httpfs.FS) + fsp.Add(datafs.WrappedFSProvider(fsys, "file", "")) + + ctx := datafs.ContextWithFSProvider(context.Background(), fsp) + + d := Data{} + + fc, err := d.readFileContent(ctx, mustParseURL("file:///foo.json"), nil) + require.NoError(t, err) + assert.Equal(t, []byte(`{"foo": "bar"}`), fc.b) + + fc, err = d.readFileContent(ctx, mustParseURL("dir/"), nil) + require.NoError(t, err) + assert.JSONEq(t, `["1.yaml", "2.yaml", "sub"]`, string(fc.b)) + + fc, err = d.readFileContent(ctx, mustParseURL(srv.URL+"/foo.json"), nil) + require.NoError(t, err) + assert.Equal(t, []byte(`{"foo": "bar"}`), fc.b) +} diff --git a/data/datasource_vault.go b/data/datasource_vault.go deleted file mode 100644 index 5f736dcb..00000000 --- a/data/datasource_vault.go +++ /dev/null @@ -1,47 +0,0 @@ -package data - -import ( - "context" - "fmt" - "strings" - - "github.com/hairyhenderson/gomplate/v4/vault" -) - -func readVault(_ context.Context, source *Source, args ...string) (data []byte, err error) { - if source.vc == nil { - source.vc, err = vault.New(source.URL) - if err != nil { - return nil, err - } - err = source.vc.Login() - if err != nil { - return nil, err - } - } - - params, p, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - source.mediaType = jsonMimetype - switch { - case len(params) > 0: - data, err = source.vc.Write(p, params) - case strings.HasSuffix(p, "/"): - source.mediaType = jsonArrayMimetype - data, err = source.vc.List(p) - default: - data, err = source.vc.Read(p) - } - if err != nil { - return nil, err - } - - if len(data) == 0 { - return nil, fmt.Errorf("no value found for path %s", p) - } - - return data, nil -} diff --git a/data/datasource_vault_test.go b/data/datasource_vault_test.go deleted file mode 100644 index 1c63d696..00000000 --- a/data/datasource_vault_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/hairyhenderson/gomplate/v4/vault" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadVault(t *testing.T) { - ctx := context.Background() - - expected := "{\"value\":\"foo\"}\n" - server, v := vault.MockServer(200, `{"data":`+expected+`}`) - defer server.Close() - - source := &Source{ - Alias: "foo", - URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, - mediaType: textMimetype, - vc: v, - } - - r, err := readVault(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "bar") - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "?param=value") - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - source.URL, _ = url.Parse("vault:///secret/foo?param1=value1¶m2=value2") - r, err = readVault(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - expected = "[\"one\",\"two\"]\n" - server, source.vc = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) - defer server.Close() - source.URL, _ = url.Parse("vault:///secret/foo/") - r, err = readVault(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) -} diff --git a/data/mimetypes.go b/data/mimetypes.go index 1f243219..24ea87de 100644 --- a/data/mimetypes.go +++ b/data/mimetypes.go @@ -1,5 +1,9 @@ package data +import ( + "mime" +) + const ( textMimetype = "text/plain" csvMimetype = "text/csv" @@ -19,6 +23,9 @@ var mimeTypeAliases = map[string]string{ } func mimeAlias(m string) string { + // normalize the type by removing any extra parameters + m, _, _ = mime.ParseMediaType(m) + if a, ok := mimeTypeAliases[m]; ok { return a } diff --git a/data/mimetypes_test.go b/data/mimetypes_test.go index 0dd1ab05..04c54439 100644 --- a/data/mimetypes_test.go +++ b/data/mimetypes_test.go @@ -3,7 +3,7 @@ package data import ( "testing" - "gotest.tools/v3/assert" + "github.com/stretchr/testify/assert" ) func TestMimeAlias(t *testing.T) { |