diff options
88 files changed, 2967 insertions, 4409 deletions
diff --git a/.golangci.yml b/.golangci.yml index 2ddcc5c7..afcd93bc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -33,7 +33,7 @@ linters: # - funlen # - gci # - gochecknoglobals - # - gochecknoinits + - gochecknoinits - gocognit - goconst - gocritic diff --git a/aws/ec2meta.go b/aws/ec2meta.go index 6c1dd76f..c6cab50d 100644 --- a/aws/ec2meta.go +++ b/aws/ec2meta.go @@ -1,6 +1,7 @@ package aws import ( + "context" "net/http" "strings" @@ -8,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/hairyhenderson/gomplate/v4/env" + "github.com/hairyhenderson/gomplate/v4/internal/deprecated" ) const ( @@ -38,6 +40,7 @@ func NewEc2Meta(options ClientOptions) *Ec2Meta { config := aws.NewConfig() config = config.WithHTTPClient(&http.Client{Timeout: options.Timeout}) if endpoint := env.Getenv("AWS_META_ENDPOINT"); endpoint != "" { + deprecated.WarnDeprecated(context.Background(), "Use AWS_EC2_METADATA_SERVICE_ENDPOINT instead of AWS_META_ENDPOINT") config = config.WithEndpoint(endpoint) } @@ -22,9 +22,18 @@ func (c *tmplctx) Env() map[string]string { } // createTmplContext reads the datasources for the given aliases -// -//nolint:staticcheck -func createTmplContext(_ context.Context, aliases []string, d *data.Data) (interface{}, error) { +func createTmplContext( + ctx context.Context, aliases []string, + //nolint:staticcheck + d *data.Data, +) (interface{}, error) { + // we need to inject the current context into the Data value, because + // the Datasource method may need it + // TODO: remove this before v4 + if d != nil { + d.Ctx = ctx + } + var err error tctx := &tmplctx{} for _, a := range aliases { diff --git a/context_test.go b/context_test.go index aa3a4468..c8885a47 100644 --- a/context_test.go +++ b/context_test.go @@ -6,7 +6,9 @@ import ( "os" "testing" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v4/data" + "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,6 +33,11 @@ func TestCreateContext(t *testing.T) { require.NoError(t, err) assert.Empty(t, c) + fsmux := fsimpl.NewMux() + fsmux.Add(datafs.EnvFS) + + ctx = datafs.ContextWithFSProvider(ctx, fsmux) + fooURL := "env:///foo?type=application/yaml" barURL := "env:///bar?type=application/yaml" uf, _ := url.Parse(fooURL) diff --git a/crypto/pbkdf2.go b/crypto/pbkdf2.go index d5217429..0cc44d7e 100644 --- a/crypto/pbkdf2.go +++ b/crypto/pbkdf2.go @@ -7,22 +7,23 @@ import ( "crypto/sha512" "fmt" "hash" + "sync" "golang.org/x/crypto/pbkdf2" ) -var hashFuncs map[crypto.Hash]func() hash.Hash +var hashFuncs = sync.OnceValue[map[crypto.Hash]func() hash.Hash](func() map[crypto.Hash]func() hash.Hash { + h := make(map[crypto.Hash]func() hash.Hash) + h[crypto.SHA1] = sha1.New + h[crypto.SHA224] = sha256.New224 + h[crypto.SHA256] = sha256.New + h[crypto.SHA384] = sha512.New384 + h[crypto.SHA512] = sha512.New + h[crypto.SHA512_224] = sha512.New512_224 + h[crypto.SHA512_256] = sha512.New512_256 -func init() { - hashFuncs = make(map[crypto.Hash]func() hash.Hash) - hashFuncs[crypto.SHA1] = sha1.New - hashFuncs[crypto.SHA224] = sha256.New224 - hashFuncs[crypto.SHA256] = sha256.New - hashFuncs[crypto.SHA384] = sha512.New384 - hashFuncs[crypto.SHA512] = sha512.New - hashFuncs[crypto.SHA512_224] = sha512.New512_224 - hashFuncs[crypto.SHA512_256] = sha512.New512_256 -} + return h +})() // StrToHash - find a hash given a certain string func StrToHash(hash string) (crypto.Hash, error) { diff --git a/data/datafuncs.go b/data/datafuncs.go new file mode 100644 index 00000000..ccd767e1 --- /dev/null +++ b/data/datafuncs.go @@ -0,0 +1,98 @@ +package data + +import ( + "github.com/hairyhenderson/gomplate/v4/internal/parsers" +) + +// temporary aliases for parser functions while I figure out if they need to be +// exported from the internal parsers package + +// JSON - Unmarshal a JSON Object. Can be ejson-encrypted. +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var JSON = parsers.JSON + +// JSONArray - Unmarshal a JSON Array +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var JSONArray = parsers.JSONArray + +// YAML - Unmarshal a YAML Object +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var YAML = parsers.YAML + +// YAMLArray - Unmarshal a YAML Array +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var YAMLArray = parsers.YAMLArray + +// TOML - Unmarshal a TOML Object +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var TOML = parsers.TOML + +// CSV - Unmarshal CSV +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CSV = parsers.CSV + +// CSVByRow - Unmarshal CSV in a row-oriented form +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CSVByRow = parsers.CSVByRow + +// CSVByColumn - Unmarshal CSV in a Columnar form +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CSVByColumn = parsers.CSVByColumn + +// ToCSV - +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToCSV = parsers.ToCSV + +// ToJSON - Stringify a struct as JSON +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToJSON = parsers.ToJSON + +// ToJSONPretty - Stringify a struct as JSON (indented) +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToJSONPretty = parsers.ToJSONPretty + +// ToYAML - Stringify a struct as YAML +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToYAML = parsers.ToYAML + +// ToTOML - Stringify a struct as TOML +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToTOML = parsers.ToTOML + +// CUE - Unmarshal a CUE expression into the appropriate type +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var CUE = parsers.CUE + +// ToCUE - Stringify a struct as CUE +// +// Deprecated: will be removed in a future version of gomplate. If you have a +// need for this, please open an issue! +var ToCUE = parsers.ToCUE diff --git a/data/datasource.go b/data/datasource.go index fe3f0877..27260ac1 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -2,98 +2,43 @@ package data import ( "context" + "encoding/json" "fmt" + "io" "io/fs" - "mime" "net/http" "net/url" - "path/filepath" + "runtime" "sort" "strings" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v4/internal/config" "github.com/hairyhenderson/gomplate/v4/internal/datafs" - "github.com/hairyhenderson/gomplate/v4/libkv" - "github.com/hairyhenderson/gomplate/v4/vault" + "github.com/hairyhenderson/gomplate/v4/internal/parsers" + "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" ) -func regExtension(ext, typ string) { - err := mime.AddExtensionType(ext, typ) - if err != nil { - panic(err) - } -} - -func init() { - // Add some types we want to be able to handle which can be missing by default - regExtension(".json", jsonMimetype) - regExtension(".yml", yamlMimetype) - regExtension(".yaml", yamlMimetype) - regExtension(".csv", csvMimetype) - regExtension(".toml", tomlMimetype) - regExtension(".env", envMimetype) - regExtension(".cue", cueMimetype) -} - -// registerReaders registers the source-reader functions -func (d *Data) registerReaders() { - d.sourceReaders = make(map[string]func(context.Context, *Source, ...string) ([]byte, error)) - - d.sourceReaders["aws+smp"] = readAWSSMP - d.sourceReaders["aws+sm"] = readAWSSecretsManager - d.sourceReaders["consul"] = readConsul - d.sourceReaders["consul+http"] = readConsul - d.sourceReaders["consul+https"] = readConsul - d.sourceReaders["env"] = readEnv - d.sourceReaders["file"] = readFile - d.sourceReaders["http"] = readHTTP - d.sourceReaders["https"] = readHTTP - d.sourceReaders["merge"] = d.readMerge - d.sourceReaders["stdin"] = readStdin - d.sourceReaders["vault"] = readVault - d.sourceReaders["vault+http"] = readVault - d.sourceReaders["vault+https"] = readVault - d.sourceReaders["s3"] = readBlob - d.sourceReaders["gs"] = readBlob - d.sourceReaders["git"] = readGit - d.sourceReaders["git+file"] = readGit - d.sourceReaders["git+http"] = readGit - d.sourceReaders["git+https"] = readGit - d.sourceReaders["git+ssh"] = readGit -} - -// lookupReader - return the reader function for the given scheme. Empty scheme -// will return the file reader. -func (d *Data) lookupReader(scheme string) (func(context.Context, *Source, ...string) ([]byte, error), error) { - if d.sourceReaders == nil { - d.registerReaders() - } - if scheme == "" { - scheme = "file" - } - - r, ok := d.sourceReaders[scheme] - if !ok { - return nil, fmt.Errorf("scheme %s not registered", scheme) - } - return r, nil -} - // Data - // // Deprecated: will be replaced in future type Data struct { Ctx context.Context + // TODO: remove this before 4.0 Sources map[string]*Source - sourceReaders map[string]func(context.Context, *Source, ...string) ([]byte, error) - cache map[string][]byte + cache map[string]*fileContent // headers from the --datasource-header/-H option that don't reference datasources from the commandline ExtraHeaders map[string]http.Header } +type fileContent struct { + contentType string + b []byte +} + // Cleanup - clean up datasources before shutting the process down - things // like Logging out happen here func (d *Data) Cleanup() { @@ -119,7 +64,7 @@ func NewData(datasourceArgs, headerArgs []string) (*Data, error) { func FromConfig(ctx context.Context, cfg *config.Config) *Data { // XXX: This is temporary, and will be replaced with something a bit cleaner // when datasources are refactored - ctx = ContextWithStdin(ctx, cfg.Stdin) + ctx = datafs.ContextWithStdin(ctx, cfg.Stdin) sources := map[string]*Source{} for alias, d := range cfg.DataSources { @@ -147,89 +92,17 @@ func FromConfig(ctx context.Context, cfg *config.Config) *Data { // // Deprecated: will be replaced in future type Source struct { - Alias string - URL *url.URL - Header http.Header // used for http[s]: URLs, nil otherwise - fs fs.FS // used for file: URLs, nil otherwise - hc *http.Client // used for http[s]: URLs, nil otherwise - vc *vault.Vault // used for vault: URLs, nil otherwise - kv *libkv.LibKV // used for consul:, etcd:, zookeeper: URLs, nil otherwise - asmpg awssmpGetter // used for aws+smp:, nil otherwise - awsSecretsManager awsSecretsManagerGetter // used for aws+sm, nil otherwise - mediaType string -} - -func (s *Source) inherit(parent *Source) { - s.fs = parent.fs - s.hc = parent.hc - s.vc = parent.vc - s.kv = parent.kv - s.asmpg = parent.asmpg + Alias string + URL *url.URL + Header http.Header // used for http[s]: URLs, nil otherwise + mediaType string } +// Deprecated: no-op func (s *Source) cleanup() { - if s.vc != nil { - s.vc.Logout() - } - if s.kv != nil { - s.kv.Logout() - } -} - -// mimeType returns the MIME type to use as a hint for parsing the datasource. -// It's expected that the datasource will have already been read before -// this function is called, and so the Source's Type property may be already set. -// -// The MIME type is determined by these rules: -// 1. the 'type' URL query parameter is used if present -// 2. otherwise, the Type property on the Source is used, if present -// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered -// 4. otherwise, the default type of 'text/plain' is used -func (s *Source) mimeType(arg string) (mimeType string, err error) { - if len(arg) > 0 { - if strings.HasPrefix(arg, "//") { - arg = arg[1:] - } - if !strings.HasPrefix(arg, "/") { - arg = "/" + arg - } - } - argURL, err := url.Parse(arg) - if err != nil { - return "", fmt.Errorf("mimeType: couldn't parse arg %q: %w", arg, err) - } - mediatype := argURL.Query().Get("type") - if mediatype == "" { - mediatype = s.URL.Query().Get("type") - } - - if mediatype == "" { - mediatype = s.mediaType - } - - // make it so + doesn't need to be escaped - mediatype = strings.ReplaceAll(mediatype, " ", "+") - - if mediatype == "" { - ext := filepath.Ext(argURL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype == "" { - ext := filepath.Ext(s.URL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype != "" { - t, _, err := mime.ParseMediaType(mediatype) - if err != nil { - return "", fmt.Errorf("MIME type was %q: %w", mediatype, err) - } - mediatype = t - return mediatype, nil - } - - return textMimetype, nil + // if s.kv != nil { + // s.kv.Logout() + // } } // String is the method to format the flag's value, part of the flag.Value interface. @@ -246,7 +119,7 @@ func (d *Data) DefineDatasource(alias, value string) (string, error) { if d.DatasourceExists(alias) { return "", nil } - srcURL, err := datafs.ParseSourceURL(value) + srcURL, err := urlhelpers.ParseSourceURL(value) if err != nil { return "", err } @@ -288,73 +161,37 @@ func (d *Data) lookupSource(alias string) (*Source, error) { return source, nil } -func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (data, mimeType string, err error) { +func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (*fileContent, error) { source, err := d.lookupSource(alias) if err != nil { - return "", "", err + return nil, err } - b, err := d.readSource(ctx, source, args...) + fc, err := d.readSource(ctx, source, args...) if err != nil { - return "", "", fmt.Errorf("couldn't read datasource '%s': %w", alias, err) + return nil, fmt.Errorf("couldn't read datasource '%s': %w", alias, err) } - subpath := "" - if len(args) > 0 { - subpath = args[0] - } - mimeType, err = source.mimeType(subpath) - if err != nil { - return "", "", err - } - return string(b), mimeType, nil + return fc, nil } // Include - func (d *Data) Include(alias string, args ...string) (string, error) { - data, _, err := d.readDataSource(d.Ctx, alias, args...) - return data, err + fc, err := d.readDataSource(d.Ctx, alias, args...) + if err != nil { + return "", err + } + + return string(fc.b), err } // Datasource - func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { - data, mimeType, err := d.readDataSource(d.Ctx, alias, args...) + fc, err := d.readDataSource(d.Ctx, alias, args...) if err != nil { return nil, err } - return parseData(mimeType, data) -} - -func parseData(mimeType, s string) (out interface{}, err error) { - switch mimeAlias(mimeType) { - case jsonMimetype: - out, err = JSON(s) - if err != nil { - // maybe it's a JSON array - out, err = JSONArray(s) - } - case jsonArrayMimetype: - out, err = JSONArray(s) - case yamlMimetype: - out, err = YAML(s) - if err != nil { - // maybe it's a YAML array - out, err = YAMLArray(s) - } - case csvMimetype: - out, err = CSV(s) - case tomlMimetype: - out, err = TOML(s) - case envMimetype: - out, err = dotEnv(s) - case textMimetype: - out = s - case cueMimetype: - out, err = CUE(s) - default: - return nil, fmt.Errorf("datasources of type %s not yet supported", mimeType) - } - return out, err + return parsers.ParseData(fc.contentType, string(fc.b)) } // DatasourceReachable - Determines if the named datasource is reachable with @@ -370,9 +207,9 @@ func (d *Data) DatasourceReachable(alias string, args ...string) bool { // readSource returns the (possibly cached) data from the given source, // as referenced by the given args -func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ([]byte, error) { +func (d *Data) readSource(ctx context.Context, source *Source, args ...string) (*fileContent, error) { if d.cache == nil { - d.cache = make(map[string][]byte) + d.cache = make(map[string]*fileContent) } cacheKey := source.Alias for _, v := range args { @@ -382,16 +219,107 @@ func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ( if ok { return cached, nil } - r, err := d.lookupReader(source.URL.Scheme) - if err != nil { - return nil, fmt.Errorf("Datasource not yet supported") + + arg := "" + if len(args) > 0 { + arg = args[0] } - data, err := r(ctx, source, args...) + u, err := resolveURL(source.URL, arg) if err != nil { return nil, err } - d.cache[cacheKey] = data - return data, nil + + fc, err := d.readFileContent(ctx, u, source.Header) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", u, err) + } + d.cache[cacheKey] = fc + return fc, nil +} + +// readFileContent returns content from the given URL +func (d Data) readFileContent(ctx context.Context, u *url.URL, hdr http.Header) (*fileContent, error) { + fsys, err := datafs.FSysForPath(ctx, u.String()) + if err != nil { + return nil, fmt.Errorf("fsys for path %v: %w", u, err) + } + + u, fname := datafs.SplitFSMuxURL(u) + + // need to support absolute paths on local filesystem too + // TODO: this is a hack, probably fix this? + if u.Scheme == "file" && runtime.GOOS != "windows" { + fname = u.Path + fname + } + + fsys = fsimpl.WithContextFS(ctx, fsys) + fsys = fsimpl.WithHeaderFS(hdr, fsys) + + // convert d.Sources to a map[string]config.DataSources + // TODO: remove this when d.Sources is removed + ds := make(map[string]config.DataSource) + for k, v := range d.Sources { + ds[k] = config.DataSource{ + URL: v.URL, + Header: v.Header, + } + } + + fsys = datafs.WithDataSourcesFS(ds, fsys) + + f, err := fsys.Open(fname) + if err != nil { + return nil, fmt.Errorf("open (url: %q, name: %q): %w", u, fname, err) + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("stat (url: %q, name: %q): %w", u, fname, err) + } + + // possible type hint in the type query param. Contrary to spec, we allow + // unescaped '+' characters to make it simpler to provide types like + // "application/array+json" + mimeType := u.Query().Get("type") + mimeType = strings.ReplaceAll(mimeType, " ", "+") + + if mimeType == "" { + mimeType = fsimpl.ContentType(fi) + } + + var data []byte + + if fi.IsDir() { + var dirents []fs.DirEntry + dirents, err = fs.ReadDir(fsys, fname) + if err != nil { + return nil, fmt.Errorf("readDir (url: %q, name: %s): %w", u, fname, err) + } + + entries := make([]string, len(dirents)) + for i, e := range dirents { + entries[i] = e.Name() + } + data, err = json.Marshal(entries) + if err != nil { + return nil, fmt.Errorf("json.Marshal: %w", err) + } + + mimeType = jsonArrayMimetype + } else { + data, err = io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("read (url: %q, name: %s): %w", u, fname, err) + } + } + + if mimeType == "" { + // default to text/plain + mimeType = textMimetype + } + + return &fileContent{contentType: mimeType, b: data}, nil } // Show all datasources - @@ -403,3 +331,62 @@ func (d *Data) ListDatasources() []string { sort.Strings(datasources) return datasources } + +// resolveURL parses the relative URL rel against base, and returns the +// resolved URL. Differs from url.ResolveReference in that query parameters are +// added. In case of duplicates, params from rel are used. +func resolveURL(base *url.URL, rel string) (*url.URL, error) { + // if there's an opaque part, there's no resolving to do - just return the + // base URL + if base.Opaque != "" { + return base, nil + } + + // git URLs are special - they have double-slashes that separate a repo + // from a path in the repo. A missing double-slash means the path is the + // root. + switch base.Scheme { + case "git", "git+file", "git+http", "git+https", "git+ssh": + if strings.Contains(base.Path, "//") && strings.Contains(rel, "//") { + return nil, fmt.Errorf("both base URL and subpath contain '//', which is not allowed in git URLs") + } + + // If there's a subpath, the base path must end with '/'. This behaviour + // is unique to git URLs - other schemes would instead drop the last + // path element and replace with the subpath. + if rel != "" && !strings.HasSuffix(base.Path, "/") { + base.Path += "/" + } + + // If subpath starts with '//', make it relative by prefixing a '.', + // otherwise it'll be treated as a schemeless URI and the first part + // will be interpreted as a hostname. + if strings.HasPrefix(rel, "//") { + rel = "." + rel + } + } + + relURL, err := url.Parse(rel) + if err != nil { + return nil, err + } + + // URL.ResolveReference requires (or assumes, at least) that the base is + // absolute. We want to support relative URLs too though, so we need to + // correct for that. + out := base.ResolveReference(relURL) + if out.Scheme == "" && out.Path[0] == '/' { + out.Path = out.Path[1:] + } + + if base.RawQuery != "" { + bq := base.Query() + rq := relURL.Query() + for k := range rq { + bq.Set(k, rq.Get(k)) + } + out.RawQuery = bq.Encode() + } + + return out, nil +} diff --git a/data/datasource_aws_sm.go b/data/datasource_aws_sm.go deleted file mode 100644 index aa42fdf9..00000000 --- a/data/datasource_aws_sm.go +++ /dev/null @@ -1,87 +0,0 @@ -package data - -import ( - "context" - "fmt" - "net/url" - "path" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - - gaws "github.com/hairyhenderson/gomplate/v4/aws" -) - -// awsSecretsManagerGetter - A subset of Secrets Manager API for use in unit testing -type awsSecretsManagerGetter interface { - GetSecretValueWithContext(ctx context.Context, input *secretsmanager.GetSecretValueInput, opts ...request.Option) (*secretsmanager.GetSecretValueOutput, error) -} - -func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { - if len(args) >= 2 { - err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", - sourceURL.Scheme, len(args)) - return nil, "", err - } - - p = sourceURL.Path - params = make(map[string]interface{}) - for key, val := range sourceURL.Query() { - params[key] = strings.Join(val, " ") - } - - if p == "" && sourceURL.Opaque != "" { - p = sourceURL.Opaque - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, "", err - } - - if parsed.Path != "" { - p = path.Join(p, parsed.Path) - if strings.HasSuffix(parsed.Path, "/") { - p += "/" - } - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - return params, p, nil -} - -func readAWSSecretsManager(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.awsSecretsManager == nil { - source.awsSecretsManager = secretsmanager.New(gaws.SDKSession()) - } - - _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - return readAWSSecretsManagerParam(ctx, source, paramPath) -} - -func readAWSSecretsManagerParam(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(paramPath), - } - - response, err := source.awsSecretsManager.GetSecretValueWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("reading aws+sm source %q: %w", source.Alias, err) - } - - if response.SecretString != nil { - return []byte(*response.SecretString), nil - } - - return response.SecretBinary, nil -} diff --git a/data/datasource_aws_sm_test.go b/data/datasource_aws_sm_test.go deleted file mode 100644 index e6e2cef3..00000000 --- a/data/datasource_aws_sm_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// DummyAWSSecretsManagerSecretGetter - test double -type DummyAWSSecretsManagerSecretGetter struct { - t *testing.T - secretValut *secretsmanager.GetSecretValueOutput - err awserr.Error - mockGetSecretValue func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) -} - -func (d DummyAWSSecretsManagerSecretGetter) GetSecretValueWithContext(_ context.Context, input *secretsmanager.GetSecretValueInput, _ ...request.Option) (*secretsmanager.GetSecretValueOutput, error) { - if d.mockGetSecretValue != nil { - output, err := d.mockGetSecretValue(input) - return output, err - } - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.secretValut, "Must provide a param if no error!") - return d.secretValut, nil -} - -func simpleAWSSecretsManagerSourceHelper(dummyGetter awsSecretsManagerGetter) *Source { - return &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "aws+sm", - Path: "/foo", - }, - awsSecretsManager: dummyGetter, - } -} - -func TestAWSSecretsManager_ParseAWSSecretsManagerArgs(t *testing.T) { - _, _, err := parseDatasourceURLArgs(mustParseURL("base"), "extra", "too many!") - assert.Error(t, err) - - tplain := map[string]interface{}{"type": "text/plain"} - - data := []struct { - eParams map[string]interface{} - u string - ePath string - args string - }{ - {u: "noddy", ePath: "noddy"}, - {u: "base", ePath: "base/extra", args: "extra"}, - {u: "/foo/", ePath: "/foo/extra", args: "/extra"}, - {u: "aws+sm:///foo", ePath: "/foo/bar", args: "bar"}, - {u: "aws+sm:foo", ePath: "foo"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar"}, - {u: "aws+sm:foo", ePath: "foo/baz", args: "baz"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar/baz", args: "baz"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar/baz", args: "baz"}, - {u: "aws+sm:///foo", ePath: "/foo/dir/", args: "dir/"}, - {u: "aws+sm:///foo/", ePath: "/foo/"}, - {u: "aws+sm:///foo/", ePath: "/foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo?type=text/plain", ePath: "foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo/bar?type=text/plain", ePath: "foo/bar/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:/foo/bar?type=text/plain", ePath: "/foo/bar/baz", args: "baz"}, - { - eParams: map[string]interface{}{ - "type": "application/json", - "param": "quux", - }, - u: "aws+sm:/foo/bar?type=text/plain", - ePath: "/foo/bar/baz/qux", - args: "baz/qux?type=application/json¶m=quux", - }, - } - - for _, d := range data { - args := []string{d.args} - if d.args == "" { - args = nil - } - params, p, err := parseDatasourceURLArgs(mustParseURL(d.u), args...) - require.NoError(t, err) - if d.eParams == nil { - assert.Empty(t, params) - } else { - assert.EqualValues(t, d.eParams, params) - } - assert.Equal(t, d.ePath, p) - } -} - -func TestAWSSecretsManager_GetParameterSetup(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - assert.Nil(t, err) -} - -func TestAWSSecretsManager_GetParameterSetupWrongArgs(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar", "/foo", "/bla") - assert.False(t, calledOk) - assert.Error(t, err) -} - -func TestAWSSecretsManager_GetParameterMissing(t *testing.T) { - expectedErr := awserr.New("ParameterNotFound", "Test of error message", nil) - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - err: expectedErr, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "") - assert.Error(t, err, "Test of error message") -} - -func TestAWSSecretsManager_ReadSecret(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - output, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - require.NoError(t, err) - assert.Equal(t, []byte("blub"), output) -} - -func TestAWSSecretsManager_ReadSecretBinary(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretBinary: []byte("supersecret")}, nil - }, - }) - - output, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - require.NoError(t, err) - assert.Equal(t, []byte("supersecret"), output) -} diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go deleted file mode 100644 index 74979129..00000000 --- a/data/datasource_awssmp.go +++ /dev/null @@ -1,77 +0,0 @@ -package data - -import ( - "context" - "fmt" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/ssm" - - gaws "github.com/hairyhenderson/gomplate/v4/aws" -) - -// awssmpGetter - A subset of SSM API for use in unit testing -type awssmpGetter interface { - GetParameterWithContext(ctx context.Context, input *ssm.GetParameterInput, opts ...request.Option) (*ssm.GetParameterOutput, error) - GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error) -} - -func readAWSSMP(ctx context.Context, source *Source, args ...string) (data []byte, err error) { - if source.asmpg == nil { - source.asmpg = ssm.New(gaws.SDKSession()) - } - - _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - source.mediaType = jsonMimetype - switch { - case strings.HasSuffix(paramPath, "/"): - source.mediaType = jsonArrayMimetype - data, err = listAWSSMPParams(ctx, source, paramPath) - default: - data, err = readAWSSMPParam(ctx, source, paramPath) - } - return data, err -} - -func readAWSSMPParam(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &ssm.GetParameterInput{ - Name: aws.String(paramPath), - WithDecryption: aws.Bool(true), - } - - response, err := source.asmpg.GetParameterWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("error reading aws+smp from AWS using GetParameter with input %v: %w", input, err) - } - - result := *response.Parameter - - output, err := ToJSON(result) - return []byte(output), err -} - -// listAWSSMPParams - supports directory semantics, returns array -func listAWSSMPParams(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &ssm.GetParametersByPathInput{ - Path: aws.String(paramPath), - } - - response, err := source.asmpg.GetParametersByPathWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("error reading aws+smp from AWS using GetParameter with input %v: %w", input, err) - } - - listing := make([]string, len(response.Parameters)) - for i, p := range response.Parameters { - listing[i] = (*p.Name)[len(paramPath):] - } - - output, err := ToJSON(listing) - return []byte(output), err -} diff --git a/data/datasource_awssmp_test.go b/data/datasource_awssmp_test.go deleted file mode 100644 index 5c5e02db..00000000 --- a/data/datasource_awssmp_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package data - -import ( - "context" - "encoding/json" - "net/url" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// DummyParamGetter - test double -type DummyParamGetter struct { - err awserr.Error - t *testing.T - param *ssm.Parameter - mockGetParameter func(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) - params []*ssm.Parameter -} - -func (d DummyParamGetter) GetParameterWithContext(_ context.Context, input *ssm.GetParameterInput, _ ...request.Option) (*ssm.GetParameterOutput, error) { - if d.mockGetParameter != nil { - output, err := d.mockGetParameter(input) - return output, err - } - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.param, "Must provide a param if no error!") - return &ssm.GetParameterOutput{ - Parameter: d.param, - }, nil -} - -func (d DummyParamGetter) GetParametersByPathWithContext(_ context.Context, _ *ssm.GetParametersByPathInput, _ ...request.Option) (*ssm.GetParametersByPathOutput, error) { - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.params, "Must provide a param if no error!") - return &ssm.GetParametersByPathOutput{ - Parameters: d.params, - }, nil -} - -func simpleAWSSourceHelper(dummy awssmpGetter) *Source { - return &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "aws+smp", - Path: "/foo", - }, - asmpg: dummy, - } -} - -func TestAWSSMP_GetParameterSetup(t *testing.T) { - calledOk := false - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - mockGetParameter: func(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { - assert.Equal(t, "/foo/bar", *input.Name) - assert.True(t, *input.WithDecryption) - calledOk = true - return &ssm.GetParameterOutput{ - Parameter: &ssm.Parameter{}, - }, nil - }, - }) - - _, err := readAWSSMP(context.Background(), s, "/bar") - assert.True(t, calledOk) - assert.Nil(t, err) -} - -func TestAWSSMP_GetParameterValidOutput(t *testing.T) { - expected := &ssm.Parameter{ - Name: aws.String("/foo"), - Type: aws.String("String"), - Value: aws.String("val"), - Version: aws.Int64(1), - } - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - param: expected, - }) - - output, err := readAWSSMP(context.Background(), s, "") - assert.Nil(t, err) - actual := &ssm.Parameter{} - err = json.Unmarshal(output, &actual) - assert.Nil(t, err) - assert.Equal(t, expected, actual) - assert.Equal(t, jsonMimetype, s.mediaType) -} - -func TestAWSSMP_GetParameterMissing(t *testing.T) { - expectedErr := awserr.New("ParameterNotFound", "Test of error message", nil) - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - err: expectedErr, - }) - - _, err := readAWSSMP(context.Background(), s, "") - assert.Error(t, err, "Test of error message") -} - -func TestAWSSMP_listAWSSMPParams(t *testing.T) { - ctx := context.Background() - s := simpleAWSSourceHelper(DummyParamGetter{ - t: t, - err: awserr.New("ParameterNotFound", "foo", nil), - }) - _, err := listAWSSMPParams(ctx, s, "") - assert.Error(t, err) - - s = simpleAWSSourceHelper(DummyParamGetter{ - t: t, - params: []*ssm.Parameter{ - {Name: aws.String("/a")}, - {Name: aws.String("/b")}, - {Name: aws.String("/c")}, - }, - }) - data, err := listAWSSMPParams(ctx, s, "/") - require.NoError(t, err) - assert.Equal(t, []byte(`["a","b","c"]`), data) - - s = simpleAWSSourceHelper(DummyParamGetter{ - t: t, - params: []*ssm.Parameter{ - {Name: aws.String("/a/a")}, - {Name: aws.String("/a/b")}, - {Name: aws.String("/a/c")}, - }, - }) - data, err = listAWSSMPParams(ctx, s, "/a/") - require.NoError(t, err) - assert.Equal(t, []byte(`["a","b","c"]`), data) -} diff --git a/data/datasource_blob.go b/data/datasource_blob.go deleted file mode 100644 index 1dac584a..00000000 --- a/data/datasource_blob.go +++ /dev/null @@ -1,173 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "mime" - "net/url" - "path" - "strings" - - gaws "github.com/hairyhenderson/gomplate/v4/aws" - "github.com/hairyhenderson/gomplate/v4/env" - - "gocloud.dev/blob" - "gocloud.dev/blob/gcsblob" - "gocloud.dev/blob/s3blob" - "gocloud.dev/gcp" -) - -func readBlob(ctx context.Context, source *Source, args ...string) (output []byte, err error) { - if len(args) >= 2 { - return nil, fmt.Errorf("maximum two arguments to blob datasource: alias, extraPath") - } - - key := source.URL.Path - if len(args) == 1 { - key = path.Join(key, args[0]) - } - - opener, err := newOpener(ctx, source.URL) - if err != nil { - return nil, err - } - - mux := blob.URLMux{} - mux.RegisterBucket(source.URL.Scheme, opener) - - u := blobURL(source.URL) - bucket, err := mux.OpenBucket(ctx, u) - if err != nil { - return nil, err - } - defer bucket.Close() - - var r func(context.Context, *blob.Bucket, string) (string, []byte, error) - if strings.HasSuffix(key, "/") { - r = listBucket - } else { - r = getBlob - } - - mediaType, data, err := r(ctx, bucket, key) - if mediaType != "" { - source.mediaType = mediaType - } - return data, err -} - -// create the correct kind of blob.BucketURLOpener for the given URL -func newOpener(ctx context.Context, u *url.URL) (opener blob.BucketURLOpener, err error) { - switch u.Scheme { - case "s3": - // set up a "regular" gomplate AWS SDK session - sess := gaws.SDKSession() - // see https://gocloud.dev/concepts/urls/#muxes - opener = &s3blob.URLOpener{ConfigProvider: sess} - case "gs": - creds, err := gcp.DefaultCredentials(ctx) - if err != nil { - return nil, fmt.Errorf("failed to retrieve GCP credentials: %w", err) - } - - client, err := gcp.NewHTTPClient( - gcp.DefaultTransport(), - gcp.CredentialsTokenSource(creds)) - if err != nil { - return nil, fmt.Errorf("failed to create GCP HTTP client: %w", err) - } - opener = &gcsblob.URLOpener{ - Client: client, - } - } - return opener, nil -} - -func getBlob(ctx context.Context, bucket *blob.Bucket, key string) (mediaType string, data []byte, err error) { - key = strings.TrimPrefix(key, "/") - attr, err := bucket.Attributes(ctx, key) - if err != nil { - return "", nil, fmt.Errorf("failed to retrieve attributes for %s: %w", key, err) - } - if attr.ContentType != "" { - mt, _, e := mime.ParseMediaType(attr.ContentType) - if e != nil { - return "", nil, e - } - mediaType = mt - } - data, err = bucket.ReadAll(ctx, key) - if err != nil { - return "", nil, fmt.Errorf("failed to read %s: %w", key, err) - } - return mediaType, data, nil -} - -// calls the bucket listing API, returning a JSON Array -func listBucket(ctx context.Context, bucket *blob.Bucket, path string) (mediaType string, data []byte, err error) { - path = strings.TrimPrefix(path, "/") - opts := &blob.ListOptions{ - Prefix: path, - Delimiter: "/", - } - li := bucket.List(opts) - keys := []string{} - for { - obj, err := li.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return "", nil, err - } - keys = append(keys, strings.TrimPrefix(obj.Key, path)) - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(keys); err != nil { - return "", nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - data = b[:len(b)-1] - return jsonArrayMimetype, data, nil -} - -// copy/sanitize the URL for the Go CDK - it doesn't like params it can't parse -func blobURL(u *url.URL) string { - out := cloneURL(u) - q := out.Query() - - for param := range q { - switch u.Scheme { - case "s3": - switch param { - case "region", "endpoint", "disableSSL", "s3ForcePathStyle": - default: - q.Del(param) - } - case "gs": - switch param { - case "access_id", "private_key_path": - default: - q.Del(param) - } - } - } - - if u.Scheme == "s3" { - // handle AWS_S3_ENDPOINT env var - endpoint := env.Getenv("AWS_S3_ENDPOINT") - if endpoint != "" { - q.Set("endpoint", endpoint) - } - } - - out.RawQuery = q.Encode() - - return out.String() -} diff --git a/data/datasource_blob_test.go b/data/datasource_blob_test.go deleted file mode 100644 index 6be8ea00..00000000 --- a/data/datasource_blob_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package data - -import ( - "bytes" - "context" - "net/http/httptest" - "net/url" - "testing" - - "github.com/johannesboyne/gofakes3" - "github.com/johannesboyne/gofakes3/backend/s3mem" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func setupTestBucket(t *testing.T) (*httptest.Server, *url.URL) { - backend := s3mem.New() - faker := gofakes3.New(backend) - ts := httptest.NewServer(faker.Server()) - - err := backend.CreateBucket("mybucket") - require.NoError(t, err) - c := "hello" - err = putFile(backend, "file1", "text/plain", c) - require.NoError(t, err) - - c = `{"value": "goodbye world"}` - err = putFile(backend, "file2", "application/json", c) - require.NoError(t, err) - - c = `value: what a world` - err = putFile(backend, "file3", "application/yaml", c) - require.NoError(t, err) - - c = `value: out of this world` - err = putFile(backend, "dir1/file1", "application/yaml", c) - require.NoError(t, err) - - c = `value: foo` - err = putFile(backend, "dir1/file2", "application/yaml", c) - require.NoError(t, err) - - u, _ := url.Parse(ts.URL) - return ts, u -} - -func putFile(backend gofakes3.Backend, file, mime, content string) error { - _, err := backend.PutObject( - "mybucket", - file, - map[string]string{"Content-Type": mime}, - bytes.NewBufferString(content), - int64(len(content)), - ) - return err -} - -func TestReadBlob(t *testing.T) { - _, err := readBlob(context.Background(), nil, "foo", "bar") - assert.Error(t, err) - - ts, u := setupTestBucket(t) - defer ts.Close() - - t.Run("no authentication", func(t *testing.T) { - t.Setenv("AWS_ANON", "true") - - d, err := NewData([]string{"-d", "data=s3://mybucket/file1?region=us-east-1&disableSSL=true&s3ForcePathStyle=true&type=text/plain&endpoint=" + u.Host}, nil) - require.NoError(t, err) - - expected := "hello" - out, err := d.Datasource("data") - require.NoError(t, err) - assert.Equal(t, expected, out) - }) - - t.Run("with authentication", func(t *testing.T) { - t.Setenv("AWS_ACCESS_KEY_ID", "fake") - t.Setenv("AWS_SECRET_ACCESS_KEY", "fake") - t.Setenv("AWS_S3_ENDPOINT", u.Host) - - d, err := NewData([]string{"-d", "data=s3://mybucket/file2?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - require.NoError(t, err) - - var expected interface{} - expected = map[string]interface{}{"value": "goodbye world"} - out, err := d.Datasource("data") - require.NoError(t, err) - assert.Equal(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - require.NoError(t, err) - - expected = []interface{}{"dir1/", "file1", "file2", "file3"} - out, err = d.Datasource("data") - require.NoError(t, err) - assert.EqualValues(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/dir1/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - require.NoError(t, err) - - expected = []interface{}{"file1", "file2"} - out, err = d.Datasource("data") - require.NoError(t, err) - assert.EqualValues(t, expected, out) - }) -} - -func TestBlobURL(t *testing.T) { - data := []struct { - in string - expected string - }{ - {"s3://foo/bar/baz", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?type=hello/world", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?region=us-east-1", "s3://foo/bar/baz?region=us-east-1"}, - {"s3://foo/bar/baz?disableSSL=true&type=text/csv", "s3://foo/bar/baz?disableSSL=true"}, - {"s3://foo/bar/baz?type=text/csv&s3ForcePathStyle=true&endpoint=1.2.3.4", "s3://foo/bar/baz?endpoint=1.2.3.4&s3ForcePathStyle=true"}, - {"gs://foo/bar/baz", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?type=foo/bar", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?access_id=123", "gs://foo/bar/baz?access_id=123"}, - {"gs://foo/bar/baz?private_key_path=/foo/bar", "gs://foo/bar/baz?private_key_path=%2Ffoo%2Fbar"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar", "gs://foo/bar/baz?private_key_path=key.json"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar&access_id=abcd", "gs://foo/bar/baz?access_id=abcd&private_key_path=key.json"}, - } - - for _, d := range data { - u, _ := url.Parse(d.in) - out := blobURL(u) - assert.Equal(t, d.expected, out) - } -} diff --git a/data/datasource_consul.go b/data/datasource_consul.go deleted file mode 100644 index ecc7e516..00000000 --- a/data/datasource_consul.go +++ /dev/null @@ -1,39 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/hairyhenderson/gomplate/v4/libkv" -) - -func readConsul(_ context.Context, source *Source, args ...string) (data []byte, err error) { - if source.kv == nil { - source.kv, err = libkv.NewConsul(source.URL) - if err != nil { - return nil, err - } - err = source.kv.Login() - if err != nil { - return nil, err - } - } - - p := source.URL.Path - if len(args) == 1 { - p = strings.TrimRight(p, "/") + "/" + args[0] - } - - if strings.HasSuffix(p, "/") { - source.mediaType = jsonArrayMimetype - data, err = source.kv.List(p) - } else { - data, err = source.kv.Read(p) - } - - if err != nil { - return nil, err - } - - return data, nil -} diff --git a/data/datasource_env.go b/data/datasource_env.go deleted file mode 100644 index e5bf180f..00000000 --- a/data/datasource_env.go +++ /dev/null @@ -1,20 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/hairyhenderson/gomplate/v4/env" -) - -//nolint:unparam -func readEnv(_ context.Context, source *Source, _ ...string) (b []byte, err error) { - n := source.URL.Path - n = strings.TrimPrefix(n, "/") - if n == "" { - n = source.URL.Opaque - } - - b = []byte(env.Getenv(n)) - return b, nil -} diff --git a/data/datasource_env_test.go b/data/datasource_env_test.go deleted file mode 100644 index 6512578c..00000000 --- a/data/datasource_env_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func mustParseURL(in string) *url.URL { - u, _ := url.Parse(in) - return u -} - -func TestReadEnv(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - t.Setenv("HELLO_WORLD", "hello world") - t.Setenv("HELLO_UNIVERSE", "hello universe") - - source := &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD")} - - actual, err := readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:/HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) -} diff --git a/data/datasource_file.go b/data/datasource_file.go deleted file mode 100644 index f5c764fe..00000000 --- a/data/datasource_file.go +++ /dev/null @@ -1,87 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/fs" - "net/url" - "path/filepath" - "strings" - - "github.com/hairyhenderson/gomplate/v4/internal/datafs" -) - -func readFile(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.fs == nil { - fsp := datafs.FSProviderFromContext(ctx) - fsys, err := fsp.New(source.URL) - if err != nil { - return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", source.URL, err) - } - source.fs = fsys - } - - p := filepath.FromSlash(source.URL.Path) - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, err - } - - if parsed.Path != "" { - p = filepath.Join(p, parsed.Path) - } - - // reset the media type - it may have been set by a parent dir read - source.mediaType = "" - } - - isDir := strings.HasSuffix(p, string(filepath.Separator)) - if strings.HasSuffix(p, string(filepath.Separator)) { - p = p[:len(p)-1] - } - - // make sure we can access the file - i, err := fs.Stat(source.fs, p) - if err != nil { - return nil, fmt.Errorf("stat %s: %w", p, err) - } - - if isDir { - source.mediaType = jsonArrayMimetype - if i.IsDir() { - return readFileDir(source, p) - } - return nil, fmt.Errorf("%s is not a directory", p) - } - - b, err := fs.ReadFile(source.fs, p) - if err != nil { - return nil, fmt.Errorf("readFile %s: %w", p, err) - } - return b, nil -} - -func readFileDir(source *Source, p string) ([]byte, error) { - names, err := fs.ReadDir(source.fs, p) - if err != nil { - return nil, err - } - - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} diff --git a/data/datasource_file_test.go b/data/datasource_file_test.go deleted file mode 100644 index 7ad1c2a0..00000000 --- a/data/datasource_file_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package data - -import ( - "context" - "io/fs" - "testing" - "testing/fstest" - - "github.com/hairyhenderson/gomplate/v4/internal/datafs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadFile(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - - fsys := datafs.WrapWdFS(fstest.MapFS{ - "tmp": {Mode: fs.ModeDir | 0o777}, - "tmp/foo": {Data: content}, - "tmp/partial": {Mode: fs.ModeDir | 0o777}, - "tmp/partial/foo.txt": {Data: content}, - "tmp/partial/bar.txt": {}, - "tmp/partial/baz.txt": {}, - }) - - source := &Source{Alias: "foo", URL: mustParseURL("file:///tmp/foo")} - source.fs = fsys - - actual, err := readFile(ctx, source) - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "bogus", URL: mustParseURL("file:///bogus")} - source.fs = fsys - _, err = readFile(ctx, source) - assert.Error(t, err) - - source = &Source{Alias: "partial", URL: mustParseURL("file:///tmp/partial")} - source.fs = fsys - actual, err = readFile(ctx, source, "foo.txt") - require.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/")} - source.fs = fsys - actual, err = readFile(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fsys - actual, err = readFile(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - mime, err := source.mimeType("") - require.NoError(t, err) - assert.Equal(t, "application/json", mime) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fsys - actual, err = readFile(ctx, source, "foo.txt") - require.NoError(t, err) - assert.Equal(t, content, actual) - mime, err = source.mimeType("") - require.NoError(t, err) - assert.Equal(t, "application/json", mime) -} diff --git a/data/datasource_git.go b/data/datasource_git.go deleted file mode 100644 index 3859d344..00000000 --- a/data/datasource_git.go +++ /dev/null @@ -1,328 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/url" - "os" - "path" - "path/filepath" - "strings" - - "github.com/hairyhenderson/gomplate/v4/base64" - "github.com/hairyhenderson/gomplate/v4/env" - "github.com/rs/zerolog" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/memory" -) - -func readGit(ctx context.Context, source *Source, args ...string) ([]byte, error) { - g := gitsource{} - - u := source.URL - repoURL, path, err := g.parseGitPath(u, args...) - if err != nil { - return nil, err - } - - depth := 1 - if u.Scheme == "git+file" { - // we can't do shallow clones for filesystem repos apparently - depth = 0 - } - - fs, _, err := g.clone(ctx, repoURL, depth) - if err != nil { - return nil, err - } - - mimeType, out, err := g.read(fs, path) - if mimeType != "" { - source.mediaType = mimeType - } - return out, err -} - -type gitsource struct{} - -func (g gitsource) parseArgURL(arg string) (u *url.URL, err error) { - if strings.HasPrefix(arg, "//") { - u, err = url.Parse(arg[1:]) - u.Path = "/" + u.Path - } else { - u, err = url.Parse(arg) - } - - if err != nil { - return nil, fmt.Errorf("failed to parse arg %s: %w", arg, err) - } - return u, err -} - -func (g gitsource) parseQuery(orig, arg *url.URL) string { - q := orig.Query() - pq := arg.Query() - for k, vs := range pq { - for _, v := range vs { - q.Add(k, v) - } - } - return q.Encode() -} - -func (g gitsource) parseArgPath(u *url.URL, arg string) (repo, p string) { - // if the source URL already specified a repo and subpath, the whole - // arg is interpreted as subpath - if strings.Contains(u.Path, "//") || strings.HasPrefix(arg, "//") { - return "", arg - } - - parts := strings.SplitN(arg, "//", 2) - repo = parts[0] - if len(parts) == 2 { - p = "/" + parts[1] - } - return repo, p -} - -// Massage the URL and args together to produce the repo to clone, -// and the path to read. -// The path is delimited from the repo by '//' -func (g gitsource) parseGitPath(u *url.URL, args ...string) (out *url.URL, p string, err error) { - if u == nil { - return nil, "", fmt.Errorf("parseGitPath: no url provided (%v)", u) - } - // copy the input url so we can modify it - out = cloneURL(u) - - parts := strings.SplitN(out.Path, "//", 2) - switch len(parts) { - case 1: - p = "/" - case 2: - p = "/" + parts[1] - - i := strings.LastIndex(out.Path, p) - out.Path = out.Path[:i-1] - } - - if len(args) > 0 { - argURL, uerr := g.parseArgURL(args[0]) - if uerr != nil { - return nil, "", uerr - } - repo, argpath := g.parseArgPath(u, argURL.Path) - out.Path = path.Join(out.Path, repo) - p = path.Join(p, argpath) - - out.RawQuery = g.parseQuery(u, argURL) - - if argURL.Fragment != "" { - out.Fragment = argURL.Fragment - } - } - return out, p, err -} - -//nolint:interfacer -func cloneURL(u *url.URL) *url.URL { - out, _ := url.Parse(u.String()) - return out -} - -// refFromURL - extract the ref from the URL fragment if present -func (g gitsource) refFromURL(u *url.URL) plumbing.ReferenceName { - switch { - case strings.HasPrefix(u.Fragment, "refs/"): - return plumbing.ReferenceName(u.Fragment) - case u.Fragment != "": - return plumbing.NewBranchReferenceName(u.Fragment) - default: - return plumbing.ReferenceName("") - } -} - -// refFromRemoteHead - extract the ref from the remote HEAD, to work around -// hard-coded 'master' default branch in go-git. -// Should be unnecessary once https://github.com/go-git/go-git/issues/249 is -// fixed. -func (g gitsource) refFromRemoteHead(ctx context.Context, u *url.URL, auth transport.AuthMethod) (plumbing.ReferenceName, error) { - e, err := transport.NewEndpoint(u.String()) - if err != nil { - return "", err - } - - cli, err := client.NewClient(e) - if err != nil { - return "", err - } - - s, err := cli.NewUploadPackSession(e, auth) - if err != nil { - return "", err - } - - info, err := s.AdvertisedReferencesContext(ctx) - if err != nil { - return "", err - } - - refs, err := info.AllReferences() - if err != nil { - return "", err - } - - headRef, ok := refs["HEAD"] - if !ok { - return "", fmt.Errorf("no HEAD ref found") - } - - return headRef.Target(), nil -} - -// clone a repo for later reading through http(s), git, or ssh. u must be the URL to the repo -// itself, and must have any file path stripped -func (g gitsource) clone(ctx context.Context, repoURL *url.URL, depth int) (billy.Filesystem, *git.Repository, error) { - fs := memfs.New() - storer := memory.NewStorage() - - // preserve repoURL by cloning it - u := cloneURL(repoURL) - - auth, err := g.auth(u) - if err != nil { - return nil, nil, err - } - - if strings.HasPrefix(u.Scheme, "git+") { - scheme := u.Scheme[len("git+"):] - u.Scheme = scheme - } - - ref := g.refFromURL(u) - u.Fragment = "" - u.RawQuery = "" - - // attempt to get the ref from the remote so we don't default to master - if ref == "" { - ref, err = g.refFromRemoteHead(ctx, u, auth) - if err != nil { - zerolog.Ctx(ctx).Warn(). - Stringer("repoURL", u). - Err(err). - Msg("failed to get ref from remote, using default") - } - } - - opts := &git.CloneOptions{ - URL: u.String(), - Auth: auth, - Depth: depth, - ReferenceName: ref, - SingleBranch: true, - Tags: git.NoTags, - } - repo, err := git.CloneContext(ctx, storer, fs, opts) - if u.Scheme == "file" && err == transport.ErrRepositoryNotFound && !strings.HasSuffix(u.Path, ".git") { - // maybe this has a `.git` subdirectory... - u = cloneURL(repoURL) - u.Path = path.Join(u.Path, ".git") - return g.clone(ctx, u, depth) - } - if err != nil { - return nil, nil, fmt.Errorf("git clone %s: %w", u, err) - } - return fs, repo, nil -} - -// read - reads the provided path out of a git repo -func (g gitsource) read(fsys billy.Filesystem, path string) (string, []byte, error) { - fi, err := fsys.Stat(path) - if err != nil { - return "", nil, fmt.Errorf("can't stat %s: %w", path, err) - } - if fi.IsDir() || strings.HasSuffix(path, string(filepath.Separator)) { - out, rerr := g.readDir(fsys, path) - return jsonArrayMimetype, out, rerr - } - - f, err := fsys.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return "", nil, fmt.Errorf("can't open %s: %w", path, err) - } - - b, err := io.ReadAll(f) - if err != nil { - return "", nil, fmt.Errorf("can't read %s: %w", path, err) - } - - return "", b, nil -} - -func (g gitsource) readDir(fs billy.Filesystem, path string) ([]byte, error) { - names, err := fs.ReadDir(path) - if err != nil { - return nil, fmt.Errorf("couldn't read dir %s: %w", path, err) - } - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} - -/* -auth methods: -- ssh named key (no password support) - - GIT_SSH_KEY (base64-encoded) or GIT_SSH_KEY_FILE (base64-encoded, or not) - -- ssh agent auth (preferred) -- http basic auth (for github, gitlab, bitbucket tokens) -- http token auth (bearer token, somewhat unusual) -*/ -func (g gitsource) auth(u *url.URL) (auth transport.AuthMethod, err error) { - user := u.User.Username() - switch u.Scheme { - case "git+http", "git+https": - if pass, ok := u.User.Password(); ok { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if pass := env.Getenv("GIT_HTTP_PASSWORD"); pass != "" { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if tok := env.Getenv("GIT_HTTP_TOKEN"); tok != "" { - // note docs on TokenAuth - this is rarely to be used - auth = &http.TokenAuth{Token: tok} - } - case "git+ssh": - k := env.Getenv("GIT_SSH_KEY") - if k != "" { - var key []byte - key, err = base64.Decode(k) - if err != nil { - key = []byte(k) - } - auth, err = ssh.NewPublicKeys(user, key, "") - } else { - auth, err = ssh.NewSSHAgentAuth(user) - } - } - return auth, err -} diff --git a/data/datasource_git_test.go b/data/datasource_git_test.go deleted file mode 100644 index 3b187ecc..00000000 --- a/data/datasource_git_test.go +++ /dev/null @@ -1,551 +0,0 @@ -package data - -import ( - "context" - "encoding/base64" - "fmt" - "io" - "net/url" - "os" - "strings" - "testing" - "time" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-billy/v5/osfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/server" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/filesystem" - - "golang.org/x/crypto/ssh/testdata" - - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -func TestParseArgPath(t *testing.T) { - t.Parallel() - g := gitsource{} - - data := []struct { - url string - arg string - repo, path string - }{ - { - "git+file:///foo//foo", - "/bar", - "", "/bar", - }, - { - "git+file:///foo//bar", - "/baz//qux", - "", "/baz//qux", - }, - { - "git+https://example.com/foo", - "/bar", - "/bar", "", - }, - { - "git+https://example.com/foo", - "//bar", - "", "//bar", - }, - { - "git+https://example.com/foo//bar", - "//baz", - "", "//baz", - }, - { - "git+https://example.com/foo", - "/bar//baz", - "/bar", "/baz", - }, - { - "git+https://example.com/foo?type=t", - "/bar//baz", - "/bar", "/baz", - }, - { - "git+https://example.com/foo#master", - "/bar//baz", - "/bar", "/baz", - }, - { - "git+https://example.com/foo", - "//bar", - "", "//bar", - }, - { - "git+https://example.com/foo?type=t", - "//baz", - "", "//baz", - }, - { - "git+https://example.com/foo?type=t#v1", - "//bar", - "", "//bar", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.arg, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - repo, path := g.parseArgPath(u, d.arg) - assert.Equal(t, d.repo, repo) - assert.Equal(t, d.path, path) - }) - } -} - -func TestParseGitPath(t *testing.T) { - t.Parallel() - g := gitsource{} - _, _, err := g.parseGitPath(nil) - assert.ErrorContains(t, err, "") - - u := mustParseURL("http://example.com//foo") - assert.Equal(t, "//foo", u.Path) - parts := strings.SplitN(u.Path, "//", 2) - assert.Equal(t, 2, len(parts)) - assert.DeepEqual(t, []string{"", "foo"}, parts) - - data := []struct { - url string - args string - repo, path string - }{ - { - "git+https://github.com/hairyhenderson/gomplate//docs-src/content/functions/aws.yml", - "", - "git+https://github.com/hairyhenderson/gomplate", - "/docs-src/content/functions/aws.yml", - }, - { - "git+ssh://github.com/hairyhenderson/gomplate.git", - "", - "git+ssh://github.com/hairyhenderson/gomplate.git", - "/", - }, - { - "https://github.com", - "", - "https://github.com", - "/", - }, - { - "git://example.com/foo//file.txt#someref", - "", - "git://example.com/foo#someref", "/file.txt", - }, - { - "git+file:///home/foo/repo//file.txt#someref", - "", - "git+file:///home/foo/repo#someref", "/file.txt", - }, - { - "git+file:///repo", - "", - "git+file:///repo", "/", - }, - { - "git+file:///foo//foo", - "", - "git+file:///foo", "/foo", - }, - { - "git+file:///foo//foo", - "/bar", - "git+file:///foo", "/foo/bar", - }, - { - "git+file:///foo//bar", - // in this case the // is meaningless - "/baz//qux", - "git+file:///foo", "/bar/baz/qux", - }, - { - "git+https://example.com/foo", - "/bar", - "git+https://example.com/foo/bar", "/", - }, - { - "git+https://example.com/foo", - "//bar", - "git+https://example.com/foo", "/bar", - }, - { - "git+https://example.com//foo", - "/bar", - "git+https://example.com", "/foo/bar", - }, - { - "git+https://example.com/foo//bar", - "//baz", - "git+https://example.com/foo", "/bar/baz", - }, - { - "git+https://example.com/foo", - "/bar//baz", - "git+https://example.com/foo/bar", "/baz", - }, - { - "git+https://example.com/foo?type=t", - "/bar//baz", - "git+https://example.com/foo/bar?type=t", "/baz", - }, - { - "git+https://example.com/foo#master", - "/bar//baz", - "git+https://example.com/foo/bar#master", "/baz", - }, - { - "git+https://example.com/foo", - "/bar//baz?type=t", - "git+https://example.com/foo/bar?type=t", "/baz", - }, - { - "git+https://example.com/foo", - "/bar//baz#master", - "git+https://example.com/foo/bar#master", "/baz", - }, - { - "git+https://example.com/foo", - "//bar?type=t", - "git+https://example.com/foo?type=t", "/bar", - }, - { - "git+https://example.com/foo", - "//bar#master", - "git+https://example.com/foo#master", "/bar", - }, - { - "git+https://example.com/foo?type=t", - "//bar#master", - "git+https://example.com/foo?type=t#master", "/bar", - }, - { - "git+https://example.com/foo?type=t#v1", - "//bar?type=j#v2", - "git+https://example.com/foo?type=t&type=j#v2", "/bar", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.args, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - args := []string{d.args} - if d.args == "" { - args = nil - } - repo, path, err := g.parseGitPath(u, args...) - assert.NilError(t, err) - assert.Equal(t, d.repo, repo.String()) - assert.Equal(t, d.path, path) - }) - } -} - -func TestReadGitRepo(t *testing.T) { - g := gitsource{} - fs := setupGitRepo(t) - fs, err := fs.Chroot("/repo") - assert.NilError(t, err) - - _, _, err = g.read(fs, "/bogus") - assert.ErrorContains(t, err, "can't stat /bogus") - - mtype, out, err := g.read(fs, "/foo") - assert.NilError(t, err) - assert.Equal(t, `["bar"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar") - assert.NilError(t, err) - assert.Equal(t, `["hi.txt"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar/hi.txt") - assert.NilError(t, err) - assert.Equal(t, `hello world`, string(out)) - assert.Equal(t, "", mtype) -} - -var testHashes = map[string]string{} - -func setupGitRepo(t *testing.T) billy.Filesystem { - fs := memfs.New() - fs.MkdirAll("/repo/.git", os.ModeDir) - repo, _ := fs.Chroot("/repo") - dot, _ := repo.Chroot("/.git") - s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - - r, err := git.Init(s, repo) - assert.NilError(t, err) - - // default to main - h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main")) - err = s.SetReference(h) - assert.NilError(t, err) - - // config needs to be created after setting up a "normal" fs repo - // this is possibly a bug in git-go? - c, err := r.Config() - assert.NilError(t, err) - - c.Init.DefaultBranch = "main" - - s.SetConfig(c) - assert.NilError(t, err) - - w, err := r.Worktree() - assert.NilError(t, err) - - repo.MkdirAll("/foo/bar", os.ModeDir) - f, err := repo.Create("/foo/bar/hi.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("hello world")) - assert.NilError(t, err) - _, err = w.Add(f.Name()) - assert.NilError(t, err) - hash, err := w.Commit("initial commit", &git.CommitOptions{Author: &object.Signature{}}) - assert.NilError(t, err) - - ref, err := r.CreateTag("v1", hash, nil) - assert.NilError(t, err) - testHashes["v1"] = hash.String() - - branchName := plumbing.NewBranchReferenceName("mybranch") - err = w.Checkout(&git.CheckoutOptions{ - Branch: branchName, - Hash: ref.Hash(), - Create: true, - }) - assert.NilError(t, err) - - f, err = repo.Create("/secondfile.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("another file\n")) - assert.NilError(t, err) - n := f.Name() - _, err = w.Add(n) - assert.NilError(t, err) - hash, err = w.Commit("second commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - }, - }) - ref = plumbing.NewHashReference(branchName, hash) - assert.NilError(t, err) - testHashes["mybranch"] = ref.Hash().String() - - // make the repo dirty - _, err = f.Write([]byte("dirty file")) - assert.NilError(t, err) - - // set up a bare repo - fs.MkdirAll("/bare.git", os.ModeDir) - fs.MkdirAll("/barewt", os.ModeDir) - repo, _ = fs.Chroot("/barewt") - dot, _ = fs.Chroot("/bare.git") - s = filesystem.NewStorage(dot, nil) - - r, err = git.Init(s, repo) - assert.NilError(t, err) - - w, err = r.Worktree() - assert.NilError(t, err) - - f, err = repo.Create("/hello.txt") - assert.NilError(t, err) - f.Write([]byte("hello world")) - w.Add(f.Name()) - _, err = w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - Email: "john@doe.org", - When: time.Now(), - }, - }) - assert.NilError(t, err) - - return fs -} - -func overrideFSLoader(fs billy.Filesystem) { - l := server.NewFilesystemLoader(fs) - client.InstallProtocol("file", server.NewClient(l)) -} - -func TestOpenFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fsys, _, err := g.clone(ctx, mustParseURL("git+file:///repo"), 0) - assert.NilError(t, err) - - f, err := fsys.Open("/foo/bar/hi.txt") - assert.NilError(t, err) - b, _ := io.ReadAll(f) - assert.Equal(t, "hello world", string(b)) - - _, repo, err := g.clone(ctx, mustParseURL("git+file:///repo#main"), 0) - assert.NilError(t, err) - - ref, err := repo.Reference(plumbing.NewBranchReferenceName("main"), true) - assert.NilError(t, err) - assert.Equal(t, "refs/heads/main", ref.Name().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo#refs/tags/v1"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, testHashes["v1"], ref.Hash().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo/#mybranch"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, "refs/heads/mybranch", ref.Name().String()) - assert.Equal(t, testHashes["mybranch"], ref.Hash().String()) -} - -func TestOpenBareFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fsys, _, err := g.clone(ctx, mustParseURL("git+file:///bare.git"), 0) - assert.NilError(t, err) - - f, err := fsys.Open("/hello.txt") - assert.NilError(t, err) - b, _ := io.ReadAll(f) - assert.Equal(t, "hello world", string(b)) -} - -func TestReadGit(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - s := &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git//hello.txt"), - } - b, err := readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "hello world", string(b)) - - s = &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git"), - } - b, err = readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "application/array+json", s.mediaType) - assert.Equal(t, `["hello.txt"]`, string(b)) -} - -func TestGitAuth(t *testing.T) { - g := gitsource{} - a, err := g.auth(mustParseURL("git+file:///bare.git")) - assert.NilError(t, err) - assert.Equal(t, nil, a) - - a, err = g.auth(mustParseURL("git+https://example.com/foo")) - assert.NilError(t, err) - assert.Assert(t, is.Nil(a)) - - a, err = g.auth(mustParseURL("git+https://user:swordfish@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - - t.Setenv("GIT_HTTP_PASSWORD", "swordfish") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - os.Unsetenv("GIT_HTTP_PASSWORD") - - t.Setenv("GIT_HTTP_TOKEN", "mytoken") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.TokenAuth{Token: "mytoken"}, a) - os.Unsetenv("GIT_HTTP_TOKEN") - - if os.Getenv("SSH_AUTH_SOCK") == "" { - t.Log("no SSH_AUTH_SOCK - skipping ssh agent test") - } else { - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - sa, ok := a.(*ssh.PublicKeysCallback) - assert.Equal(t, true, ok) - assert.Equal(t, "git", sa.User) - } - - t.Run("plain string ed25519", func(t *testing.T) { - key := string(testdata.PEMBytes["ed25519"]) - t.Setenv("GIT_SSH_KEY", key) - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok := a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - }) - - t.Run("base64 ed25519", func(t *testing.T) { - key := base64.StdEncoding.EncodeToString(testdata.PEMBytes["ed25519"]) - t.Setenv("GIT_SSH_KEY", key) - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok := a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - os.Unsetenv("GIT_SSH_KEY") - }) -} - -func TestRefFromURL(t *testing.T) { - t.Parallel() - g := gitsource{} - data := []struct { - url, expected string - }{ - {"git://localhost:1234/foo/bar.git//baz", ""}, - {"git+http://localhost:1234/foo/bar.git//baz", ""}, - {"git+ssh://localhost:1234/foo/bar.git//baz", ""}, - {"git+file:///foo/bar.git//baz", ""}, - {"git://localhost:1234/foo/bar.git//baz#master", "refs/heads/master"}, - {"git+http://localhost:1234/foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - {"git+ssh://localhost:1234/foo/bar.git//baz#refs/tags/foo", "refs/tags/foo"}, - {"git+file:///foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - } - - for _, d := range data { - out := g.refFromURL(mustParseURL(d.url)) - assert.Equal(t, plumbing.ReferenceName(d.expected), out) - } -} diff --git a/data/datasource_http.go b/data/datasource_http.go deleted file mode 100644 index 23c7dc36..00000000 --- a/data/datasource_http.go +++ /dev/null @@ -1,62 +0,0 @@ -package data - -import ( - "context" - "fmt" - "io" - "mime" - "net/http" - "net/url" - "time" -) - -func buildURL(base *url.URL, args ...string) (*url.URL, error) { - if len(args) == 0 { - return base, nil - } - p, err := url.Parse(args[0]) - if err != nil { - return nil, fmt.Errorf("bad sub-path %s: %w", args[0], err) - } - return base.ResolveReference(p), nil -} - -func readHTTP(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.hc == nil { - source.hc = &http.Client{Timeout: time.Second * 5} - } - u, err := buildURL(source.URL, args...) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - req.Header = source.Header - res, err := source.hc.Do(req) - if err != nil { - return nil, err - } - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - err = res.Body.Close() - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - err := fmt.Errorf("unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body)) - return nil, err - } - ctypeHdr := res.Header.Get("Content-Type") - if ctypeHdr != "" { - mediatype, _, e := mime.ParseMediaType(ctypeHdr) - if e != nil { - return nil, e - } - source.mediaType = mediatype - } - return body, nil -} diff --git a/data/datasource_http_test.go b/data/datasource_http_test.go deleted file mode 100644 index 90c4a7f9..00000000 --- a/data/datasource_http_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package data - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func must(r interface{}, err error) interface{} { - if err != nil { - panic(err) - } - return r -} - -func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", mimetype) - w.WriteHeader(code) - if body == "" { - // mirror back the headers - fmt.Fprintln(w, must(marshalObj(r.Header, json.Marshal))) - } else { - fmt.Fprintln(w, body) - } - })) - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - }, - } - - return server, client -} - -func TestHTTPFile(t *testing.T) { - server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`) - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - } - data := &Data{ - Ctx: context.Background(), - Sources: sources, - } - - expected := map[string]interface{}{ - "hello": "world", - } - - actual, err := data.Datasource("foo") - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - actual, err = data.Datasource(server.URL) - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestHTTPFileWithHeaders(t *testing.T) { - server, client := setupHTTP(200, jsonMimetype, "") - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - Header: http.Header{ - "Foo": {"bar"}, - "foo": {"baz"}, - "User-Agent": {}, - "Accept-Encoding": {"test"}, - }, - } - data := &Data{ - Ctx: context.Background(), - Sources: sources, - } - expected := http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - } - actual, err := data.Datasource("foo") - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - expected = http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - "User-Agent": {"Go-http-client/1.1"}, - } - data = &Data{ - Ctx: context.Background(), - Sources: sources, - ExtraHeaders: map[string]http.Header{server.URL: expected}, - } - actual, err = data.Datasource(server.URL) - require.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestBuildURL(t *testing.T) { - expected := "https://example.com/index.html" - base := mustParseURL(expected) - u, err := buildURL(base) - require.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/index.html" - base = mustParseURL("https://example.com") - u, err = buildURL(base, "index.html") - require.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/a/b/c/index.html" - base = mustParseURL("https://example.com/a/") - u, err = buildURL(base, "b/c/index.html") - require.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/bar/baz/index.html" - base = mustParseURL("https://example.com/foo") - u, err = buildURL(base, "bar/baz/index.html") - require.NoError(t, err) - assert.Equal(t, expected, u.String()) -} diff --git a/data/datasource_merge.go b/data/datasource_merge.go deleted file mode 100644 index b0d8b6bd..00000000 --- a/data/datasource_merge.go +++ /dev/null @@ -1,100 +0,0 @@ -package data - -import ( - "context" - "fmt" - "strings" - - "github.com/hairyhenderson/gomplate/v4/coll" - "github.com/hairyhenderson/gomplate/v4/internal/datafs" -) - -// readMerge demultiplexes a `merge:` datasource. The 'args' parameter currently -// has no meaning for this source. -// -// URI format is 'merge:<source 1>|<source 2>[|<source n>...]' where `<source #>` -// is a supported URI or a pre-defined alias name. -// -// Query strings and fragments are interpreted relative to the merged data, not -// the source data. To merge datasources with query strings or fragments, define -// separate sources first and specify the alias names. HTTP headers are also not -// supported directly. -func (d *Data) readMerge(ctx context.Context, source *Source, _ ...string) ([]byte, error) { - opaque := source.URL.Opaque - parts := strings.Split(opaque, "|") - if len(parts) < 2 { - return nil, fmt.Errorf("need at least 2 datasources to merge") - } - data := make([]map[string]interface{}, len(parts)) - for i, part := range parts { - // supports either URIs or aliases - subSource, err := d.lookupSource(part) - if err != nil { - // maybe it's a relative filename? - u, uerr := datafs.ParseSourceURL(part) - if uerr != nil { - return nil, uerr - } - subSource = &Source{ - Alias: part, - URL: u, - } - } - subSource.inherit(source) - - b, err := d.readSource(ctx, subSource) - if err != nil { - return nil, fmt.Errorf("couldn't read datasource '%s': %w", part, err) - } - - mimeType, err := subSource.mimeType("") - if err != nil { - return nil, fmt.Errorf("failed to read datasource %s: %w", subSource.URL, err) - } - - data[i], err = parseMap(mimeType, string(b)) - if err != nil { - return nil, err - } - } - - // Merge the data together - b, err := mergeData(data) - if err != nil { - return nil, err - } - - source.mediaType = yamlMimetype - return b, nil -} - -func mergeData(data []map[string]interface{}) (out []byte, err error) { - dst := data[0] - data = data[1:] - - dst, err = coll.Merge(dst, data...) - if err != nil { - return nil, err - } - - s, err := ToYAML(dst) - if err != nil { - return nil, err - } - return []byte(s), nil -} - -func parseMap(mimeType, data string) (map[string]interface{}, error) { - datum, err := parseData(mimeType, data) - if err != nil { - return nil, err - } - var m map[string]interface{} - switch datum := datum.(type) { - case map[string]interface{}: - m = datum - default: - return nil, fmt.Errorf("unexpected data type '%T' for datasource (type %s); merge: can only merge maps", datum, mimeType) - } - return m, nil -} diff --git a/data/datasource_merge_test.go b/data/datasource_merge_test.go deleted file mode 100644 index 48d1f85e..00000000 --- a/data/datasource_merge_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package data - -import ( - "context" - "io/fs" - "net/url" - "os" - "path" - "path/filepath" - "testing" - "testing/fstest" - - "github.com/hairyhenderson/gomplate/v4/internal/datafs" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadMerge(t *testing.T) { - ctx := context.Background() - - jsonContent := `{"hello": "world"}` - yamlContent := "hello: earth\ngoodnight: moon\n" - arrayContent := `["hello", "world"]` - - mergedContent := "goodnight: moon\nhello: world\n" - - wd, _ := os.Getwd() - - // MapFS doesn't support windows path separators, so we use / exclusively - // in this test - vol := filepath.VolumeName(wd) - if vol != "" && wd != vol { - wd = wd[len(vol)+1:] - } else if wd[0] == '/' { - wd = wd[1:] - } - wd = filepath.ToSlash(wd) - - fsys := datafs.WrapWdFS(fstest.MapFS{ - "tmp": {Mode: fs.ModeDir | 0o777}, - "tmp/jsonfile.json": {Data: []byte(jsonContent)}, - "tmp/array.json": {Data: []byte(arrayContent)}, - "tmp/yamlfile.yaml": {Data: []byte(yamlContent)}, - "tmp/textfile.txt": {Data: []byte(`plain text...`)}, - path.Join(wd, "jsonfile.json"): {Data: []byte(jsonContent)}, - path.Join(wd, "array.json"): {Data: []byte(arrayContent)}, - path.Join(wd, "yamlfile.yaml"): {Data: []byte(yamlContent)}, - path.Join(wd, "textfile.txt"): {Data: []byte(`plain text...`)}, - }) - - source := &Source{Alias: "foo", URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml")} - source.fs = fsys - d := &Data{ - Sources: map[string]*Source{ - "foo": source, - "bar": {Alias: "bar", URL: mustParseURL("file:///tmp/jsonfile.json")}, - "baz": {Alias: "baz", URL: mustParseURL("file:///tmp/yamlfile.yaml")}, - "text": {Alias: "text", URL: mustParseURL("file:///tmp/textfile.txt")}, - "badscheme": {Alias: "badscheme", URL: mustParseURL("bad:///scheme.json")}, - "badtype": {Alias: "badtype", URL: mustParseURL("file:///tmp/textfile.txt?type=foo/bar")}, - "array": {Alias: "array", URL: mustParseURL("file:///tmp/array.json?type=" + url.QueryEscape(jsonArrayMimetype))}, - }, - } - - actual, err := d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:bar|baz") - actual, err = d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:jsonfile.json|baz") - actual, err = d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:./jsonfile.json|baz") - actual, err = d.readMerge(ctx, source) - require.NoError(t, err) - assert.Equal(t, mergedContent, string(actual)) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:bogusalias|file:///tmp/jsonfile.json") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|badscheme") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|badtype") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) - - source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|array") - _, err = d.readMerge(ctx, source) - assert.Error(t, err) -} - -func TestMergeData(t *testing.T) { - def := map[string]interface{}{ - "f": true, - "t": false, - "z": "def", - } - out, err := mergeData([]map[string]interface{}{def}) - require.NoError(t, err) - assert.Equal(t, "f: true\nt: false\nz: def\n", string(out)) - - over := map[string]interface{}{ - "f": false, - "t": true, - "z": "over", - } - out, err = mergeData([]map[string]interface{}{over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nt: true\nz: over\n", string(out)) - - over = map[string]interface{}{ - "f": false, - "t": true, - "z": "over", - "m": map[string]interface{}{ - "a": "aaa", - }, - } - out, err = mergeData([]map[string]interface{}{over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm:\n a: aaa\nt: true\nz: over\n", string(out)) - - uber := map[string]interface{}{ - "z": "über", - } - out, err = mergeData([]map[string]interface{}{uber, over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm:\n a: aaa\nt: true\nz: über\n", string(out)) - - uber = map[string]interface{}{ - "m": "notamap", - "z": map[string]interface{}{ - "b": "bbb", - }, - } - out, err = mergeData([]map[string]interface{}{uber, over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm: notamap\nt: true\nz:\n b: bbb\n", string(out)) - - uber = map[string]interface{}{ - "m": map[string]interface{}{ - "b": "bbb", - }, - } - out, err = mergeData([]map[string]interface{}{uber, over, def}) - require.NoError(t, err) - assert.Equal(t, "f: false\nm:\n a: aaa\n b: bbb\nt: true\nz: over\n", string(out)) -} diff --git a/data/datasource_stdin.go b/data/datasource_stdin.go deleted file mode 100644 index 13bb5fa4..00000000 --- a/data/datasource_stdin.go +++ /dev/null @@ -1,32 +0,0 @@ -package data - -import ( - "context" - "fmt" - "io" - "os" -) - -func readStdin(ctx context.Context, _ *Source, _ ...string) ([]byte, error) { - stdin := stdinFromContext(ctx) - - b, err := io.ReadAll(stdin) - if err != nil { - return nil, fmt.Errorf("can't read %s: %w", stdin, err) - } - return b, nil -} - -type stdinCtxKey struct{} - -func ContextWithStdin(ctx context.Context, r io.Reader) context.Context { - return context.WithValue(ctx, stdinCtxKey{}, r) -} - -func stdinFromContext(ctx context.Context) io.Reader { - if r, ok := ctx.Value(stdinCtxKey{}).(io.Reader); ok { - return r - } - - return os.Stdin -} diff --git a/data/datasource_stdin_test.go b/data/datasource_stdin_test.go deleted file mode 100644 index 8cef6827..00000000 --- a/data/datasource_stdin_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package data - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadStdin(t *testing.T) { - ctx := context.Background() - - ctx = ContextWithStdin(ctx, strings.NewReader("foo")) - out, err := readStdin(ctx, nil) - require.NoError(t, err) - assert.Equal(t, []byte("foo"), out) - - ctx = ContextWithStdin(ctx, errorReader{}) - _, err = readStdin(ctx, nil) - assert.Error(t, err) -} diff --git a/data/datasource_test.go b/data/datasource_test.go index a77f2645..da3d3094 100644 --- a/data/datasource_test.go +++ b/data/datasource_test.go @@ -2,13 +2,16 @@ package data import ( "context" - "fmt" "net/http" + "net/http/httptest" "net/url" + "os" "runtime" "testing" "testing/fstest" + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/httpfs" "github.com/hairyhenderson/gomplate/v4/internal/config" "github.com/hairyhenderson/gomplate/v4/internal/datafs" @@ -18,6 +21,11 @@ import ( const osWindows = "windows" +func mustParseURL(in string) *url.URL { + u, _ := url.Parse(in) + return u +} + func TestNewData(t *testing.T) { d, err := NewData(nil, nil) require.NoError(t, err) @@ -56,21 +64,22 @@ func TestDatasource(t *testing.T) { fsys := datafs.WrapWdFS(fstest.MapFS{ "tmp/" + fname: &fstest.MapFile{Data: contents}, }) + ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "file", "")) sources := map[string]*Source{ "foo": { Alias: "foo", URL: &url.URL{Scheme: "file", Path: uPath}, mediaType: mime, - fs: fsys, }, } - return &Data{Sources: sources} + return &Data{Sources: sources, Ctx: ctx} } + test := func(ext, mime string, contents []byte, expected interface{}) { data := setup(ext, mime, contents) - actual, err := data.Datasource("foo") + actual, err := data.Datasource("foo", "?type="+mime) require.NoError(t, err) assert.Equal(t, expected, actual) } @@ -110,21 +119,20 @@ func TestDatasourceReachable(t *testing.T) { fsys := datafs.WrapWdFS(fstest.MapFS{ "tmp/" + fname: &fstest.MapFile{Data: []byte("{}")}, }) + ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "file", "")) sources := map[string]*Source{ "foo": { Alias: "foo", URL: &url.URL{Scheme: "file", Path: uPath}, mediaType: jsonMimetype, - fs: fsys, }, "bar": { Alias: "bar", URL: &url.URL{Scheme: "file", Path: "/bogus"}, - fs: fsys, }, } - data := &Data{Sources: sources} + data := &Data{Sources: sources, Ctx: ctx} assert.True(t, data.DatasourceReachable("foo")) assert.False(t, data.DatasourceReachable("bar")) @@ -154,29 +162,21 @@ func TestInclude(t *testing.T) { fsys := datafs.WrapWdFS(fstest.MapFS{ "tmp/" + fname: &fstest.MapFile{Data: []byte(contents)}, }) + ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "file", "")) sources := map[string]*Source{ "foo": { Alias: "foo", URL: &url.URL{Scheme: "file", Path: uPath}, mediaType: textMimetype, - fs: fsys, }, } - data := &Data{ - Sources: sources, - } + data := &Data{Sources: sources, Ctx: ctx} actual, err := data.Include("foo") require.NoError(t, err) assert.Equal(t, contents, actual) } -type errorReader struct{} - -func (e errorReader) Read(_ []byte) (n int, err error) { - return 0, fmt.Errorf("error") -} - func TestDefineDatasource(t *testing.T) { d := &Data{} _, err := d.DefineDatasource("", "foo.json") @@ -231,134 +231,6 @@ func TestDefineDatasource(t *testing.T) { s = d.Sources["data"] require.NoError(t, err) assert.Equal(t, "data", s.Alias) - m, err := s.mimeType("") - require.NoError(t, err) - assert.Equal(t, "application/x-env", m) -} - -func TestMimeType(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com/list?type=a/b/c")} - _, err := s.mimeType("") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - expected string - }{ - { - "http://example.com/foo.json", - "", - jsonMimetype, - }, - { - "http://example.com/foo.json", - "text/foo", - "text/foo", - }, - { - "http://example.com/foo.json?type=application/yaml", - "text/foo", - "application/yaml", - }, - { - "http://example.com/list?type=application/array%2Bjson", - "text/foo", - "application/array+json", - }, - { - "http://example.com/list?type=application/array+json", - "", - "application/array+json", - }, - { - "http://example.com/unknown", - "", - "text/plain", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q==%q", i, d.url, d.mediaType, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType("") - require.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } -} - -func TestMimeTypeWithArg(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com")} - _, err := s.mimeType("h\nttp://foo") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - arg string - expected string - }{ - { - "http://example.com/unknown", - "", - "/foo.json", - "application/json", - }, - { - "http://example.com/unknown", - "", - "foo.json", - "application/json", - }, - { - "http://example.com/", - "text/foo", - "/foo.json", - "text/foo", - }, - { - "git+https://example.com/myrepo", - "", - "//foo.yaml", - "application/yaml", - }, - { - "http://example.com/foo.json", - "", - "/foo.yaml", - "application/yaml", - }, - { - "http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml", - "application/array+yaml", - }, - { - "http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml?type=application/yaml", - "application/yaml", - }, - { - "http://example.com/foo.json?type=application/array+yaml", - "text/plain", - "/foo.yaml?type=application/yaml", - "application/yaml", - }, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q,%q==%q", i, d.url, d.mediaType, d.arg, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType(d.arg) - require.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } } func TestFromConfig(t *testing.T) { @@ -445,3 +317,94 @@ func TestListDatasources(t *testing.T) { assert.Equal(t, []string{"bar", "foo"}, data.ListDatasources()) } + +func TestResolveURL(t *testing.T) { + out, err := resolveURL(mustParseURL("http://example.com/foo.json"), "bar.json") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/bar.json", out.String()) + + out, err = resolveURL(mustParseURL("http://example.com/a/b/?n=2"), "bar.json?q=1") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/a/b/bar.json?n=2&q=1", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo"), "//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/myrepo//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/foo/bar/"), "//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/foo/bar//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo/"), ".//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/myrepo//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/repo//foo.txt"), "") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/repo//foo.txt", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo"), ".//myfile?type=application/json") + assert.NoError(t, err) + assert.Equal(t, "git+file:///tmp/myrepo//myfile?type=application/json", out.String()) + + out, err = resolveURL(mustParseURL("git+file:///tmp/myrepo//foo/?type=application/json"), "bar/myfile") + assert.NoError(t, err) + // note that the '/' in the query string is encoded to %2F - that's OK + assert.Equal(t, "git+file:///tmp/myrepo//foo/bar/myfile?type=application%2Fjson", out.String()) + + // both base and relative may not contain "//" + _, err = resolveURL(mustParseURL("git+ssh://git@example.com/foo//bar"), ".//myfile") + assert.Error(t, err) + + _, err = resolveURL(mustParseURL("git+ssh://git@example.com/foo//bar"), "baz//myfile") + assert.Error(t, err) + + // relative urls must remain relative + out, err = resolveURL(mustParseURL("tmp/foo.json"), "") + require.NoError(t, err) + assert.Equal(t, "tmp/foo.json", out.String()) +} + +func TestReadFileContent(t *testing.T) { + wd, _ := os.Getwd() + t.Cleanup(func() { + _ = os.Chdir(wd) + }) + _ = os.Chdir("/") + + mux := http.NewServeMux() + mux.HandleFunc("/foo.json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", jsonMimetype) + w.Write([]byte(`{"foo": "bar"}`)) + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + fsys := datafs.WrapWdFS(fstest.MapFS{ + "foo.json": &fstest.MapFile{Data: []byte(`{"foo": "bar"}`)}, + "dir/1.yaml": &fstest.MapFile{Data: []byte(`foo: bar`)}, + "dir/2.yaml": &fstest.MapFile{Data: []byte(`baz: qux`)}, + "dir/sub/sub1.yaml": &fstest.MapFile{Data: []byte(`quux: corge`)}, + }) + + fsp := fsimpl.NewMux() + fsp.Add(httpfs.FS) + fsp.Add(datafs.WrappedFSProvider(fsys, "file", "")) + + ctx := datafs.ContextWithFSProvider(context.Background(), fsp) + + d := Data{} + + fc, err := d.readFileContent(ctx, mustParseURL("file:///foo.json"), nil) + require.NoError(t, err) + assert.Equal(t, []byte(`{"foo": "bar"}`), fc.b) + + fc, err = d.readFileContent(ctx, mustParseURL("dir/"), nil) + require.NoError(t, err) + assert.JSONEq(t, `["1.yaml", "2.yaml", "sub"]`, string(fc.b)) + + fc, err = d.readFileContent(ctx, mustParseURL(srv.URL+"/foo.json"), nil) + require.NoError(t, err) + assert.Equal(t, []byte(`{"foo": "bar"}`), fc.b) +} diff --git a/data/datasource_vault.go b/data/datasource_vault.go deleted file mode 100644 index 5f736dcb..00000000 --- a/data/datasource_vault.go +++ /dev/null @@ -1,47 +0,0 @@ -package data - -import ( - "context" - "fmt" - "strings" - - "github.com/hairyhenderson/gomplate/v4/vault" -) - -func readVault(_ context.Context, source *Source, args ...string) (data []byte, err error) { - if source.vc == nil { - source.vc, err = vault.New(source.URL) - if err != nil { - return nil, err - } - err = source.vc.Login() - if err != nil { - return nil, err - } - } - - params, p, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - source.mediaType = jsonMimetype - switch { - case len(params) > 0: - data, err = source.vc.Write(p, params) - case strings.HasSuffix(p, "/"): - source.mediaType = jsonArrayMimetype - data, err = source.vc.List(p) - default: - data, err = source.vc.Read(p) - } - if err != nil { - return nil, err - } - - if len(data) == 0 { - return nil, fmt.Errorf("no value found for path %s", p) - } - - return data, nil -} diff --git a/data/datasource_vault_test.go b/data/datasource_vault_test.go deleted file mode 100644 index 1c63d696..00000000 --- a/data/datasource_vault_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/hairyhenderson/gomplate/v4/vault" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadVault(t *testing.T) { - ctx := context.Background() - - expected := "{\"value\":\"foo\"}\n" - server, v := vault.MockServer(200, `{"data":`+expected+`}`) - defer server.Close() - - source := &Source{ - Alias: "foo", - URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, - mediaType: textMimetype, - vc: v, - } - - r, err := readVault(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "bar") - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "?param=value") - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - source.URL, _ = url.Parse("vault:///secret/foo?param1=value1¶m2=value2") - r, err = readVault(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - expected = "[\"one\",\"two\"]\n" - server, source.vc = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) - defer server.Close() - source.URL, _ = url.Parse("vault:///secret/foo/") - r, err = readVault(ctx, source) - require.NoError(t, err) - assert.Equal(t, []byte(expected), r) -} diff --git a/data/mimetypes.go b/data/mimetypes.go index 1f243219..24ea87de 100644 --- a/data/mimetypes.go +++ b/data/mimetypes.go @@ -1,5 +1,9 @@ package data +import ( + "mime" +) + const ( textMimetype = "text/plain" csvMimetype = "text/csv" @@ -19,6 +23,9 @@ var mimeTypeAliases = map[string]string{ } func mimeAlias(m string) string { + // normalize the type by removing any extra parameters + m, _, _ = mime.ParseMediaType(m) + if a, ok := mimeTypeAliases[m]; ok { return a } diff --git a/data/mimetypes_test.go b/data/mimetypes_test.go index 0dd1ab05..04c54439 100644 --- a/data/mimetypes_test.go +++ b/data/mimetypes_test.go @@ -3,7 +3,7 @@ package data import ( "testing" - "gotest.tools/v3/assert" + "github.com/stretchr/testify/assert" ) func TestMimeAlias(t *testing.T) { diff --git a/docs-src/content/functions/aws.yml b/docs-src/content/functions/aws.yml index 749397c3..9b02d7c5 100644 --- a/docs-src/content/functions/aws.yml +++ b/docs-src/content/functions/aws.yml @@ -16,7 +16,8 @@ preamble: | | `AWS_TIMEOUT` | _(Default `500`)_ Adjusts timeout for API requests, in milliseconds. Not part of the AWS SDK. | | `AWS_PROFILE` | Profile name the SDK should use when loading shared config from the configuration files. If not provided `default` will be used as the profile name. | | `AWS_REGION` | Specifies where to send requests. See [this list](https://docs.aws.amazon.com/general/latest/gr/rande.html). Note that the region must be set for AWS functions to work correctly, either through this variable, through a configuration profile, or by running on an EC2 instance. | - | `AWS_META_ENDPOINT` | _(Default `http://169.254.169.254`)_ Sets the base address of the instance metadata service. | + | `AWS_EC2_METADATA_SERVICE_ENDPOINT` | _(Default `http://169.254.169.254`)_ Sets the base address of the instance metadata service. | + | `AWS_META_ENDPOINT` _(Deprecated)_ | _(Default `http://169.254.169.254`)_ Sets the base address of the instance metadata service. Use `AWS_EC2_METADATA_SERVICE_ENDPOINT` instead. | funcs: - name: aws.EC2Meta alias: ec2meta diff --git a/docs/content/functions/aws.md b/docs/content/functions/aws.md index c93cc764..5ad2ee18 100644 --- a/docs/content/functions/aws.md +++ b/docs/content/functions/aws.md @@ -21,7 +21,8 @@ for details. | `AWS_TIMEOUT` | _(Default `500`)_ Adjusts timeout for API requests, in milliseconds. Not part of the AWS SDK. | | `AWS_PROFILE` | Profile name the SDK should use when loading shared config from the configuration files. If not provided `default` will be used as the profile name. | | `AWS_REGION` | Specifies where to send requests. See [this list](https://docs.aws.amazon.com/general/latest/gr/rande.html). Note that the region must be set for AWS functions to work correctly, either through this variable, through a configuration profile, or by running on an EC2 instance. | -| `AWS_META_ENDPOINT` | _(Default `http://169.254.169.254`)_ Sets the base address of the instance metadata service. | +| `AWS_EC2_METADATA_SERVICE_ENDPOINT` | _(Default `http://169.254.169.254`)_ Sets the base address of the instance metadata service. | +| `AWS_META_ENDPOINT` _(Deprecated)_ | _(Default `http://169.254.169.254`)_ Sets the base address of the instance metadata service. Use `AWS_EC2_METADATA_SERVICE_ENDPOINT` instead. | ## `aws.EC2Meta` @@ -2,10 +2,6 @@ package env import ( - "io/fs" - "os" - "strings" - osfs "github.com/hack-pad/hackpadfs/os" "github.com/hairyhenderson/gomplate/v4/internal/datafs" ) @@ -16,55 +12,11 @@ import ( // Otherwise the provided default (or an emptry string) is returned. func Getenv(key string, def ...string) string { fsys := datafs.WrapWdFS(osfs.NewFS()) - return getenvVFS(fsys, key, def...) + return datafs.GetenvFsys(fsys, key, def...) } // ExpandEnv - like os.ExpandEnv, except supports `_FILE` vars as well func ExpandEnv(s string) string { fsys := datafs.WrapWdFS(osfs.NewFS()) - return expandEnvVFS(fsys, s) -} - -// expandEnvVFS - -func expandEnvVFS(fsys fs.FS, s string) string { - return os.Expand(s, func(s string) string { - return getenvVFS(fsys, s) - }) -} - -// getenvVFS - a convenience function intended for internal use only! -func getenvVFS(fsys fs.FS, key string, def ...string) string { - val := getenvFile(fsys, key) - if val == "" && len(def) > 0 { - return def[0] - } - - return val -} - -func getenvFile(fsys fs.FS, key string) string { - val := os.Getenv(key) - if val != "" { - return val - } - - p := os.Getenv(key + "_FILE") - if p != "" { - val, err := readFile(fsys, p) - if err != nil { - return "" - } - return strings.TrimSpace(val) - } - - return "" -} - -func readFile(fsys fs.FS, p string) (string, error) { - b, err := fs.ReadFile(fsys, p) - if err != nil { - return "", err - } - - return string(b), nil + return datafs.ExpandEnvFsys(fsys, s) } diff --git a/env/env_test.go b/env/env_test.go index faaaa603..9f637944 100644 --- a/env/env_test.go +++ b/env/env_test.go @@ -1,14 +1,8 @@ package env import ( - "errors" - "io/fs" "os" "testing" - "testing/fstest" - - "github.com/hack-pad/hackpadfs" - "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/stretchr/testify/assert" ) @@ -19,25 +13,6 @@ func TestGetenv(t *testing.T) { assert.Equal(t, "default value", Getenv("BLAHBLAHBLAH", "default value")) } -func TestGetenvFile(t *testing.T) { - fsys := fs.FS(fstest.MapFS{ - "tmp": &fstest.MapFile{Mode: fs.ModeDir | 0o777}, - "tmp/foo": &fstest.MapFile{Data: []byte("foo")}, - "tmp/unreadable": &fstest.MapFile{Data: []byte("foo"), Mode: 0o000}, - }) - fsys = datafs.WrapWdFS(fsys) - - t.Setenv("FOO_FILE", "/tmp/foo") - assert.Equal(t, "foo", getenvVFS(fsys, "FOO", "bar")) - - t.Setenv("FOO_FILE", "/tmp/missing") - assert.Equal(t, "bar", getenvVFS(fsys, "FOO", "bar")) - - fsys = writeOnly(fsys) - t.Setenv("FOO_FILE", "/tmp/unreadable") - assert.Equal(t, "bar", getenvVFS(fsys, "FOO", "bar")) -} - func TestExpandEnv(t *testing.T) { assert.Empty(t, ExpandEnv("${FOOBARBAZ}")) assert.Equal(t, os.Getenv("USER"), ExpandEnv("$USER")) @@ -45,69 +20,3 @@ func TestExpandEnv(t *testing.T) { assert.Equal(t, os.Getenv("USER")+": "+os.Getenv("HOME"), ExpandEnv("$USER: ${HOME}")) } - -func TestExpandEnvFile(t *testing.T) { - fsys := fs.FS(fstest.MapFS{ - "tmp": &fstest.MapFile{Mode: fs.ModeDir | 0o777}, - "tmp/foo": &fstest.MapFile{Data: []byte("foo")}, - "tmp/unreadable": &fstest.MapFile{Data: []byte("foo"), Mode: 0o000}, - }) - fsys = datafs.WrapWdFS(fsys) - - t.Setenv("FOO_FILE", "/tmp/foo") - assert.Equal(t, "foo is foo", expandEnvVFS(fsys, "foo is $FOO")) - - t.Setenv("FOO_FILE", "/tmp/missing") - assert.Equal(t, "empty", expandEnvVFS(fsys, "${FOO}empty")) - - fsys = writeOnly(fsys) - t.Setenv("FOO_FILE", "/tmp/unreadable") - assert.Equal(t, "", expandEnvVFS(fsys, "${FOO}")) -} - -// Maybe extract this into a separate package sometime... -// writeOnly - represents a filesystem that's writeable, but read operations fail -func writeOnly(fsys fs.FS) fs.FS { - return &woFS{fsys} -} - -type woFS struct { - fsys fs.FS -} - -func (fsys woFS) Open(name string) (fs.File, error) { - f, err := fsys.fsys.Open(name) - return writeOnlyFile(f), err -} - -func (fsys woFS) ReadDir(_ string) ([]fs.DirEntry, error) { - return nil, ErrWriteOnly -} - -func (fsys woFS) Stat(_ string) (fs.FileInfo, error) { - return nil, ErrWriteOnly -} - -func writeOnlyFile(f fs.File) fs.File { - if f == nil { - return nil - } - - return &woFile{f} -} - -type woFile struct { - fs.File -} - -// Write - -func (f woFile) Write(p []byte) (n int, err error) { - return hackpadfs.WriteFile(f.File, p) -} - -// Read is disabled and returns ErrWriteOnly -func (f woFile) Read([]byte) (n int, err error) { - return 0, ErrWriteOnly -} - -var ErrWriteOnly = errors.New("filesystem is write-only") diff --git a/funcs/data.go b/funcs/data.go index e9ce7fa5..32f5b2da 100644 --- a/funcs/data.go +++ b/funcs/data.go @@ -5,6 +5,7 @@ import ( "github.com/hairyhenderson/gomplate/v4/conv" "github.com/hairyhenderson/gomplate/v4/data" + "github.com/hairyhenderson/gomplate/v4/internal/parsers" ) // DataNS - @@ -65,75 +66,75 @@ type DataFuncs struct { // JSON - func (f *DataFuncs) JSON(in interface{}) (map[string]interface{}, error) { - return data.JSON(conv.ToString(in)) + return parsers.JSON(conv.ToString(in)) } // JSONArray - func (f *DataFuncs) JSONArray(in interface{}) ([]interface{}, error) { - return data.JSONArray(conv.ToString(in)) + return parsers.JSONArray(conv.ToString(in)) } // YAML - func (f *DataFuncs) YAML(in interface{}) (map[string]interface{}, error) { - return data.YAML(conv.ToString(in)) + return parsers.YAML(conv.ToString(in)) } // YAMLArray - func (f *DataFuncs) YAMLArray(in interface{}) ([]interface{}, error) { - return data.YAMLArray(conv.ToString(in)) + return parsers.YAMLArray(conv.ToString(in)) } // TOML - func (f *DataFuncs) TOML(in interface{}) (interface{}, error) { - return data.TOML(conv.ToString(in)) + return parsers.TOML(conv.ToString(in)) } // CSV - func (f *DataFuncs) CSV(args ...string) ([][]string, error) { - return data.CSV(args...) + return parsers.CSV(args...) } // CSVByRow - func (f *DataFuncs) CSVByRow(args ...string) (rows []map[string]string, err error) { - return data.CSVByRow(args...) + return parsers.CSVByRow(args...) } // CSVByColumn - func (f *DataFuncs) CSVByColumn(args ...string) (cols map[string][]string, err error) { - return data.CSVByColumn(args...) + return parsers.CSVByColumn(args...) } // CUE - func (f *DataFuncs) CUE(in interface{}) (interface{}, error) { - return data.CUE(conv.ToString(in)) + return parsers.CUE(conv.ToString(in)) } // ToCSV - func (f *DataFuncs) ToCSV(args ...interface{}) (string, error) { - return data.ToCSV(args...) + return parsers.ToCSV(args...) } // ToCUE - func (f *DataFuncs) ToCUE(in interface{}) (string, error) { - return data.ToCUE(in) + return parsers.ToCUE(in) } // ToJSON - func (f *DataFuncs) ToJSON(in interface{}) (string, error) { - return data.ToJSON(in) + return parsers.ToJSON(in) } // ToJSONPretty - func (f *DataFuncs) ToJSONPretty(indent string, in interface{}) (string, error) { - return data.ToJSONPretty(indent, in) + return parsers.ToJSONPretty(indent, in) } // ToYAML - func (f *DataFuncs) ToYAML(in interface{}) (string, error) { - return data.ToYAML(in) + return parsers.ToYAML(in) } // ToTOML - func (f *DataFuncs) ToTOML(in interface{}) (string, error) { - return data.ToTOML(in) + return parsers.ToTOML(in) } @@ -8,32 +8,31 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Shopify/ejson v1.4.1 github.com/aws/aws-sdk-go v1.49.20 - github.com/docker/libkv v0.2.2-0.20180912205406-458977154600 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa github.com/google/uuid v1.5.0 github.com/gosimple/slug v1.13.1 github.com/hack-pad/hackpadfs v0.2.1 - github.com/hairyhenderson/go-fsimpl v0.0.0-20230121155226-8aa24800449d + github.com/hairyhenderson/go-fsimpl v0.0.0-20240117024222-dd2398b673d3 github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf github.com/hairyhenderson/xignore v0.3.3-0.20230403012150-95fe86932830 // iofs-port branch - github.com/hashicorp/consul/api v1.26.1 github.com/hashicorp/go-sockaddr v1.0.6 github.com/hashicorp/vault/api v1.10.0 + github.com/hashicorp/vault/api/auth/aws v0.5.0 github.com/itchyny/gojq v0.12.14 - github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 + github.com/johannesboyne/gofakes3 v0.0.0-20230914150226-f005f5cc03aa github.com/joho/godotenv v1.5.1 github.com/rs/zerolog v1.31.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 github.com/ugorji/go/codec v1.2.12 - go4.org/netipx v0.0.0-20230125063823-8449b0a6169f - gocloud.dev v0.36.0 + go4.org/netipx v0.0.0-20230303233057-f1b76eb4bb35 golang.org/x/crypto v0.18.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sys v0.16.0 golang.org/x/term v0.16.0 golang.org/x/text v0.14.0 gotest.tools/v3 v3.5.1 - inet.af/netaddr v0.0.0-20220811202034-502d2d690317 + inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a k8s.io/client-go v0.29.0 ) @@ -42,43 +41,47 @@ require ( // cherry-pick from the PR on top of v5.11.0 replace github.com/go-git/go-git/v5 => github.com/hairyhenderson/go-git/v5 v5.0.0-20240112193603-9068a607f23a -require ( - github.com/go-git/go-billy/v5 v5.5.0 - github.com/go-git/go-git/v5 v5.11.0 -) - // TODO: replace with gopkg.in/yaml.v3 after https://github.com/go-yaml/yaml/pull/862 // is merged require github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce require ( - cloud.google.com/go v0.110.10 // indirect + cloud.google.com/go v0.111.0 // indirect cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.5 // indirect - cloud.google.com/go/storage v1.35.1 // indirect + cloud.google.com/go/storage v1.36.0 // indirect dario.cat/mergo v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/armon/go-metrics v0.4.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.24.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/config v1.26.1 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.26.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.26.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect github.com/aws/smithy-go v1.19.0 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -87,9 +90,15 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -98,6 +107,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect + github.com/hashicorp/consul/api v1.26.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect @@ -105,47 +115,59 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/serf v0.10.1 // indirect + github.com/hashicorp/vault/api/auth/approle v0.5.0 // indirect + github.com/hashicorp/vault/api/auth/userpass v0.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sergi/go-diff v1.2.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go4.org/intern v0.0.0-20220617035311-6925f38cc365 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + gocloud.dev v0.36.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.20.0 // indirect - golang.org/x/oauth2 v0.14.0 // indirect + golang.org/x/oauth2 v0.15.0 // indirect golang.org/x/sync v0.5.0 // indirect - golang.org/x/time v0.4.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/api v0.151.0 // indirect + google.golang.org/api v0.154.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect - google.golang.org/grpc v1.59.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect + google.golang.org/grpc v1.60.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -1,20 +1,38 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= -cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= +cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= +cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= -cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= -cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= +cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 h1:zkiIe8AxZ/kDjqQN+mDKc5BxoVJOqioSdqApjc+eB1I= cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13/go.mod h1:XGKYSMtsJWfqQYPwq51ZygxAPqpEUj/9bdg16iDPTAA= cuelang.org/go v0.7.0 h1:gMztinxuKfJwMIxtboFsNc6s8AxwJGgsJV+3CuLffHI= cuelang.org/go v0.7.0/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -42,25 +60,26 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.49.20 h1:VgEUq2/ZbUkLbqPyDcxrirfXB+PgiZUUF5XbsgWe2S0= github.com/aws/aws-sdk-go v1.49.20/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= -github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= -github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= -github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= -github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= -github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= +github.com/aws/aws-sdk-go-v2/config v1.26.3/go.mod h1:Bxgi+DeeswYofcYO0XyGClwlrq3DZEXli0kLf4hkGA0= +github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE= +github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.7 h1:FnLf60PtjXp8ZOzQfhJVsqF0OtYKQZWQfqOLshh8YXg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.7/go.mod h1:tDVvl8hyU6E9B8TrnNrZQEVkQlB8hjJwcgpPhgtlnNg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= @@ -69,18 +88,22 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.26.2 h1:A5sGOT/mukuU+4At1vkSIWAN8tPwPCoYZBp7aruR540= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.26.2/go.mod h1:qutL00aW8GSo2D0I6UEOqMvRS3ZyuBrOC1BLe5D2jPc= +github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7 h1:a8HvP/+ew3tKwSXqL3BCSjiuicr+XTU2eFYeogV9GJE= +github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7/go.mod h1:Q7XIWsMo0JcMpI/6TGD6XXcXcV1DbTj6e9BKNntIMIM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -88,6 +111,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -99,18 +123,21 @@ github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUK github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/libkv v0.2.2-0.20180912205406-458977154600 h1:x0AMRhackzbivKKiEeSMzH6gZmbALPXCBG0ecBmRlco= -github.com/docker/libkv v0.2.2-0.20180912205406-458977154600/go.mod h1:r5hEwHwW8dr0TFBYGCarMNbrQOiwL1xoqDYZ/JqoTK0= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad h1:Qk76DOWdOp+GlyDKBAG3Klr9cn7N+LcYc82AZ2S7+cA= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad/go.mod h1:mPKfmRa823oBIgl2r20LeMSpTAteW5j7FLkc0vjmzyQ= github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= @@ -124,11 +151,17 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsouza/fake-gcs-server v1.47.7 h1:56/U4rKY081TaNbq0gHWi7/71UxC2KROqcnrD9BRJhs= +github.com/fsouza/fake-gcs-server v1.47.7/go.mod h1:4vPUynN8/zZlxk5Jpy6LvvTTxItdTAObK4DYnp89Jys= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gliderlabs/ssh v0.3.6 h1:ZzjlDa05TcFRICb3anf/dSPN3ewz1Zx6CMLPWgkm3b8= @@ -139,19 +172,28 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-jwt/jwt/v5 v5.1.0 h1:UGKbA/IPjtS6zLcdB7i5TyACMgSbOTiR8qzXgw8HWQU= +github.com/golang-jwt/jwt/v5 v5.1.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -181,6 +223,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= @@ -190,6 +233,8 @@ github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= @@ -204,14 +249,18 @@ github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= +github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/hack-pad/hackpadfs v0.2.1 h1:FelFhIhv26gyjujoA/yeFO+6YGlqzmc9la/6iKMIxMw= github.com/hack-pad/hackpadfs v0.2.1/go.mod h1:khQBuCEwGXWakkmq8ZiFUvUZz84ZkJ2KNwKvChs4OrU= -github.com/hairyhenderson/go-fsimpl v0.0.0-20230121155226-8aa24800449d h1:RqyRRWUi3ftiPrEuO9ZbaILf6C8TDRw90fLM8pqbzGs= -github.com/hairyhenderson/go-fsimpl v0.0.0-20230121155226-8aa24800449d/go.mod h1:2k9HLXToBp8dOXrbgZUTaZfm03JZlkCGkrgvp4ip/JQ= +github.com/hairyhenderson/go-fsimpl v0.0.0-20240117024222-dd2398b673d3 h1:hEAU1+bbqbGTUIQfoKcMiRgza+47nByeZ8jEHlOLDac= +github.com/hairyhenderson/go-fsimpl v0.0.0-20240117024222-dd2398b673d3/go.mod h1:/8Yhp1ir+A+t3n4iYIL/O47X7c35QrkTi39ZjJpfoYw= github.com/hairyhenderson/go-git/v5 v5.0.0-20240112193603-9068a607f23a h1:tY0NASRMFiqOxJgT5CngoUft1ZZGTDK9jCWNljFrGMs= github.com/hairyhenderson/go-git/v5 v5.0.0-20240112193603-9068a607f23a/go.mod h1:rGg8aLOxsDU5LgK8kiSJUzXaTPmwYZQYCj4+ouohMuM= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf h1:I1sbT4ZbIt9i+hB1zfKw2mE8C12TuGxPiW7YmtLbPa4= @@ -228,9 +277,11 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -244,12 +295,16 @@ github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 h1:W9WN8p6moV1fjKLkeqEgkAMu5rauy9QeYDAmIaPuuiA= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6/go.mod h1:MpCPSPGLDILGb4JMm94/mMi3YysIqsXzGCzkEZjcjXg= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= @@ -260,10 +315,11 @@ github.com/hashicorp/go-sockaddr v1.0.6/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3ly github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.5.0 h1:O293SZ2Eg+AAYijkVK3jR786Am1bhDEh2GHT0tIVE5E= -github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -277,6 +333,12 @@ github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/vault/api v1.10.0 h1:/US7sIjWN6Imp4o/Rj1Ce2Nr5bki/AXi9vAW3p2tOJQ= github.com/hashicorp/vault/api v1.10.0/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8= +github.com/hashicorp/vault/api/auth/approle v0.5.0 h1:a1TK6VGwYqSAfkmX4y4dJ4WBxMU5dStIZqScW4EPXR8= +github.com/hashicorp/vault/api/auth/approle v0.5.0/go.mod h1:CHOQIA1AZACfjTzHggmyfiOZ+xCSKNRFqe48FTCzH0k= +github.com/hashicorp/vault/api/auth/aws v0.5.0 h1:IKf0W3A2tXEtw9KrooslBWw72Ld63V+fUHkkSmm+2T0= +github.com/hashicorp/vault/api/auth/aws v0.5.0/go.mod h1:U2Y6Ci/kDsUkDTzUXq0OKG2/GQkEtqzQjTY1YYSQFnk= +github.com/hashicorp/vault/api/auth/userpass v0.5.0 h1:u//BC15YJviWSpeTlxsmt96FPULsCF7dYhPHg5oOAzo= +github.com/hashicorp/vault/api/auth/userpass v0.5.0/go.mod h1:TNxl3X6ZaeILi1rfxP/mhGnWuiCiP7SNv2qeZ5aSAMQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc= @@ -285,13 +347,13 @@ github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 h1:V5q1Mx2WTE5coXLG2QpkRZ7LsJvgkedm6Ib4AwC1Lfg= -github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= +github.com/johannesboyne/gofakes3 v0.0.0-20230914150226-f005f5cc03aa h1:a6Hc6Hlq6MxPNBW53/S/HnVwVXKc0nbdD/vgnQYuxG0= +github.com/johannesboyne/gofakes3 v0.0.0-20230914150226-f005f5cc03aa/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -304,12 +366,15 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -321,6 +386,7 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -361,10 +427,14 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -384,6 +454,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4= github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c h1:fPpdjePK1atuOg28PXfNSqgwf9I/qD1Hlo39JFwKBXk= github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -398,9 +469,8 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -425,6 +495,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -443,11 +514,23 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= go4.org/intern v0.0.0-20220617035311-6925f38cc365 h1:t9hFvR102YlOqU0fQn1wgwhNvSbHGBbbJxX9JKfU3l0= go4.org/intern v0.0.0-20220617035311-6925f38cc365/go.mod h1:WXRv3p7T6gzt0CcJm43AAKdKVZmcQbwwC7EwquU5BZU= -go4.org/netipx v0.0.0-20230125063823-8449b0a6169f h1:ketMxHg+vWm3yccyYiq+uK8D3fRmna2Fcj+awpQp84s= -go4.org/netipx v0.0.0-20230125063823-8449b0a6169f/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc= +go4.org/netipx v0.0.0-20230303233057-f1b76eb4bb35 h1:nJAwRlGWZZDOD+6wni9KVUNHMpHko/OnRwsrCYeAzPo= +go4.org/netipx v0.0.0-20230303233057-f1b76eb4bb35/go.mod h1:TQvodOM+hJTioNQJilmLXu08JNb8i+ccq418+KWu1/Y= go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 h1:WJhcL4p+YeDxmZWg141nRm7XC8IDmhz7lk5GpadO1Sg= @@ -463,6 +546,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= @@ -475,32 +559,36 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= -golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -522,11 +610,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -535,6 +625,7 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -543,18 +634,22 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -566,14 +661,15 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= -golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -583,6 +679,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -591,8 +688,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.151.0 h1:FhfXLO/NFdJIzQtCqjpysWwqKk8AzGWBUhMIx67cVDU= -google.golang.org/api v0.151.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg= +google.golang.org/api v0.154.0 h1:X7QkVKZBskztmpPKWQXgjJRPA2dJYrL6r+sYPRLj050= +google.golang.org/api v0.154.0/go.mod h1:qhSMkM85hgqiokIYsrRyKxrjfBeIhgl4Z2JmeRkYylc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= @@ -604,15 +701,15 @@ google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f/go.mod h1:nWSwAFPb+qfNJXsoeO3Io7zf4tMSfN8EA8RlDA04GhY= google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY= google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -628,9 +725,11 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -648,7 +747,7 @@ gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -inet.af/netaddr v0.0.0-20220811202034-502d2d690317 h1:U2fwK6P2EqmopP/hFLTOAjWTki0qgd4GMJn5X8wOleU= -inet.af/netaddr v0.0.0-20220811202034-502d2d690317/go.mod h1:OIezDfdzOgFhuw4HuWapWq2e9l0H9tK4F1j+ETRtF3k= +inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a h1:1XCVEdxrvL6c0TGOhecLuB7U9zYNdxZEjvOqJreKZiM= +inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a/go.mod h1:e83i32mAQOW1LAqEIweALsuK2Uw4mhQadA5r7b0Wobo= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= diff --git a/gomplate.go b/gomplate.go index f95dc2a3..b1312a34 100644 --- a/gomplate.go +++ b/gomplate.go @@ -11,8 +11,8 @@ import ( "text/template" "time" - "github.com/hairyhenderson/gomplate/v4/data" "github.com/hairyhenderson/gomplate/v4/internal/config" + "github.com/hairyhenderson/gomplate/v4/internal/datafs" ) // RunTemplates - run all gomplate templates specified by the given configuration @@ -46,7 +46,7 @@ func Run(ctx context.Context, cfg *config.Config) error { } // if a custom Stdin is set in the config, inject it into the context now - ctx = data.ContextWithStdin(ctx, cfg.Stdin) + ctx = datafs.ContextWithStdin(ctx, cfg.Stdin) opts := optionsFromConfig(cfg) opts.Funcs = funcMap @@ -87,7 +87,6 @@ func simpleNamer(outDir string) func(ctx context.Context, inPath string) (string func mappingNamer(outMap string, tr *Renderer) func(context.Context, string) (string, error) { return func(ctx context.Context, inPath string) (string, error) { - tr.data.Ctx = ctx tcontext, err := createTmplContext(ctx, tr.tctxAliases, tr.data) if err != nil { return "", err diff --git a/gomplate_test.go b/gomplate_test.go index 2485e7af..62ae4525 100644 --- a/gomplate_test.go +++ b/gomplate_test.go @@ -12,6 +12,7 @@ import ( "github.com/hairyhenderson/gomplate/v4/conv" "github.com/hairyhenderson/gomplate/v4/data" "github.com/hairyhenderson/gomplate/v4/env" + "github.com/hairyhenderson/gomplate/v4/internal/parsers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -73,7 +74,7 @@ func TestEc2MetaTemplates_WithJSON(t *testing.T) { Funcs: template.FuncMap{ "ec2meta": ec2meta.Meta, "ec2dynamic": ec2meta.Dynamic, - "json": data.JSON, + "json": parsers.JSON, }, }) @@ -84,7 +85,7 @@ func TestEc2MetaTemplates_WithJSON(t *testing.T) { func TestJSONArrayTemplates(t *testing.T) { g := NewRenderer(Options{ Funcs: template.FuncMap{ - "jsonArray": data.JSONArray, + "jsonArray": parsers.JSONArray, }, }) @@ -95,8 +96,8 @@ func TestJSONArrayTemplates(t *testing.T) { func TestYAMLTemplates(t *testing.T) { g := NewRenderer(Options{ Funcs: template.FuncMap{ - "yaml": data.YAML, - "yamlArray": data.YAMLArray, + "yaml": parsers.YAML, + "yamlArray": parsers.YAMLArray, }, }) @@ -108,7 +109,7 @@ func TestYAMLTemplates(t *testing.T) { func TestHasTemplate(t *testing.T) { g := NewRenderer(Options{ Funcs: template.FuncMap{ - "yaml": data.YAML, + "yaml": parsers.YAML, "has": conv.Has, }, }) diff --git a/internal/cmd/config.go b/internal/cmd/config.go index 7f071417..7dc53911 100644 --- a/internal/cmd/config.go +++ b/internal/cmd/config.go @@ -73,7 +73,7 @@ func readConfigFile(ctx context.Context, cmd *cobra.Command) (cfg *config.Config // we only support loading configs from the local filesystem for now fsys, err := datafs.FSysForPath(ctx, cfgFile) if err != nil { - return nil, err + return nil, fmt.Errorf("fsys for path %v: %w", cfgFile, err) } f, err := fsys.Open(cfgFile) diff --git a/internal/cmd/main.go b/internal/cmd/main.go index 3cce83e2..7a3dea98 100644 --- a/internal/cmd/main.go +++ b/internal/cmd/main.go @@ -7,7 +7,6 @@ import ( "os/exec" "os/signal" - "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v4" "github.com/hairyhenderson/gomplate/v4/env" "github.com/hairyhenderson/gomplate/v4/internal/datafs" @@ -169,13 +168,10 @@ func InitFlags(command *cobra.Command) { func Main(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { ctx = initLogger(ctx, stderr) - // inject a default filesystem provider for file:// URLs + // inject default filesystem provider if it hasn't already been provided in + // the context if datafs.FSProviderFromContext(ctx) == nil { - // TODO: expand this to support other schemes! - mux := fsimpl.NewMux() - mux.Add(datafs.WdFS) - - ctx = datafs.ContextWithFSProvider(ctx, mux) + ctx = datafs.ContextWithFSProvider(ctx, gomplate.DefaultFSProvider) } command := NewGomplateCmd() diff --git a/internal/config/configfile.go b/internal/config/configfile.go index 1ed266b3..1fcccc20 100644 --- a/internal/config/configfile.go +++ b/internal/config/configfile.go @@ -15,8 +15,8 @@ import ( "golang.org/x/exp/slices" - "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" + "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" "github.com/hairyhenderson/yaml" ) @@ -115,7 +115,7 @@ func (d *DataSource) UnmarshalYAML(value *yaml.Node) error { if err != nil { return err } - u, err := datafs.ParseSourceURL(r.URL) + u, err := urlhelpers.ParseSourceURL(r.URL) if err != nil { return fmt.Errorf("could not parse datasource URL %q: %w", r.URL, err) } @@ -378,7 +378,7 @@ func parseDatasourceArg(value string) (alias string, ds DataSource, err error) { } } - ds.URL, err = datafs.ParseSourceURL(u) + ds.URL, err = urlhelpers.ParseSourceURL(u) return alias, ds, err } diff --git a/internal/config/types.go b/internal/config/types.go index 57901f0d..648ad2b9 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - "github.com/hairyhenderson/gomplate/v4/internal/datafs" + "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" "github.com/hairyhenderson/yaml" ) @@ -53,7 +53,7 @@ func (t *Templates) unmarshalYAMLArray(value *yaml.Node) error { pth = alias } - u, err := datafs.ParseSourceURL(pth) + u, err := urlhelpers.ParseSourceURL(pth) if err != nil { return fmt.Errorf("could not parse template URL %q: %w", pth, err) } @@ -90,7 +90,7 @@ func parseTemplateArg(value string) (alias string, ds DataSource, err error) { u = alias } - ds.URL, err = datafs.ParseSourceURL(u) + ds.URL, err = urlhelpers.ParseSourceURL(u) return alias, ds, err } diff --git a/internal/datafs/context.go b/internal/datafs/context.go new file mode 100644 index 00000000..7f1235bf --- /dev/null +++ b/internal/datafs/context.go @@ -0,0 +1,45 @@ +package datafs + +import ( + "context" + "io" + "io/fs" + "os" + + "github.com/hairyhenderson/gomplate/v4/internal/config" +) + +// withContexter is an fs.FS that can be configured with a custom context +// copied from go-fsimpl - see internal/types.go +type withContexter interface { + WithContext(ctx context.Context) fs.FS +} + +type withDataSourceser interface { + WithDataSources(sources map[string]config.DataSource) fs.FS +} + +// WithDataSourcesFS injects a datasource map into the filesystem fs, if the +// filesystem supports it (i.e. has a WithDataSources method). This is used for +// the mergefs filesystem. +func WithDataSourcesFS(sources map[string]config.DataSource, fsys fs.FS) fs.FS { + if fsys, ok := fsys.(withDataSourceser); ok { + return fsys.WithDataSources(sources) + } + + return fsys +} + +type stdinCtxKey struct{} + +func ContextWithStdin(ctx context.Context, r io.Reader) context.Context { + return context.WithValue(ctx, stdinCtxKey{}, r) +} + +func StdinFromContext(ctx context.Context) io.Reader { + if r, ok := ctx.Value(stdinCtxKey{}).(io.Reader); ok { + return r + } + + return os.Stdin +} diff --git a/internal/datafs/envfs.go b/internal/datafs/envfs.go new file mode 100644 index 00000000..41b3928f --- /dev/null +++ b/internal/datafs/envfs.go @@ -0,0 +1,221 @@ +package datafs + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "strings" + "time" + + "github.com/hairyhenderson/go-fsimpl" +) + +// NewEnvFS returns a filesystem (an fs.FS) that can be used to read data from +// environment variables. +func NewEnvFS(_ *url.URL) (fs.FS, error) { + return &envFS{locfs: os.DirFS("/")}, nil +} + +type envFS struct { + locfs fs.FS +} + +//nolint:gochecknoglobals +var EnvFS = fsimpl.FSProviderFunc(NewEnvFS, "env") + +var _ fs.FS = (*envFS)(nil) + +func (f *envFS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + + return &envFile{locfs: f.locfs, name: name}, nil +} + +type envFile struct { + locfs fs.FS + body io.Reader + name string + + dirents []fs.DirEntry + diroff int +} + +var ( + _ fs.File = (*envFile)(nil) + _ fs.ReadDirFile = (*envFile)(nil) +) + +// overridable env functions +var ( + lookupEnv = os.LookupEnv + environ = os.Environ +) + +func (e *envFile) Close() error { + e.body = nil + return nil +} + +func (e *envFile) envReader() (int, io.Reader, error) { + v, found := lookupEnv(e.name) + if found { + return len(v), bytes.NewBufferString(v), nil + } + + fname, found := lookupEnv(e.name + "_FILE") + if found && fname != "" { + fname = strings.TrimPrefix(fname, "/") + + b, err := fs.ReadFile(e.locfs, fname) + if err != nil { + return 0, nil, err + } + + b = bytes.TrimSpace(b) + + return len(b), bytes.NewBuffer(b), nil + } + + return 0, nil, fs.ErrNotExist +} + +func (e *envFile) Stat() (fs.FileInfo, error) { + n, _, err := e.envReader() + if err != nil { + return nil, err + } + + return FileInfo(e.name, int64(n), 0o444, time.Time{}, ""), nil +} + +func (e *envFile) Read(p []byte) (int, error) { + if e.body == nil { + _, r, err := e.envReader() + if err != nil { + return 0, err + } + e.body = r + } + + return e.body.Read(p) +} + +func (e *envFile) ReadDir(n int) ([]fs.DirEntry, error) { + // envFS has no concept of subdirectories, but we can support a root + // directory by listing all environment variables. + if e.name != "." { + return nil, fmt.Errorf("%s: not a directory", e.name) + } + + if e.dirents == nil { + envs := environ() + e.dirents = make([]fs.DirEntry, 0, len(envs)) + for _, env := range envs { + parts := strings.SplitN(env, "=", 2) + name, value := parts[0], parts[1] + + if name == "" { + // this might be a Windows =C: style env var, so skip it + continue + } + + e.dirents = append(e.dirents, FileInfoDirEntry( + FileInfo(name, int64(len(value)), 0o444, time.Time{}, ""), + )) + } + } + + if n > 0 && e.diroff >= len(e.dirents) { + return nil, io.EOF + } + + low := e.diroff + high := e.diroff + n + + // clamp high at the max, and ensure it's higher than low + if high >= len(e.dirents) || high <= low { + high = len(e.dirents) + } + + entries := make([]fs.DirEntry, high-low) + copy(entries, e.dirents[e.diroff:]) + + e.diroff = high + + return entries, nil +} + +// FileInfo/DirInfo/FileInfoDirEntry/etc are taken from go-fsimpl's internal +// package, and may be exported in the future... + +// FileInfo creates a static fs.FileInfo with the given properties. +// The result is also a fs.DirEntry and can be safely cast. +func FileInfo(name string, size int64, mode fs.FileMode, modTime time.Time, contentType string) fs.FileInfo { + return &staticFileInfo{ + name: name, + size: size, + mode: mode, + modTime: modTime, + contentType: contentType, + } +} + +// DirInfo creates a fs.FileInfo for a directory with the given name. Use +// FileInfo to set other values. +func DirInfo(name string, modTime time.Time) fs.FileInfo { + return FileInfo(name, 0, fs.ModeDir, modTime, "") +} + +type staticFileInfo struct { + modTime time.Time + name string + contentType string + size int64 + mode fs.FileMode +} + +var ( + _ fs.FileInfo = (*staticFileInfo)(nil) + _ fs.DirEntry = (*staticFileInfo)(nil) +) + +func (fi staticFileInfo) ContentType() string { return fi.contentType } +func (fi staticFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi staticFileInfo) Mode() fs.FileMode { return fi.mode } +func (fi *staticFileInfo) ModTime() time.Time { return fi.modTime } +func (fi staticFileInfo) Name() string { return fi.name } +func (fi staticFileInfo) Size() int64 { return fi.size } +func (fi staticFileInfo) Sys() interface{} { return nil } +func (fi *staticFileInfo) Info() (fs.FileInfo, error) { return fi, nil } +func (fi staticFileInfo) Type() fs.FileMode { return fi.Mode().Type() } + +// FileInfoDirEntry adapts a fs.FileInfo into a fs.DirEntry. If it doesn't +// already implement fs.DirEntry, it will be wrapped to always return the +// same fs.FileInfo. +func FileInfoDirEntry(fi fs.FileInfo) fs.DirEntry { + de, ok := fi.(fs.DirEntry) + if ok { + return de + } + + return &fileinfoDirEntry{fi} +} + +// a wrapper to make a fs.FileInfo into an fs.DirEntry +type fileinfoDirEntry struct { + fs.FileInfo +} + +var _ fs.DirEntry = (*fileinfoDirEntry)(nil) + +func (fi *fileinfoDirEntry) Info() (fs.FileInfo, error) { return fi, nil } +func (fi *fileinfoDirEntry) Type() fs.FileMode { return fi.Mode().Type() } diff --git a/internal/datafs/envfs_test.go b/internal/datafs/envfs_test.go new file mode 100644 index 00000000..d8bfd092 --- /dev/null +++ b/internal/datafs/envfs_test.go @@ -0,0 +1,140 @@ +package datafs + +import ( + "io/fs" + "net/url" + "os" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnvFS_Open(t *testing.T) { + fsys, err := NewEnvFS(nil) + assert.NoError(t, err) + assert.IsType(t, &envFS{}, fsys) + + f, err := fsys.Open("foo") + assert.NoError(t, err) + assert.IsType(t, &envFile{}, f) +} + +func TestEnvFile_Read(t *testing.T) { + content := `hello world` + t.Setenv("HELLO_WORLD", "hello world") + + f := &envFile{name: "HELLO_WORLD"} + b := make([]byte, len(content)) + n, err := f.Read(b) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) + + fsys := fstest.MapFS{} + fsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello world\n")} + + t.Setenv("FOO_FILE", "/foo/bar/baz.txt") + + f = &envFile{name: "FOO", locfs: fsys} + + b = make([]byte, len(content)) + t.Logf("b len is %d", len(b)) + n, err = f.Read(b) + t.Logf("b len is %d", len(b)) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) +} + +func TestEnvFile_Stat(t *testing.T) { + content := []byte(`hello world`) + t.Setenv("HELLO_WORLD", "hello world") + + f := &envFile{name: "HELLO_WORLD"} + + fi, err := f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) + + fsys := fstest.MapFS{} + fsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello world\n")} + + t.Setenv("FOO_FILE", "/foo/bar/baz.txt") + + f = &envFile{name: "FOO", locfs: fsys} + + fi, err = f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) +} + +func TestEnvFS(t *testing.T) { + t.Cleanup(func() { environ = os.Environ }) + + u, _ := url.Parse("env:") + + lfsys := fstest.MapFS{} + lfsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello file\n")} + + fsys, err := NewEnvFS(u) + assert.NoError(t, err) + assert.IsType(t, &envFS{}, fsys) + + envfs, ok := fsys.(*envFS) + assert.True(t, ok) + envfs.locfs = lfsys + + t.Setenv("FOO_FILE", "/foo/bar/baz.txt") + + b, err := fs.ReadFile(fsys, "FOO") + assert.NoError(t, err) + assert.Equal(t, "hello file", string(b)) + + t.Setenv("FOO", "hello world") + + b, err = fs.ReadFile(fsys, "FOO") + assert.NoError(t, err) + assert.Equal(t, "hello world", string(b)) + + assert.NoError(t, fstest.TestFS(fsys, "FOO", "FOO_FILE")) +} + +func TestEnvFile_ReadDir(t *testing.T) { + t.Cleanup(func() { environ = os.Environ }) + + t.Run("name must be .", func(t *testing.T) { + f := &envFile{name: "foo"} + _, err := f.ReadDir(-1) + require.Error(t, err) + }) + + t.Run("empty env should return empty dir", func(t *testing.T) { + f := &envFile{name: "."} + environ = func() []string { return []string{} } + des, err := f.ReadDir(-1) + require.NoError(t, err) + assert.Empty(t, des) + }) + + t.Run("non-empty env should return dir with entries", func(t *testing.T) { + f := &envFile{name: "."} + environ = func() []string { return []string{"FOO=bar", "BAR=quux"} } + des, err := f.ReadDir(-1) + require.NoError(t, err) + require.Len(t, des, 2) + assert.Equal(t, "FOO", des[0].Name()) + assert.Equal(t, "BAR", des[1].Name()) + }) + + t.Run("deal with odd Windows env vars like '=C:=C:\tmp'", func(t *testing.T) { + f := &envFile{name: "."} + environ = func() []string { return []string{"FOO=bar", "=C:=C:\\tmp", "BAR=quux"} } + des, err := f.ReadDir(-1) + require.NoError(t, err) + require.Len(t, des, 2) + assert.Equal(t, "FOO", des[0].Name()) + assert.Equal(t, "BAR", des[1].Name()) + }) +} diff --git a/internal/datafs/fsurl.go b/internal/datafs/fsurl.go new file mode 100644 index 00000000..a1cf5cbc --- /dev/null +++ b/internal/datafs/fsurl.go @@ -0,0 +1,42 @@ +package datafs + +import ( + "net/url" + "strings" +) + +// SplitFSMuxURL splits a URL into a filesystem URL and a relative file path +func SplitFSMuxURL(in *url.URL) (*url.URL, string) { + u := *in + + // git URLs are special - they have double-slashes that separate a repo + // from a path in the repo. A missing double-slash means the path is the + // root. + switch u.Scheme { + case "git", "git+file", "git+http", "git+https", "git+ssh": + repo, base, _ := strings.Cut(u.Path, "//") + u.Path = repo + if base == "" { + base = "." + } + + return &u, base + } + + // trim leading and trailing slashes - they are not part of a valid path + // according to [io/fs.ValidPath] + base := strings.Trim(u.Path, "/") + + if base == "" && u.Opaque != "" { + base = u.Opaque + u.Opaque = "" + } + + if base == "" { + base = "." + } + + u.Path = "/" + + return &u, base +} diff --git a/internal/datafs/fsurl_test.go b/internal/datafs/fsurl_test.go new file mode 100644 index 00000000..a44edaff --- /dev/null +++ b/internal/datafs/fsurl_test.go @@ -0,0 +1,106 @@ +package datafs + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSplitFSMuxURL(t *testing.T) { + testdata := []struct { + in string + url string + file string + }{ + { + "http://example.com/foo.json", + "http://example.com/", + "foo.json", + }, + { + "http://example.com/foo.json?type=application/array+yaml", + "http://example.com/?type=application/array+yaml", + "foo.json", + }, + { + "vault:///secret/a/b/c", + "vault:///", + "secret/a/b/c", + }, + { + "vault:///secret/a/b/", + "vault:///", + "secret/a/b", + }, + { + "s3://bucket/a/b/", + "s3://bucket/", + "a/b", + }, + { + "vault:///foo/bar", + "vault:///", + "foo/bar", + }, + { + "consul://myhost/foo/bar/baz?q=1", + "consul://myhost/?q=1", + "foo/bar/baz", + }, + { + "git+https://example.com/myrepo//foo.yaml", + "git+https://example.com/myrepo", + "foo.yaml", + }, + { + "git+https://example.com/myrepo//", + "git+https://example.com/myrepo", + ".", + }, + { + // git repos are special - no double-slash means the root + "git+https://example.com/myrepo", + "git+https://example.com/myrepo", + ".", + }, + { + "git+ssh://git@github.com/hairyhenderson/go-which.git//a/b/c/d?q=1", + "git+ssh://git@github.com/hairyhenderson/go-which.git?q=1", + "a/b/c/d", + }, + { + "merge:file:///tmp/jsonfile.json", + "merge:///", + "file:///tmp/jsonfile.json", + }, + { + "merge:a|b", + "merge:///", + "a|b", + }, + { + "merge:a|b|c|d|e", + "merge:///", + "a|b|c|d|e", + }, + { + "merge:foo/bar/baz.json|qux", + "merge:///", + "foo/bar/baz.json|qux", + }, + { + "merge:vault:///foo/bar|foo|git+ssh://git@github.com/hairyhenderson/go-which.git//a/b/c/d", + "merge:///", + "vault:///foo/bar|foo|git+ssh://git@github.com/hairyhenderson/go-which.git//a/b/c/d", + }, + } + + for _, d := range testdata { + u, err := url.Parse(d.in) + assert.NoError(t, err) + url, file := SplitFSMuxURL(u) + assert.Equal(t, d.url, url.String()) + assert.Equal(t, d.file, file) + } +} diff --git a/internal/datafs/fsys.go b/internal/datafs/fsys.go index fb88b0da..08f52b88 100644 --- a/internal/datafs/fsys.go +++ b/internal/datafs/fsys.go @@ -5,11 +5,11 @@ import ( "fmt" "io/fs" "net/url" - "path" - "path/filepath" "strings" "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/vaultfs/vaultauth" + "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" ) type fsProviderCtxKey struct{} @@ -29,50 +29,11 @@ func FSProviderFromContext(ctx context.Context) fsimpl.FSProvider { return nil } -// ParseSourceURL parses a datasource URL value, which may be '-' (for stdin://), -// or it may be a Windows path (with driver letter and back-slash separators) or -// UNC, or it may be relative. It also might just be a regular absolute URL... -// In all cases it returns a correct URL for the value. It may be a relative URL -// in which case the scheme should be assumed to be 'file' -func ParseSourceURL(value string) (*url.URL, error) { - if value == "-" { - value = "stdin://" - } - value = filepath.ToSlash(value) - // handle absolute Windows paths - volName := "" - if volName = filepath.VolumeName(value); volName != "" { - // handle UNCs - if len(volName) > 2 { - value = "file:" + value - } else { - value = "file:///" + value - } - } - srcURL, err := url.Parse(value) - if err != nil { - return nil, err - } - - if volName != "" && len(srcURL.Path) >= 3 { - if srcURL.Path[0] == '/' && srcURL.Path[2] == ':' { - srcURL.Path = srcURL.Path[1:] - } - } - - // if it's an absolute path with no scheme, assume it's a file - if srcURL.Scheme == "" && path.IsAbs(srcURL.Path) { - srcURL.Scheme = "file" - } - - return srcURL, nil -} - // FSysForPath returns an [io/fs.FS] for the given path (which may be an URL), // rooted at /. A [fsimpl.FSProvider] is required to be present in ctx, // otherwise an error is returned. func FSysForPath(ctx context.Context, path string) (fs.FS, error) { - u, err := ParseSourceURL(path) + u, err := urlhelpers.ParseSourceURL(path) if err != nil { return nil, err } @@ -82,18 +43,47 @@ func FSysForPath(ctx context.Context, path string) (fs.FS, error) { return nil, fmt.Errorf("no filesystem provider in context") } - // default to "/" so we have a rooted filesystem for all schemes, but also - // support volumes on Windows origPath := u.Path - if u.Scheme == "file" || strings.HasSuffix(u.Scheme, "+file") || u.Scheme == "" { - u.Path, _, err = ResolveLocalPath(origPath) - if err != nil { - return nil, fmt.Errorf("resolve local path %q: %w", origPath, err) + + switch u.Scheme { + case "git+file", "git+http", "git+https", "git+ssh", "git": + // git URLs are special - they have double-slashes that separate a repo from + // a path in the repo. A missing double-slash means the path is the root. + u.Path, _, _ = strings.Cut(u.Path, "//") + } + + switch u.Scheme { + case "git+http", "git+https", "git+ssh", "git": + // no-op, these are handled + case "", "file", "git+file": + // default to "/" so we have a rooted filesystem for all schemes, but also + // support volumes on Windows + root, name, rerr := ResolveLocalPath(nil, u.Path) + if rerr != nil { + return nil, fmt.Errorf("resolve local path %q: %w", origPath, rerr) + } + + // windows absolute paths need a slash between the volume and path + if root != "" && root[0] != '/' { + u.Path = root + "/" + name + } else { + u.Path = root + name } + // if this is a drive letter, add a trailing slash - if u.Path[0] != '/' { + if len(u.Path) == 2 && u.Path[0] != '/' && u.Path[1] == ':' { + u.Path += "/" + } else if u.Path[0] != '/' { u.Path += "/" } + + // if this starts with a drive letter, add a leading slash + // NOPE - this breaks lots of things + // if len(u.Path) > 2 && u.Path[0] != '/' && u.Path[1] == ':' { + // u.Path = "/" + u.Path + // } + default: + u.Path = "/" } fsys, err := fsp.New(u) @@ -101,6 +91,16 @@ func FSysForPath(ctx context.Context, path string) (fs.FS, error) { return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", path, err) } + // inject vault auth methods if needed + switch u.Scheme { + case "vault", "vault+http", "vault+https": + fileFsys, err := fsp.New(&url.URL{Scheme: "file", Path: "/"}) + if err != nil { + return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", path, err) + } + fsys = vaultauth.WithAuthMethod(compositeVaultAuthMethod(fileFsys), fsys) + } + return fsys, nil } diff --git a/internal/datafs/fsys_test.go b/internal/datafs/fsys_test.go index 80ff5ca1..14e2a0bd 100644 --- a/internal/datafs/fsys_test.go +++ b/internal/datafs/fsys_test.go @@ -1,42 +1,78 @@ package datafs import ( + "context" + "io/fs" "net/url" + "os" + "runtime" "testing" + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/gitfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestParseSourceURL(t *testing.T) { - expected := &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo.json", - RawQuery: "bar", - } - u, err := ParseSourceURL("http://example.com/foo.json?bar") - require.NoError(t, err) - assert.EqualValues(t, expected, u) - - expected = &url.URL{Scheme: "", Path: ""} - u, err = ParseSourceURL("") - require.NoError(t, err) - assert.EqualValues(t, expected, u) - - expected = &url.URL{Scheme: "stdin"} - u, err = ParseSourceURL("-") - require.NoError(t, err) - assert.EqualValues(t, expected, u) - - // behviour change in v4 - return relative if it's relative - expected = &url.URL{Path: "./foo/bar.json"} - u, err = ParseSourceURL("./foo/bar.json") - require.NoError(t, err) - assert.EqualValues(t, expected, u) - - expected = &url.URL{Scheme: "file", Path: "/absolute/bar.json"} - u, err = ParseSourceURL("/absolute/bar.json") - require.NoError(t, err) - assert.EqualValues(t, expected, u) +func TestFSysForPath(t *testing.T) { + vol, _ := workingVolume() + + t.Run("no provider", func(t *testing.T) { + ctx := ContextWithFSProvider(context.Background(), nil) + _, err := FSysForPath(ctx, "foo") + require.Error(t, err) + + _, err = FSysForPath(ctx, "foo://bar") + require.Error(t, err) + }) + + t.Run("file url", func(t *testing.T) { + fsp := fsimpl.FSProviderFunc(func(u *url.URL) (fs.FS, error) { + assert.Equal(t, "file", u.Scheme) + + if runtime.GOOS == "windows" { + assert.Equal(t, vol+"/tmp/foo/", u.Path) + return os.DirFS(vol + "/"), nil + } + + assert.Equal(t, "/tmp/foo", u.Path) + return os.DirFS("/"), nil + }, "file") + + ctx := ContextWithFSProvider(context.Background(), fsp) + fsys, err := FSysForPath(ctx, "file:///tmp/foo") + require.NoError(t, err) + require.NotNil(t, fsys) + }) + + t.Run("git url", func(t *testing.T) { + fsp := fsimpl.FSProviderFunc(func(u *url.URL) (fs.FS, error) { + assert.Equal(t, "git://github.com/hairyhenderson/gomplate", u.String()) + return gitfs.New(u) + }, "git") + + ctx := ContextWithFSProvider(context.Background(), fsp) + + fsys, err := FSysForPath(ctx, "git://github.com/hairyhenderson/gomplate//README.md") + require.NoError(t, err) + require.NotNil(t, fsys) + }) + + t.Run("git+file url", func(t *testing.T) { + fsp := fsimpl.FSProviderFunc(func(u *url.URL) (fs.FS, error) { + assert.Equal(t, "git+file", u.Scheme) + if runtime.GOOS == "windows" { + assert.Equal(t, vol+"/tmp/repo/", u.Path) + } else { + assert.Equal(t, "/tmp/repo", u.Path) + } + + return gitfs.New(u) + }, "git+file") + + ctx := ContextWithFSProvider(context.Background(), fsp) + fsys, err := FSysForPath(ctx, "git+file:///tmp/repo//README.md") + require.NoError(t, err) + require.NotNil(t, fsys) + }) } diff --git a/internal/datafs/getenv.go b/internal/datafs/getenv.go new file mode 100644 index 00000000..5f9761d4 --- /dev/null +++ b/internal/datafs/getenv.go @@ -0,0 +1,51 @@ +package datafs + +import ( + "io/fs" + "os" + "strings" +) + +// ExpandEnvFsys - a convenience function intended for internal use only! +func ExpandEnvFsys(fsys fs.FS, s string) string { + return os.Expand(s, func(s string) string { + return GetenvFsys(fsys, s) + }) +} + +// GetenvFsys - a convenience function intended for internal use only! +func GetenvFsys(fsys fs.FS, key string, def ...string) string { + val := getenvFile(fsys, key) + if val == "" && len(def) > 0 { + return def[0] + } + + return val +} + +func getenvFile(fsys fs.FS, key string) string { + val := os.Getenv(key) + if val != "" { + return val + } + + p := os.Getenv(key + "_FILE") + if p != "" { + val, err := readFile(fsys, p) + if err != nil { + return "" + } + return strings.TrimSpace(val) + } + + return "" +} + +func readFile(fsys fs.FS, p string) (string, error) { + b, err := fs.ReadFile(fsys, p) + if err != nil { + return "", err + } + + return string(b), nil +} diff --git a/internal/datafs/getenv_test.go b/internal/datafs/getenv_test.go new file mode 100644 index 00000000..c05a4c4b --- /dev/null +++ b/internal/datafs/getenv_test.go @@ -0,0 +1,97 @@ +package datafs + +import ( + "errors" + "io/fs" + "testing" + "testing/fstest" + + "github.com/hack-pad/hackpadfs" + + "github.com/stretchr/testify/assert" +) + +func TestGetenvFsys(t *testing.T) { + fsys := fs.FS(fstest.MapFS{ + "tmp": &fstest.MapFile{Mode: fs.ModeDir | 0o777}, + "tmp/foo": &fstest.MapFile{Data: []byte("foo")}, + "tmp/unreadable": &fstest.MapFile{Data: []byte("foo"), Mode: 0o000}, + }) + fsys = WrapWdFS(fsys) + + t.Setenv("FOO_FILE", "/tmp/foo") + assert.Equal(t, "foo", GetenvFsys(fsys, "FOO", "bar")) + + t.Setenv("FOO_FILE", "/tmp/missing") + assert.Equal(t, "bar", GetenvFsys(fsys, "FOO", "bar")) + + fsys = writeOnly(fsys) + t.Setenv("FOO_FILE", "/tmp/unreadable") + assert.Equal(t, "bar", GetenvFsys(fsys, "FOO", "bar")) +} + +func TestExpandEnvFsys(t *testing.T) { + fsys := fs.FS(fstest.MapFS{ + "tmp": &fstest.MapFile{Mode: fs.ModeDir | 0o777}, + "tmp/foo": &fstest.MapFile{Data: []byte("foo")}, + "tmp/unreadable": &fstest.MapFile{Data: []byte("foo"), Mode: 0o000}, + }) + fsys = WrapWdFS(fsys) + + t.Setenv("FOO_FILE", "/tmp/foo") + assert.Equal(t, "foo is foo", ExpandEnvFsys(fsys, "foo is $FOO")) + + t.Setenv("FOO_FILE", "/tmp/missing") + assert.Equal(t, "empty", ExpandEnvFsys(fsys, "${FOO}empty")) + + fsys = writeOnly(fsys) + t.Setenv("FOO_FILE", "/tmp/unreadable") + assert.Equal(t, "", ExpandEnvFsys(fsys, "${FOO}")) +} + +// Maybe extract this into a separate package sometime... +// writeOnly - represents a filesystem that's writeable, but read operations fail +func writeOnly(fsys fs.FS) fs.FS { + return &woFS{fsys} +} + +type woFS struct { + fsys fs.FS +} + +func (fsys woFS) Open(name string) (fs.File, error) { + f, err := fsys.fsys.Open(name) + return writeOnlyFile(f), err +} + +func (fsys woFS) ReadDir(_ string) ([]fs.DirEntry, error) { + return nil, ErrWriteOnly +} + +func (fsys woFS) Stat(_ string) (fs.FileInfo, error) { + return nil, ErrWriteOnly +} + +func writeOnlyFile(f fs.File) fs.File { + if f == nil { + return nil + } + + return &woFile{f} +} + +type woFile struct { + fs.File +} + +// Write - +func (f woFile) Write(p []byte) (n int, err error) { + return hackpadfs.WriteFile(f.File, p) +} + +// Read is disabled and returns ErrWriteOnly +func (f woFile) Read([]byte) (n int, err error) { + return 0, ErrWriteOnly +} + +var ErrWriteOnly = errors.New("filesystem is write-only") diff --git a/internal/datafs/mergefs.go b/internal/datafs/mergefs.go new file mode 100644 index 00000000..7c612382 --- /dev/null +++ b/internal/datafs/mergefs.go @@ -0,0 +1,280 @@ +package datafs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "runtime" + "strings" + "sync" + "time" + + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/gomplate/v4/coll" + "github.com/hairyhenderson/gomplate/v4/internal/config" + "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" + "github.com/hairyhenderson/gomplate/v4/internal/parsers" + "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" +) + +// NewMergeFS returns a new filesystem that merges the contents of multiple +// paths. Only a URL like "merge:" or "merge:///" makes sense here - the +// piped-separated lists of sub-sources to merge must be given to Open. +// +// Usually you'll want to use WithDataSourcesFS to provide the map of +// datasources that can be referenced. Otherwise, only URLs will be supported. +// +// An FSProvider will also be needed, which can be provided with a context +// using ContextWithFSProvider. Provide that context with fsimpl.WithContextFS. +func NewMergeFS(u *url.URL) (fs.FS, error) { + if u.Scheme != "merge" { + return nil, fmt.Errorf("unsupported scheme %q", u.Scheme) + } + + return &mergeFS{ + ctx: context.Background(), + sources: map[string]config.DataSource{}, + }, nil +} + +type mergeFS struct { + ctx context.Context + httpClient *http.Client + sources map[string]config.DataSource +} + +//nolint:gochecknoglobals +var MergeFS = fsimpl.FSProviderFunc(NewMergeFS, "merge") + +var ( + _ fs.FS = (*mergeFS)(nil) + _ withContexter = (*mergeFS)(nil) + _ withDataSourceser = (*mergeFS)(nil) +) + +func (f *mergeFS) WithContext(ctx context.Context) fs.FS { + if ctx == nil { + return f + } + + fsys := *f + fsys.ctx = ctx + + return &fsys +} + +func (f *mergeFS) WithHTTPClient(client *http.Client) fs.FS { + if client == nil { + return f + } + + fsys := *f + fsys.httpClient = client + + return &fsys +} + +func (f *mergeFS) WithDataSources(sources map[string]config.DataSource) fs.FS { + if sources == nil { + return f + } + + fsys := *f + fsys.sources = sources + + return &fsys +} + +func (f *mergeFS) Open(name string) (fs.File, error) { + parts := strings.Split(name, "|") + if len(parts) < 2 { + return nil, &fs.PathError{ + Op: "open", Path: name, + Err: fmt.Errorf("need at least 2 datasources to merge"), + } + } + + // now open each of the sub-files + subFiles := make([]subFile, len(parts)) + + modTime := time.Time{} + + for i, part := range parts { + // if this is a datasource, look it up + subSource, ok := f.sources[part] + if !ok { + // maybe it's a relative filename? + u, uerr := urlhelpers.ParseSourceURL(part) + if uerr != nil { + return nil, fmt.Errorf("unknown datasource %q, and couldn't parse URL: %w", part, uerr) + } + subSource = config.DataSource{URL: u} + } + + u := subSource.URL + + fsURL, base := SplitFSMuxURL(u) + + // need to support absolute paths on local filesystem too + // TODO: this is a hack, probably fix this? + if fsURL.Scheme == "file" && runtime.GOOS != "windows" { + base = fsURL.Path + base + } + + fsys, err := FSysForPath(f.ctx, fsURL.String()) + if err != nil { + return nil, &fs.PathError{ + Op: "open", Path: name, + Err: fmt.Errorf("lookup for %s: %w", u.String(), err), + } + } + + // pass in the context and other bits + fsys = fsimpl.WithContextFS(f.ctx, fsys) + fsys = fsimpl.WithHeaderFS(subSource.Header, fsys) + + fsys = fsimpl.WithHTTPClientFS(f.httpClient, fsys) + + // find the content type + fi, err := fs.Stat(fsys, base) + if err != nil { + return nil, &fs.PathError{ + Op: "open", Path: name, + Err: fmt.Errorf("stat merge part %q: %w", part, err), + } + } + + if fi.ModTime().After(modTime) { + modTime = fi.ModTime() + } + + // possible type hint in the type query param. Contrary to spec, we allow + // unescaped '+' characters to make it simpler to provide types like + // "application/array+json" + mimeType := u.Query().Get("type") + mimeType = strings.ReplaceAll(mimeType, " ", "+") + + if mimeType == "" { + mimeType = fsimpl.ContentType(fi) + } + + f, err := fsys.Open(base) + if err != nil { + return nil, &fs.PathError{ + Op: "open", Path: name, + Err: fmt.Errorf("opening merge part %q: %w", part, err), + } + } + + subFiles[i] = subFile{f, mimeType} + } + + return &mergeFile{ + name: name, + subFiles: subFiles, + modTime: modTime, + }, nil +} + +type subFile struct { + fs.File + contentType string +} + +type mergeFile struct { + name string + merged io.Reader // the file's contents, post-merge - buffered here to enable partial reads + fi fs.FileInfo + modTime time.Time // the modTime of the most recently modified sub-file + subFiles []subFile + readMux sync.Mutex +} + +var _ fs.File = (*mergeFile)(nil) + +func (f *mergeFile) Close() error { + for _, f := range f.subFiles { + f.Close() + } + return nil +} + +func (f *mergeFile) Stat() (fs.FileInfo, error) { + if f.merged == nil { + p := make([]byte, 0) + _, err := f.Read(p) + if err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("read: %w", err) + } + } + + return f.fi, nil +} + +func (f *mergeFile) Read(p []byte) (int, error) { + // read from all and merge, then return the requested amount + if f.merged == nil { + f.readMux.Lock() + defer f.readMux.Unlock() + // read from all and merge + data := make([]map[string]interface{}, len(f.subFiles)) + for i, sf := range f.subFiles { + b, err := io.ReadAll(sf) + if err != nil && !errors.Is(err, io.EOF) { + return 0, fmt.Errorf("readAll: %w", err) + } + + data[i], err = parseMap(sf.contentType, string(b)) + if err != nil { + return 0, fmt.Errorf("parsing map with content type %s: %w", sf.contentType, err) + } + } + + md, err := mergeData(data) + if err != nil { + return 0, fmt.Errorf("mergeData: %w", err) + } + + f.merged = bytes.NewReader(md) + + f.fi = FileInfo(f.name, int64(len(md)), 0o400, f.modTime, iohelpers.YAMLMimetype) + } + + return f.merged.Read(p) +} + +func mergeData(data []map[string]interface{}) ([]byte, error) { + dst := data[0] + data = data[1:] + + dst, err := coll.Merge(dst, data...) + if err != nil { + return nil, err + } + + s, err := parsers.ToYAML(dst) + if err != nil { + return nil, err + } + return []byte(s), nil +} + +func parseMap(mimeType, data string) (map[string]interface{}, error) { + datum, err := parsers.ParseData(mimeType, data) + if err != nil { + return nil, fmt.Errorf("parseData: %w", err) + } + var m map[string]interface{} + switch datum := datum.(type) { + case map[string]interface{}: + m = datum + default: + return nil, fmt.Errorf("unexpected data type '%T' for datasource (type %s); merge: can only merge maps", datum, mimeType) + } + return m, nil +} diff --git a/internal/datafs/mergefs_test.go b/internal/datafs/mergefs_test.go new file mode 100644 index 00000000..1e2c777b --- /dev/null +++ b/internal/datafs/mergefs_test.go @@ -0,0 +1,358 @@ +package datafs + +import ( + "context" + "io" + "io/fs" + "mime" + "net/url" + "os" + "path" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/gomplate/v4/internal/config" + "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func mustParseURL(in string) *url.URL { + u, _ := url.Parse(in) + return u +} + +func setupMergeFsys(ctx context.Context, t *testing.T) fs.FS { + t.Helper() + + jsonContent := `{"hello": "world"}` + yamlContent := "hello: earth\ngoodnight: moon\n" + arrayContent := `["hello", "world"]` + + wd, _ := os.Getwd() + + // MapFS doesn't support windows path separators, so we use / exclusively + // in this test + vol := filepath.VolumeName(wd) + if vol != "" && wd != vol { + wd = wd[len(vol)+1:] + } else if wd[0] == '/' { + wd = wd[1:] + } + wd = filepath.ToSlash(wd) + + t.Logf("wd: %s", wd) + + fsys := WrapWdFS(fstest.MapFS{ + "tmp": {Mode: fs.ModeDir | 0o777}, + "tmp/jsonfile.json": {Data: []byte(jsonContent)}, + "tmp/array.json": {Data: []byte(arrayContent)}, + "tmp/yamlfile.yaml": {Data: []byte(yamlContent)}, + "tmp/textfile.txt": {Data: []byte(`plain text...`)}, + path.Join(wd, "jsonfile.json"): {Data: []byte(jsonContent)}, + path.Join(wd, "array.json"): {Data: []byte(arrayContent)}, + path.Join(wd, "yamlfile.yaml"): {Data: []byte(yamlContent)}, + path.Join(wd, "textfile.txt"): {Data: []byte(`plain text...`)}, + path.Join(wd, "tmp/jsonfile.json"): {Data: []byte(jsonContent)}, + path.Join(wd, "tmp/array.json"): {Data: []byte(arrayContent)}, + path.Join(wd, "tmp/yamlfile.yaml"): {Data: []byte(yamlContent)}, + path.Join(wd, "tmp/textfile.txt"): {Data: []byte(`plain text...`)}, + }) + + source := config.DataSource{ + URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml"), + } + sources := map[string]config.DataSource{ + "foo": source, + "bar": {URL: mustParseURL("file:///tmp/jsonfile.json")}, + "baz": {URL: mustParseURL("file:///tmp/yamlfile.yaml")}, + "text": {URL: mustParseURL("file:///tmp/textfile.txt")}, + "badscheme": {URL: mustParseURL("bad:///scheme.json")}, + // mime type overridden by URL query, should fail to parse + "badtype": {URL: mustParseURL("file:///tmp/jsonfile.json?type=foo/bar")}, + "array": { + URL: mustParseURL("file:///tmp/array.json?type=" + url.QueryEscape(iohelpers.JSONArrayMimetype)), + }, + } + + mux := fsimpl.NewMux() + mux.Add(MergeFS) + mux.Add(WrappedFSProvider(fsys, "file", "")) + + ctx = ContextWithFSProvider(ctx, mux) + + fsys, err := NewMergeFS(mustParseURL("merge:///")) + require.NoError(t, err) + + fsys = WithDataSourcesFS(sources, fsys) + fsys = fsimpl.WithContextFS(ctx, fsys) + + return fsys +} + +// func TestReadMerge(t *testing.T) { +// ctx := context.Background() + +// jsonContent := `{"hello": "world"}` +// yamlContent := "hello: earth\ngoodnight: moon\n" +// arrayContent := `["hello", "world"]` + +// mergedContent := "goodnight: moon\nhello: world\n" + +// fsys := fstest.MapFS{} +// fsys["tmp"] = &fstest.MapFile{Mode: fs.ModeDir | 0777} +// fsys["tmp/jsonfile.json"] = &fstest.MapFile{Data: []byte(jsonContent)} +// fsys["tmp/array.json"] = &fstest.MapFile{Data: []byte(arrayContent)} +// fsys["tmp/yamlfile.yaml"] = &fstest.MapFile{Data: []byte(yamlContent)} +// fsys["tmp/textfile.txt"] = &fstest.MapFile{Data: []byte(`plain text...`)} + +// // workding dir with volume name trimmed +// wd, _ := os.Getwd() +// vol := filepath.VolumeName(wd) +// wd = wd[len(vol)+1:] + +// fsys[path.Join(wd, "jsonfile.json")] = &fstest.MapFile{Data: []byte(jsonContent)} +// fsys[path.Join(wd, "array.json")] = &fstest.MapFile{Data: []byte(arrayContent)} +// fsys[path.Join(wd, "yamlfile.yaml")] = &fstest.MapFile{Data: []byte(yamlContent)} +// fsys[path.Join(wd, "textfile.txt")] = &fstest.MapFile{Data: []byte(`plain text...`)} + +// fsmux := fsimpl.NewMux() +// fsmux.Add(fsimpl.WrappedFSProvider(&fsys, "file")) +// ctx = datafs.ContextWithFSProvider(ctx, fsmux) + +// source := &Source{Alias: "foo", URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml")} +// d := &Data{ +// Sources: map[string]*Source{ +// "foo": source, +// "bar": {Alias: "bar", URL: mustParseURL("file:///tmp/jsonfile.json")}, +// "baz": {Alias: "baz", URL: mustParseURL("file:///tmp/yamlfile.yaml")}, +// "text": {Alias: "text", URL: mustParseURL("file:///tmp/textfile.txt")}, +// "badscheme": {Alias: "badscheme", URL: mustParseURL("bad:///scheme.json")}, +// "badtype": {Alias: "badtype", URL: mustParseURL("file:///tmp/textfile.txt?type=foo/bar")}, +// "array": {Alias: "array", URL: mustParseURL("file:///tmp/array.json?type=" + url.QueryEscape(jsonArrayMimetype))}, +// }, +// Ctx: ctx, +// } + +// actual, err := d.readMerge(ctx, source) +// assert.NoError(t, err) +// assert.Equal(t, mergedContent, string(actual)) + +// source.URL = mustParseURL("merge:bar|baz") +// actual, err = d.readMerge(ctx, source) +// assert.NoError(t, err) +// assert.Equal(t, mergedContent, string(actual)) + +// source.URL = mustParseURL("merge:./jsonfile.json|baz") +// actual, err = d.readMerge(ctx, source) +// assert.NoError(t, err) +// assert.Equal(t, mergedContent, string(actual)) + +// source.URL = mustParseURL("merge:file:///tmp/jsonfile.json") +// _, err = d.readMerge(ctx, source) +// assert.Error(t, err) + +// source.URL = mustParseURL("merge:bogusalias|file:///tmp/jsonfile.json") +// _, err = d.readMerge(ctx, source) +// assert.Error(t, err) + +// source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|badscheme") +// _, err = d.readMerge(ctx, source) +// assert.Error(t, err) + +// source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|badtype") +// _, err = d.readMerge(ctx, source) +// assert.Error(t, err) + +// source.URL = mustParseURL("merge:file:///tmp/jsonfile.json|array") +// _, err = d.readMerge(ctx, source) +// assert.Error(t, err) +// } + +func TestMergeData(t *testing.T) { + def := map[string]interface{}{ + "f": true, + "t": false, + "z": "def", + } + out, err := mergeData([]map[string]interface{}{def}) + assert.NoError(t, err) + assert.Equal(t, "f: true\nt: false\nz: def\n", string(out)) + + over := map[string]interface{}{ + "f": false, + "t": true, + "z": "over", + } + out, err = mergeData([]map[string]interface{}{over, def}) + assert.NoError(t, err) + assert.Equal(t, "f: false\nt: true\nz: over\n", string(out)) + + over = map[string]interface{}{ + "f": false, + "t": true, + "z": "over", + "m": map[string]interface{}{ + "a": "aaa", + }, + } + out, err = mergeData([]map[string]interface{}{over, def}) + assert.NoError(t, err) + assert.Equal(t, "f: false\nm:\n a: aaa\nt: true\nz: over\n", string(out)) + + uber := map[string]interface{}{ + "z": "über", + } + out, err = mergeData([]map[string]interface{}{uber, over, def}) + assert.NoError(t, err) + assert.Equal(t, "f: false\nm:\n a: aaa\nt: true\nz: über\n", string(out)) + + uber = map[string]interface{}{ + "m": "notamap", + "z": map[string]interface{}{ + "b": "bbb", + }, + } + out, err = mergeData([]map[string]interface{}{uber, over, def}) + assert.NoError(t, err) + assert.Equal(t, "f: false\nm: notamap\nt: true\nz:\n b: bbb\n", string(out)) + + uber = map[string]interface{}{ + "m": map[string]interface{}{ + "b": "bbb", + }, + } + out, err = mergeData([]map[string]interface{}{uber, over, def}) + assert.NoError(t, err) + assert.Equal(t, "f: false\nm:\n a: aaa\n b: bbb\nt: true\nz: over\n", string(out)) +} + +func TestMergeFS_Open(t *testing.T) { + // u, _ := url.Parse("merge:") + fsys := setupMergeFsys(context.Background(), t) + assert.IsType(t, &mergeFS{}, fsys) + + _, err := fsys.Open("/") + assert.Error(t, err) + + _, err = fsys.Open("just/one/part") + assert.Error(t, err) + assert.ErrorContains(t, err, "need at least 2 datasources to merge") + + // missing aliases, fallback to relative files, but there's no FS registered + // for the empty scheme + _, err = fsys.Open("a|b") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // missing alias + _, err = fsys.Open("bogusalias|file:///tmp/jsonfile.json") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // unregistered scheme + _, err = fsys.Open("file:///tmp/jsonfile.json|badscheme") + assert.ErrorContains(t, err, "no filesystem registered for scheme \"bad\"") +} + +func TestMergeFile_Read(t *testing.T) { + fsys := fstest.MapFS{ + "one.yml": {Data: []byte("one: 1\n")}, + "two.json": {Data: []byte(`{"one": false, "two": 2}`)}, + "three.toml": {Data: []byte("one = 999\nthree = 3\n")}, + } + + files := make([]subFile, 3) + for i, fn := range []string{"one.yml", "two.json", "three.toml"} { + f, _ := fsys.Open(fn) + defer f.Close() + + ct := mime.TypeByExtension(filepath.Ext(fn)) + + files[i] = subFile{f, ct} + } + + mf := &mergeFile{name: "one.yml|two.json|three.toml", subFiles: files} + + b, err := io.ReadAll(mf) + require.NoError(t, err) + assert.Equal(t, "one: 1\nthree: 3\ntwo: 2\n", string(b)) + + // now try with partial reads + for i, fn := range []string{"one.yml", "two.json", "three.toml"} { + f, _ := fsys.Open(fn) + defer f.Close() + + ct := mime.TypeByExtension(filepath.Ext(fn)) + + files[i] = subFile{f, ct} + } + + mf = &mergeFile{name: "one.yml|two.json|three.toml", subFiles: files} + + p := make([]byte, 10) + n, err := mf.Read(p) + require.NoError(t, err) + assert.Equal(t, 10, n) + assert.Equal(t, "one: 1\nthr", string(p)) + + n, err = mf.Read(p) + require.NoError(t, err) + assert.Equal(t, 10, n) + assert.Equal(t, "ee: 3\ntwo:", string(p)) + + n, err = mf.Read(p) + require.NoError(t, err) + assert.Equal(t, 3, n) + assert.Equal(t, " 2\n 3\ntwo:", string(p)) +} + +func TestMergeFS_ReadFile(t *testing.T) { + mergedContent := "goodnight: moon\nhello: world\n" + + fsys := setupMergeFsys(context.Background(), t) + + testdata := []string{ + // absolute URLs + "file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml", + // aliases + "bar|baz", + // mixed relative file and alias + "jsonfile.json|baz", + // relative file with ./ and alias + "./jsonfile.json|baz", + } + + for _, td := range testdata { + t.Run(td, func(t *testing.T) { + f, err := fsys.Open(td) + require.NoError(t, err) + defer f.Close() + + b, err := io.ReadAll(f) + require.NoError(t, err) + assert.Equal(t, mergedContent, string(b)) + }) + } + + // read errors + errortests := []struct { + in string + expectedError string + }{ + {"file:///tmp/jsonfile.json|badtype", "data of type \"foo/bar\" not yet supported"}, + {"file:///tmp/jsonfile.json|array", "can only merge maps"}, + } + + for _, td := range errortests { + t.Run(td.in, func(t *testing.T) { + f, err := fsys.Open(td.in) + require.NoError(t, err) + defer f.Close() + + _, err = io.ReadAll(f) + require.Error(t, err) + assert.Contains(t, err.Error(), td.expectedError) + }) + } +} diff --git a/internal/datafs/stdinfs.go b/internal/datafs/stdinfs.go new file mode 100644 index 00000000..46cb030a --- /dev/null +++ b/internal/datafs/stdinfs.go @@ -0,0 +1,110 @@ +package datafs + +import ( + "bytes" + "context" + "io" + "io/fs" + "net/url" + "time" + + "github.com/hairyhenderson/go-fsimpl" +) + +// NewStdinFS returns a filesystem (an fs.FS) that can be used to read data from +// standard input (os.Stdin). +func NewStdinFS(_ *url.URL) (fs.FS, error) { + return &stdinFS{ctx: context.Background()}, nil +} + +type stdinFS struct { + ctx context.Context +} + +//nolint:gochecknoglobals +var StdinFS = fsimpl.FSProviderFunc(NewStdinFS, "stdin") + +var ( + _ fs.FS = (*stdinFS)(nil) + _ fs.ReadFileFS = (*stdinFS)(nil) + _ withContexter = (*stdinFS)(nil) +) + +func (f stdinFS) WithContext(ctx context.Context) fs.FS { + fsys := f + fsys.ctx = ctx + + return &fsys +} + +func (f *stdinFS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + + stdin := StdinFromContext(f.ctx) + + return &stdinFile{name: name, body: stdin}, nil +} + +func (f *stdinFS) ReadFile(name string) ([]byte, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "readFile", + Path: name, + Err: fs.ErrInvalid, + } + } + + stdin := StdinFromContext(f.ctx) + + return io.ReadAll(stdin) +} + +type stdinFile struct { + body io.Reader + name string +} + +var _ fs.File = (*stdinFile)(nil) + +func (f *stdinFile) Close() error { + if f.body == nil { + return &fs.PathError{Op: "close", Path: f.name, Err: fs.ErrClosed} + } + + f.body = nil + return nil +} + +func (f *stdinFile) stdinReader() (int, error) { + b, err := io.ReadAll(f.body) + if err != nil { + return 0, err + } + + f.body = bytes.NewReader(b) + + return len(b), err +} + +func (f *stdinFile) Stat() (fs.FileInfo, error) { + n, err := f.stdinReader() + if err != nil { + return nil, err + } + + return FileInfo(f.name, int64(n), 0o444, time.Time{}, ""), nil +} + +func (f *stdinFile) Read(p []byte) (int, error) { + if f.body == nil { + return 0, io.EOF + } + + return f.body.Read(p) +} diff --git a/internal/datafs/stdinfs_test.go b/internal/datafs/stdinfs_test.go new file mode 100644 index 00000000..2de477cc --- /dev/null +++ b/internal/datafs/stdinfs_test.go @@ -0,0 +1,109 @@ +package datafs + +import ( + "bytes" + "context" + "io" + "io/fs" + "net/url" + "testing" + + "github.com/hairyhenderson/go-fsimpl" + "github.com/stretchr/testify/assert" +) + +func TestStdinFS_Open(t *testing.T) { + fsys, err := NewStdinFS(nil) + assert.NoError(t, err) + assert.IsType(t, &stdinFS{}, fsys) + + f, err := fsys.Open("foo") + assert.NoError(t, err) + assert.IsType(t, &stdinFile{}, f) +} + +func TestStdinFile_Read(t *testing.T) { + content := `hello world` + + f := &stdinFile{name: "foo", body: bytes.NewBufferString(content)} + b := make([]byte, len(content)) + n, err := f.Read(b) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) +} + +func TestStdinFile_Stat(t *testing.T) { + content := []byte(`hello world`) + + f := &stdinFile{name: "hello", body: bytes.NewReader(content)} + + fi, err := f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) + + f = &stdinFile{name: "hello", body: &errorReader{err: fs.ErrPermission}} + + _, err = f.Stat() + assert.ErrorIs(t, err, fs.ErrPermission) +} + +func TestStdinFS(t *testing.T) { + u, _ := url.Parse("stdin:") + + content := []byte("\nhello file\n") + + ctx := ContextWithStdin(context.Background(), bytes.NewReader(content)) + + fsys, err := NewStdinFS(u) + assert.NoError(t, err) + assert.IsType(t, &stdinFS{}, fsys) + + _, ok := fsys.(*stdinFS) + assert.True(t, ok) + + fsys = fsimpl.WithContextFS(ctx, fsys) + + b, err := fs.ReadFile(fsys, "foo") + assert.NoError(t, err) + assert.Equal(t, "\nhello file\n", string(b)) + + ctx = ContextWithStdin(context.Background(), bytes.NewReader(content)) + fsys = fsimpl.WithContextFS(ctx, fsys) + + _, err = fsys.Open("..") + assert.ErrorIs(t, err, fs.ErrInvalid) + + _, err = fs.ReadFile(fsys, "/foo") + assert.ErrorIs(t, err, fs.ErrInvalid) + + f, err := fsys.Open("doesn't matter what it's named.txt") + assert.NoError(t, err) + + fi, err := f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) + + b, err = io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, content, b) + + err = f.Close() + assert.NoError(t, err) + + err = f.Close() + assert.ErrorIs(t, err, fs.ErrClosed) + + p := make([]byte, 5) + _, err = f.Read(p) + assert.Error(t, err) + assert.ErrorIs(t, err, io.EOF) +} + +type errorReader struct { + err error +} + +func (r *errorReader) Read(_ []byte) (int, error) { + return 0, r.err +} diff --git a/internal/datafs/vaultauth.go b/internal/datafs/vaultauth.go new file mode 100644 index 00000000..3ad733a0 --- /dev/null +++ b/internal/datafs/vaultauth.go @@ -0,0 +1,89 @@ +package datafs + +import ( + "context" + "fmt" + "io/fs" + "os" + + "github.com/hairyhenderson/go-fsimpl/vaultfs/vaultauth" + "github.com/hairyhenderson/gomplate/v4/internal/deprecated" + "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/api/auth/aws" +) + +// compositeVaultAuthMethod configures the auth method based on environment +// variables. It extends [vaultfs.EnvAuthMethod] by falling back to AWS EC2 +// authentication if the other methods fail. +func compositeVaultAuthMethod(envFsys fs.FS) api.AuthMethod { + return vaultauth.CompositeAuthMethod( + vaultauth.EnvAuthMethod(), + envEC2AuthAdapter(envFsys), + ) +} + +// func CompositeVaultAuthMethod() api.AuthMethod { +// return compositeVaultAuthMethod(WrapWdFS(osfs.NewFS())) +// } + +// envEC2AuthAdapter builds an AWS EC2 authentication method from environment +// variables, for use only with [CompositeVaultAuthMethod] +func envEC2AuthAdapter(envFS fs.FS) api.AuthMethod { + mountPath := GetenvFsys(envFS, "VAULT_AUTH_AWS_MOUNT", "aws") + + nonce := GetenvFsys(envFS, "VAULT_AUTH_AWS_NONCE") + role := GetenvFsys(envFS, "VAULT_AUTH_AWS_ROLE") + + // temporary workaround while we wait to deprecate AWS_META_ENDPOINT + if endpoint := os.Getenv("AWS_META_ENDPOINT"); endpoint != "" { + deprecated.WarnDeprecated(context.Background(), "Use AWS_EC2_METADATA_SERVICE_ENDPOINT instead of AWS_META_ENDPOINT") + if os.Getenv("AWS_EC2_METADATA_SERVICE_ENDPOINT") == "" { + os.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", endpoint) + } + } + + awsauth, err := aws.NewAWSAuth( + aws.WithEC2Auth(), + aws.WithMountPath(mountPath), + aws.WithNonce(nonce), + aws.WithRole(role), + ) + if err != nil { + return nil + } + + output := GetenvFsys(envFS, "VAULT_AUTH_AWS_NONCE_OUTPUT") + if output == "" { + return awsauth + } + + return &ec2AuthNonceWriter{AWSAuth: awsauth, nonce: nonce, output: output} +} + +// ec2AuthNonceWriter - wraps an AWSAuth, and writes the nonce to the nonce +// output file +type ec2AuthNonceWriter struct { + *aws.AWSAuth + nonce string + output string +} + +func (a *ec2AuthNonceWriter) Login(ctx context.Context, client *api.Client) (*api.Secret, error) { + secret, err := a.AWSAuth.Login(ctx, client) + if err != nil { + return nil, err + } + + nonce := a.nonce + if val, ok := secret.Auth.Metadata["nonce"]; ok { + nonce = val + } + + err = os.WriteFile(a.output, []byte(nonce+"\n"), iohelpers.NormalizeFileMode(0o600)) + if err != nil { + return nil, fmt.Errorf("error writing nonce output file: %w", err) + } + + return secret, nil +} diff --git a/internal/datafs/wdfs.go b/internal/datafs/wdfs.go index e696e7d1..e10fb0f5 100644 --- a/internal/datafs/wdfs.go +++ b/internal/datafs/wdfs.go @@ -13,23 +13,38 @@ import ( "github.com/hairyhenderson/go-fsimpl" ) -// ResolveLocalPath resolves a path on the local filesystem, relative to the +// ResolveLocalPath resolves a path on the given filesystem, relative to the // current working directory, and returns both the root (/ or a volume name on // Windows) and the resolved path. If the path is absolute (e.g. starts with a `/` or // volume name on Windows), it is split and returned as-is. +// If fsys is nil, the current working directory is used. // The output is suitable for use with [io/fs] functions. -// -// TODO: maybe take fsys as an argument, and if it's a wdFS, use its vol instead -// of calling os.Getwd? -func ResolveLocalPath(name string) (root, resolved string, err error) { +func ResolveLocalPath(fsys fs.FS, name string) (root, resolved string, err error) { // ignore empty names if len(name) == 0 { return "", "", nil } + switch fsys := fsys.(type) { + case *wdFS: + return resolveLocalPath(fsys.vol, name) + default: + } + + vol, err := workingVolume() + if err != nil { + return "", "", err + } + + return resolveLocalPath(vol, name) +} + +// workingVolume - returns the current working directory's volume name, or "/" if +// the current working directory has no volume name (e.g. on Unix). +func workingVolume() (string, error) { wd, err := os.Getwd() if err != nil { - return "", "", fmt.Errorf("getwd: %w", err) + return "", fmt.Errorf("getwd: %w", err) } vol := filepath.VolumeName(wd) @@ -37,11 +52,10 @@ func ResolveLocalPath(name string) (root, resolved string, err error) { vol = "/" } - f := &wdFS{vol: vol} - return f.resolveLocalPath(name) + return vol, nil } -func (w *wdFS) resolveLocalPath(name string) (root, resolved string, err error) { +func resolveLocalPath(wvol, name string) (root, resolved string, err error) { // ignore empty names if len(name) == 0 { return "", "", nil @@ -53,15 +67,11 @@ func (w *wdFS) resolveLocalPath(name string) (root, resolved string, err error) // special-case for (Windows) paths that start with '/' but have no volume // name (e.g. "/foo/bar"). UNC paths (beginning with "//") are ignored. if name[0] == '/' && (len(name) == 1 || (name[1] != '/' && name[1] != '?')) { - name = filepath.Join(w.vol, name) + name = filepath.Join(wvol, name) // TODO: maybe this can be reduced to just '!filepath.IsAbs(name)'? } else if name[0] != '/' && !filepath.IsAbs(name) { - abs := "" - abs, err = filepath.Abs(name) - if err != nil { - return "", "", fmt.Errorf("abs %q: %w", name, err) - } - name = abs + wd, _ := os.Getwd() + name = filepath.Join(wd, name) } name, err = normalizeWindowsPath(name) @@ -200,7 +210,7 @@ var WdFS = fsimpl.FSProviderFunc( return nil, fmt.Errorf("unsupported path %q: %w", u.Path, fs.ErrInvalid) } - vol, _, err := ResolveLocalPath(u.Path) + vol, _, err := ResolveLocalPath(nil, u.Path) if err != nil { return nil, fmt.Errorf("resolve %q: %w", u.Path, err) } @@ -232,31 +242,37 @@ func WrapWdFS(fsys fs.FS) fs.FS { return fsys } - return &wdFS{fsys: fsys} + vol, _ := workingVolume() + + return &wdFS{vol: vol, fsys: fsys} } // wdFS is a filesystem wrapper that assumes non-absolute paths are relative to // the current working directory (as reported by [os.Getwd]). -// It only works in a meaningful way when used with a local filesystem (e.g. +// It only works in a meaningful way when used with a local filesystem (e.g. // [os.DirFS] or [hackpadfs/os.FS]). type wdFS struct { fsys fs.FS - vol string + + // volume name used for drive-relative paths on Windows for cases when they + // shouldn't be relative to the current working directory's volume + // TODO: validate that this is actually needed + vol string } var ( - _ fs.FS = &wdFS{} - _ fs.StatFS = &wdFS{} - _ fs.ReadFileFS = &wdFS{} - _ fs.ReadDirFS = &wdFS{} - _ fs.SubFS = &wdFS{} - _ fs.GlobFS = &wdFS{} - _ hackpadfs.CreateFS = &wdFS{} - _ hackpadfs.OpenFileFS = &wdFS{} - _ hackpadfs.MkdirFS = &wdFS{} - _ hackpadfs.MkdirAllFS = &wdFS{} - _ hackpadfs.RemoveFS = &wdFS{} - _ hackpadfs.ChmodFS = &wdFS{} + _ fs.FS = (*wdFS)(nil) + _ fs.StatFS = (*wdFS)(nil) + _ fs.ReadFileFS = (*wdFS)(nil) + _ fs.ReadDirFS = (*wdFS)(nil) + _ fs.SubFS = (*wdFS)(nil) + _ fs.GlobFS = (*wdFS)(nil) + _ hackpadfs.CreateFS = (*wdFS)(nil) + _ hackpadfs.OpenFileFS = (*wdFS)(nil) + _ hackpadfs.MkdirFS = (*wdFS)(nil) + _ hackpadfs.MkdirAllFS = (*wdFS)(nil) + _ hackpadfs.RemoveFS = (*wdFS)(nil) + _ hackpadfs.ChmodFS = (*wdFS)(nil) ) func (w *wdFS) fsysFor(vol string) (fs.FS, error) { @@ -280,7 +296,7 @@ func (w *wdFS) fsysFor(vol string) (fs.FS, error) { } func (w *wdFS) Open(name string) (fs.File, error) { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return nil, fmt.Errorf("resolve: %w", err) } @@ -292,7 +308,7 @@ func (w *wdFS) Open(name string) (fs.File, error) { } func (w *wdFS) Stat(name string) (fs.FileInfo, error) { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return nil, fmt.Errorf("resolve: %w", err) } @@ -304,7 +320,7 @@ func (w *wdFS) Stat(name string) (fs.FileInfo, error) { } func (w *wdFS) ReadFile(name string) ([]byte, error) { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return nil, fmt.Errorf("resolve: %w", err) } @@ -316,7 +332,7 @@ func (w *wdFS) ReadFile(name string) ([]byte, error) { } func (w *wdFS) ReadDir(name string) ([]fs.DirEntry, error) { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return nil, fmt.Errorf("resolve: %w", err) } @@ -344,7 +360,7 @@ func (w *wdFS) Glob(_ string) ([]string, error) { } func (w *wdFS) Create(name string) (fs.File, error) { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return nil, fmt.Errorf("resolve: %w", err) } @@ -356,7 +372,7 @@ func (w *wdFS) Create(name string) (fs.File, error) { } func (w *wdFS) OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error) { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return nil, fmt.Errorf("resolve: %w", err) } @@ -368,7 +384,7 @@ func (w *wdFS) OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error } func (w *wdFS) Mkdir(name string, perm fs.FileMode) error { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return fmt.Errorf("resolve: %w", err) } @@ -384,7 +400,7 @@ func (w *wdFS) Mkdir(name string, perm fs.FileMode) error { } func (w *wdFS) MkdirAll(name string, perm fs.FileMode) error { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return fmt.Errorf("resolve: %w", err) } @@ -396,7 +412,7 @@ func (w *wdFS) MkdirAll(name string, perm fs.FileMode) error { } func (w *wdFS) Remove(name string) error { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return fmt.Errorf("resolve: %w", err) } @@ -408,7 +424,7 @@ func (w *wdFS) Remove(name string) error { } func (w *wdFS) Chmod(name string, mode fs.FileMode) error { - root, resolved, err := w.resolveLocalPath(name) + root, resolved, err := resolveLocalPath(w.vol, name) if err != nil { return fmt.Errorf("resolve: %w", err) } diff --git a/internal/datafs/wdfs_test.go b/internal/datafs/wdfs_test.go index 768ec386..f14ebb19 100644 --- a/internal/datafs/wdfs_test.go +++ b/internal/datafs/wdfs_test.go @@ -140,6 +140,20 @@ func TestWDFS_WriteOps(t *testing.T) { // and check that it's gone _, err = fsys.Stat("/tmp/foo") assert.ErrorIs(t, err, fs.ErrNotExist) + + // make sure we can write to a subfs + subfs, err := fs.Sub(fsys, "tmp") + require.NoError(t, err) + require.NotNil(t, subfs) + + // this is no longer a wdFS so we need to make sure not to use absolute + // paths - the path is relative to the root of the subfs + err = hackpadfs.WriteFullFile(subfs, "foo", []byte("hello world"), 0o600) + require.NoError(t, err) + + b, err = fs.ReadFile(subfs, "foo") + require.NoError(t, err) + assert.Equal(t, "hello world", string(b)) } func skipWindows(t *testing.T) { @@ -160,6 +174,8 @@ func TestResolveLocalPath_NonWindows(t *testing.T) { skipWindows(t) wd, _ := os.Getwd() + fsys := &wdFS{vol: "/", fsys: osfs.NewFS()} + wd = wd[1:] testdata := []struct { @@ -176,7 +192,7 @@ func TestResolveLocalPath_NonWindows(t *testing.T) { for _, td := range testdata { td := td t.Run(td.path, func(t *testing.T) { - root, path, err := ResolveLocalPath(td.path) + root, path, err := ResolveLocalPath(fsys, td.path) require.NoError(t, err) assert.Equal(t, "/", root) assert.Equal(t, td.expected, path) @@ -189,9 +205,12 @@ func TestResolveLocalPath_Windows(t *testing.T) { wd, _ := os.Getwd() volname := filepath.VolumeName(wd) - wd = wd[len(volname)+1:] wd = filepath.ToSlash(wd) + fsys := &wdFS{vol: volname, fsys: osfs.NewFS()} + + wd = wd[len(volname)+1:] + testdata := []struct { path string expRoot string @@ -208,7 +227,7 @@ func TestResolveLocalPath_Windows(t *testing.T) { for _, td := range testdata { td := td t.Run(td.path, func(t *testing.T) { - root, path, err := ResolveLocalPath(td.path) + root, path, err := ResolveLocalPath(fsys, td.path) require.NoError(t, err) assert.Equal(t, td.expRoot, root) assert.Equal(t, td.expected, path) @@ -233,10 +252,8 @@ func TestWdFS_ResolveLocalPath_NonWindows(t *testing.T) { {"/", "."}, } - fsys := &wdFS{} - for _, td := range testdata { - root, path, err := fsys.resolveLocalPath(td.path) + root, path, err := resolveLocalPath("/", td.path) require.NoError(t, err) assert.Equal(t, "/", root) assert.Equal(t, td.expected, path) @@ -248,8 +265,8 @@ func TestWdFS_ResolveLocalPath_Windows(t *testing.T) { wd, _ := os.Getwd() volname := filepath.VolumeName(wd) - wd = wd[len(volname)+1:] wd = filepath.ToSlash(wd) + wd = wd[len(volname)+1:] testdata := []struct { path string @@ -268,12 +285,10 @@ func TestWdFS_ResolveLocalPath_Windows(t *testing.T) { {`//somehost/share/foo/bar`, "//somehost/share", "foo/bar"}, } - fsys := &wdFS{vol: volname} - for _, td := range testdata { td := td t.Run(td.path, func(t *testing.T) { - root, path, err := fsys.resolveLocalPath(td.path) + root, path, err := resolveLocalPath(volname, td.path) require.NoError(t, err) assert.Equal(t, td.expRoot, root) assert.Equal(t, td.expected, path) diff --git a/internal/deprecated/deprecated.go b/internal/deprecated/deprecated.go index c453ebbe..6f9ef317 100644 --- a/internal/deprecated/deprecated.go +++ b/internal/deprecated/deprecated.go @@ -2,6 +2,8 @@ package deprecated import ( "context" + "fmt" + "log/slog" "github.com/rs/zerolog" ) @@ -10,5 +12,10 @@ import ( // datasources func WarnDeprecated(ctx context.Context, msg string) { logger := zerolog.Ctx(ctx) + if !logger.Warn().Enabled() { + // we'll flip to slog soon, but in the meantime if we don't have a + // logger in the context, just log it + slog.WarnContext(ctx, fmt.Sprintf("Deprecated: %s", msg)) + } logger.Warn().Msgf("Deprecated: %s", msg) } diff --git a/internal/iohelpers/mimetypes.go b/internal/iohelpers/mimetypes.go new file mode 100644 index 00000000..f0678fe3 --- /dev/null +++ b/internal/iohelpers/mimetypes.go @@ -0,0 +1,33 @@ +package iohelpers + +import ( + "mime" +) + +const ( + TextMimetype = "text/plain" + CSVMimetype = "text/csv" + JSONMimetype = "application/json" + JSONArrayMimetype = "application/array+json" + TOMLMimetype = "application/toml" + YAMLMimetype = "application/yaml" + EnvMimetype = "application/x-env" + CUEMimetype = "application/cue" +) + +// mimeTypeAliases defines a mapping for non-canonical mime types that are +// sometimes seen in the wild +var mimeTypeAliases = map[string]string{ + "application/x-yaml": YAMLMimetype, + "application/text": TextMimetype, +} + +func MimeAlias(m string) string { + // normalize the type by removing any extra parameters + m, _, _ = mime.ParseMediaType(m) + + if a, ok := mimeTypeAliases[m]; ok { + return a + } + return m +} diff --git a/internal/iohelpers/write_test.go b/internal/iohelpers/write_test.go new file mode 100644 index 00000000..e11b583f --- /dev/null +++ b/internal/iohelpers/write_test.go @@ -0,0 +1,71 @@ +// this is in a separate package so WriteFile can be more thoroughly tested +// without involving an import cycle with datafs +package iohelpers_test + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/hack-pad/hackpadfs" + osfs "github.com/hack-pad/hackpadfs/os" + "github.com/hairyhenderson/gomplate/v4/internal/datafs" + "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + tfs "gotest.tools/v3/fs" +) + +func TestWrite(t *testing.T) { + oldwd, _ := os.Getwd() + defer os.Chdir(oldwd) + + rootDir := tfs.NewDir(t, "gomplate-test") + t.Cleanup(rootDir.Remove) + + // we want to use a real filesystem here, so we can test interactions with + // the current working directory + fsys := datafs.WrapWdFS(osfs.NewFS()) + + newwd := rootDir.Join("the", "path", "we", "want") + badwd := rootDir.Join("some", "other", "dir") + hackpadfs.MkdirAll(fsys, newwd, 0o755) + hackpadfs.MkdirAll(fsys, badwd, 0o755) + newwd, _ = filepath.EvalSymlinks(newwd) + badwd, _ = filepath.EvalSymlinks(badwd) + + err := os.Chdir(newwd) + require.NoError(t, err) + + err = iohelpers.WriteFile(fsys, "/foo", []byte("Hello world")) + assert.Error(t, err) + + rel, err := filepath.Rel(newwd, badwd) + require.NoError(t, err) + err = iohelpers.WriteFile(fsys, rel, []byte("Hello world")) + assert.Error(t, err) + + foopath := filepath.Join(newwd, "foo") + err = iohelpers.WriteFile(fsys, foopath, []byte("Hello world")) + require.NoError(t, err) + + out, err := fs.ReadFile(fsys, foopath) + require.NoError(t, err) + assert.Equal(t, "Hello world", string(out)) + + err = iohelpers.WriteFile(fsys, foopath, []byte("truncate")) + require.NoError(t, err) + + out, err = fs.ReadFile(fsys, foopath) + require.NoError(t, err) + assert.Equal(t, "truncate", string(out)) + + foopath = filepath.Join(newwd, "nonexistant", "subdir", "foo") + err = iohelpers.WriteFile(fsys, foopath, []byte("Hello subdirranean world!")) + require.NoError(t, err) + + out, err = fs.ReadFile(fsys, foopath) + require.NoError(t, err) + assert.Equal(t, "Hello subdirranean world!", string(out)) +} diff --git a/internal/iohelpers/writers_test.go b/internal/iohelpers/writers_test.go index 8a9c68e0..3dbce93b 100644 --- a/internal/iohelpers/writers_test.go +++ b/internal/iohelpers/writers_test.go @@ -4,17 +4,12 @@ import ( "bytes" "fmt" "io" - "io/fs" "os" "path/filepath" "testing" - "github.com/hack-pad/hackpadfs" - osfs "github.com/hack-pad/hackpadfs/os" - "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tfs "gotest.tools/v3/fs" ) func TestAllWhitespace(t *testing.T) { @@ -166,58 +161,59 @@ func TestLazyWriteCloser(t *testing.T) { assert.Error(t, err) } -func TestWrite(t *testing.T) { - oldwd, _ := os.Getwd() - defer os.Chdir(oldwd) +// TODO: uncomment this and fix the import cycle! +// func TestWrite(t *testing.T) { +// oldwd, _ := os.Getwd() +// defer os.Chdir(oldwd) - rootDir := tfs.NewDir(t, "gomplate-test") - t.Cleanup(rootDir.Remove) +// rootDir := tfs.NewDir(t, "gomplate-test") +// t.Cleanup(rootDir.Remove) - // we want to use a real filesystem here, so we can test interactions with - // the current working directory - fsys := datafs.WrapWdFS(osfs.NewFS()) +// // we want to use a real filesystem here, so we can test interactions with +// // the current working directory +// fsys := datafs.WrapWdFS(osfs.NewFS()) - newwd := rootDir.Join("the", "path", "we", "want") - badwd := rootDir.Join("some", "other", "dir") - hackpadfs.MkdirAll(fsys, newwd, 0o755) - hackpadfs.MkdirAll(fsys, badwd, 0o755) - newwd, _ = filepath.EvalSymlinks(newwd) - badwd, _ = filepath.EvalSymlinks(badwd) +// newwd := rootDir.Join("the", "path", "we", "want") +// badwd := rootDir.Join("some", "other", "dir") +// hackpadfs.MkdirAll(fsys, newwd, 0o755) +// hackpadfs.MkdirAll(fsys, badwd, 0o755) +// newwd, _ = filepath.EvalSymlinks(newwd) +// badwd, _ = filepath.EvalSymlinks(badwd) - err := os.Chdir(newwd) - require.NoError(t, err) +// err := os.Chdir(newwd) +// require.NoError(t, err) - err = WriteFile(fsys, "/foo", []byte("Hello world")) - assert.Error(t, err) +// err = WriteFile(fsys, "/foo", []byte("Hello world")) +// assert.Error(t, err) - rel, err := filepath.Rel(newwd, badwd) - require.NoError(t, err) - err = WriteFile(fsys, rel, []byte("Hello world")) - assert.Error(t, err) +// rel, err := filepath.Rel(newwd, badwd) +// require.NoError(t, err) +// err = WriteFile(fsys, rel, []byte("Hello world")) +// assert.Error(t, err) - foopath := filepath.Join(newwd, "foo") - err = WriteFile(fsys, foopath, []byte("Hello world")) - require.NoError(t, err) +// foopath := filepath.Join(newwd, "foo") +// err = WriteFile(fsys, foopath, []byte("Hello world")) +// require.NoError(t, err) - out, err := fs.ReadFile(fsys, foopath) - require.NoError(t, err) - assert.Equal(t, "Hello world", string(out)) +// out, err := fs.ReadFile(fsys, foopath) +// require.NoError(t, err) +// assert.Equal(t, "Hello world", string(out)) - err = WriteFile(fsys, foopath, []byte("truncate")) - require.NoError(t, err) +// err = WriteFile(fsys, foopath, []byte("truncate")) +// require.NoError(t, err) - out, err = fs.ReadFile(fsys, foopath) - require.NoError(t, err) - assert.Equal(t, "truncate", string(out)) +// out, err = fs.ReadFile(fsys, foopath) +// require.NoError(t, err) +// assert.Equal(t, "truncate", string(out)) - foopath = filepath.Join(newwd, "nonexistant", "subdir", "foo") - err = WriteFile(fsys, foopath, []byte("Hello subdirranean world!")) - require.NoError(t, err) +// foopath = filepath.Join(newwd, "nonexistant", "subdir", "foo") +// err = WriteFile(fsys, foopath, []byte("Hello subdirranean world!")) +// require.NoError(t, err) - out, err = fs.ReadFile(fsys, foopath) - require.NoError(t, err) - assert.Equal(t, "Hello subdirranean world!", string(out)) -} +// out, err = fs.ReadFile(fsys, foopath) +// require.NoError(t, err) +// assert.Equal(t, "Hello subdirranean world!", string(out)) +// } func TestAssertPathInWD(t *testing.T) { oldwd, _ := os.Getwd() diff --git a/data/data.go b/internal/parsers/parsefuncs.go index d2fc32fb..c16f86e7 100644 --- a/data/data.go +++ b/internal/parsers/parsefuncs.go @@ -1,8 +1,5 @@ -// Package data contains functions that parse and produce data structures in -// different formats. -// -// Supported formats are: JSON, YAML, TOML, and CSV. -package data +// Package parsers has internal parsers for various formats. +package parsers import ( "bytes" @@ -10,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "os" "strings" "cuelang.org/go/cue" @@ -18,7 +16,6 @@ import ( "github.com/Shopify/ejson" ejsonJson "github.com/Shopify/ejson/json" "github.com/hairyhenderson/gomplate/v4/conv" - "github.com/hairyhenderson/gomplate/v4/env" "github.com/joho/godotenv" // XXX: replace once https://github.com/BurntSushi/toml/pull/179 is merged @@ -61,8 +58,8 @@ func JSON(in string) (map[string]interface{}, error) { // decryptEJSON - decrypts an ejson input, and unmarshals it, stripping the _public_key field. func decryptEJSON(in string) (map[string]interface{}, error) { - keyDir := env.Getenv("EJSON_KEYDIR", "/opt/ejson/keys") - key := env.Getenv("EJSON_KEY") + keyDir := getenv("EJSON_KEYDIR", "/opt/ejson/keys") + key := getenv("EJSON_KEY") rIn := bytes.NewBufferString(in) rOut := &bytes.Buffer{} @@ -79,6 +76,30 @@ func decryptEJSON(in string) (map[string]interface{}, error) { return out, nil } +// a reimplementation of env.Getenv to avoid import cycles +func getenv(key string, def ...string) string { + val := os.Getenv(key) + if val != "" { + return val + } + + p := os.Getenv(key + "_FILE") + if p != "" { + b, err := os.ReadFile(p) + if err != nil { + return "" + } + + val = strings.TrimSpace(string(b)) + } + + if val == "" && len(def) > 0 { + return def[0] + } + + return val +} + // JSONArray - Unmarshal a JSON Array func JSONArray(in string) ([]interface{}, error) { obj := make([]interface{}, 1) @@ -190,8 +211,8 @@ func TOML(in string) (interface{}, error) { return unmarshalObj(obj, in, toml.Unmarshal) } -// dotEnv - Unmarshal a dotenv file -func dotEnv(in string) (interface{}, error) { +// DotEnv - Unmarshal a dotenv file +func DotEnv(in string) (interface{}, error) { env, err := godotenv.Unmarshal(in) if err != nil { return nil, err diff --git a/data/data_test.go b/internal/parsers/parsefuncs_test.go index e4f0f555..caf83fba 100644 --- a/data/data_test.go +++ b/internal/parsers/parsefuncs_test.go @@ -1,4 +1,4 @@ -package data +package parsers import ( "fmt" @@ -562,7 +562,7 @@ QUX='single quotes ignore $variables' "BAZ": "variable expansion: a regular unquoted value", "QUX": "single quotes ignore $variables", } - out, err := dotEnv(in) + out, err := DotEnv(in) require.NoError(t, err) assert.EqualValues(t, expected, out) } diff --git a/internal/parsers/parser.go b/internal/parsers/parser.go new file mode 100644 index 00000000..b01972d8 --- /dev/null +++ b/internal/parsers/parser.go @@ -0,0 +1,39 @@ +package parsers + +import ( + "fmt" + + "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" +) + +func ParseData(mimeType, s string) (out any, err error) { + switch iohelpers.MimeAlias(mimeType) { + case iohelpers.JSONMimetype: + out, err = JSON(s) + if err != nil { + // maybe it's a JSON array + out, err = JSONArray(s) + } + case iohelpers.JSONArrayMimetype: + out, err = JSONArray(s) + case iohelpers.YAMLMimetype: + out, err = YAML(s) + if err != nil { + // maybe it's a YAML array + out, err = YAMLArray(s) + } + case iohelpers.CSVMimetype: + out, err = CSV(s) + case iohelpers.TOMLMimetype: + out, err = TOML(s) + case iohelpers.EnvMimetype: + out, err = DotEnv(s) + case iohelpers.TextMimetype: + out = s + case iohelpers.CUEMimetype: + out, err = CUE(s) + default: + return nil, fmt.Errorf("data of type %q not yet supported", mimeType) + } + return out, err +} diff --git a/internal/tests/integration/basic_test.go b/internal/tests/integration/basic_test.go index 49a3b168..f8aec7cf 100644 --- a/internal/tests/integration/basic_test.go +++ b/internal/tests/integration/basic_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" + "github.com/stretchr/testify/require" "gotest.tools/v3/assert" "gotest.tools/v3/assert/cmp" tfs "gotest.tools/v3/fs" @@ -29,7 +30,7 @@ func setupBasicTest(t *testing.T) *tfs.Dir { func TestBasic_ReportsVersion(t *testing.T) { o, e, err := cmd(t, "-v").run() - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, "", e) assert.Assert(t, cmp.Contains(o, "gomplate version ")) } @@ -88,11 +89,11 @@ func TestBasic_RoutesInputsToProperOutputs(t *testing.T) { } for _, v := range testdata { info, err := os.Stat(v.path) - assert.NilError(t, err) + require.NoError(t, err) m := iohelpers.NormalizeFileMode(v.mode) assert.Equal(t, m, info.Mode(), v.path) content, err := os.ReadFile(v.path) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, v.content, string(content)) } } @@ -209,10 +210,10 @@ func TestBasic_RoutesInputsToProperOutputsWithChmod(t *testing.T) { } for _, v := range testdata { info, err := os.Stat(v.path) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, iohelpers.NormalizeFileMode(v.mode), info.Mode()) content, err := os.ReadFile(v.path) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, v.content, string(content)) } } @@ -237,10 +238,10 @@ func TestBasic_OverridesOutputModeWithChmod(t *testing.T) { } for _, v := range testdata { info, err := os.Stat(v.path) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, iohelpers.NormalizeFileMode(v.mode), info.Mode()) content, err := os.ReadFile(v.path) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, v.content, string(content)) } } @@ -254,13 +255,13 @@ func TestBasic_AppliesChmodBeforeWrite(t *testing.T) { "-f", tmpDir.Join("one"), "-o", out, "--chmod", "0644").run() - assert.NilError(t, err) + require.NoError(t, err) info, err := os.Stat(out) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, iohelpers.NormalizeFileMode(0o644), info.Mode()) content, err := os.ReadFile(out) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, "hi\n", string(content)) } @@ -271,10 +272,10 @@ func TestBasic_CreatesMissingDirectory(t *testing.T) { assertSuccess(t, o, e, err, "") info, err := os.Stat(out) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, iohelpers.NormalizeFileMode(0o640), info.Mode()) content, err := os.ReadFile(out) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, "hi\n", string(content)) out = tmpDir.Join("outdir") @@ -285,7 +286,7 @@ func TestBasic_CreatesMissingDirectory(t *testing.T) { assertSuccess(t, o, e, err, "") info, err = os.Stat(out) - assert.NilError(t, err) + require.NoError(t, err) assert.Equal(t, iohelpers.NormalizeFileMode(0o755|fs.ModeDir), info.Mode()) assert.Equal(t, true, info.IsDir()) diff --git a/internal/tests/integration/datasources_consul_test.go b/internal/tests/integration/datasources_consul_test.go index cbf154d6..f04d5020 100644 --- a/internal/tests/integration/datasources_consul_test.go +++ b/internal/tests/integration/datasources_consul_test.go @@ -139,23 +139,28 @@ func TestDatasources_Consul_ListKeys(t *testing.T) { consulPut(t, consulAddr, "list-of-keys/foo2", "bar2") // Get a list of keys using the ds args - expectedResult := `[{"key":"foo1","value":"{\"bar1\": \"bar1\"}"},{"key":"foo2","value":"bar2"}]` + // expectedResult := `[{"key":"foo1","value":"{\"bar1\": \"bar1\"}"},{"key":"foo2","value":"bar2"}]` + expectedResult := `["foo1","foo2"]` o, e, err := cmd(t, "-d", "consul=consul://", "-i", `{{(ds "consul" "list-of-keys/") | data.ToJSON }}`). withEnv("CONSUL_HTTP_ADDR", "http://"+consulAddr).run() assertSuccess(t, o, e, err, expectedResult) // Get a list of keys using the ds uri - expectedResult = `[{"key":"foo1","value":"{\"bar1\": \"bar1\"}"},{"key":"foo2","value":"bar2"}]` + // expectedResult = `[{"key":"foo1","value":"{\"bar1\": \"bar1\"}"},{"key":"foo2","value":"bar2"}]` + expectedResult = `["foo1","foo2"]` o, e, err = cmd(t, "-d", "consul=consul+http://"+consulAddr+"/list-of-keys/", "-i", `{{(ds "consul" ) | data.ToJSON }}`).run() assertSuccess(t, o, e, err, expectedResult) - // Get a specific value from the list of Consul keys - expectedResult = `{"bar1": "bar1"}` - o, e, err = cmd(t, "-d", "consul=consul+http://"+consulAddr+"/list-of-keys/", - "-i", `{{ $data := ds "consul" }}{{ (index $data 0).value }}`).run() - assertSuccess(t, o, e, err, expectedResult) + // TODO: this doesn't work anymore because consulfs returns a directory + // listing now. + // + // // Get a specific value from the list of Consul keys + // expectedResult = `{"bar1": "bar1"}` + // o, e, err = cmd(t, "-d", "consul=consul+http://"+consulAddr+"/list-of-keys/", + // "-i", `{{ $data := ds "consul" }}{{ (index $data 0).value }}`).run() + // assertSuccess(t, o, e, err, expectedResult) } func TestDatasources_Consul_WithVaultAuth(t *testing.T) { diff --git a/internal/tests/integration/datasources_file_test.go b/internal/tests/integration/datasources_file_test.go index 9af8503b..37e65a0f 100644 --- a/internal/tests/integration/datasources_file_test.go +++ b/internal/tests/integration/datasources_file_test.go @@ -1,6 +1,7 @@ package integration import ( + "path/filepath" "testing" "gotest.tools/v3/fs" @@ -83,7 +84,7 @@ func TestDatasources_File(t *testing.T) { "-i", `{{(datasource "config").foo.bar}}`).run() assertSuccess(t, o, e, err, "baz") - o, e, err = cmd(t, "-d", "dir="+tmpDir.Path(), + o, e, err = cmd(t, "-d", "dir="+tmpDir.Path()+string(filepath.Separator), "-i", `{{ (datasource "dir" "config.json").foo.bar }}`).run() assertSuccess(t, o, e, err, "baz") diff --git a/internal/tests/integration/datasources_git_test.go b/internal/tests/integration/datasources_git_test.go index fbc8e85d..cb8c78a9 100644 --- a/internal/tests/integration/datasources_git_test.go +++ b/internal/tests/integration/datasources_git_test.go @@ -15,6 +15,8 @@ import ( ) func setupDatasourcesGitTest(t *testing.T) *fs.Dir { + t.Helper() + tmpDir := fs.NewDir(t, "gomplate-inttests", fs.WithDir("repo", fs.WithFiles(map[string]string{ @@ -44,6 +46,8 @@ func setupDatasourcesGitTest(t *testing.T) *fs.Dir { } func startGitDaemon(t *testing.T) string { + t.Helper() + tmpDir := setupDatasourcesGitTest(t) pidDir := fs.NewDir(t, "gomplate-inttests-pid") diff --git a/internal/tests/integration/datasources_vault_ec2_test.go b/internal/tests/integration/datasources_vault_ec2_test.go index fb34f0eb..9e4a5192 100644 --- a/internal/tests/integration/datasources_vault_ec2_test.go +++ b/internal/tests/integration/datasources_vault_ec2_test.go @@ -35,6 +35,10 @@ func setupDatasourcesVaultEc2Test(t *testing.T) (*fs.Dir, *vaultClient, *httptes w.Write([]byte("testtoken")) })) + mux.HandleFunc("/latest/meta-data/instance-id", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Logf("IMDS request: %s %s", r.Method, r.URL) + w.Write([]byte("i-00000000")) + })) mux.HandleFunc("/sts/", stsHandler) mux.HandleFunc("/ec2/", ec2Handler) mux.HandleFunc("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -74,6 +78,7 @@ func TestDatasources_VaultEc2(t *testing.T) { "endpoint": srv.URL + "/ec2", "iam_endpoint": srv.URL + "/iam", "sts_endpoint": srv.URL + "/sts", + "sts_region": "us-east-1", }) require.NoError(t, err) @@ -88,11 +93,11 @@ func TestDatasources_VaultEc2(t *testing.T) { }) require.NoError(t, err) - o, e, err := cmd(t, "-d", "vault=vault:///secret", + o, e, err := cmd(t, "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("HOME", tmpDir.Join("home")). withEnv("VAULT_ADDR", "http://"+v.addr). - withEnv("AWS_META_ENDPOINT", srv.URL). + withEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL). run() assertSuccess(t, o, e, err, "bar") } diff --git a/internal/tests/integration/datasources_vault_test.go b/internal/tests/integration/datasources_vault_test.go index 39ccb549..448fa759 100644 --- a/internal/tests/integration/datasources_vault_test.go +++ b/internal/tests/integration/datasources_vault_test.go @@ -20,6 +20,8 @@ import ( const vaultRootToken = "00000000-1111-2222-3333-444455556666" func setupDatasourcesVaultTest(t *testing.T) *vaultClient { + t.Helper() + _, vaultClient := startVault(t) err := vaultClient.vc.Sys().PutPolicy("writepol", `path "*" { @@ -30,7 +32,7 @@ func setupDatasourcesVaultTest(t *testing.T) *vaultClient { capabilities = ["read","delete"] }`) require.NoError(t, err) - err = vaultClient.vc.Sys().PutPolicy("listPol", `path "*" { + err = vaultClient.vc.Sys().PutPolicy("listpol", `path "*" { capabilities = ["read","list","delete"] }`) require.NoError(t, err) @@ -39,6 +41,8 @@ func setupDatasourcesVaultTest(t *testing.T) *vaultClient { } func startVault(t *testing.T) (*fs.Dir, *vaultClient) { + t.Helper() + pidDir := fs.NewDir(t, "gomplate-inttests-vaultpid") t.Cleanup(pidDir.Remove) @@ -85,6 +89,8 @@ func startVault(t *testing.T) (*fs.Dir, *vaultClient) { result.Assert(t, icmd.Expected{ExitCode: 0}) + t.Log(result.Combined()) + // restore old token if it was backed up u, _ := user.Current() homeDir := u.HomeDir @@ -106,30 +112,32 @@ func TestDatasources_Vault_TokenAuth(t *testing.T) { tok, err := v.tokenCreate("readpol", 5) require.NoError(t, err) - o, e, err := cmd(t, "-d", "vault=vault:///secret", + o, e, err := cmd(t, "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_TOKEN", tok). run() assertSuccess(t, o, e, err, "bar") - o, e, err = cmd(t, "-d", "vault=vault+http://"+v.addr+"/secret", + o, e, err = cmd(t, "-d", "vault=vault+http://"+v.addr+"/secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_TOKEN", tok). run() assertSuccess(t, o, e, err, "bar") - _, _, err = cmd(t, "-d", "vault=vault:///secret", + _, _, err = cmd(t, "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "bar").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_TOKEN", tok). run() - assert.ErrorContains(t, err, "error calling ds: couldn't read datasource 'vault': no value found for path /secret/bar") + assert.ErrorContains(t, err, "error calling ds: couldn't read datasource 'vault':") + assert.ErrorContains(t, err, "stat secret/bar") + assert.ErrorContains(t, err, "file does not exist") tokFile := fs.NewFile(t, "test-vault-token", fs.WithContent(tok)) defer tokFile.Remove() - o, e, err = cmd(t, "-d", "vault=vault:///secret", + o, e, err = cmd(t, "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_TOKEN_FILE", tokFile.Path()). @@ -157,7 +165,7 @@ func TestDatasources_Vault_UserPassAuth(t *testing.T) { }) require.NoError(t, err) - o, e, err := cmd(t, "-d", "vault=vault:///secret", + o, e, err := cmd(t, "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_AUTH_USERNAME", "dave"). @@ -170,7 +178,7 @@ func TestDatasources_Vault_UserPassAuth(t *testing.T) { defer userFile.Remove() defer passFile.Remove() o, e, err = cmd(t, - "-d", "vault=vault:///secret", + "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_AUTH_USERNAME_FILE", userFile.Path()). @@ -179,7 +187,7 @@ func TestDatasources_Vault_UserPassAuth(t *testing.T) { assertSuccess(t, o, e, err, "bar") o, e, err = cmd(t, - "-d", "vault=vault:///secret", + "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_AUTH_USERNAME", "dave"). @@ -216,7 +224,7 @@ func TestDatasources_Vault_AppRoleAuth(t *testing.T) { sid, _ := v.vc.Logical().Write("auth/approle/role/testrole/secret-id", nil) secretID := sid.Data["secret_id"].(string) o, e, err := cmd(t, - "-d", "vault=vault:///secret", + "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_ROLE_ID", roleID). @@ -229,7 +237,7 @@ func TestDatasources_Vault_AppRoleAuth(t *testing.T) { sid, _ = v.vc.Logical().Write("auth/approle2/role/testrole/secret-id", nil) secretID = sid.Data["secret_id"].(string) o, e, err = cmd(t, - "-d", "vault=vault:///secret", + "-d", "vault=vault:///secret/", "-i", `{{(ds "vault" "foo").value}}`). withEnv("VAULT_ADDR", "http://"+v.addr). withEnv("VAULT_ROLE_ID", roleID). @@ -258,7 +266,8 @@ func TestDatasources_Vault_DynamicAuth(t *testing.T) { {"vault=vault:///ssh/creds/test?ip=10.1.2.3&username=user", `{{(ds "vault").ip}}`}, {"vault=vault:///?ip=10.1.2.3&username=user", `{{(ds "vault" "ssh/creds/test").ip}}`}, } - tok, err := v.tokenCreate("writepol", len(testCommands)*2) + + tok, err := v.tokenCreate("writepol", len(testCommands)*4) require.NoError(t, err) for _, tc := range testCommands { @@ -277,7 +286,7 @@ func TestDatasources_Vault_List(t *testing.T) { v.vc.Logical().Write("secret/dir/bar", map[string]interface{}{"value": "two"}) defer v.vc.Logical().Delete("secret/dir/foo") defer v.vc.Logical().Delete("secret/dir/bar") - tok, err := v.tokenCreate("listpol", 5) + tok, err := v.tokenCreate("listpol", 15) require.NoError(t, err) o, e, err := cmd(t, @@ -289,7 +298,7 @@ func TestDatasources_Vault_List(t *testing.T) { assertSuccess(t, o, e, err, "bar: two foo: one ") o, e, err = cmd(t, - "-d", "vault=vault+http://"+v.addr+"/secret", + "-d", "vault=vault+http://"+v.addr+"/secret/", "-i", `{{ range (ds "vault" "dir/" ) }}{{ . }} {{end}}`). withEnv("VAULT_TOKEN", tok). run() diff --git a/internal/tests/integration/integration_test.go b/internal/tests/integration/integration_test.go index a57d8d0b..e80ed350 100644 --- a/internal/tests/integration/integration_test.go +++ b/internal/tests/integration/integration_test.go @@ -256,7 +256,9 @@ func (c *command) runInProcess() (o, e string, err error) { stdin := strings.NewReader(c.stdin) - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} err = gcmd.Main(ctx, c.args, stdin, stdout, stderr) return stdout.String(), stderr.String(), err diff --git a/internal/urlhelpers/urlhelpers.go b/internal/urlhelpers/urlhelpers.go new file mode 100644 index 00000000..d3de4ce8 --- /dev/null +++ b/internal/urlhelpers/urlhelpers.go @@ -0,0 +1,46 @@ +package urlhelpers + +import ( + "net/url" + "path" + "path/filepath" +) + +// ParseSourceURL parses a datasource URL value, which may be '-' (for stdin://), +// or it may be a Windows path (with driver letter and back-slash separators) or +// UNC, or it may be relative. It also might just be a regular absolute URL... +// In all cases it returns a correct URL for the value. It may be a relative URL +// in which case the scheme should be assumed to be 'file' +func ParseSourceURL(value string) (*url.URL, error) { + if value == "-" { + value = "stdin://" + } + value = filepath.ToSlash(value) + // handle absolute Windows paths + volName := "" + if volName = filepath.VolumeName(value); volName != "" { + // handle UNCs + if len(volName) > 2 { + value = "file:" + value + } else { + value = "file:///" + value + } + } + srcURL, err := url.Parse(value) + if err != nil { + return nil, err + } + + if volName != "" && len(srcURL.Path) >= 3 { + if srcURL.Path[0] == '/' && srcURL.Path[2] == ':' { + srcURL.Path = srcURL.Path[1:] + } + } + + // if it's an absolute path with no scheme, assume it's a file + if srcURL.Scheme == "" && path.IsAbs(srcURL.Path) { + srcURL.Scheme = "file" + } + + return srcURL, nil +} diff --git a/internal/urlhelpers/urlhelpers_test.go b/internal/urlhelpers/urlhelpers_test.go new file mode 100644 index 00000000..9a0df386 --- /dev/null +++ b/internal/urlhelpers/urlhelpers_test.go @@ -0,0 +1,42 @@ +package urlhelpers + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseSourceURL(t *testing.T) { + expected := &url.URL{ + Scheme: "http", + Host: "example.com", + Path: "/foo.json", + RawQuery: "bar", + } + u, err := ParseSourceURL("http://example.com/foo.json?bar") + require.NoError(t, err) + assert.EqualValues(t, expected, u) + + expected = &url.URL{Scheme: "", Path: ""} + u, err = ParseSourceURL("") + require.NoError(t, err) + assert.EqualValues(t, expected, u) + + expected = &url.URL{Scheme: "stdin"} + u, err = ParseSourceURL("-") + require.NoError(t, err) + assert.EqualValues(t, expected, u) + + // behviour change in v4 - return relative if it's relative + expected = &url.URL{Path: "./foo/bar.json"} + u, err = ParseSourceURL("./foo/bar.json") + require.NoError(t, err) + assert.EqualValues(t, expected, u) + + expected = &url.URL{Scheme: "file", Path: "/absolute/bar.json"} + u, err = ParseSourceURL("/absolute/bar.json") + require.NoError(t, err) + assert.EqualValues(t, expected, u) +} diff --git a/libkv/consul.go b/libkv/consul.go deleted file mode 100644 index c47fb798..00000000 --- a/libkv/consul.go +++ /dev/null @@ -1,193 +0,0 @@ -package libkv - -import ( - "fmt" - "net/url" - "os" - "strings" - "time" - - "github.com/hairyhenderson/yaml" - - "github.com/docker/libkv" - "github.com/docker/libkv/store" - "github.com/docker/libkv/store/consul" - "github.com/hairyhenderson/gomplate/v4/conv" - "github.com/hairyhenderson/gomplate/v4/env" - "github.com/hairyhenderson/gomplate/v4/vault" - consulapi "github.com/hashicorp/consul/api" -) - -const ( - http = "http" - https = "https" - - // environment variables which aren't used by the consul client - consulVaultRoleEnv = "CONSUL_VAULT_ROLE" - consulVaultMountEnv = "CONSUL_VAULT_MOUNT" - consulTimeoutEnv = "CONSUL_TIMEOUT" -) - -// NewConsul - instantiate a new Consul datasource handler -func NewConsul(u *url.URL) (*LibKV, error) { - consul.Register() - - c, err := consulURL(u) - if err != nil { - return nil, err - } - config, err := consulConfig(c.Scheme == https) - if err != nil { - return nil, err - } - - token, err := consulTokenFromVault() - if err != nil { - return nil, fmt.Errorf("failed to set Consul Vault token: %w", err) - } - - if token != "" { - // set CONSUL_HTTP_TOKEN before creating the client - _ = os.Setenv(consulapi.HTTPTokenEnvName, token) - } - - kv, err := libkv.NewStore(store.CONSUL, []string{c.String()}, config) - if err != nil { - return nil, fmt.Errorf("consul setup failed: %w", err) - } - - return &LibKV{kv}, nil -} - -func consulTokenFromVault() (string, error) { - role := env.Getenv(consulVaultRoleEnv) - if role == "" { - return "", nil - } - - client, err := vault.New(nil) - if err != nil { - return "", err - } - - err = client.Login() - defer client.Logout() - if err != nil { - return "", err - } - - mount := env.Getenv(consulVaultMountEnv, "consul") - path := fmt.Sprintf("%s/creds/%s", mount, role) - - data, err := client.Read(path) - if err != nil { - return "", fmt.Errorf("vault auth failed: %w", err) - } - - decoded := make(map[string]interface{}) - err = yaml.Unmarshal(data, &decoded) - if err != nil { - return "", fmt.Errorf("YAML unmarshal: %w", err) - } - - token := decoded["token"].(string) - - return token, nil -} - -// consulAddrFromEnv parses the given address as either a URL or a host:port -// pair. Given no schema, the URL will need to have a schema set separately. -func consulAddrFromEnv(addr string) (*url.URL, error) { - parts := strings.SplitN(addr, "://", 2) - if len(parts) < 2 { - // temporary schema so it parses correctly - addr = "temp://" + addr - } - - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - - if u.Scheme == "temp" { - u.Scheme = "" - } - - return u, nil -} - -// consulURL gets the Consul URL from either the given URL or the -// CONSUL_HTTP_ADDR environment variable. The given URL takes precedence. -func consulURL(u *url.URL) (c *url.URL, err error) { - if u.Host == "" { - addrEnv := env.Getenv(consulapi.HTTPAddrEnvName) - c, err = consulAddrFromEnv(addrEnv) - if err != nil { - return nil, fmt.Errorf("invalid URL %q: %w", addrEnv, err) - } - - if c.Scheme == "" { - c.Scheme = u.Scheme - } - } else { - // We don't want the full URL here, just the scheme and host - c = &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - } - } - - switch c.Scheme { - case "consul+http", http: - c.Scheme = http - case "consul+https", https: - c.Scheme = https - case "consul": - if conv.ToBool(env.Getenv(consulapi.HTTPSSLEnvName)) { - c.Scheme = https - } else { - c.Scheme = http - } - } - - if c.Host == "" && u.Host == "" { - c.Host = "localhost:8500" - } - - return c, nil -} - -func consulConfig(useTLS bool) (*store.Config, error) { - t := conv.MustAtoi(env.Getenv(consulTimeoutEnv)) - config := &store.Config{ - ConnectionTimeout: time.Duration(t) * time.Second, - } - - if useTLS { - tconf := setupTLS() - - var err error - config.TLS, err = consulapi.SetupTLSConfig(tconf) - if err != nil { - return nil, fmt.Errorf("TLS config setup failed: %w", err) - } - } - - return config, nil -} - -func setupTLS() *consulapi.TLSConfig { - tlsConfig := consulapi.TLSConfig{ - Address: env.Getenv(consulapi.HTTPTLSServerName), - CAFile: env.Getenv(consulapi.HTTPCAFile), - CAPath: env.Getenv(consulapi.HTTPCAPath), - CertFile: env.Getenv(consulapi.HTTPClientCert), - KeyFile: env.Getenv(consulapi.HTTPClientKey), - } - - if v := env.Getenv(consulapi.HTTPSSLVerifyEnvName); v != "" { - verify := conv.ToBool(v) - tlsConfig.InsecureSkipVerify = !verify - } - return &tlsConfig -} diff --git a/libkv/consul_test.go b/libkv/consul_test.go deleted file mode 100644 index 5f128aa7..00000000 --- a/libkv/consul_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package libkv - -import ( - "crypto/tls" - "net/url" - "os" - "testing" - "time" - - "github.com/docker/libkv/store" - consulapi "github.com/hashicorp/consul/api" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConsulURL(t *testing.T) { - os.Unsetenv("CONSUL_HTTP_SSL") - - t.Run("consul scheme, CONSUL_HTTP_SSL set to true", func(t *testing.T) { - t.Setenv("CONSUL_HTTP_SSL", "true") - - u, _ := url.Parse("consul://") - expected := &url.URL{Host: "localhost:8500", Scheme: "https"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("consul+http scheme", func(t *testing.T) { - u, _ := url.Parse("consul+http://myconsul.server") - expected := &url.URL{Host: "myconsul.server", Scheme: "http"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("consul+https scheme, CONSUL_HTTP_SSL set to false", func(t *testing.T) { - t.Setenv("CONSUL_HTTP_SSL", "false") - - u, _ := url.Parse("consul+https://myconsul.server:1234") - expected := &url.URL{Host: "myconsul.server:1234", Scheme: "https"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("consul scheme, CONSUL_HTTP_SSL unset", func(t *testing.T) { - u, _ := url.Parse("consul://myconsul.server:2345") - expected := &url.URL{Host: "myconsul.server:2345", Scheme: "http"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("consul scheme, ignore path", func(t *testing.T) { - u, _ := url.Parse("consul://myconsul.server:3456/foo/bar/baz") - expected := &url.URL{Host: "myconsul.server:3456", Scheme: "http"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("given URL takes precedence over env var", func(t *testing.T) { - t.Setenv("CONSUL_HTTP_ADDR", "https://foo:8500") - - u, _ := url.Parse("consul://myconsul.server:3456/foo/bar/baz") - expected := &url.URL{Host: "myconsul.server:3456", Scheme: "http"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("TLS enabled, HTTP_ADDR is set, URL has no host and ambiguous scheme", func(t *testing.T) { - t.Setenv("CONSUL_HTTP_ADDR", "https://foo:8500") - t.Setenv("CONSUL_HTTP_SSL", "true") - - u, _ := url.Parse("consul://") - expected := &url.URL{Host: "foo:8500", Scheme: "https"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) - - t.Run("TLS enabled, HTTP_ADDR is set without scheme, URL has no host and ambiguous scheme", func(t *testing.T) { - t.Setenv("CONSUL_HTTP_ADDR", "localhost:8501") - t.Setenv("CONSUL_HTTP_SSL", "true") - - u, _ := url.Parse("consul://") - expected := &url.URL{Host: "localhost:8501", Scheme: "https"} - actual, err := consulURL(u) - require.NoError(t, err) - assert.Equal(t, expected, actual) - }) -} - -func TestConsulAddrFromEnv(t *testing.T) { - in := "" - - _, err := consulAddrFromEnv("bogus:url:xxx") - assert.Error(t, err) - - addr, err := consulAddrFromEnv(in) - require.NoError(t, err) - assert.Empty(t, addr) - - addr, err = consulAddrFromEnv("https://foo:8500") - require.NoError(t, err) - assert.Equal(t, &url.URL{Scheme: "https", Host: "foo:8500"}, addr) - - addr, err = consulAddrFromEnv("foo:8500") - require.NoError(t, err) - assert.Equal(t, &url.URL{Host: "foo:8500"}, addr) -} - -func TestSetupTLS(t *testing.T) { - expected := &consulapi.TLSConfig{ - Address: "address", - CAFile: "cafile", - CAPath: "ca/path", - CertFile: "certfile", - KeyFile: "keyfile", - } - - t.Setenv("CONSUL_TLS_SERVER_NAME", expected.Address) - t.Setenv("CONSUL_CACERT", expected.CAFile) - t.Setenv("CONSUL_CAPATH", expected.CAPath) - t.Setenv("CONSUL_CLIENT_CERT", expected.CertFile) - t.Setenv("CONSUL_CLIENT_KEY", expected.KeyFile) - - assert.Equal(t, expected, setupTLS()) - - t.Run("CONSUL_HTTP_SSL_VERIFY is true", func(t *testing.T) { - expected.InsecureSkipVerify = false - t.Setenv("CONSUL_HTTP_SSL_VERIFY", "true") - assert.Equal(t, expected, setupTLS()) - }) - - t.Run("CONSUL_HTTP_SSL_VERIFY is false", func(t *testing.T) { - expected.InsecureSkipVerify = true - t.Setenv("CONSUL_HTTP_SSL_VERIFY", "false") - assert.Equal(t, expected, setupTLS()) - }) -} - -func TestConsulConfig(t *testing.T) { - t.Run("default ", func(t *testing.T) { - expectedConfig := &store.Config{} - - actualConfig, err := consulConfig(false) - require.NoError(t, err) - - assert.Equal(t, expectedConfig, actualConfig) - }) - - t.Run("with CONSUL_TIMEOUT", func(t *testing.T) { - t.Setenv("CONSUL_TIMEOUT", "10") - expectedConfig := &store.Config{ - ConnectionTimeout: 10 * time.Second, - } - - actualConfig, err := consulConfig(false) - require.NoError(t, err) - assert.Equal(t, expectedConfig, actualConfig) - }) - - t.Run("with TLS", func(t *testing.T) { - expectedConfig := &store.Config{ - TLS: &tls.Config{MinVersion: tls.VersionTLS13}, - } - actualConfig, err := consulConfig(true) - require.NoError(t, err) - assert.NotNil(t, actualConfig.TLS) - actualConfig.TLS = &tls.Config{MinVersion: tls.VersionTLS13} - assert.Equal(t, expectedConfig, actualConfig) - }) -} diff --git a/libkv/libkv.go b/libkv/libkv.go deleted file mode 100644 index 881bfbf0..00000000 --- a/libkv/libkv.go +++ /dev/null @@ -1,64 +0,0 @@ -package libkv - -import ( - "bytes" - "encoding/json" - "strings" - - "github.com/docker/libkv/store" -) - -// LibKV - -type LibKV struct { - store store.Store -} - -// Login - -func (kv *LibKV) Login() error { - return nil -} - -// Logout - -func (kv *LibKV) Logout() { -} - -// Read - -func (kv *LibKV) Read(path string) ([]byte, error) { - data, err := kv.store.Get(path) - if err != nil { - return nil, err - } - - return data.Value, nil -} - -// List - -func (kv *LibKV) List(path string) ([]byte, error) { - data, err := kv.store.List(path) - if err != nil { - return nil, err - } - - result := []map[string]string{} - for _, pair := range data { - // Remove the path from the key - key := strings.TrimPrefix( - pair.Key, - strings.TrimLeft(path, "/"), - ) - result = append( - result, - map[string]string{ - "key": key, - "value": string(pair.Value), - }, - ) - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(result); err != nil { - return nil, err - } - return buf.Bytes(), nil -} diff --git a/libkv/libkv_test.go b/libkv/libkv_test.go deleted file mode 100644 index a0f9da68..00000000 --- a/libkv/libkv_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package libkv - -import ( - "errors" - "testing" - - "github.com/docker/libkv/store" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRead(t *testing.T) { - s := &FakeStore{data: []*store.KVPair{ - {Key: "foo", Value: []byte("bar")}, - }} - kv := &LibKV{s} - _, err := kv.Read("foo") - - require.NoError(t, err) - - s = &FakeStore{err: errors.New("fail")} - kv = &LibKV{s} - _, err = kv.Read("foo") - - assert.Error(t, err) -} - -type FakeStore struct { - err error - data []*store.KVPair -} - -func (s *FakeStore) Put(_ string, _ []byte, _ *store.WriteOptions) error { - return nil -} - -func (s *FakeStore) Get(key string) (*store.KVPair, error) { - if s.err != nil { - return nil, s.err - } - - for _, v := range s.data { - if v.Key == key { - return v, nil - } - } - return nil, nil -} - -func (s *FakeStore) Delete(_ string) error { - return nil -} - -func (s *FakeStore) Exists(_ string) (bool, error) { - return false, nil -} - -func (s *FakeStore) Watch(_ string, _ <-chan struct{}) (<-chan *store.KVPair, error) { - return nil, nil -} - -func (s *FakeStore) WatchTree(_ string, _ <-chan struct{}) (<-chan []*store.KVPair, error) { - return nil, nil -} - -func (s *FakeStore) NewLock(_ string, _ *store.LockOptions) (store.Locker, error) { - return nil, nil -} - -func (s *FakeStore) List(_ string) ([]*store.KVPair, error) { - return nil, nil -} - -func (s *FakeStore) DeleteTree(_ string) error { - return nil -} - -func (s *FakeStore) AtomicPut(_ string, _ []byte, _ *store.KVPair, _ *store.WriteOptions) (bool, *store.KVPair, error) { - return false, nil, nil -} - -func (s *FakeStore) AtomicDelete(_ string, _ *store.KVPair) (bool, error) { - return false, nil -} - -func (s *FakeStore) Close() {} @@ -7,17 +7,25 @@ import ( "net/http" "net/url" "os" + "sync" "text/template" "time" + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/autofs" "github.com/hairyhenderson/gomplate/v4/data" "github.com/hairyhenderson/gomplate/v4/internal/config" + "github.com/hairyhenderson/gomplate/v4/internal/datafs" ) // Options for template rendering. // // Experimental: subject to breaking changes before the next major release type Options struct { + // FSProvider - allows lookups of data source filesystems. Defaults to + // [DefaultFSProvider]. + FSProvider fsimpl.FSProvider + // Datasources - map of datasources to be read on demand when the // 'datasource'/'ds'/'include' functions are used. Datasources map[string]Datasource @@ -103,6 +111,7 @@ type Datasource struct { type Renderer struct { //nolint:staticcheck data *data.Data + fsp fsimpl.FSProvider nested config.Templates funcs template.FuncMap lDelim string @@ -171,6 +180,10 @@ func NewRenderer(opts Options) *Renderer { missingKey = "error" } + if opts.FSProvider == nil { + opts.FSProvider = DefaultFSProvider + } + return &Renderer{ nested: nested, data: d, @@ -179,6 +192,7 @@ func NewRenderer(opts Options) *Renderer { lDelim: opts.LDelim, rDelim: opts.RDelim, missingKey: missingKey, + fsp: opts.FSProvider, } } @@ -202,10 +216,9 @@ type Template struct { // // Experimental: subject to breaking changes before the next major release func (t *Renderer) RenderTemplates(ctx context.Context, templates []Template) error { - // we need to inject the current context into the Data value, because - // the Datasource method may need it - // TODO: remove this in v4 - t.data.Ctx = ctx + if datafs.FSProviderFromContext(ctx) == nil { + ctx = datafs.ContextWithFSProvider(ctx, t.fsp) + } // configure the template context with the refreshed Data value // only done here because the data context may have changed @@ -273,3 +286,22 @@ func (t *Renderer) Render(ctx context.Context, name, text string, wr io.Writer) {Name: name, Text: text, Writer: wr}, }) } + +// DefaultFSProvider is the default filesystem provider used by gomplate +var DefaultFSProvider = sync.OnceValue[fsimpl.FSProvider]( + func() fsimpl.FSProvider { + fsp := fsimpl.NewMux() + + // start with all go-fsimpl filesystems + fsp.Add(autofs.FS) + + // override go-fsimpl's filefs with wdfs to handle working directories + fsp.Add(datafs.WdFS) + + // gomplate-only filesystem + fsp.Add(datafs.EnvFS) + fsp.Add(datafs.StdinFS) + fsp.Add(datafs.MergeFS) + + return fsp + })() diff --git a/render_test.go b/render_test.go index 45b22067..ab6f58f2 100644 --- a/render_test.go +++ b/render_test.go @@ -10,7 +10,7 @@ import ( "testing" "testing/fstest" - "github.com/hairyhenderson/gomplate/v4/data" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,7 +24,11 @@ func TestRenderTemplate(t *testing.T) { _ = os.Chdir("/") fsys := fstest.MapFS{} - ctx := datafs.ContextWithFSProvider(context.Background(), datafs.WrappedFSProvider(fsys, "mem")) + fsp := fsimpl.NewMux() + fsp.Add(datafs.EnvFS) + fsp.Add(datafs.StdinFS) + fsp.Add(datafs.WrappedFSProvider(fsys, "mem", "")) + ctx := datafs.ContextWithFSProvider(context.Background(), fsp) // no options - built-in function tr := NewRenderer(Options{}) @@ -47,7 +51,7 @@ func TestRenderTemplate(t *testing.T) { "world": {URL: wu}, }, }) - ctx = data.ContextWithStdin(ctx, strings.NewReader("hello")) + ctx = datafs.ContextWithStdin(ctx, strings.NewReader("hello")) out = &bytes.Buffer{} err = tr.Render(ctx, "test", `{{ .hi | toUpper }} {{ (ds "world") | toUpper }}`, out) require.NoError(t, err) diff --git a/template.go b/template.go index 71796ab4..8c0aa64a 100644 --- a/template.go +++ b/template.go @@ -100,7 +100,7 @@ func parseNestedTemplates(ctx context.Context, nested config.Templates, tmpl *te } // TODO: maybe need to do something with root here? - _, reldir, err := datafs.ResolveLocalPath(u.Path) + _, reldir, err := datafs.ResolveLocalPath(fsys, u.Path) if err != nil { return fmt.Errorf("resolveLocalPath: %w", err) } @@ -227,20 +227,21 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer func(context.Context, string) (string, error), excludeGlob []string, mode os.FileMode, modeOverride bool) ([]Template, error) { dir = filepath.ToSlash(filepath.Clean(dir)) - // we want a filesystem rooted at dir, for relative matching + // get a filesystem rooted in the same volume as dir (or / on non-Windows) fsys, err := datafs.FSysForPath(ctx, dir) if err != nil { - return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", dir, err) + return nil, err } // we need dir to be relative to the root of fsys // TODO: maybe need to do something with root here? - _, reldir, err := datafs.ResolveLocalPath(dir) + _, resolvedDir, err := datafs.ResolveLocalPath(fsys, dir) if err != nil { return nil, fmt.Errorf("resolveLocalPath: %w", err) } - subfsys, err := fs.Sub(fsys, reldir) + // we need to sub the filesystem to the dir + subfsys, err := fs.Sub(fsys, resolvedDir) if err != nil { return nil, fmt.Errorf("sub: %w", err) } @@ -248,7 +249,7 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f // just check . because fsys is subbed to dir already dirStat, err := fs.Stat(subfsys, ".") if err != nil { - return nil, fmt.Errorf("stat %q (%q): %w", dir, reldir, err) + return nil, fmt.Errorf("stat %q (%q): %w", dir, resolvedDir, err) } dirMode := dirStat.Mode() diff --git a/vault/auth.go b/vault/auth.go deleted file mode 100644 index 188978ed..00000000 --- a/vault/auth.go +++ /dev/null @@ -1,205 +0,0 @@ -package vault - -import ( - "fmt" - "io" - "os" - "path" - "strings" - "time" - - "github.com/hairyhenderson/gomplate/v4/aws" - "github.com/hairyhenderson/gomplate/v4/conv" - "github.com/hairyhenderson/gomplate/v4/env" - "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" -) - -// GetToken - -func (v *Vault) GetToken() (string, error) { - // sorted in order of precedence - authFuncs := []func() (string, error){ - v.AppRoleLogin, - v.GitHubLogin, - v.UserPassLogin, - v.TokenLogin, - v.EC2Login, - } - for _, f := range authFuncs { - if token, err := f(); token != "" || err != nil { - return token, err - } - } - return "", fmt.Errorf("no vault auth methods succeeded") -} - -// AppRoleLogin - approle auth backend -func (v *Vault) AppRoleLogin() (string, error) { - roleID := env.Getenv("VAULT_ROLE_ID") - secretID := env.Getenv("VAULT_SECRET_ID") - - if roleID == "" || secretID == "" { - return "", nil - } - - mount := env.Getenv("VAULT_AUTH_APPROLE_MOUNT", "approle") - - vars := map[string]interface{}{ - "role_id": roleID, - "secret_id": secretID, - } - - path := fmt.Sprintf("auth/%s/login", mount) - secret, err := v.client.Logical().Write(path, vars) - if err != nil { - return "", fmt.Errorf("appRole logon failed: %w", err) - } - if secret == nil { - return "", fmt.Errorf("empty response from AppRole logon") - } - - return secret.Auth.ClientToken, nil -} - -// GitHubLogin - github auth backend -func (v *Vault) GitHubLogin() (string, error) { - githubToken := env.Getenv("VAULT_AUTH_GITHUB_TOKEN") - - if githubToken == "" { - return "", nil - } - - mount := env.Getenv("VAULT_AUTH_GITHUB_MOUNT", "github") - - vars := map[string]interface{}{ - "token": githubToken, - } - - path := fmt.Sprintf("auth/%s/login", mount) - secret, err := v.client.Logical().Write(path, vars) - if err != nil { - return "", fmt.Errorf("appRole logon failed: %w", err) - } - if secret == nil { - return "", fmt.Errorf("empty response from AppRole logon") - } - - return secret.Auth.ClientToken, nil -} - -// UserPassLogin - userpass auth backend -func (v *Vault) UserPassLogin() (string, error) { - username := env.Getenv("VAULT_AUTH_USERNAME") - password := env.Getenv("VAULT_AUTH_PASSWORD") - - if username == "" || password == "" { - return "", nil - } - - mount := env.Getenv("VAULT_AUTH_USERPASS_MOUNT", "userpass") - - vars := map[string]interface{}{ - "password": password, - } - - path := fmt.Sprintf("auth/%s/login/%s", mount, username) - secret, err := v.client.Logical().Write(path, vars) - if err != nil { - return "", fmt.Errorf("userPass logon failed: %w", err) - } - if secret == nil { - return "", fmt.Errorf("empty response from UserPass logon") - } - - return secret.Auth.ClientToken, nil -} - -// EC2Login - AWS EC2 auth backend -func (v *Vault) EC2Login() (string, error) { - mount := env.Getenv("VAULT_AUTH_AWS_MOUNT", "aws") - output := env.Getenv("VAULT_AUTH_AWS_NONCE_OUTPUT") - - nonce := env.Getenv("VAULT_AUTH_AWS_NONCE") - - vars, err := createEc2LoginVars(nonce) - if err != nil { - return "", err - } - if vars["pkcs7"] == "" { - return "", nil - } - - path := fmt.Sprintf("auth/%s/login", mount) - secret, err := v.client.Logical().Write(path, vars) - if err != nil { - return "", fmt.Errorf("AWS EC2 logon failed: %w", err) - } - if secret == nil { - return "", fmt.Errorf("empty response from AWS EC2 logon") - } - - if output != "" { - if val, ok := secret.Auth.Metadata["nonce"]; ok { - nonce = val - } - f, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, iohelpers.NormalizeFileMode(0o600)) - if err != nil { - return "", fmt.Errorf("error opening nonce output file: %w", err) - } - n, err := f.Write([]byte(nonce + "\n")) - if err != nil { - return "", fmt.Errorf("error writing nonce output file: %w", err) - } - if n == 0 { - return "", fmt.Errorf("no bytes written to nonce output file: %w", err) - } - } - - return secret.Auth.ClientToken, nil -} - -func createEc2LoginVars(nonce string) (map[string]interface{}, error) { - role := env.Getenv("VAULT_AUTH_AWS_ROLE") - - vars := map[string]interface{}{} - - if role != "" { - vars["role"] = role - } - - if nonce != "" { - vars["nonce"] = nonce - } - - opts := aws.ClientOptions{ - Timeout: time.Duration(conv.MustAtoi(env.Getenv("AWS_TIMEOUT"))) * time.Millisecond, - } - - meta := aws.NewEc2Meta(opts) - - doc, err := meta.Dynamic("instance-identity/pkcs7") - if err != nil { - return nil, err - } - vars["pkcs7"] = strings.ReplaceAll(strings.TrimSpace(doc), "\n", "") - return vars, nil -} - -// TokenLogin - -func (v *Vault) TokenLogin() (string, error) { - if token := env.Getenv("VAULT_TOKEN"); token != "" { - return token, nil - } - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - f, err := os.OpenFile(path.Join(homeDir, ".vault-token"), os.O_RDONLY, 0) - if err != nil { - return "", nil - } - b, err := io.ReadAll(f) - if err != nil { - return "", err - } - return string(b), nil -} diff --git a/vault/auth_test.go b/vault/auth_test.go deleted file mode 100644 index fa8f29cc..00000000 --- a/vault/auth_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package vault - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLogin(t *testing.T) { - server, v := MockServer(404, "Not Found") - defer server.Close() - t.Setenv("VAULT_TOKEN", "foo") - v.Login() - assert.Equal(t, "foo", v.client.Token()) -} - -func TestTokenLogin(t *testing.T) { - server, v := MockServer(404, "Not Found") - defer server.Close() - t.Setenv("VAULT_TOKEN", "foo") - - token, err := v.TokenLogin() - require.NoError(t, err) - assert.Equal(t, "foo", token) -} diff --git a/vault/testutils.go b/vault/testutils.go deleted file mode 100644 index 479a669c..00000000 --- a/vault/testutils.go +++ /dev/null @@ -1,32 +0,0 @@ -package vault - -import ( - "fmt" - "net/http" - "net/http/httptest" - "net/url" - - "github.com/hashicorp/vault/api" -) - -// MockServer - -func MockServer(code int, body string) (*httptest.Server, *Vault) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(code) - fmt.Fprintln(w, body) - })) - - tr := &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - } - httpClient := &http.Client{Transport: tr} - config := &api.Config{ - Address: server.URL, - HttpClient: httpClient, - } - - c, _ := api.NewClient(config) - return server, &Vault{c} -} diff --git a/vault/vault.go b/vault/vault.go deleted file mode 100644 index 456e8820..00000000 --- a/vault/vault.go +++ /dev/null @@ -1,118 +0,0 @@ -package vault - -import ( - "bytes" - "encoding/json" - "fmt" - "net/url" - - vaultapi "github.com/hashicorp/vault/api" -) - -// Vault - -type Vault struct { - client *vaultapi.Client -} - -// New - -func New(u *url.URL) (*Vault, error) { - vaultConfig := vaultapi.DefaultConfig() - - err := vaultConfig.ReadEnvironment() - if err != nil { - return nil, fmt.Errorf("vault setup failed: %w", err) - } - - setVaultURL(vaultConfig, u) - - client, err := vaultapi.NewClient(vaultConfig) - if err != nil { - return nil, fmt.Errorf("vault setup failed: %w", err) - } - - return &Vault{client}, nil -} - -func setVaultURL(c *vaultapi.Config, u *url.URL) { - if u != nil && u.Host != "" { - scheme := "https" - if u.Scheme == "vault+http" { - scheme = "http" - } - c.Address = scheme + "://" + u.Host - } -} - -// Login - -func (v *Vault) Login() error { - token, err := v.GetToken() - if err != nil { - return err - } - v.client.SetToken(token) - return nil -} - -// Logout - -func (v *Vault) Logout() { - v.client.ClearToken() -} - -// Read - returns the value of a given path. If no value is found at the given -// path, returns empty slice. -func (v *Vault) Read(path string) ([]byte, error) { - secret, err := v.client.Logical().Read(path) - if err != nil { - return nil, err - } - if secret == nil { - return []byte{}, nil - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(secret.Data); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func (v *Vault) Write(path string, data map[string]interface{}) ([]byte, error) { - secret, err := v.client.Logical().Write(path, data) - if secret == nil { - return []byte{}, err - } - if err != nil { - return nil, err - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(secret.Data); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// List - -func (v *Vault) List(path string) ([]byte, error) { - secret, err := v.client.Logical().List(path) - if err != nil { - return nil, err - } - if secret == nil { - return nil, nil - } - - keys, ok := secret.Data["keys"] - if !ok { - return nil, fmt.Errorf("keys param missing from vault list") - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(keys); err != nil { - return nil, err - } - return buf.Bytes(), nil -} diff --git a/vault/vault_test.go b/vault/vault_test.go deleted file mode 100644 index 1cc5e31d..00000000 --- a/vault/vault_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package vault - -import ( - "net/url" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNew(t *testing.T) { - os.Unsetenv("VAULT_ADDR") - v, err := New(nil) - require.NoError(t, err) - assert.Equal(t, "https://127.0.0.1:8200", v.client.Address()) - - t.Setenv("VAULT_ADDR", "http://example.com:1234") - v, err = New(nil) - require.NoError(t, err) - assert.Equal(t, "http://example.com:1234", v.client.Address()) - os.Unsetenv("VAULT_ADDR") - - u, _ := url.Parse("vault://vault.rocks:8200/secret/foo/bar") - v, err = New(u) - require.NoError(t, err) - assert.Equal(t, "https://vault.rocks:8200", v.client.Address()) - - u, _ = url.Parse("vault+https://vault.rocks:8200/secret/foo/bar") - v, err = New(u) - require.NoError(t, err) - assert.Equal(t, "https://vault.rocks:8200", v.client.Address()) - - u, _ = url.Parse("vault+http://vault.rocks:8200/secret/foo/bar") - v, err = New(u) - require.NoError(t, err) - assert.Equal(t, "http://vault.rocks:8200", v.client.Address()) -} - -func TestRead(t *testing.T) { - server, v := MockServer(404, "") - defer server.Close() - val, err := v.Read("secret/bogus") - assert.Empty(t, val) - require.NoError(t, err) - - expected := "{\"value\":\"foo\"}\n" - server, v = MockServer(200, `{"data":`+expected+`}`) - defer server.Close() - val, err = v.Read("s") - assert.Equal(t, expected, string(val)) - require.NoError(t, err) -} - -func TestWrite(t *testing.T) { - server, v := MockServer(404, "Not Found") - defer server.Close() - val, err := v.Write("secret/bogus", nil) - assert.Empty(t, val) - assert.Error(t, err) - - expected := "{\"value\":\"foo\"}\n" - server, v = MockServer(200, `{"data":`+expected+`}`) - defer server.Close() - val, err = v.Write("s", nil) - assert.Equal(t, expected, string(val)) - require.NoError(t, err) -} |