diff options
| -rw-r--r-- | docs/content/datasources.md | 7 | ||||
| -rw-r--r-- | internal/datafs/mergefs.go | 17 | ||||
| -rw-r--r-- | internal/datafs/reader.go | 36 | ||||
| -rw-r--r-- | internal/tests/integration/datasources_http_test.go | 7 | ||||
| -rw-r--r-- | internal/tests/integration/datasources_merge_test.go | 28 | ||||
| -rw-r--r-- | internal/tests/integration/integration_test.go | 12 |
6 files changed, 95 insertions, 12 deletions
diff --git a/docs/content/datasources.md b/docs/content/datasources.md index 3782b427..351b1a50 100644 --- a/docs/content/datasources.md +++ b/docs/content/datasources.md @@ -127,6 +127,13 @@ $ gomplate -d data=file:///tmp/data.txt?type=application/json -i '{{ (ds "data") bar ``` +If you need to provide a query parameter named `type` to the data source, set the `GOMPLATE_TYPE_PARAM` environment variable to another value: + +```console +$ GOMPLATE_TYPE_PARAM=content-type gomplate -d data=https://example.com/mydata?content-type=application/json -i '{{ (ds "data").foo }}' +bar +``` + ### The `.env` file format Many applications and frameworks support the use of a ".env" file for providing environment variables. It can also be considerd a simple key/value file format, and as such can be used as a datasource in gomplate. diff --git a/internal/datafs/mergefs.go b/internal/datafs/mergefs.go index 38302cf2..b056aff7 100644 --- a/internal/datafs/mergefs.go +++ b/internal/datafs/mergefs.go @@ -118,6 +118,17 @@ func (f *mergeFS) Open(name string) (fs.File, error) { u := subSource.URL + // possible type hint in the type query param. Contrary to spec, we allow + // unescaped '+' characters to make it simpler to provide types like + // "application/array+json" + overrideType := typeOverrideParam() + mimeType := u.Query().Get(overrideType) + mimeType = strings.ReplaceAll(mimeType, " ", "+") + + // now that we have the hint, remove it from the URL - we can't have it + // leaking into the filesystem layer + u = removeQueryParam(u, overrideType) + fsURL, base := SplitFSMuxURL(u) // need to support absolute paths on local filesystem too @@ -153,12 +164,6 @@ func (f *mergeFS) Open(name string) (fs.File, error) { modTime = fi.ModTime() } - // possible type hint in the type query param. Contrary to spec, we allow - // unescaped '+' characters to make it simpler to provide types like - // "application/array+json" - mimeType := u.Query().Get("type") - mimeType = strings.ReplaceAll(mimeType, " ", "+") - if mimeType == "" { mimeType = fsimpl.ContentType(fi) } diff --git a/internal/datafs/reader.go b/internal/datafs/reader.go index f1af07cf..a7c00d34 100644 --- a/internal/datafs/reader.go +++ b/internal/datafs/reader.go @@ -8,6 +8,7 @@ import ( "io/fs" "net/http" "net/url" + "os" "runtime" "strings" @@ -16,6 +17,17 @@ import ( "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" ) +// typeOverrideParam gets the query parameter used to override the content type +// used to parse a given datasource - use GOMPLATE_TYPE_PARAM to use a different +// parameter name. +func typeOverrideParam() string { + if v := os.Getenv("GOMPLATE_TYPE_PARAM"); v != "" { + return v + } + + return "type" +} + // DataSourceReader reads content from a datasource type DataSourceReader interface { // ReadSource reads the content of a datasource, given an alias and optional @@ -91,7 +103,25 @@ func (d *dsReader) ReadSource(ctx context.Context, alias string, args ...string) return fc.contentType, fc.b, nil } +func removeQueryParam(u *url.URL, key string) *url.URL { + q := u.Query() + q.Del(key) + u.RawQuery = q.Encode() + return u +} + func (d *dsReader) readFileContent(ctx context.Context, u *url.URL, hdr http.Header) (*content, error) { + // possible type hint in the type query param. Contrary to spec, we allow + // unescaped '+' characters to make it simpler to provide types like + // "application/array+json" + overrideType := typeOverrideParam() + mimeType := u.Query().Get(overrideType) + mimeType = strings.ReplaceAll(mimeType, " ", "+") + + // now that we have the hint, remove it from the URL - we can't have it + // leaking into the filesystem layer + u = removeQueryParam(u, overrideType) + fsys, err := FSysForPath(ctx, u.String()) if err != nil { return nil, fmt.Errorf("fsys for path %v: %w", u, err) @@ -120,12 +150,6 @@ func (d *dsReader) readFileContent(ctx context.Context, u *url.URL, hdr http.Hea return nil, fmt.Errorf("stat (url: %q, name: %q): %w", u, fname, err) } - // possible type hint in the type query param. Contrary to spec, we allow - // unescaped '+' characters to make it simpler to provide types like - // "application/array+json" - mimeType := u.Query().Get("type") - mimeType = strings.ReplaceAll(mimeType, " ", "+") - if mimeType == "" { mimeType = fsimpl.ContentType(fi) } diff --git a/internal/tests/integration/datasources_http_test.go b/internal/tests/integration/datasources_http_test.go index 78679ca1..2c4390de 100644 --- a/internal/tests/integration/datasources_http_test.go +++ b/internal/tests/integration/datasources_http_test.go @@ -15,6 +15,7 @@ func setupDatasourcesHTTPTest(t *testing.T) *httptest.Server { mux.HandleFunc("/actually.json", typeHandler("", `{"value": "json"}`)) mux.HandleFunc("/bogus.csv", typeHandler("text/plain", `{"value": "json"}`)) mux.HandleFunc("/list", typeHandler("application/array+json", `[1, 2, 3, 4, 5]`)) + mux.HandleFunc("/params", paramHandler(t)) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) @@ -68,6 +69,12 @@ func TestDatasources_HTTP_TypeOverridePrecedence(t *testing.T) { "-c", ".="+srv.URL+"/list?type=application/array+json", "-i", "{{ range . }}{{ . }}{{ end }}").run() assertSuccess(t, o, e, err, "12345") + + o, e, err = cmd(t, + "-c", ".="+srv.URL+"/params?foo=bar&type=http&_type=application/json", + "-i", "{{ . | toJSON }}"). + withEnv("GOMPLATE_TYPE_PARAM", "_type").run() + assertSuccess(t, o, e, err, `{"foo":["bar"],"type":["http"]}`) } func TestDatasources_HTTP_AppendQueryAfterSubPaths(t *testing.T) { diff --git a/internal/tests/integration/datasources_merge_test.go b/internal/tests/integration/datasources_merge_test.go index e4a14c97..4317ef80 100644 --- a/internal/tests/integration/datasources_merge_test.go +++ b/internal/tests/integration/datasources_merge_test.go @@ -21,6 +21,10 @@ func setupDatasourcesMergeTest(t *testing.T) (*fs.Dir, *httptest.Server) { mux.HandleFunc("/foo.json", typeHandler("application/json", `{"foo": "bar"}`)) mux.HandleFunc("/1.env", typeHandler("application/x-env", "FOO=1\nBAR=2\n")) mux.HandleFunc("/2.env", typeHandler("application/x-env", "FOO=3\n")) + // this file is served with a misleading content type and extension, for + // testing overriding the type + mux.HandleFunc("/wrongtype.txt", typeHandler("text/html", `{"foo": "bar"}`)) + mux.HandleFunc("/params", paramHandler(t)) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) @@ -65,4 +69,28 @@ func TestDatasources_Merge(t *testing.T) { {{ ds "merged" | toJSON }}`, ).run() assertSuccess(t, o, e, err, `{"foo":"bar","isDefault":true,"isOverride":false,"other":true}`) + + o, e, err = cmd(t, + "-d", "default="+tmpDir.Join("default.yml"), + "-d", "wrongtype="+srv.URL+"/wrongtype.txt?type=application/json", + "-d", "config=merge:wrongtype|default", + "-i", `{{ ds "config" | toJSON }}`, + ).run() + assertSuccess(t, o, e, err, `{"foo":"bar","isDefault":true,"isOverride":false,"other":true}`) + + o, e, err = cmd(t, + "-d", "default="+tmpDir.Join("default.yml"), + "-d", "wrongtype="+srv.URL+"/wrongtype.txt?_=application/json", + "-d", "config=merge:wrongtype|default", + "-i", `{{ ds "config" | toJSON }}`, + ).withEnv("GOMPLATE_TYPE_PARAM", "_").run() + assertSuccess(t, o, e, err, `{"foo":"bar","isDefault":true,"isOverride":false,"other":true}`) + + o, e, err = cmd(t, + "-c", "default="+tmpDir.Join("default.yml"), + "-c", "params="+srv.URL+"/params?foo=bar&type=http&_type=application/json", + "-c", "merged=merge:params|default", + "-i", `{{ .merged | toJSON }}`, + ).withEnv("GOMPLATE_TYPE_PARAM", "_type").run() + assertSuccess(t, o, e, err, `{"foo":["bar"],"isDefault":true,"isOverride":false,"other":true,"type":["http"]}`) } diff --git a/internal/tests/integration/integration_test.go b/internal/tests/integration/integration_test.go index d12bbeaf..8557a155 100644 --- a/internal/tests/integration/integration_test.go +++ b/internal/tests/integration/integration_test.go @@ -95,6 +95,18 @@ func typeHandler(t, body string) func(http.ResponseWriter, *http.Request) { } } +func paramHandler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // just returns params as JSON + w.Header().Set("Content-Type", "application/json") + + enc := json.NewEncoder(w) + if err := enc.Encode(r.URL.Query()); err != nil { + t.Fatalf("error encoding: %v", err) + } + } +} + // freeport - find a free TCP port for immediate use. No guarantees! func freeport(t *testing.T) (port int, addr string) { l, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1")}) |