summaryrefslogtreecommitdiff
path: root/data
diff options
context:
space:
mode:
Diffstat (limited to 'data')
-rw-r--r--data/datasource.go15
-rw-r--r--data/datasource_file.go37
-rw-r--r--data/datasource_file_test.go36
-rw-r--r--data/datasource_git.go8
-rw-r--r--data/datasource_git_test.go8
-rw-r--r--data/datasource_merge.go4
-rw-r--r--data/datasource_merge_test.go55
-rw-r--r--data/datasource_test.go48
8 files changed, 108 insertions, 103 deletions
diff --git a/data/datasource.go b/data/datasource.go
index bebef803..d16fe09a 100644
--- a/data/datasource.go
+++ b/data/datasource.go
@@ -3,6 +3,7 @@ package data
import (
"context"
"fmt"
+ "io/fs"
"mime"
"net/http"
"net/url"
@@ -10,9 +11,8 @@ import (
"sort"
"strings"
- "github.com/spf13/afero"
-
"github.com/hairyhenderson/gomplate/v4/internal/config"
+ "github.com/hairyhenderson/gomplate/v4/internal/datafs"
"github.com/hairyhenderson/gomplate/v4/libkv"
"github.com/hairyhenderson/gomplate/v4/vault"
)
@@ -61,11 +61,16 @@ func (d *Data) registerReaders() {
d.sourceReaders["git+ssh"] = readGit
}
-// lookupReader - return the reader function for the given scheme
+// lookupReader - return the reader function for the given scheme. Empty scheme
+// will return the file reader.
func (d *Data) lookupReader(scheme string) (func(context.Context, *Source, ...string) ([]byte, error), error) {
if d.sourceReaders == nil {
d.registerReaders()
}
+ if scheme == "" {
+ scheme = "file"
+ }
+
r, ok := d.sourceReaders[scheme]
if !ok {
return nil, fmt.Errorf("scheme %s not registered", scheme)
@@ -144,7 +149,7 @@ type Source struct {
Alias string
URL *url.URL
Header http.Header // used for http[s]: URLs, nil otherwise
- fs afero.Fs // used for file: URLs, nil otherwise
+ fs fs.FS // used for file: URLs, nil otherwise
hc *http.Client // used for http[s]: URLs, nil otherwise
vc *vault.Vault // used for vault: URLs, nil otherwise
kv *libkv.LibKV // used for consul:, etcd:, zookeeper: URLs, nil otherwise
@@ -240,7 +245,7 @@ func (d *Data) DefineDatasource(alias, value string) (string, error) {
if d.DatasourceExists(alias) {
return "", nil
}
- srcURL, err := config.ParseSourceURL(value)
+ srcURL, err := datafs.ParseSourceURL(value)
if err != nil {
return "", err
}
diff --git a/data/datasource_file.go b/data/datasource_file.go
index 156ab276..f5c764fe 100644
--- a/data/datasource_file.go
+++ b/data/datasource_file.go
@@ -5,18 +5,22 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
+ "io/fs"
"net/url"
- "os"
"path/filepath"
"strings"
- "github.com/spf13/afero"
+ "github.com/hairyhenderson/gomplate/v4/internal/datafs"
)
-func readFile(_ context.Context, source *Source, args ...string) ([]byte, error) {
+func readFile(ctx context.Context, source *Source, args ...string) ([]byte, error) {
if source.fs == nil {
- source.fs = afero.NewOsFs()
+ fsp := datafs.FSProviderFromContext(ctx)
+ fsys, err := fsp.New(source.URL)
+ if err != nil {
+ return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", source.URL, err)
+ }
+ source.fs = fsys
}
p := filepath.FromSlash(source.URL.Path)
@@ -35,13 +39,18 @@ func readFile(_ context.Context, source *Source, args ...string) ([]byte, error)
source.mediaType = ""
}
+ isDir := strings.HasSuffix(p, string(filepath.Separator))
+ if strings.HasSuffix(p, string(filepath.Separator)) {
+ p = p[:len(p)-1]
+ }
+
// make sure we can access the file
- i, err := source.fs.Stat(p)
+ i, err := fs.Stat(source.fs, p)
if err != nil {
return nil, fmt.Errorf("stat %s: %w", p, err)
}
- if strings.HasSuffix(p, string(filepath.Separator)) {
+ if isDir {
source.mediaType = jsonArrayMimetype
if i.IsDir() {
return readFileDir(source, p)
@@ -49,25 +58,19 @@ func readFile(_ context.Context, source *Source, args ...string) ([]byte, error)
return nil, fmt.Errorf("%s is not a directory", p)
}
- f, err := source.fs.OpenFile(p, os.O_RDONLY, 0)
+ b, err := fs.ReadFile(source.fs, p)
if err != nil {
- return nil, fmt.Errorf("openFile %s: %w", p, err)
- }
-
- defer f.Close()
-
- b, err := io.ReadAll(f)
- if err != nil {
- return nil, fmt.Errorf("readAll %s: %w", p, err)
+ return nil, fmt.Errorf("readFile %s: %w", p, err)
}
return b, nil
}
func readFileDir(source *Source, p string) ([]byte, error) {
- names, err := afero.ReadDir(source.fs, p)
+ names, err := fs.ReadDir(source.fs, p)
if err != nil {
return nil, err
}
+
files := make([]string, len(names))
for i, v := range names {
files[i] = v.Name()
diff --git a/data/datasource_file_test.go b/data/datasource_file_test.go
index 420ca380..7ad1c2a0 100644
--- a/data/datasource_file_test.go
+++ b/data/datasource_file_test.go
@@ -2,10 +2,11 @@ package data
import (
"context"
+ "io/fs"
"testing"
+ "testing/fstest"
- "github.com/spf13/afero"
-
+ "github.com/hairyhenderson/gomplate/v4/internal/datafs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -14,45 +15,42 @@ func TestReadFile(t *testing.T) {
ctx := context.Background()
content := []byte(`hello world`)
- fs := afero.NewMemMapFs()
-
- _ = fs.Mkdir("/tmp", 0777)
- f, _ := fs.Create("/tmp/foo")
- _, _ = f.Write(content)
- _ = fs.Mkdir("/tmp/partial", 0777)
- f, _ = fs.Create("/tmp/partial/foo.txt")
- _, _ = f.Write(content)
- _, _ = fs.Create("/tmp/partial/bar.txt")
- _, _ = fs.Create("/tmp/partial/baz.txt")
- _ = f.Close()
+ fsys := datafs.WrapWdFS(fstest.MapFS{
+ "tmp": {Mode: fs.ModeDir | 0o777},
+ "tmp/foo": {Data: content},
+ "tmp/partial": {Mode: fs.ModeDir | 0o777},
+ "tmp/partial/foo.txt": {Data: content},
+ "tmp/partial/bar.txt": {},
+ "tmp/partial/baz.txt": {},
+ })
source := &Source{Alias: "foo", URL: mustParseURL("file:///tmp/foo")}
- source.fs = fs
+ source.fs = fsys
actual, err := readFile(ctx, source)
require.NoError(t, err)
assert.Equal(t, content, actual)
source = &Source{Alias: "bogus", URL: mustParseURL("file:///bogus")}
- source.fs = fs
+ source.fs = fsys
_, err = readFile(ctx, source)
assert.Error(t, err)
source = &Source{Alias: "partial", URL: mustParseURL("file:///tmp/partial")}
- source.fs = fs
+ source.fs = fsys
actual, err = readFile(ctx, source, "foo.txt")
require.NoError(t, err)
assert.Equal(t, content, actual)
source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/")}
- source.fs = fs
+ source.fs = fsys
actual, err = readFile(ctx, source)
require.NoError(t, err)
assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual)
source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")}
- source.fs = fs
+ source.fs = fsys
actual, err = readFile(ctx, source)
require.NoError(t, err)
assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual)
@@ -61,7 +59,7 @@ func TestReadFile(t *testing.T) {
assert.Equal(t, "application/json", mime)
source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")}
- source.fs = fs
+ source.fs = fsys
actual, err = readFile(ctx, source, "foo.txt")
require.NoError(t, err)
assert.Equal(t, content, actual)
diff --git a/data/datasource_git.go b/data/datasource_git.go
index c5e5adf8..5c12da4c 100644
--- a/data/datasource_git.go
+++ b/data/datasource_git.go
@@ -248,17 +248,17 @@ func (g gitsource) clone(ctx context.Context, repoURL *url.URL, depth int) (bill
}
// read - reads the provided path out of a git repo
-func (g gitsource) read(fs billy.Filesystem, path string) (string, []byte, error) {
- fi, err := fs.Stat(path)
+func (g gitsource) read(fsys billy.Filesystem, path string) (string, []byte, error) {
+ fi, err := fsys.Stat(path)
if err != nil {
return "", nil, fmt.Errorf("can't stat %s: %w", path, err)
}
if fi.IsDir() || strings.HasSuffix(path, string(filepath.Separator)) {
- out, rerr := g.readDir(fs, path)
+ out, rerr := g.readDir(fsys, path)
return jsonArrayMimetype, out, rerr
}
- f, err := fs.OpenFile(path, os.O_RDONLY, 0)
+ f, err := fsys.OpenFile(path, os.O_RDONLY, 0)
if err != nil {
return "", nil, fmt.Errorf("can't open %s: %w", path, err)
}
diff --git a/data/datasource_git_test.go b/data/datasource_git_test.go
index 157e14a8..a5de9bc4 100644
--- a/data/datasource_git_test.go
+++ b/data/datasource_git_test.go
@@ -331,10 +331,10 @@ func TestOpenFileRepo(t *testing.T) {
overrideFSLoader(repoFS)
defer overrideFSLoader(osfs.New(""))
- fs, _, err := g.clone(ctx, mustParseURL("git+file:///repo"), 0)
+ fsys, _, err := g.clone(ctx, mustParseURL("git+file:///repo"), 0)
assert.NilError(t, err)
- f, err := fs.Open("/foo/bar/hi.txt")
+ f, err := fsys.Open("/foo/bar/hi.txt")
assert.NilError(t, err)
b, _ := io.ReadAll(f)
assert.Equal(t, "hello world", string(b))
@@ -370,10 +370,10 @@ func TestOpenBareFileRepo(t *testing.T) {
overrideFSLoader(repoFS)
defer overrideFSLoader(osfs.New(""))
- fs, _, err := g.clone(ctx, mustParseURL("git+file:///bare.git"), 0)
+ fsys, _, err := g.clone(ctx, mustParseURL("git+file:///bare.git"), 0)
assert.NilError(t, err)
- f, err := fs.Open("/hello.txt")
+ f, err := fsys.Open("/hello.txt")
assert.NilError(t, err)
b, _ := io.ReadAll(f)
assert.Equal(t, "hello world", string(b))
diff --git a/data/datasource_merge.go b/data/datasource_merge.go
index f2c28399..b0d8b6bd 100644
--- a/data/datasource_merge.go
+++ b/data/datasource_merge.go
@@ -6,7 +6,7 @@ import (
"strings"
"github.com/hairyhenderson/gomplate/v4/coll"
- "github.com/hairyhenderson/gomplate/v4/internal/config"
+ "github.com/hairyhenderson/gomplate/v4/internal/datafs"
)
// readMerge demultiplexes a `merge:` datasource. The 'args' parameter currently
@@ -31,7 +31,7 @@ func (d *Data) readMerge(ctx context.Context, source *Source, _ ...string) ([]by
subSource, err := d.lookupSource(part)
if err != nil {
// maybe it's a relative filename?
- u, uerr := config.ParseSourceURL(part)
+ u, uerr := datafs.ParseSourceURL(part)
if uerr != nil {
return nil, uerr
}
diff --git a/data/datasource_merge_test.go b/data/datasource_merge_test.go
index 66365f36..48d1f85e 100644
--- a/data/datasource_merge_test.go
+++ b/data/datasource_merge_test.go
@@ -2,12 +2,15 @@ package data
import (
"context"
+ "io/fs"
"net/url"
"os"
+ "path"
"path/filepath"
"testing"
+ "testing/fstest"
- "github.com/spf13/afero"
+ "github.com/hairyhenderson/gomplate/v4/internal/datafs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -22,31 +25,32 @@ func TestReadMerge(t *testing.T) {
mergedContent := "goodnight: moon\nhello: world\n"
- fs := afero.NewMemMapFs()
-
- _ = fs.Mkdir("/tmp", 0777)
- f, _ := fs.Create("/tmp/jsonfile.json")
- _, _ = f.WriteString(jsonContent)
- f, _ = fs.Create("/tmp/array.json")
- _, _ = f.WriteString(arrayContent)
- f, _ = fs.Create("/tmp/yamlfile.yaml")
- _, _ = f.WriteString(yamlContent)
- f, _ = fs.Create("/tmp/textfile.txt")
- _, _ = f.WriteString(`plain text...`)
-
wd, _ := os.Getwd()
- _ = fs.Mkdir(wd, 0777)
- f, _ = fs.Create(filepath.Join(wd, "jsonfile.json"))
- _, _ = f.WriteString(jsonContent)
- f, _ = fs.Create(filepath.Join(wd, "array.json"))
- _, _ = f.WriteString(arrayContent)
- f, _ = fs.Create(filepath.Join(wd, "yamlfile.yaml"))
- _, _ = f.WriteString(yamlContent)
- f, _ = fs.Create(filepath.Join(wd, "textfile.txt"))
- _, _ = f.WriteString(`plain text...`)
+
+ // MapFS doesn't support windows path separators, so we use / exclusively
+ // in this test
+ vol := filepath.VolumeName(wd)
+ if vol != "" && wd != vol {
+ wd = wd[len(vol)+1:]
+ } else if wd[0] == '/' {
+ wd = wd[1:]
+ }
+ wd = filepath.ToSlash(wd)
+
+ fsys := datafs.WrapWdFS(fstest.MapFS{
+ "tmp": {Mode: fs.ModeDir | 0o777},
+ "tmp/jsonfile.json": {Data: []byte(jsonContent)},
+ "tmp/array.json": {Data: []byte(arrayContent)},
+ "tmp/yamlfile.yaml": {Data: []byte(yamlContent)},
+ "tmp/textfile.txt": {Data: []byte(`plain text...`)},
+ path.Join(wd, "jsonfile.json"): {Data: []byte(jsonContent)},
+ path.Join(wd, "array.json"): {Data: []byte(arrayContent)},
+ path.Join(wd, "yamlfile.yaml"): {Data: []byte(yamlContent)},
+ path.Join(wd, "textfile.txt"): {Data: []byte(`plain text...`)},
+ })
source := &Source{Alias: "foo", URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml")}
- source.fs = fs
+ source.fs = fsys
d := &Data{
Sources: map[string]*Source{
"foo": source,
@@ -68,6 +72,11 @@ func TestReadMerge(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, mergedContent, string(actual))
+ source.URL = mustParseURL("merge:jsonfile.json|baz")
+ actual, err = d.readMerge(ctx, source)
+ require.NoError(t, err)
+ assert.Equal(t, mergedContent, string(actual))
+
source.URL = mustParseURL("merge:./jsonfile.json|baz")
actual, err = d.readMerge(ctx, source)
require.NoError(t, err)
diff --git a/data/datasource_test.go b/data/datasource_test.go
index c0d0c70a..43d59c85 100644
--- a/data/datasource_test.go
+++ b/data/datasource_test.go
@@ -7,9 +7,10 @@ import (
"net/url"
"runtime"
"testing"
+ "testing/fstest"
"github.com/hairyhenderson/gomplate/v4/internal/config"
- "github.com/spf13/afero"
+ "github.com/hairyhenderson/gomplate/v4/internal/datafs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -45,26 +46,23 @@ func TestNewData(t *testing.T) {
func TestDatasource(t *testing.T) {
setup := func(ext, mime string, contents []byte) *Data {
fname := "foo." + ext
- fs := afero.NewMemMapFs()
var uPath string
- var f afero.File
if runtime.GOOS == osWindows {
- _ = fs.Mkdir("C:\\tmp", 0777)
- f, _ = fs.Create("C:\\tmp\\" + fname)
uPath = "C:/tmp/" + fname
} else {
- _ = fs.Mkdir("/tmp", 0777)
- f, _ = fs.Create("/tmp/" + fname)
uPath = "/tmp/" + fname
}
- _, _ = f.Write(contents)
+
+ fsys := datafs.WrapWdFS(fstest.MapFS{
+ "tmp/" + fname: &fstest.MapFile{Data: contents},
+ })
sources := map[string]*Source{
"foo": {
Alias: "foo",
URL: &url.URL{Scheme: "file", Path: uPath},
mediaType: mime,
- fs: fs,
+ fs: fsys,
},
}
return &Data{Sources: sources}
@@ -102,31 +100,28 @@ func TestDatasource(t *testing.T) {
func TestDatasourceReachable(t *testing.T) {
fname := "foo.json"
- fs := afero.NewMemMapFs()
var uPath string
- var f afero.File
if runtime.GOOS == osWindows {
- _ = fs.Mkdir("C:\\tmp", 0777)
- f, _ = fs.Create("C:\\tmp\\" + fname)
uPath = "C:/tmp/" + fname
} else {
- _ = fs.Mkdir("/tmp", 0777)
- f, _ = fs.Create("/tmp/" + fname)
uPath = "/tmp/" + fname
}
- _, _ = f.Write([]byte("{}"))
+
+ fsys := datafs.WrapWdFS(fstest.MapFS{
+ "tmp/" + fname: &fstest.MapFile{Data: []byte("{}")},
+ })
sources := map[string]*Source{
"foo": {
Alias: "foo",
URL: &url.URL{Scheme: "file", Path: uPath},
mediaType: jsonMimetype,
- fs: fs,
+ fs: fsys,
},
"bar": {
Alias: "bar",
URL: &url.URL{Scheme: "file", Path: "/bogus"},
- fs: fs,
+ fs: fsys,
},
}
data := &Data{Sources: sources}
@@ -148,27 +143,24 @@ func TestInclude(t *testing.T) {
ext := "txt"
contents := "hello world"
fname := "foo." + ext
- fs := afero.NewMemMapFs()
var uPath string
- var f afero.File
if runtime.GOOS == osWindows {
- _ = fs.Mkdir("C:\\tmp", 0777)
- f, _ = fs.Create("C:\\tmp\\" + fname)
uPath = "C:/tmp/" + fname
} else {
- _ = fs.Mkdir("/tmp", 0777)
- f, _ = fs.Create("/tmp/" + fname)
uPath = "/tmp/" + fname
}
- _, _ = f.Write([]byte(contents))
+
+ fsys := datafs.WrapWdFS(fstest.MapFS{
+ "tmp/" + fname: &fstest.MapFile{Data: []byte(contents)},
+ })
sources := map[string]*Source{
"foo": {
Alias: "foo",
URL: &url.URL{Scheme: "file", Path: uPath},
mediaType: textMimetype,
- fs: fs,
+ fs: fsys,
},
}
data := &Data{
@@ -185,7 +177,6 @@ func (e errorReader) Read(_ []byte) (n int, err error) {
return 0, fmt.Errorf("error")
}
-// nolint: megacheck
func TestDefineDatasource(t *testing.T) {
d := &Data{}
_, err := d.DefineDatasource("", "foo.json")
@@ -204,8 +195,7 @@ func TestDefineDatasource(t *testing.T) {
s := d.Sources["data"]
require.NoError(t, err)
assert.Equal(t, "data", s.Alias)
- assert.Equal(t, "file", s.URL.Scheme)
- assert.True(t, s.URL.IsAbs())
+ assert.EqualValues(t, &url.URL{Path: "foo.json"}, s.URL)
d = &Data{}
_, err = d.DefineDatasource("data", "/otherdir/foo.json")
generated by cgit v1.2.3 (git 2.25.1) at 2026-02-12 10:14:45 +0000