New datasource function - works for JSON files

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

4 files changed, 329 insertions, 2 deletions

diff --git a/README.md b/README.md
index a625ddf3..81dc6734 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,22 @@ $ echo "Hello, {{.Env.USER}}" | gomplate
 Hello, hairyhenderson
 ```
 
+### Commandline Arguments
+
+#### `--datasource`/`-d`
+
+Add a data source in `name=URL` form. Specify multiple times to add multiple sources. The data can then be used by the [`datasource`](#datasource) function.
+
+A few different forms are valid:
+- `mydata=file:///tmp/my/file.json`
+  - Create a data source named `mydata` which is read from `/tmp/my/file.json`. This form is valid for any file in any path.
+- `mydata=file.json`
+  - Create a data source named `mydata` which is read from `file.json` (in the current working directory). This form is only valid for files in the current directory.
+- `mydata.json`
+  - This form infers the name from the file name (without extension). Only valid for files in the current directory.
+
+## Syntax
+
 #### About `.Env`
 
 You can easily access environment variables with `.Env`, but there's a catch:
@@ -130,6 +146,31 @@ $ gomplate < input.tmpl
 Hello world
 ```
 
+#### `datasource`
+
+Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument).
+
+Currently, only `file://` URLs are supported, and only the JSON format  can be parsed. More support is coming.
+
+##### Example
+
+_`person.json`:_
+```json
+{
+  "name": "Dave"
+}
+```
+
+_`input.tmpl`:_
+```
+Hello {{ (datasource "person").name }}
+```
+
+```console
+$ gomplate -d person.json < input.tmpl
+Hello Dave
+```
+
 #### `ec2meta`
 
 Queries AWS [EC2 Instance Metadata](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for information. This only retrieves data in the `meta-data` path -- for data in the `dynamic` path use `ec2dynamic`.
diff --git a/data.go b/data.go
new file mode 100644
index 00000000..b053faa8
--- /dev/null
+++ b/data.go
@@ -0,0 +1,180 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"mime"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/blang/vfs"
+)
+
+func init() {
+	// Add some types we want to be able to handle which can be missing by default
+	mime.AddExtensionType(".json", "application/json")
+	mime.AddExtensionType(".yml", "application/yaml")
+	mime.AddExtensionType(".yaml", "application/yaml")
+}
+
+// Data -
+type Data struct {
+	Sources map[string]*Source
+	cache   map[string][]byte
+}
+
+// NewData - constructor for Data
+func NewData(datasourceArgs []string) *Data {
+	sources := make(map[string]*Source)
+	for _, v := range datasourceArgs {
+		s, err := ParseSource(v)
+		if err != nil {
+			log.Fatalf("error parsing datasource %v", err)
+			return nil
+		}
+		sources[s.Alias] = s
+	}
+	return &Data{
+		Sources: sources,
+	}
+}
+
+// Source - a data source
+type Source struct {
+	Alias string
+	URL   *url.URL
+	Ext   string
+	Type  string
+	FS    vfs.Filesystem
+}
+
+// NewSource - builds a &Source
+func NewSource(alias string, URL *url.URL) (s *Source) {
+	ext := filepath.Ext(URL.Path)
+
+	var t string
+	if ext != "" {
+		t = mime.TypeByExtension(ext)
+	}
+
+	s = &Source{
+		Alias: alias,
+		URL:   URL,
+		Ext:   ext,
+		Type:  t,
+	}
+	return
+}
+
+// String is the method to format the flag's value, part of the flag.Value interface.
+// The String method's output will be used in diagnostics.
+func (s *Source) String() string {
+	return fmt.Sprintf("%s=%s (%s)", s.Alias, s.URL.String(), s.Type)
+}
+
+// ParseSource -
+func ParseSource(value string) (*Source, error) {
+	var (
+		alias  string
+		srcURL *url.URL
+	)
+	parts := strings.SplitN(value, "=", 2)
+	if len(parts) == 1 {
+		f := parts[0]
+		alias = strings.SplitN(value, ".", 2)[0]
+		if path.Base(f) != f {
+			err := fmt.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory.", value)
+			return nil, err
+		}
+		srcURL = absURL(f)
+	} else if len(parts) == 2 {
+		alias = parts[0]
+		var err error
+		srcURL, err = url.Parse(parts[1])
+		if err != nil {
+			return nil, err
+		}
+
+		if !srcURL.IsAbs() {
+			srcURL = absURL(parts[1])
+		}
+	}
+
+	s := NewSource(alias, srcURL)
+	return s, nil
+}
+
+func absURL(value string) *url.URL {
+	cwd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Can't get working directory: %s", err)
+	}
+	baseURL := &url.URL{
+		Scheme: "file",
+		Path:   cwd + "/",
+	}
+	relURL := &url.URL{
+		Path: value,
+	}
+	return baseURL.ResolveReference(relURL)
+}
+
+// Datasource -
+func (d *Data) Datasource(alias string) map[string]interface{} {
+	source := d.Sources[alias]
+	b, err := d.ReadSource(source.FS, source)
+	if err != nil {
+		log.Fatalf("Couldn't read datasource '%s': %#v", alias, err)
+	}
+	if source.Type == "application/json" {
+		ty := &TypeConv{}
+		return ty.JSON(string(b))
+	}
+	log.Fatalf("Datasources of type %s not yet supported", source.Type)
+	return nil
+}
+
+// ReadSource -
+func (d *Data) ReadSource(fs vfs.Filesystem, source *Source) ([]byte, error) {
+	if d.cache == nil {
+		d.cache = make(map[string][]byte)
+	}
+	cached, ok := d.cache[source.Alias]
+	if ok {
+		return cached, nil
+	}
+	if source.URL.Scheme == "file" {
+		if fs == nil {
+			fs = vfs.OS()
+			source.FS = fs
+		}
+
+		// make sure we can access the file
+		_, err := fs.Stat(source.URL.Path)
+		if err != nil {
+			log.Fatalf("Can't stat %s: %#v", source.URL.Path, err)
+			return nil, err
+		}
+
+		f, err := fs.OpenFile(source.URL.Path, os.O_RDWR, 0)
+		if err != nil {
+			log.Fatalf("Can't open %s: %#v", source.URL.Path, err)
+			return nil, err
+		}
+
+		b, err := ioutil.ReadAll(f)
+		if err != nil {
+			log.Fatalf("Can't read %s: %#v", source.URL.Path, err)
+			return nil, err
+		}
+		d.cache[source.Alias] = b
+		return b, nil
+	}
+
+	log.Fatalf("Datasources with scheme %s not yet supported", source.URL.Scheme)
+	return nil, nil
+}
diff --git a/data_test.go b/data_test.go
new file mode 100644
index 00000000..be60430c
--- /dev/null
+++ b/data_test.go
@@ -0,0 +1,96 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/blang/vfs"
+	"github.com/blang/vfs/memfs"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewSource(t *testing.T) {
+	s := NewSource("foo", &url.URL{
+		Scheme: "file",
+		Path:   "/foo.json",
+	})
+	assert.Equal(t, "application/json", s.Type)
+	assert.Equal(t, ".json", s.Ext)
+
+	s = NewSource("foo", &url.URL{
+		Scheme: "http",
+		Host:   "example.com",
+		Path:   "/foo.json",
+	})
+	assert.Equal(t, "application/json", s.Type)
+	assert.Equal(t, ".json", s.Ext)
+
+	s = NewSource("foo", &url.URL{
+		Scheme: "ftp",
+		Host:   "example.com",
+		Path:   "/foo.json",
+	})
+	assert.Equal(t, "application/json", s.Type)
+	assert.Equal(t, ".json", s.Ext)
+}
+
+func TestParseSourceNoAlias(t *testing.T) {
+	s, err := ParseSource("foo.json")
+	assert.NoError(t, err)
+	assert.Equal(t, "foo", s.Alias)
+
+	_, err = ParseSource("../foo.json")
+	assert.Error(t, err)
+
+	_, err = ParseSource("ftp://example.com/foo.yml")
+	assert.Error(t, err)
+}
+
+func TestParseSourceWithAlias(t *testing.T) {
+	s, err := ParseSource("data=foo.json")
+	assert.NoError(t, err)
+	assert.Equal(t, "data", s.Alias)
+	assert.Equal(t, "file", s.URL.Scheme)
+	assert.Equal(t, "application/json", s.Type)
+	assert.True(t, s.URL.IsAbs())
+
+	s, err = ParseSource("data=/otherdir/foo.json")
+	assert.NoError(t, err)
+	assert.Equal(t, "data", s.Alias)
+	assert.Equal(t, "file", s.URL.Scheme)
+	assert.True(t, s.URL.IsAbs())
+	assert.Equal(t, "/otherdir/foo.json", s.URL.Path)
+
+	s, err = ParseSource("data=sftp://example.com/blahblah/foo.json")
+	assert.NoError(t, err)
+	assert.Equal(t, "data", s.Alias)
+	assert.Equal(t, "sftp", s.URL.Scheme)
+	assert.True(t, s.URL.IsAbs())
+	assert.Equal(t, "/blahblah/foo.json", s.URL.Path)
+}
+
+func TestDatasource(t *testing.T) {
+	fs := memfs.Create()
+	fs.Mkdir("/tmp", 0777)
+	f, _ := vfs.Create(fs, "/tmp/foo.json")
+	f.Write([]byte(`{"hello":"world"}`))
+
+	sources := make(map[string]*Source)
+	sources["foo"] = &Source{
+		Alias: "foo",
+		URL: &url.URL{
+			Scheme: "file",
+			Path:   "/tmp/foo.json",
+		},
+		Ext:  "json",
+		Type: "application/json",
+		FS:   fs,
+	}
+	data := &Data{
+		Sources: sources,
+	}
+	expected := make(map[string]interface{})
+	expected["hello"] = "world"
+	actual := data.Datasource("foo")
+	assert.Equal(t, expected["hello"], actual["hello"])
+}
diff --git a/main.go b/main.go
index 38f1c5f3..50fa9a2f 100644
--- a/main.go
+++ b/main.go
@@ -42,7 +42,7 @@ func (g *Gomplate) RunTemplate(in io.Reader, out io.Writer) {
 }
 
 // NewGomplate -
-func NewGomplate() *Gomplate {
+func NewGomplate(data *Data) *Gomplate {
 	env := &Env{}
 	typeconv := &TypeConv{}
 	ec2meta := &aws.Ec2Meta{}
@@ -62,12 +62,15 @@ func NewGomplate() *Gomplate {
 			"title":      strings.Title,
 			"toUpper":    strings.ToUpper,
 			"toLower":    strings.ToLower,
+			"datasource": data.Datasource,
 		},
 	}
 }
 
 func runTemplate(c *cli.Context) error {
-	g := NewGomplate()
+	data := NewData(c.StringSlice("datasource"))
+
+	g := NewGomplate(data)
 	g.RunTemplate(os.Stdin, os.Stdout)
 	return nil
 }
@@ -79,5 +82,12 @@ func main() {
 	app.Version = version.Version
 	app.Action = runTemplate
 
+	app.Flags = []cli.Flag{
+		cli.StringSliceFlag{
+			Name:  "datasource, d",
+			Usage: "Data source in alias=URL form. Specify multiple times to add multiple sources.",
+		},
+	}
+
 	app.Run(os.Args)
 }