Support subpaths for http datasources (#442)

* bug: subpaths were ignored in http datasources

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

* fixup! bug: subpaths were ignored in http datasources

4 files changed, 332 insertions, 228 deletions

diff --git a/data/datasource.go b/data/datasource.go
index 2e4d845d..5ceadd11 100644
--- a/data/datasource.go
+++ b/data/datasource.go
@@ -11,7 +11,6 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 
@@ -373,42 +372,6 @@ func readStdin(source *Source, args ...string) ([]byte, error) {
 	return b, nil
 }
 
-func readHTTP(source *Source, args ...string) ([]byte, error) {
-	if source.hc == nil {
-		source.hc = &http.Client{Timeout: time.Second * 5}
-	}
-	req, err := http.NewRequest("GET", source.URL.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header = source.header
-	res, err := source.hc.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	err = res.Body.Close()
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		err := errors.Errorf("Unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body))
-		return nil, err
-	}
-	ctypeHdr := res.Header.Get("Content-Type")
-	if ctypeHdr != "" {
-		mediatype, _, e := mime.ParseMediaType(ctypeHdr)
-		if e != nil {
-			return nil, e
-		}
-		source.mediaType = mediatype
-	}
-	return body, nil
-}
-
 func readConsul(source *Source, args ...string) (data []byte, err error) {
 	if source.kv == nil {
 		source.kv, err = libkv.NewConsul(source.URL)
@@ -454,40 +417,3 @@ func readBoltDB(source *Source, args ...string) (data []byte, err error) {
 
 	return data, nil
 }
-
-func parseHeaderArgs(headerArgs []string) (map[string]http.Header, error) {
-	headers := make(map[string]http.Header)
-	for _, v := range headerArgs {
-		ds, name, value, err := splitHeaderArg(v)
-		if err != nil {
-			return nil, err
-		}
-		if _, ok := headers[ds]; !ok {
-			headers[ds] = make(http.Header)
-		}
-		headers[ds][name] = append(headers[ds][name], strings.TrimSpace(value))
-	}
-	return headers, nil
-}
-
-func splitHeaderArg(arg string) (datasourceAlias, name, value string, err error) {
-	parts := strings.SplitN(arg, "=", 2)
-	if len(parts) != 2 {
-		err = errors.Errorf("Invalid datasource-header option '%s'", arg)
-		return "", "", "", err
-	}
-	datasourceAlias = parts[0]
-	name, value, err = splitHeader(parts[1])
-	return datasourceAlias, name, value, err
-}
-
-func splitHeader(header string) (name, value string, err error) {
-	parts := strings.SplitN(header, ":", 2)
-	if len(parts) != 2 {
-		err = errors.Errorf("Invalid HTTP Header format '%s'", header)
-		return "", "", err
-	}
-	name = http.CanonicalHeaderKey(parts[0])
-	value = parts[1]
-	return name, value, nil
-}
diff --git a/data/datasource_http.go b/data/datasource_http.go
new file mode 100644
index 00000000..5424fbd0
--- /dev/null
+++ b/data/datasource_http.go
@@ -0,0 +1,100 @@
+package data
+
+import (
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+func buildURL(base *url.URL, args ...string) (*url.URL, error) {
+	if len(args) == 0 {
+		return base, nil
+	}
+	p, err := url.Parse(args[0])
+	if err != nil {
+		return nil, errors.Wrapf(err, "bad sub-path %s", args[0])
+	}
+	return base.ResolveReference(p), nil
+}
+
+func readHTTP(source *Source, args ...string) ([]byte, error) {
+	if source.hc == nil {
+		source.hc = &http.Client{Timeout: time.Second * 5}
+	}
+	u, err := buildURL(source.URL, args...)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = source.header
+	res, err := source.hc.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = res.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		err := errors.Errorf("Unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body))
+		return nil, err
+	}
+	ctypeHdr := res.Header.Get("Content-Type")
+	if ctypeHdr != "" {
+		mediatype, _, e := mime.ParseMediaType(ctypeHdr)
+		if e != nil {
+			return nil, e
+		}
+		source.mediaType = mediatype
+	}
+	return body, nil
+}
+
+func parseHeaderArgs(headerArgs []string) (map[string]http.Header, error) {
+	headers := make(map[string]http.Header)
+	for _, v := range headerArgs {
+		ds, name, value, err := splitHeaderArg(v)
+		if err != nil {
+			return nil, err
+		}
+		if _, ok := headers[ds]; !ok {
+			headers[ds] = make(http.Header)
+		}
+		headers[ds][name] = append(headers[ds][name], strings.TrimSpace(value))
+	}
+	return headers, nil
+}
+
+func splitHeaderArg(arg string) (datasourceAlias, name, value string, err error) {
+	parts := strings.SplitN(arg, "=", 2)
+	if len(parts) != 2 {
+		err = errors.Errorf("Invalid datasource-header option '%s'", arg)
+		return "", "", "", err
+	}
+	datasourceAlias = parts[0]
+	name, value, err = splitHeader(parts[1])
+	return datasourceAlias, name, value, err
+}
+
+func splitHeader(header string) (name, value string, err error) {
+	parts := strings.SplitN(header, ":", 2)
+	if len(parts) != 2 {
+		err = errors.Errorf("Invalid HTTP Header format '%s'", header)
+		return "", "", err
+	}
+	name = http.CanonicalHeaderKey(parts[0])
+	value = parts[1]
+	return name, value, nil
+}
diff --git a/data/datasource_http_test.go b/data/datasource_http_test.go
new file mode 100644
index 00000000..62531241
--- /dev/null
+++ b/data/datasource_http_test.go
@@ -0,0 +1,220 @@
+package data
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func must(r interface{}, err error) interface{} {
+	if err != nil {
+		panic(err)
+	}
+	return r
+}
+
+func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		w.Header().Set("Content-Type", mimetype)
+		w.WriteHeader(code)
+		if body == "" {
+			// mirror back the headers
+			fmt.Fprintln(w, must(marshalObj(r.Header, json.Marshal)))
+		} else {
+			fmt.Fprintln(w, body)
+		}
+	}))
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: func(req *http.Request) (*url.URL, error) {
+				return url.Parse(server.URL)
+			},
+		},
+	}
+
+	return server, client
+}
+
+func TestHTTPFile(t *testing.T) {
+	server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`)
+	defer server.Close()
+
+	sources := make(map[string]*Source)
+	sources["foo"] = &Source{
+		Alias: "foo",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   "example.com",
+			Path:   "/foo",
+		},
+		hc: client,
+	}
+	data := &Data{
+		Sources: sources,
+	}
+
+	expected := map[string]interface{}{
+		"hello": "world",
+	}
+
+	actual, err := data.Datasource("foo")
+	assert.NoError(t, err)
+	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
+
+	actual, err = data.Datasource(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
+}
+
+func TestHTTPFileWithHeaders(t *testing.T) {
+	server, client := setupHTTP(200, jsonMimetype, "")
+	defer server.Close()
+
+	sources := make(map[string]*Source)
+	sources["foo"] = &Source{
+		Alias: "foo",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   "example.com",
+			Path:   "/foo",
+		},
+		hc: client,
+		header: http.Header{
+			"Foo":             {"bar"},
+			"foo":             {"baz"},
+			"User-Agent":      {},
+			"Accept-Encoding": {"test"},
+		},
+	}
+	data := &Data{
+		Sources: sources,
+	}
+	expected := http.Header{
+		"Accept-Encoding": {"test"},
+		"Foo":             {"bar", "baz"},
+	}
+	actual, err := data.Datasource("foo")
+	assert.NoError(t, err)
+	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
+
+	expected = http.Header{
+		"Accept-Encoding": {"test"},
+		"Foo":             {"bar", "baz"},
+		"User-Agent":      {"Go-http-client/1.1"},
+	}
+	data = &Data{
+		Sources:      sources,
+		extraHeaders: map[string]http.Header{server.URL: expected},
+	}
+	actual, err = data.Datasource(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
+}
+
+func TestParseHeaderArgs(t *testing.T) {
+	args := []string{
+		"foo=Accept: application/json",
+		"bar=Authorization: Bearer supersecret",
+	}
+	expected := map[string]http.Header{
+		"foo": {
+			"Accept": {jsonMimetype},
+		},
+		"bar": {
+			"Authorization": {"Bearer supersecret"},
+		},
+	}
+	parsed, err := parseHeaderArgs(args)
+	assert.NoError(t, err)
+	assert.Equal(t, expected, parsed)
+
+	_, err = parseHeaderArgs([]string{"foo"})
+	assert.Error(t, err)
+
+	_, err = parseHeaderArgs([]string{"foo=bar"})
+	assert.Error(t, err)
+
+	args = []string{
+		"foo=Accept: application/json",
+		"foo=Foo: bar",
+		"foo=foo: baz",
+		"foo=fOO: qux",
+		"bar=Authorization: Bearer  supersecret",
+	}
+	expected = map[string]http.Header{
+		"foo": {
+			"Accept": {jsonMimetype},
+			"Foo":    {"bar", "baz", "qux"},
+		},
+		"bar": {
+			"Authorization": {"Bearer  supersecret"},
+		},
+	}
+	parsed, err = parseHeaderArgs(args)
+	assert.NoError(t, err)
+	assert.Equal(t, expected, parsed)
+}
+
+func TestHTTPFileWithSubPath(t *testing.T) {
+	server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`)
+	defer server.Close()
+
+	sources := make(map[string]*Source)
+	sources["foo"] = &Source{
+		Alias: "foo",
+		URL: &url.URL{
+			Scheme: "http",
+			Host:   "example.com",
+			Path:   "/foo",
+		},
+		hc: client,
+	}
+	data := &Data{
+		Sources: sources,
+	}
+
+	expected := map[string]interface{}{
+		"hello": "world",
+	}
+
+	actual, err := data.Datasource("foo")
+	assert.NoError(t, err)
+	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
+
+	actual, err = data.Datasource(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
+}
+
+func TestBuildURL(t *testing.T) {
+	expected := "https://example.com/index.html"
+	base := mustParseURL(expected)
+	u, err := buildURL(base)
+	assert.NoError(t, err)
+	assert.Equal(t, expected, u.String())
+
+	expected = "https://example.com/index.html"
+	base = mustParseURL("https://example.com")
+	u, err = buildURL(base, "index.html")
+	assert.NoError(t, err)
+	assert.Equal(t, expected, u.String())
+
+	expected = "https://example.com/a/b/c/index.html"
+	base = mustParseURL("https://example.com/a/")
+	u, err = buildURL(base, "b/c/index.html")
+	assert.NoError(t, err)
+	assert.Equal(t, expected, u.String())
+
+	expected = "https://example.com/bar/baz/index.html"
+	base = mustParseURL("https://example.com/foo")
+	u, err = buildURL(base, "bar/baz/index.html")
+	assert.NoError(t, err)
+	assert.Equal(t, expected, u.String())
+}
diff --git a/data/datasource_test.go b/data/datasource_test.go
index 2d966f52..a3075945 100644
--- a/data/datasource_test.go
+++ b/data/datasource_test.go
@@ -3,10 +3,7 @@
 package data
 
 import (
-	"encoding/json"
 	"fmt"
-	"net/http"
-	"net/http/httptest"
 	"net/url"
 	"strings"
 	"testing"
@@ -148,157 +145,6 @@ func TestDatasourceExists(t *testing.T) {
 	assert.False(t, data.DatasourceExists("bar"))
 }
 
-func must(r interface{}, err error) interface{} {
-	if err != nil {
-		panic(err)
-	}
-	return r
-}
-
-func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) {
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		w.Header().Set("Content-Type", mimetype)
-		w.WriteHeader(code)
-		if body == "" {
-			// mirror back the headers
-			fmt.Fprintln(w, must(marshalObj(r.Header, json.Marshal)))
-		} else {
-			fmt.Fprintln(w, body)
-		}
-	}))
-
-	client := &http.Client{
-		Transport: &http.Transport{
-			Proxy: func(req *http.Request) (*url.URL, error) {
-				return url.Parse(server.URL)
-			},
-		},
-	}
-
-	return server, client
-}
-
-func TestHTTPFile(t *testing.T) {
-	server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`)
-	defer server.Close()
-
-	sources := make(map[string]*Source)
-	sources["foo"] = &Source{
-		Alias: "foo",
-		URL: &url.URL{
-			Scheme: "http",
-			Host:   "example.com",
-			Path:   "/foo",
-		},
-		hc: client,
-	}
-	data := &Data{
-		Sources: sources,
-	}
-
-	expected := map[string]interface{}{
-		"hello": "world",
-	}
-
-	actual, err := data.Datasource("foo")
-	assert.NoError(t, err)
-	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
-
-	actual, err = data.Datasource(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
-}
-
-func TestHTTPFileWithHeaders(t *testing.T) {
-	server, client := setupHTTP(200, jsonMimetype, "")
-	defer server.Close()
-
-	sources := make(map[string]*Source)
-	sources["foo"] = &Source{
-		Alias: "foo",
-		URL: &url.URL{
-			Scheme: "http",
-			Host:   "example.com",
-			Path:   "/foo",
-		},
-		hc: client,
-		header: http.Header{
-			"Foo":             {"bar"},
-			"foo":             {"baz"},
-			"User-Agent":      {},
-			"Accept-Encoding": {"test"},
-		},
-	}
-	data := &Data{
-		Sources: sources,
-	}
-	expected := http.Header{
-		"Accept-Encoding": {"test"},
-		"Foo":             {"bar", "baz"},
-	}
-	actual, err := data.Datasource("foo")
-	assert.NoError(t, err)
-	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
-
-	expected = http.Header{
-		"Accept-Encoding": {"test"},
-		"Foo":             {"bar", "baz"},
-		"User-Agent":      {"Go-http-client/1.1"},
-	}
-	data = &Data{
-		Sources:      sources,
-		extraHeaders: map[string]http.Header{server.URL: expected},
-	}
-	actual, err = data.Datasource(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal)))
-}
-
-func TestParseHeaderArgs(t *testing.T) {
-	args := []string{
-		"foo=Accept: application/json",
-		"bar=Authorization: Bearer supersecret",
-	}
-	expected := map[string]http.Header{
-		"foo": {
-			"Accept": {jsonMimetype},
-		},
-		"bar": {
-			"Authorization": {"Bearer supersecret"},
-		},
-	}
-	parsed, err := parseHeaderArgs(args)
-	assert.NoError(t, err)
-	assert.Equal(t, expected, parsed)
-
-	_, err = parseHeaderArgs([]string{"foo"})
-	assert.Error(t, err)
-
-	_, err = parseHeaderArgs([]string{"foo=bar"})
-	assert.Error(t, err)
-
-	args = []string{
-		"foo=Accept: application/json",
-		"foo=Foo: bar",
-		"foo=foo: baz",
-		"foo=fOO: qux",
-		"bar=Authorization: Bearer  supersecret",
-	}
-	expected = map[string]http.Header{
-		"foo": {
-			"Accept": {jsonMimetype},
-			"Foo":    {"bar", "baz", "qux"},
-		},
-		"bar": {
-			"Authorization": {"Bearer  supersecret"},
-		},
-	}
-	parsed, err = parseHeaderArgs(args)
-	assert.NoError(t, err)
-	assert.Equal(t, expected, parsed)
-}
-
 func TestInclude(t *testing.T) {
 	ext := "txt"
 	contents := "hello world"
@@ -417,3 +263,15 @@ func TestMimeType(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "application/yaml", mt)
 }
+
+func TestQueryParse(t *testing.T) {
+	expected := &url.URL{
+		Scheme:   "http",
+		Host:     "example.com",
+		Path:     "/foo.json",
+		RawQuery: "bar",
+	}
+	u, err := parseSourceURL("http://example.com/foo.json?bar")
+	assert.NoError(t, err)
+	assert.EqualValues(t, expected, u)
+}