Skip to content

Commit

Permalink
Merge pull request #19 from jumppad-labs/registry
Browse files Browse the repository at this point in the history
Add support for a module registry
  • Loading branch information
nicholasjackson committed Apr 24, 2024
2 parents 851845c + 4c82d67 commit 013770a
Show file tree
Hide file tree
Showing 4 changed files with 261 additions and 7 deletions.
3 changes: 3 additions & 0 deletions getter.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ func (g *GoGetter) Get(src, dest string, ignoreCache bool) (string, error) {
output, err := filenamify.Filenamify(src, filenamify.Options{
Replacement: "_",
})
if err != nil {
return "", err
}

downloadPath := path.Join(dest, output)

Expand Down
113 changes: 107 additions & 6 deletions parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/hashicorp/hcl/v2/hclparse"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/jumppad-labs/hclconfig/errors"
"github.com/jumppad-labs/hclconfig/registry"
"github.com/jumppad-labs/hclconfig/resources"
"github.com/jumppad-labs/hclconfig/types"
"github.com/zclconf/go-cty/cty"
Expand All @@ -43,6 +44,10 @@ type ParserOptions struct {
VariableEnvPrefix string
// location of any downloaded modules
ModuleCache string
// default registry to use when fetching modules
DefaultRegistry string
// credentials to use with the registries
RegistryCredentials map[string]string
// Callback executed when the parser reads a resource stanza, callbacks are
// executed based on a directed acyclic graph. If resource 'a' references
// a property defined in resource 'b', i.e 'resource.a.myproperty' then the
Expand All @@ -68,9 +73,12 @@ func DefaultOptions() *ParserOptions {
cacheDir = filepath.Join(cacheDir, ".hclconfig", "cache")
os.MkdirAll(cacheDir, os.ModePerm)

registryCredentials := map[string]string{}

return &ParserOptions{
ModuleCache: cacheDir,
VariableEnvPrefix: "HCL_VAR_",
ModuleCache: cacheDir,
VariableEnvPrefix: "HCL_VAR_",
RegistryCredentials: registryCredentials,
}
}

Expand Down Expand Up @@ -593,7 +601,7 @@ func (p *Parser) parseModule(ctx *hcl.EvalContext, c *Config, file string, b *hc
setDisabled(ctx, rt, b.Body, false)

derr := setDependsOn(ctx, rt, b.Body, dependsOn)
if err != nil {
if derr != nil {
de := errors.ParserError{}
de.Line = b.TypeRange.Start.Line
de.Column = b.TypeRange.Start.Column
Expand All @@ -618,24 +626,117 @@ func (p *Parser) parseModule(ctx *hcl.EvalContext, c *Config, file string, b *hc
return []error{&de}
}

// src could be a github module or a relative folder
version := "latest"
if b.Body.Attributes["version"] != nil {
v, diags := b.Body.Attributes["version"].Expr.Value(ctx)
if diags.HasErrors() {
de := errors.ParserError{}
de.Line = b.TypeRange.Start.Line
de.Column = b.TypeRange.Start.Column
de.Filename = file
de.Level = errors.ParserErrorLevelError
de.Message = fmt.Sprintf("unable to read version from module: %s", diags.Error())

return []error{&de}
}
version = v.AsString()
}

// src could be a registry url, github repository or a relative folder
// first check if it is a folder, we need to make it absolute relative to the current file
dir := path.Dir(file)
moduleSrc := path.Join(dir, src.AsString())

fi, serr := os.Stat(moduleSrc)
if serr != nil || !fi.IsDir() {
moduleURL := src.AsString()

parts := strings.Split(moduleURL, "/")

// if there are 2 parts (namespace, module), check if the default registry is set
if len(parts) == 2 && p.options.DefaultRegistry != "" {
parts = append([]string{p.options.DefaultRegistry}, parts...)
}

// if there are 3 parts (registry, namespace, module) it could be a registry
if len(parts) == 3 {
host := parts[0]
namespace := parts[1]
name := parts[2]

// check if the registry has credentials
var token string
if _, ok := p.options.RegistryCredentials[host]; ok {
token = p.options.RegistryCredentials[host]
}

// if we can't create a registry, it is not a module registry so we can ignore the error
r, err := registry.New(host, token)
if err == nil {
// get all available versions of the module from the registry
// check if the requested version exists
versions, err := r.GetModuleVersions(namespace, name)
if err != nil {
de := errors.ParserError{}
de.Line = b.TypeRange.Start.Line
de.Column = b.TypeRange.Start.Column
de.Filename = file
de.Message = err.Error()

return []error{&de}
}

// if no version is set, use latest
if version == "latest" {
version = versions.Latest
} else {
// otherwise check the version exists
versionExists := false
for _, v := range versions.Versions {
if v.Version == version {
versionExists = true
break
}
}

if !versionExists {
de := errors.ParserError{}
de.Line = b.TypeRange.Start.Line
de.Column = b.TypeRange.Start.Column
de.Filename = file
de.Message = fmt.Sprintf(`version "%s" does not exist for module "%s/%s" in registry "%s"`, version, namespace, name, host)

return []error{&de}
}
}

module, err := r.GetModule(namespace, name, version)
if err == nil {
// if we get back a module url from the registry,
// set the source to the returned url
moduleURL = module.DownloadURL
} else {
de := errors.ParserError{}
de.Line = b.TypeRange.Start.Line
de.Column = b.TypeRange.Start.Column
de.Filename = file
de.Message = fmt.Sprintf(`unable to fetch module "%s/%s" from registry "%s": %s`, namespace, name, host, err)

return []error{&de}
}
}
}

// is not a directory fetch from source using go getter
gg := NewGoGetter()

mp, err := gg.Get(src.AsString(), p.options.ModuleCache, false)
mp, err := gg.Get(moduleURL, p.options.ModuleCache, false)
if err != nil {
de := errors.ParserError{}
de.Line = b.TypeRange.Start.Line
de.Column = b.TypeRange.Start.Column
de.Filename = file
de.Message = fmt.Sprintf(`unable to fetch remote module "%s" %s`, src.AsString(), err)
de.Message = fmt.Sprintf(`unable to fetch remote module "%s": %s`, src.AsString(), err)

return []error{&de}
}
Expand Down
149 changes: 149 additions & 0 deletions registry/registry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package registry

import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"time"
)

type Registry interface {
GetModuleVersions(organization string, module string) (*Versions, error)
GetModule(organization string, name string, version string) (*Module, error)
}

type TransportWithCredentials struct {
token string
T http.RoundTripper
}

func (t *TransportWithCredentials) RoundTrip(req *http.Request) (*http.Response, error) {
if t.token != "" {
req.Header.Set("Authorization", "Bearer "+t.token)
}
return t.T.RoundTrip(req)
}

type RegistryImpl struct {
client http.Client
Host string
Modules string
}

type Config struct {
Capabilities map[string]string `json:"capabilities"`
}

type Credential struct {
Token string `hcl:"token,optional" json:"token,omitempty"`
}

type Module struct {
ID string `json:"id"`
Name string `json:"name"`
Organization string `json:"organization"`
Namespace string `json:"namespace"` // implement later? public/private modules
Version string `json:"version"`
SourceURL string `json:"source_url"`
DownloadURL string `json:"download_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}

type Versions struct {
Latest string `json:"latest"`
Versions []Version `json:"versions"`
}

type Version struct {
Version string `json:"version"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}

func New(host string, token string) (Registry, error) {
client := http.Client{
Timeout: 5 * time.Second,
Transport: &TransportWithCredentials{
token: token,
T: http.DefaultTransport,
},
}

req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/.well-known/registry.json", host), nil)
if err != nil {
return nil, err
}

resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf(`"%s" is not a valid registry`, host)
}

var config Config
err = json.NewDecoder(resp.Body).Decode(&config)
if err != nil {
return nil, err
}

if config.Capabilities["modules.v1"] == "" {
return nil, fmt.Errorf(`registry "%s" does not support modules`, host)
}

parsedURL, err := url.Parse(config.Capabilities["modules.v1"])
if err != nil {
return nil, err
}

// if the modules url also contains a host, use that instead
if parsedURL.Host != "" {
host = parsedURL.Host
}

return &RegistryImpl{
client: client,
Host: host,
Modules: host + parsedURL.Path,
}, nil
}

func (r *RegistryImpl) GetModuleVersions(organization string, module string) (*Versions, error) {
req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s/%s/versions", r.Modules, organization, module), nil)
if err != nil {
return nil, err
}

resp, err := r.client.Do(req)
if err != nil {
return nil, err
}

var versions Versions
err = json.NewDecoder(resp.Body).Decode(&versions)
if err != nil {
return nil, err
}

return &versions, nil
}

func (r *RegistryImpl) GetModule(organization string, name string, version string) (*Module, error) {
req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s/%s/%s", r.Modules, organization, name, version), nil)
if err != nil {
return nil, err
}

resp, err := r.client.Do(req)
if err != nil {
return nil, err
}

var module Module
err = json.NewDecoder(resp.Body).Decode(&module)
if err != nil {
return nil, err
}

return &module, nil
}
3 changes: 2 additions & 1 deletion resources/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ const TypeModule = "module"
type Module struct {
types.ResourceBase `hcl:",remain"`

Source string `hcl:"source" json:"source"`
Source string `hcl:"source" json:"source"`
Version string `hcl:"version,optional" json:"version,omitempty"`

Variables interface{} `hcl:"variables,optional" json:"variables,omitempty"`

Expand Down

0 comments on commit 013770a

Please sign in to comment.