Support aws s3 (#15)

Author: huykingsofm

CHANGELOG.md

@@ -1,3 +1,6 @@
+# v1.4.0 (Jan 16, 2023)
+1. Support AWS S3.
+
 # v1.3.2 (Jan 11, 2023)
 
 1. Fix read non-existed directory in watching mode.

README.md

@@ -19,13 +19,18 @@ configuration files.
 var config = xyconfig.GetConfig("app")
 
 // Read config from a string.
-config.Read(xyconfig.JSON, `{"general": {"timeout": 3.14}}`)
+config.ReadBytes(xyconfig.JSON, []byte(`{"general": {"timeout": 3.14}}`))
 
-// Read config from default.ini but do not watch the file.
-config.ReadFile("config/default.ini", false)
+// Read from files.
+config.Read("config/default.ini")
+config.Read("config/override.yml")
+config.Read(".env")
 
-// Read config from override.ini and watch the file change.
-config.ReadFile("config/override.yml", true)
+// Load global environment variables to config files.
+config.Read("env")
+
+// Read config from aws s3 bucket.
+config.Read("s3://bucket/item.ini")
 
 fmt.Println(config.MustGet("general.timeout").MustFloat())
 

config.go

@@ -29,6 +29,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/fsnotify/fsnotify"
 	"github.com/go-ini/ini"
 	"github.com/joho/godotenv"
@@ -87,8 +91,12 @@ type Config struct {
 	// watcher tracks changes of files.
 	watcher *fsnotify.Watcher
 
-	// envWatcher tracks the waching of environment variables.
-	envWatcher *time.Timer
+	// timerWatchers tracks the waching of non-inotify instances.
+	timerWatchers map[string]*time.Timer
+
+	// watchInterval is used to choose the time interval to watch changes when
+	// using Read method.
+	watchInterval time.Duration
 
 	// lock avoids race condition.
 	lock *xylock.RWLock
@@ -124,9 +132,11 @@ func GetConfig(name string) *Config {
 	}
 
 	var cfg = &Config{
-		config: make(map[string]Value),
-		hook:   make(map[string]func(Event)),
-		lock:   &xylock.RWLock{},
+		config:        make(map[string]Value),
+		hook:          make(map[string]func(Event)),
+		timerWatchers: make(map[string]*time.Timer),
+		watchInterval: 5 * time.Minute,
+		lock:          &xylock.RWLock{},
 	}
 
 	if name == "" {
@@ -151,22 +161,41 @@ func (c *Config) CloseWatcher() error {
 		c.watcher = nil
 	}
 
-	if c.envWatcher != nil {
-		c.envWatcher.Stop()
-		c.envWatcher = nil
+	for k, w := range c.timerWatchers {
+		w.Stop()
+		delete(c.timerWatchers, k)
 	}
 
 	return err
 }
 
-// UnWatch removes a filename from the watcher.
+// SetWatchInterval sets the time interval to watch the change when using Read()
+// method.
+func (c *Config) SetWatchInterval(d time.Duration) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.watchInterval = d
+}
+
+// UnWatch removes a filename from the watcher. This method also works with s3
+// url. Put "env" as parameter if you want to stop watching environment
+// variables of LoadEnv().
 func (c *Config) UnWatch(filename string) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
+	if w, ok := c.timerWatchers[filename]; ok {
+		w.Stop()
+		delete(c.timerWatchers, filename)
+		return nil
+	}
+
 	if c.watcher != nil {
-		return c.watcher.Remove(filename)
+		if err := c.watcher.Remove(filename); err != nil {
+			return ConfigError.New(err)
+		}
 	}
+
 	return nil
 }
 
@@ -329,11 +358,6 @@ func (c *Config) ReadBytes(format Format, b []byte) error {
 	}
 }
 
-// Read reads the config values from a string under any format.
-func (c *Config) Read(format Format, s string) error {
-	return c.ReadBytes(format, []byte(s))
-}
-
 // ReadFile reads the config values from a file. If watch is true, it will
 // reload config when the file is changed.
 func (c *Config) ReadFile(filename string, watch bool) error {
@@ -344,16 +368,16 @@ func (c *Config) ReadFile(filename string, watch bool) error {
 		}
 	}
 
+	if fileFormat == UnknownFormat {
+		return FormatError.Newf("unknown extension: %s", filename)
+	}
+
 	if watch {
 		if err := c.watchFile(filename); err != nil {
 			return err
 		}
 	}
 
-	if fileFormat == UnknownFormat {
-		return ExtensionError.Newf("unknown extension: %s", filename)
-	}
-
 	if data, err := ioutil.ReadFile(filename); err != nil {
 		if !os.IsNotExist(err) || !watch {
 			return ConfigError.New(err)
@@ -365,6 +389,66 @@ func (c *Config) ReadFile(filename string, watch bool) error {
 	return nil
 }
 
+// ReadS3 reads a file from AWS S3 bucket and watch for their changes every
+// duration. Set the duration as zero if no need to watch the change.
+//
+// You must provide the aws credentials in ~/.aws/credentials. The AWS_REGION
+// is required.
+func (c *Config) ReadS3(url string, d time.Duration) error {
+	var fileFormat = UnknownFormat
+	for ext, format := range extensions {
+		if strings.HasSuffix(url, ext) {
+			fileFormat = format
+		}
+	}
+
+	if fileFormat == UnknownFormat {
+		return FormatError.Newf("unknown extension: %s", url)
+	}
+
+	if !strings.HasPrefix(url, "s3://") {
+		return FormatError.Newf("can not parse the s3 url %s", url)
+	}
+
+	var path = url[5:]
+	var bucket, item, found = strings.Cut(path, "/")
+	if !found {
+		return FormatError.Newf("not found item in path %s", path)
+	}
+
+	var sess, err = session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	})
+
+	if err != nil {
+		return ConfigError.New(err)
+	}
+
+	var downloader = s3manager.NewDownloader(sess)
+	var buf = aws.NewWriteAtBuffer([]byte{})
+	_, err = downloader.Download(
+		buf,
+		&s3.GetObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(item),
+		})
+
+	if d != 0 {
+		c.lock.Lock()
+		c.timerWatchers[url] = time.AfterFunc(d, func() { c.ReadS3(url, d) })
+		c.lock.Unlock()
+	}
+
+	if err != nil {
+		if d == 0 {
+			return ConfigError.New(err)
+		}
+		return nil
+	}
+
+	return c.ReadBytes(fileFormat, buf.Bytes())
+}
+
 // LoadEnv loads all environment variables and watch for their changes every
 // duration. Set the duration as zero if no need to watch the change.
 func (c *Config) LoadEnv(d time.Duration) error {
@@ -379,13 +463,31 @@ func (c *Config) LoadEnv(d time.Duration) error {
 
 	if d != 0 {
 		c.lock.Lock()
-		c.envWatcher = time.AfterFunc(d, func() { c.LoadEnv(d) })
+		c.timerWatchers["env"] = time.AfterFunc(d, func() { c.LoadEnv(d) })
 		c.lock.Unlock()
 	}
 
 	return nil
 }
 
+// Read reads the config with any instance. If the instance is s3 url or
+// environment variable, the watchInterval is used to choose the time interval
+// for watching changes. If the instance is file path, it will watch the change
+// if watchInterval > 0.
+func (c *Config) Read(path string) error {
+	switch {
+	case path == "env":
+		return c.LoadEnv(c.watchInterval)
+	case strings.HasPrefix(path, "s3://"):
+		return c.ReadS3(path, c.watchInterval)
+	default:
+		if c.watchInterval > 0 {
+			return c.ReadFile(path, true)
+		}
+		return c.ReadFile(path, false)
+	}
+}
+
 // Get returns the value assigned with the key. The latter returned value is
 // false if they key doesn't exist.
 func (c *Config) Get(key string) (Value, bool) {

config_test.go

@@ -167,18 +167,11 @@ func TestConfigReadByteUnknown(t *testing.T) {
 	xycond.ExpectError(err, xyconfig.FormatError).Test(t)
 }
 
-func TestConfigRead(t *testing.T) {
-	var cfg = xyconfig.GetConfig(t.Name())
-	var err = cfg.Read(xyconfig.UnknownFormat, "")
-
-	xycond.ExpectError(err, xyconfig.FormatError).Test(t)
-}
-
 func TestConfigReadFileUnknownExt(t *testing.T) {
 	var cfg = xyconfig.GetConfig(t.Name())
 	var err = cfg.ReadFile("foo.bar", false)
 
-	xycond.ExpectError(err, xyconfig.ExtensionError).Test(t)
+	xycond.ExpectError(err, xyconfig.FormatError).Test(t)
 }
 
 func TestConfigReadFileNotExist(t *testing.T) {
@@ -215,6 +208,36 @@ func TestConfigReadFileWithChange(t *testing.T) {
 	xycond.ExpectEqual(cfg.MustGet("foo").MustString(), "buzz").Test(t)
 }
 
+func TestConfigReadS3UnknownExt(t *testing.T) {
+	var cfg = xyconfig.GetConfig(t.Name())
+	var err = cfg.ReadS3("s3://bucket/abc.unk", 0)
+	xycond.ExpectError(err, xyconfig.FormatError).Test(t)
+}
+
+func TestConfigReadS3InvalidPrefix(t *testing.T) {
+	var cfg = xyconfig.GetConfig(t.Name())
+	var err = cfg.ReadS3("s3:/bucket/abc.ini", 0)
+	xycond.ExpectError(err, xyconfig.FormatError).Test(t)
+}
+
+func TestConfigReadS3InvalidFormat(t *testing.T) {
+	var cfg = xyconfig.GetConfig(t.Name())
+	var err = cfg.ReadS3("s3://abc.ini", 0)
+	xycond.ExpectError(err, xyconfig.FormatError).Test(t)
+}
+
+func TestConfigReadInvalidAndWatch(t *testing.T) {
+	var cfg = xyconfig.GetConfig(t.Name())
+	var err = cfg.ReadS3("s3://bucket/abc.ini", time.Second)
+	xycond.ExpectNil(err).Test(t)
+}
+
+func TestConfigReadInvalidAndNotWatch(t *testing.T) {
+	var cfg = xyconfig.GetConfig(t.Name())
+	var err = cfg.ReadS3("s3://bucket/abc.ini", 0)
+	xycond.ExpectError(err, xyconfig.ConfigError).Test(t)
+}
+
 func TestConfigLoadEnvWithChange(t *testing.T) {
 	os.Setenv("foo", "bar")
 
@@ -275,3 +298,18 @@ func TestConfigToMap(t *testing.T) {
 	xycond.ExpectEqual(cfg.ToMap()["foo"], "bar").Test(t)
 	xycond.ExpectIn("buzz", cfg.ToMap()["subcfg"]).Test(t)
 }
+
+func TestConfigUnWatch(t *testing.T) {
+	ioutil.WriteFile(t.Name()+".json", []byte(`{"error":""}`), 0644)
+	var cfg = xyconfig.GetConfig(t.Name())
+
+	cfg.SetWatchInterval(time.Second)
+
+	xycond.ExpectNil(cfg.Read("env")).Test(t)
+	xycond.ExpectNil(cfg.Read(t.Name() + ".json")).Test(t)
+	xycond.ExpectError(cfg.Read(""), xyconfig.FormatError).Test(t)
+
+	xycond.ExpectNil(cfg.UnWatch("env")).Test(t)
+	xycond.ExpectNil(cfg.UnWatch(t.Name() + ".json")).Test(t)
+	xycond.ExpectError(cfg.UnWatch("foo.json"), xyconfig.ConfigError).Test(t)
+}

error.go

@@ -23,14 +23,11 @@ package xyconfig
 import "github.com/xybor-x/xyerror"
 
 // ConfigError represents for all error in xyconfig.
-var ConfigError = xyerror.NewException("BaseError")
+var ConfigError = xyerror.NewException("ConfigError")
 
 // CastError happens when the value cannot cast to a specific type.
 var CastError = ConfigError.NewException("CastError")
 
-// ExtensionError represents for file extension error.
-var ExtensionError = ConfigError.NewException("ExtensionError")
-
 // FormatError represents for file format error.
 var FormatError = ConfigError.NewException("FormatError")
 

go.mod

@@ -3,16 +3,18 @@ module github.com/xybor-x/xyconfig
 go 1.18
 
 require (
+	github.com/aws/aws-sdk-go v1.44.180
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/go-ini/ini v1.67.0
 	github.com/joho/godotenv v1.4.0
 	github.com/xybor-x/xycond v1.0.0
 	github.com/xybor-x/xyerror v1.0.5
 	github.com/xybor-x/xylock v0.0.1
-	github.com/xybor-x/xylog v0.3.0
+	github.com/xybor-x/xylog v0.5.0
 )
 
 require (
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect