From b8af9fe60f27353bdd5933ed37508b30d4290046 Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Sun, 5 Jun 2016 22:18:24 +0100 Subject: Add AWS S3 and Google GCS virtual filesystems. This allows the signing key to be read directly from S3 using a path like /s3// or /gcs//. --- server/config/config.go | 15 +++++ server/fs/s3.go | 152 ++++++++++++++++++++++++++++++++++++++++++++++++ server/signer/signer.go | 6 +- 3 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 server/fs/s3.go (limited to 'server') diff --git a/server/config/config.go b/server/config/config.go index fae823b..648cf46 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -13,6 +13,7 @@ type Config struct { Server `mapstructure:"server"` Auth `mapstructure:"auth"` SSH `mapstructure:"ssh"` + AWS `mapstructure:"aws"` } // unmarshalled holds the raw config. @@ -20,6 +21,7 @@ type unmarshalled struct { Server []Server `mapstructure:"server"` Auth []Auth `mapstructure:"auth"` SSH []SSH `mapstructure:"ssh"` + AWS []AWS `mapstructure:"aws"` } // Server holds the configuration specific to the web server and sessions. @@ -48,6 +50,14 @@ type SSH struct { Permissions []string `mapstructure:"permissions"` } +// AWS holds Amazon AWS configuration. +// AWS can also be configured using SDK methods. +type AWS struct { + Region string `mapstructure:"region"` + AccessKey string `mapstructure:"access_key"` + SecretKey string `mapstructure:"secret_key"` +} + func verifyConfig(u *unmarshalled) error { var err error if len(u.SSH) == 0 { @@ -59,6 +69,10 @@ func verifyConfig(u *unmarshalled) error { if len(u.Server) == 0 { err = multierror.Append(errors.New("missing server config block")) } + if len(u.AWS) == 0 { + // AWS config is optional + u.AWS = append(u.AWS, AWS{}) + } return err } @@ -80,5 +94,6 @@ func ReadConfig(r io.Reader) (*Config, error) { Server: u.Server[0], Auth: u.Auth[0], SSH: u.SSH[0], + AWS: u.AWS[0], }, nil } diff --git a/server/fs/s3.go b/server/fs/s3.go new file mode 100644 index 0000000..4e82bb4 --- /dev/null +++ b/server/fs/s3.go @@ -0,0 +1,152 @@ +package fs + +import ( + "bytes" + "errors" + "io/ioutil" + "os" + "path" + "strings" + "time" + + "go4.org/wkfs" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/nsheridan/cashier/server/config" +) + +func Register(config *config.AWS) { + ac := &aws.Config{} + // If region is unset the SDK will attempt to read the region from the environment. + if config.Region != "" { + ac.Region = aws.String(config.Region) + } + // Attempt to get credentials from the cashier config. + // Otherwise check for standard credentials. If neither are present register the fs as broken. + // TODO: implement this as a provider. + if config.AccessKey != "" && config.SecretKey != "" { + ac.Credentials = credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, "") + } else { + _, err := session.New().Config.Credentials.Get() + if err != nil { + registerBrokenFS(errors.New("aws credentials not found")) + return + } + } + sc := s3.New(session.New(ac)) + if aws.StringValue(sc.Config.Region) == "" { + registerBrokenFS(errors.New("aws region configuration not found")) + return + } + wkfs.RegisterFS("/s3/", &s3FS{ + sc: sc, + }) +} + +func registerBrokenFS(err error) { + wkfs.RegisterFS("/s3/", &s3FS{ + err: err, + }) +} + +type s3FS struct { + sc *s3.S3 + err error +} + +func (fs *s3FS) parseName(name string) (bucket, fileName string, err error) { + if fs.err != nil { + return "", "", fs.err + } + name = strings.TrimPrefix(name, "/s3/") + i := strings.Index(name, "/") + if i < 0 { + return name, "", nil + } + return name[:i], name[i+1:], nil +} + +// Open opens the named file for reading. +func (fs *s3FS) Open(name string) (wkfs.File, error) { + bucket, fileName, err := fs.parseName(name) + if err != nil { + return nil, err + } + obj, err := fs.sc.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &fileName, + }) + if err != nil { + return nil, err + } + defer obj.Body.Close() + slurp, err := ioutil.ReadAll(obj.Body) + if err != nil { + return nil, err + } + return &file{ + name: name, + Reader: bytes.NewReader(slurp), + }, nil +} + +func (fs *s3FS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) } +func (fs *s3FS) Lstat(name string) (os.FileInfo, error) { + bucket, fileName, err := fs.parseName(name) + if err != nil { + return nil, err + } + obj, err := fs.sc.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &fileName, + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "NoSuchKey" { + return nil, os.ErrNotExist + } + } + } + if err != nil { + return nil, err + } + return &statInfo{ + name: path.Base(fileName), + size: *obj.ContentLength, + }, nil +} + +func (fs *s3FS) MkdirAll(path string, perm os.FileMode) error { return nil } + +func (fs *s3FS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) { + return nil, errors.New("not implemented") +} + +type statInfo struct { + name string + size int64 + isDir bool + modtime time.Time +} + +func (si *statInfo) IsDir() bool { return si.isDir } +func (si *statInfo) ModTime() time.Time { return si.modtime } +func (si *statInfo) Mode() os.FileMode { return 0644 } +func (si *statInfo) Name() string { return path.Base(si.name) } +func (si *statInfo) Size() int64 { return si.size } +func (si *statInfo) Sys() interface{} { return nil } + +type file struct { + name string + *bytes.Reader +} + +func (*file) Close() error { return nil } +func (f *file) Name() string { return path.Base(f.name) } +func (f *file) Stat() (os.FileInfo, error) { + panic("Stat not implemented on /s3/ files yet") +} diff --git a/server/signer/signer.go b/server/signer/signer.go index 8be5cad..1be6d75 100644 --- a/server/signer/signer.go +++ b/server/signer/signer.go @@ -4,11 +4,13 @@ import ( "crypto/md5" "crypto/rand" "fmt" - "io/ioutil" "log" "strings" "time" + "go4.org/wkfs" + _ "go4.org/wkfs/gcs" // Register "/gcs/" as a wkfs. + "github.com/nsheridan/cashier/lib" "github.com/nsheridan/cashier/server/config" "golang.org/x/crypto/ssh" @@ -71,7 +73,7 @@ func makeperms(perms []string) map[string]string { // New creates a new KeySigner from the supplied configuration. func New(conf config.SSH) (*KeySigner, error) { - data, err := ioutil.ReadFile(conf.SigningKey) + data, err := wkfs.ReadFile(conf.SigningKey) if err != nil { return nil, fmt.Errorf("unable to read CA key %s: %v", conf.SigningKey, err) } -- cgit v1.2.3