From 8066efd45861e7c024fc1daabc6d002266a527e7 Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Thu, 29 Dec 2016 21:53:26 +0000 Subject: Use vendored s3 wkfs --- cmd/cashierd/main.go | 53 ++++---- server/wkfs/s3fs/s3.go | 157 --------------------- vendor/github.com/nsheridan/wkfs/s3/README.md | 41 ++++++ vendor/github.com/nsheridan/wkfs/s3/s3.go | 180 +++++++++++++++++++++++++ vendor/github.com/nsheridan/wkfs/s3/s3_file.go | 59 ++++++++ vendor/vendor.json | 6 + 6 files changed, 316 insertions(+), 180 deletions(-) delete mode 100644 server/wkfs/s3fs/s3.go create mode 100644 vendor/github.com/nsheridan/wkfs/s3/README.md create mode 100644 vendor/github.com/nsheridan/wkfs/s3/s3.go create mode 100644 vendor/github.com/nsheridan/wkfs/s3/s3_file.go diff --git a/cmd/cashierd/main.go b/cmd/cashierd/main.go index 12d744d..31ee240 100644 --- a/cmd/cashierd/main.go +++ b/cmd/cashierd/main.go @@ -35,8 +35,8 @@ import ( "github.com/nsheridan/cashier/server/store" "github.com/nsheridan/cashier/server/templates" "github.com/nsheridan/cashier/server/util" - "github.com/nsheridan/cashier/server/wkfs/s3fs" "github.com/nsheridan/cashier/server/wkfs/vaultfs" + "github.com/nsheridan/wkfs/s3" "github.com/sid77/drop" ) @@ -313,46 +313,53 @@ func loadCerts(certFile, keyFile string) (tls.Certificate, error) { func main() { // Privileged section flag.Parse() - config, err := readConfig(*cfg) + conf, err := readConfig(*cfg) if err != nil { log.Fatal(err) } // Register well-known filesystems. - s3fs.Register(config.AWS) - vaultfs.Register(config.Vault) + if conf.AWS == nil { + conf.AWS = &config.AWS{} + } + s3.Register(&s3.Options{ + Region: conf.AWS.Region, + AccessKey: conf.AWS.AccessKey, + SecretKey: conf.AWS.SecretKey, + }) + vaultfs.Register(conf.Vault) - signer, err := signer.New(config.SSH) + signer, err := signer.New(conf.SSH) if err != nil { log.Fatal(err) } logfile := os.Stderr - if config.Server.HTTPLogFile != "" { - logfile, err = os.OpenFile(config.Server.HTTPLogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640) + if conf.Server.HTTPLogFile != "" { + logfile, err = os.OpenFile(conf.Server.HTTPLogFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640) if err != nil { log.Fatal(err) } } - laddr := fmt.Sprintf("%s:%d", config.Server.Addr, config.Server.Port) + laddr := fmt.Sprintf("%s:%d", conf.Server.Addr, conf.Server.Port) l, err := net.Listen("tcp", laddr) if err != nil { log.Fatal(err) } tlsConfig := &tls.Config{} - if config.Server.UseTLS { - if config.Server.LetsEncryptServername != "" { + if conf.Server.UseTLS { + if conf.Server.LetsEncryptServername != "" { m := autocert.Manager{ Prompt: autocert.AcceptTOS, - Cache: autocert.DirCache(config.Server.LetsEncryptCache), - HostPolicy: autocert.HostWhitelist(config.Server.LetsEncryptServername), + Cache: autocert.DirCache(conf.Server.LetsEncryptCache), + HostPolicy: autocert.HostWhitelist(conf.Server.LetsEncryptServername), } tlsConfig.GetCertificate = m.GetCertificate } else { tlsConfig.Certificates = make([]tls.Certificate, 1) - tlsConfig.Certificates[0], err = loadCerts(config.Server.TLSCert, config.Server.TLSKey) + tlsConfig.Certificates[0], err = loadCerts(conf.Server.TLSCert, conf.Server.TLSKey) if err != nil { log.Fatal(err) } @@ -360,33 +367,33 @@ func main() { l = tls.NewListener(l, tlsConfig) } - if config.Server.User != "" { + if conf.Server.User != "" { log.Print("Dropping privileges...") - if err := drop.DropPrivileges(config.Server.User); err != nil { + if err := drop.DropPrivileges(conf.Server.User); err != nil { log.Fatal(err) } } // Unprivileged section var authprovider auth.Provider - switch config.Auth.Provider { + switch conf.Auth.Provider { case "google": - authprovider, err = google.New(config.Auth) + authprovider, err = google.New(conf.Auth) case "github": - authprovider, err = github.New(config.Auth) + authprovider, err = github.New(conf.Auth) default: - log.Fatalf("Unknown provider %s\n", config.Auth.Provider) + log.Fatalf("Unknown provider %s\n", conf.Auth.Provider) } if err != nil { log.Fatal(err) } - certstore, err := store.New(config.Server.Database) + certstore, err := store.New(conf.Server.Database) if err != nil { log.Fatal(err) } ctx := &appContext{ - cookiestore: sessions.NewCookieStore([]byte(config.Server.CookieSecret)), + cookiestore: sessions.NewCookieStore([]byte(conf.Server.CookieSecret)), authprovider: authprovider, sshKeySigner: signer, certstore: certstore, @@ -394,11 +401,11 @@ func main() { ctx.cookiestore.Options = &sessions.Options{ MaxAge: 900, Path: "/", - Secure: config.Server.UseTLS, + Secure: conf.Server.UseTLS, HttpOnly: true, } - CSRF := csrf.Protect([]byte(config.Server.CSRFSecret), csrf.Secure(config.Server.UseTLS)) + CSRF := csrf.Protect([]byte(conf.Server.CSRFSecret), csrf.Secure(conf.Server.UseTLS)) r := mux.NewRouter() r.Methods("GET").Path("/").Handler(appHandler{ctx, rootHandler}) r.Methods("GET").Path("/auth/login").Handler(appHandler{ctx, loginHandler}) diff --git a/server/wkfs/s3fs/s3.go b/server/wkfs/s3fs/s3.go deleted file mode 100644 index 331b55f..0000000 --- a/server/wkfs/s3fs/s3.go +++ /dev/null @@ -1,157 +0,0 @@ -package s3fs - -import ( - "bytes" - "errors" - "io/ioutil" - "os" - "path" - "strings" - "time" - - "go4.org/wkfs" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/nsheridan/cashier/server/config" -) - -// Register the /s3/ filesystem as a well-known filesystem. -func Register(config *config.AWS) { - if config == nil { - registerBrokenFS(errors.New("aws credentials not found")) - return - } - ac := &aws.Config{} - // If region is unset the SDK will attempt to read the region from the environment. - if config.Region != "" { - ac.Region = aws.String(config.Region) - } - // Attempt to get credentials from the cashier config. - // Otherwise check for standard credentials. If neither are present register the fs as broken. - // TODO: implement this as a provider. - if config.AccessKey != "" && config.SecretKey != "" { - ac.Credentials = credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, "") - } else { - _, err := session.New().Config.Credentials.Get() - if err != nil { - registerBrokenFS(errors.New("aws credentials not found")) - return - } - } - sc := s3.New(session.New(ac)) - if aws.StringValue(sc.Config.Region) == "" { - registerBrokenFS(errors.New("aws region configuration not found")) - return - } - wkfs.RegisterFS("/s3/", &s3FS{ - sc: sc, - }) -} - -func registerBrokenFS(err error) { - wkfs.RegisterFS("/s3/", &s3FS{ - err: err, - }) -} - -type s3FS struct { - sc *s3.S3 - err error -} - -func (fs *s3FS) parseName(name string) (bucket, fileName string, err error) { - if fs.err != nil { - return "", "", fs.err - } - name = strings.TrimPrefix(name, "/s3/") - i := strings.Index(name, "/") - if i < 0 { - return name, "", nil - } - return name[:i], name[i+1:], nil -} - -// Open opens the named file for reading. -func (fs *s3FS) Open(name string) (wkfs.File, error) { - bucket, fileName, err := fs.parseName(name) - if err != nil { - return nil, err - } - obj, err := fs.sc.GetObject(&s3.GetObjectInput{ - Bucket: &bucket, - Key: &fileName, - }) - if err != nil { - return nil, err - } - defer obj.Body.Close() - slurp, err := ioutil.ReadAll(obj.Body) - if err != nil { - return nil, err - } - return &file{ - name: name, - Reader: bytes.NewReader(slurp), - }, nil -} - -func (fs *s3FS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) } -func (fs *s3FS) Lstat(name string) (os.FileInfo, error) { - bucket, fileName, err := fs.parseName(name) - if err != nil { - return nil, err - } - obj, err := fs.sc.GetObject(&s3.GetObjectInput{ - Bucket: &bucket, - Key: &fileName, - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - if awsErr.Code() == "NoSuchKey" { - return nil, os.ErrNotExist - } - } - } - if err != nil { - return nil, err - } - return &statInfo{ - name: path.Base(fileName), - size: *obj.ContentLength, - }, nil -} - -func (fs *s3FS) MkdirAll(path string, perm os.FileMode) error { return nil } - -func (fs *s3FS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) { - return nil, errors.New("not implemented") -} - -type statInfo struct { - name string - size int64 - isDir bool - modtime time.Time -} - -func (si *statInfo) IsDir() bool { return si.isDir } -func (si *statInfo) ModTime() time.Time { return si.modtime } -func (si *statInfo) Mode() os.FileMode { return 0644 } -func (si *statInfo) Name() string { return path.Base(si.name) } -func (si *statInfo) Size() int64 { return si.size } -func (si *statInfo) Sys() interface{} { return nil } - -type file struct { - name string - *bytes.Reader -} - -func (*file) Close() error { return nil } -func (f *file) Name() string { return path.Base(f.name) } -func (f *file) Stat() (os.FileInfo, error) { - panic("Stat not implemented on /s3/ files yet") -} diff --git a/vendor/github.com/nsheridan/wkfs/s3/README.md b/vendor/github.com/nsheridan/wkfs/s3/README.md new file mode 100644 index 0000000..177f738 --- /dev/null +++ b/vendor/github.com/nsheridan/wkfs/s3/README.md @@ -0,0 +1,41 @@ +## S3 plugin for WKFS + + + +Package `s3` registers an AWS S3 filesystem at the well-known `/s3/` filesystem path. + +Sample usage: + +```go +package main + +import ( + "fmt" + "io" + "log" + + "github.com/nsheridan/wkfs/s3" + "go4.org/wkfs" +) + +func main() { + opts := &s3.Options{ + Region: "us-east-1" + AccessKey: "abcdef" + SecretKey: "secret" + } + s3.Register(opts) + f, err := wkfs.Create("/s3/some-bucket/hello.txt") + if err != nil { + log.Fatal(err) + } + _, err := io.WriteString(f, "hello, world") + if err != nil { + log.Fatal(err) + } +} +``` + + + +`Options` are completely optional as the AWS SDK will attempt to obtain credentials from a number of locations - see [the documentation for details](http://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) - e.g. if you're using environment variables you can register the filesystem with `s3.Register(nil)`. diff --git a/vendor/github.com/nsheridan/wkfs/s3/s3.go b/vendor/github.com/nsheridan/wkfs/s3/s3.go new file mode 100644 index 0000000..19e72a9 --- /dev/null +++ b/vendor/github.com/nsheridan/wkfs/s3/s3.go @@ -0,0 +1,180 @@ +package s3 + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "time" + + "go4.org/wkfs" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Options for registering the S3 wkfs. +// None of these are required and can be supplied to the aws client by other means. +type Options struct { + Region string + AccessKey string + SecretKey string +} + +// Register the /s3/ filesystem as a well-known filesystem. +func Register(opts *Options) { + if opts == nil { + opts = &Options{} + } + config := &aws.Config{} + // If region is unset the SDK will attempt to read the region from the environment. + if opts.Region != "" { + config.Region = aws.String(opts.Region) + } + // Attempt to use supplied credentials, otherwise fall back to the SDK. + if opts.AccessKey != "" && opts.SecretKey != "" { + config.Credentials = credentials.NewStaticCredentials(opts.AccessKey, opts.SecretKey, "") + } + s, err := session.NewSession(config) + if err != nil { + registerBrokenFS(err) + return + } + sc := s3.New(s) + if aws.StringValue(sc.Config.Region) == "" { + registerBrokenFS(errors.New("could not find region configuration")) + return + } + wkfs.RegisterFS("/s3/", &s3FS{ + sc: sc, + }) +} + +func registerBrokenFS(err error) { + wkfs.RegisterFS("/s3/", &s3FS{ + err: err, + }) +} + +type s3FS struct { + sc *s3.S3 + err error +} + +func (fs *s3FS) parseName(name string) (bucket, fileName string, err error) { + if fs.err != nil { + return "", "", fs.err + } + name = strings.TrimPrefix(name, "/s3/") + i := strings.Index(name, "/") + if i < 0 { + return name, "", nil + } + return name[:i], name[i+1:], nil +} + +// Open opens the named file for reading. +func (fs *s3FS) Open(name string) (wkfs.File, error) { + bucket, fileName, err := fs.parseName(name) + if err != nil { + return nil, err + } + obj, err := fs.sc.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &fileName, + }) + if err != nil { + return nil, err + } + defer obj.Body.Close() + slurp, err := ioutil.ReadAll(obj.Body) + if err != nil { + return nil, err + } + return &file{ + name: name, + Reader: bytes.NewReader(slurp), + }, nil +} + +func (fs *s3FS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) } +func (fs *s3FS) Lstat(name string) (os.FileInfo, error) { + bucket, fileName, err := fs.parseName(name) + if err != nil { + return nil, err + } + obj, err := fs.sc.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &fileName, + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "NoSuchKey" { + return nil, os.ErrNotExist + } + } + } + if err != nil { + return nil, err + } + return &statInfo{ + name: path.Base(fileName), + size: *obj.ContentLength, + }, nil +} + +func (fs *s3FS) MkdirAll(path string, perm os.FileMode) error { + _, err := fs.OpenFile(fmt.Sprintf("%s/", filepath.Clean(path)), os.O_CREATE, perm) + return err +} + +func (fs *s3FS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) { + bucket, filename, err := fs.parseName(name) + if err != nil { + return nil, err + } + switch flag { + case os.O_WRONLY | os.O_CREATE | os.O_EXCL: + case os.O_WRONLY | os.O_CREATE | os.O_TRUNC: + default: + return nil, fmt.Errorf("Unsupported OpenFlag flag mode %d on S3", flag) + } + if flag&os.O_EXCL != 0 { + if _, err := fs.Stat(name); err == nil { + return nil, os.ErrExist + } + } + return NewS3file(bucket, filename, fs.sc) +} + +type statInfo struct { + name string + size int64 + isDir bool + modtime time.Time +} + +func (si *statInfo) IsDir() bool { return si.isDir } +func (si *statInfo) ModTime() time.Time { return si.modtime } +func (si *statInfo) Mode() os.FileMode { return 0644 } +func (si *statInfo) Name() string { return path.Base(si.name) } +func (si *statInfo) Size() int64 { return si.size } +func (si *statInfo) Sys() interface{} { return nil } + +type file struct { + name string + *bytes.Reader +} + +func (*file) Close() error { return nil } +func (f *file) Name() string { return path.Base(f.name) } +func (f *file) Stat() (os.FileInfo, error) { + panic("Stat not implemented on /s3/ files yet") +} diff --git a/vendor/github.com/nsheridan/wkfs/s3/s3_file.go b/vendor/github.com/nsheridan/wkfs/s3/s3_file.go new file mode 100644 index 0000000..c04597e --- /dev/null +++ b/vendor/github.com/nsheridan/wkfs/s3/s3_file.go @@ -0,0 +1,59 @@ +package s3 + +import ( + "bytes" + "errors" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// S3file represents a file in S3. +type S3file struct { + bucket string + name string + offset int + closed bool + + s3api *s3.S3 +} + +// NewS3file initializes an S3file. +func NewS3file(bucket, name string, s3api *s3.S3) (*S3file, error) { + return &S3file{ + bucket: bucket, + name: name, + offset: 0, + closed: false, + s3api: s3api, + }, nil +} + +// Write len(p) bytes to the file in S3. +// It returns the number of bytes written and an error, if any. +func (f *S3file) Write(p []byte) (n int, err error) { + if f.closed { + panic("read after close") + } + if f.offset != 0 { + return 0, errors.New("Offset cannot be > 0") + } + readSeeker := bytes.NewReader(p) + size := int(readSeeker.Size()) + obj := &s3.PutObjectInput{ + Bucket: aws.String(f.bucket), + Key: aws.String(f.name), + Body: readSeeker, + } + if _, err := f.s3api.PutObject(obj); err != nil { + return 0, err + } + f.offset += size + return size, nil +} + +// Close the file, rendering it unusable. +func (f *S3file) Close() error { + f.closed = true + return nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index bb753f3..27fa85e 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -392,6 +392,12 @@ "revision": "bfdb1a85537d60bc7e954e600c250219ea497417", "revisionTime": "2016-12-11T22:23:15Z" }, + { + "checksumSHA1": "Ywe06VqOCpwDNjipGTMO0oOG/Yg=", + "path": "github.com/nsheridan/wkfs/s3", + "revision": "60e6f1760f59568e4ce95080d08cd4a90c3c50c7", + "revisionTime": "2016-12-29T20:48:42Z" + }, { "checksumSHA1": "8Y05Pz7onrQPcVWW6JStSsYRh6E=", "path": "github.com/pelletier/go-buffruneio", -- cgit v1.2.3