From 8066efd45861e7c024fc1daabc6d002266a527e7 Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Thu, 29 Dec 2016 21:53:26 +0000 Subject: Use vendored s3 wkfs --- vendor/github.com/nsheridan/wkfs/s3/README.md | 41 ++++++ vendor/github.com/nsheridan/wkfs/s3/s3.go | 180 +++++++++++++++++++++++++ vendor/github.com/nsheridan/wkfs/s3/s3_file.go | 59 ++++++++ vendor/vendor.json | 6 + 4 files changed, 286 insertions(+) create mode 100644 vendor/github.com/nsheridan/wkfs/s3/README.md create mode 100644 vendor/github.com/nsheridan/wkfs/s3/s3.go create mode 100644 vendor/github.com/nsheridan/wkfs/s3/s3_file.go (limited to 'vendor') diff --git a/vendor/github.com/nsheridan/wkfs/s3/README.md b/vendor/github.com/nsheridan/wkfs/s3/README.md new file mode 100644 index 0000000..177f738 --- /dev/null +++ b/vendor/github.com/nsheridan/wkfs/s3/README.md @@ -0,0 +1,41 @@ +## S3 plugin for WKFS + + + +Package `s3` registers an AWS S3 filesystem at the well-known `/s3/` filesystem path. + +Sample usage: + +```go +package main + +import ( + "fmt" + "io" + "log" + + "github.com/nsheridan/wkfs/s3" + "go4.org/wkfs" +) + +func main() { + opts := &s3.Options{ + Region: "us-east-1" + AccessKey: "abcdef" + SecretKey: "secret" + } + s3.Register(opts) + f, err := wkfs.Create("/s3/some-bucket/hello.txt") + if err != nil { + log.Fatal(err) + } + _, err := io.WriteString(f, "hello, world") + if err != nil { + log.Fatal(err) + } +} +``` + + + +`Options` are completely optional as the AWS SDK will attempt to obtain credentials from a number of locations - see [the documentation for details](http://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) - e.g. if you're using environment variables you can register the filesystem with `s3.Register(nil)`. diff --git a/vendor/github.com/nsheridan/wkfs/s3/s3.go b/vendor/github.com/nsheridan/wkfs/s3/s3.go new file mode 100644 index 0000000..19e72a9 --- /dev/null +++ b/vendor/github.com/nsheridan/wkfs/s3/s3.go @@ -0,0 +1,180 @@ +package s3 + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "time" + + "go4.org/wkfs" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +// Options for registering the S3 wkfs. +// None of these are required and can be supplied to the aws client by other means. +type Options struct { + Region string + AccessKey string + SecretKey string +} + +// Register the /s3/ filesystem as a well-known filesystem. +func Register(opts *Options) { + if opts == nil { + opts = &Options{} + } + config := &aws.Config{} + // If region is unset the SDK will attempt to read the region from the environment. + if opts.Region != "" { + config.Region = aws.String(opts.Region) + } + // Attempt to use supplied credentials, otherwise fall back to the SDK. + if opts.AccessKey != "" && opts.SecretKey != "" { + config.Credentials = credentials.NewStaticCredentials(opts.AccessKey, opts.SecretKey, "") + } + s, err := session.NewSession(config) + if err != nil { + registerBrokenFS(err) + return + } + sc := s3.New(s) + if aws.StringValue(sc.Config.Region) == "" { + registerBrokenFS(errors.New("could not find region configuration")) + return + } + wkfs.RegisterFS("/s3/", &s3FS{ + sc: sc, + }) +} + +func registerBrokenFS(err error) { + wkfs.RegisterFS("/s3/", &s3FS{ + err: err, + }) +} + +type s3FS struct { + sc *s3.S3 + err error +} + +func (fs *s3FS) parseName(name string) (bucket, fileName string, err error) { + if fs.err != nil { + return "", "", fs.err + } + name = strings.TrimPrefix(name, "/s3/") + i := strings.Index(name, "/") + if i < 0 { + return name, "", nil + } + return name[:i], name[i+1:], nil +} + +// Open opens the named file for reading. +func (fs *s3FS) Open(name string) (wkfs.File, error) { + bucket, fileName, err := fs.parseName(name) + if err != nil { + return nil, err + } + obj, err := fs.sc.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &fileName, + }) + if err != nil { + return nil, err + } + defer obj.Body.Close() + slurp, err := ioutil.ReadAll(obj.Body) + if err != nil { + return nil, err + } + return &file{ + name: name, + Reader: bytes.NewReader(slurp), + }, nil +} + +func (fs *s3FS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) } +func (fs *s3FS) Lstat(name string) (os.FileInfo, error) { + bucket, fileName, err := fs.parseName(name) + if err != nil { + return nil, err + } + obj, err := fs.sc.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &fileName, + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "NoSuchKey" { + return nil, os.ErrNotExist + } + } + } + if err != nil { + return nil, err + } + return &statInfo{ + name: path.Base(fileName), + size: *obj.ContentLength, + }, nil +} + +func (fs *s3FS) MkdirAll(path string, perm os.FileMode) error { + _, err := fs.OpenFile(fmt.Sprintf("%s/", filepath.Clean(path)), os.O_CREATE, perm) + return err +} + +func (fs *s3FS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) { + bucket, filename, err := fs.parseName(name) + if err != nil { + return nil, err + } + switch flag { + case os.O_WRONLY | os.O_CREATE | os.O_EXCL: + case os.O_WRONLY | os.O_CREATE | os.O_TRUNC: + default: + return nil, fmt.Errorf("Unsupported OpenFlag flag mode %d on S3", flag) + } + if flag&os.O_EXCL != 0 { + if _, err := fs.Stat(name); err == nil { + return nil, os.ErrExist + } + } + return NewS3file(bucket, filename, fs.sc) +} + +type statInfo struct { + name string + size int64 + isDir bool + modtime time.Time +} + +func (si *statInfo) IsDir() bool { return si.isDir } +func (si *statInfo) ModTime() time.Time { return si.modtime } +func (si *statInfo) Mode() os.FileMode { return 0644 } +func (si *statInfo) Name() string { return path.Base(si.name) } +func (si *statInfo) Size() int64 { return si.size } +func (si *statInfo) Sys() interface{} { return nil } + +type file struct { + name string + *bytes.Reader +} + +func (*file) Close() error { return nil } +func (f *file) Name() string { return path.Base(f.name) } +func (f *file) Stat() (os.FileInfo, error) { + panic("Stat not implemented on /s3/ files yet") +} diff --git a/vendor/github.com/nsheridan/wkfs/s3/s3_file.go b/vendor/github.com/nsheridan/wkfs/s3/s3_file.go new file mode 100644 index 0000000..c04597e --- /dev/null +++ b/vendor/github.com/nsheridan/wkfs/s3/s3_file.go @@ -0,0 +1,59 @@ +package s3 + +import ( + "bytes" + "errors" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +// S3file represents a file in S3. +type S3file struct { + bucket string + name string + offset int + closed bool + + s3api *s3.S3 +} + +// NewS3file initializes an S3file. +func NewS3file(bucket, name string, s3api *s3.S3) (*S3file, error) { + return &S3file{ + bucket: bucket, + name: name, + offset: 0, + closed: false, + s3api: s3api, + }, nil +} + +// Write len(p) bytes to the file in S3. +// It returns the number of bytes written and an error, if any. +func (f *S3file) Write(p []byte) (n int, err error) { + if f.closed { + panic("read after close") + } + if f.offset != 0 { + return 0, errors.New("Offset cannot be > 0") + } + readSeeker := bytes.NewReader(p) + size := int(readSeeker.Size()) + obj := &s3.PutObjectInput{ + Bucket: aws.String(f.bucket), + Key: aws.String(f.name), + Body: readSeeker, + } + if _, err := f.s3api.PutObject(obj); err != nil { + return 0, err + } + f.offset += size + return size, nil +} + +// Close the file, rendering it unusable. +func (f *S3file) Close() error { + f.closed = true + return nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index bb753f3..27fa85e 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -392,6 +392,12 @@ "revision": "bfdb1a85537d60bc7e954e600c250219ea497417", "revisionTime": "2016-12-11T22:23:15Z" }, + { + "checksumSHA1": "Ywe06VqOCpwDNjipGTMO0oOG/Yg=", + "path": "github.com/nsheridan/wkfs/s3", + "revision": "60e6f1760f59568e4ce95080d08cd4a90c3c50c7", + "revisionTime": "2016-12-29T20:48:42Z" + }, { "checksumSHA1": "8Y05Pz7onrQPcVWW6JStSsYRh6E=", "path": "github.com/pelletier/go-buffruneio", -- cgit v1.2.3