From 1b8b6e218c7a70cb61c5449a204e38738b7bd945 Mon Sep 17 00:00:00 2001 From: Ben Burwell Date: Thu, 11 Jul 2019 09:44:51 -0400 Subject: Add maildir backend worker Add the initial implementation of a backend for Maildir accounts. Much of the functionality required is implemented in the go-message and go-maildir libraries, so we use them as much as possible. The maildir worker hooks into a new maildir:// URL scheme in the accounts.conf file which points to a container of several maildir directories. From there, the OpenDirectory, FetchDirectoryContents, etc messages work on subdirectories. This is implemented as a Container struct which handles mapping between the symbolic email folder names and UIDs to the concrete directories and file names. --- worker/maildir/container.go | 105 +++++++++++++ worker/maildir/message.go | 322 ++++++++++++++++++++++++++++++++++++++++ worker/maildir/worker.go | 353 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 780 insertions(+) create mode 100644 worker/maildir/container.go create mode 100644 worker/maildir/message.go create mode 100644 worker/maildir/worker.go (limited to 'worker/maildir') diff --git a/worker/maildir/container.go b/worker/maildir/container.go new file mode 100644 index 0000000..351afed --- /dev/null +++ b/worker/maildir/container.go @@ -0,0 +1,105 @@ +package maildir + +import ( + "fmt" + "io/ioutil" + "log" + "path/filepath" + "sort" + + "github.com/emersion/go-maildir" + + "git.sr.ht/~sircmpwn/aerc/lib/uidstore" +) + +// A Container is a directory which contains other directories which adhere to +// the Maildir spec +type Container struct { + dir string + log *log.Logger + uids *uidstore.Store +} + +// NewContainer creates a new container at the specified directory +// TODO: return an error if the provided directory is not accessible +func NewContainer(dir string, l *log.Logger) *Container { + return &Container{dir: dir, uids: uidstore.NewStore(), log: l} +} + +// ListFolders returns a list of maildir folders in the container +func (c *Container) ListFolders() ([]string, error) { + files, err := ioutil.ReadDir(c.dir) + if err != nil { + return nil, fmt.Errorf("error reading folders: %v", err) + } + dirnames := []string{} + for _, f := range files { + if f.IsDir() { + dirnames = append(dirnames, f.Name()) + } + } + return dirnames, nil +} + +// OpenDirectory opens an existing maildir in the container by name, moves new +// messages into cur, and registers the new keys in the UIDStore. +func (c *Container) OpenDirectory(name string) (maildir.Dir, error) { + dir := c.Dir(name) + keys, err := dir.Unseen() + if err != nil { + return dir, err + } + for _, key := range keys { + c.uids.GetOrInsert(key) + } + return dir, nil +} + +// Dir returns a maildir.Dir with the specified name inside the container +func (c *Container) Dir(name string) maildir.Dir { + return maildir.Dir(filepath.Join(c.dir, name)) +} + +// UIDs fetches the unique message identifiers for the maildir +func (c *Container) UIDs(d maildir.Dir) ([]uint32, error) { + keys, err := d.Keys() + if err != nil { + return nil, fmt.Errorf("could not get keys for %s: %v", d, err) + } + sort.Strings(keys) + var uids []uint32 + for _, key := range keys { + uids = append(uids, c.uids.GetOrInsert(key)) + } + return uids, nil +} + +// Message returns a Message struct for the given UID and maildir +func (c *Container) Message(d maildir.Dir, uid uint32) (*Message, error) { + if key, ok := c.uids.GetKey(uid); ok { + return &Message{ + dir: d, + uid: uid, + key: key, + }, nil + } + return nil, fmt.Errorf("could not find message with uid %d in maildir %s", + uid, d) +} + +// DeleteAll deletes a set of messages by UID and returns the subset of UIDs +// which were successfully deleted, stopping upon the first error. +func (c *Container) DeleteAll(d maildir.Dir, uids []uint32) ([]uint32, error) { + var success []uint32 + for _, uid := range uids { + msg, err := c.Message(d, uid) + if err != nil { + return success, err + } + if err := msg.Remove(); err != nil { + return success, err + } + success = append(success, uid) + } + return success, nil +} diff --git a/worker/maildir/message.go b/worker/maildir/message.go new file mode 100644 index 0000000..b95ec98 --- /dev/null +++ b/worker/maildir/message.go @@ -0,0 +1,322 @@ +package maildir + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "mime/quotedprintable" + gomail "net/mail" + "strings" + + "github.com/emersion/go-maildir" + "github.com/emersion/go-message" + _ "github.com/emersion/go-message/charset" + "github.com/emersion/go-message/mail" + + "git.sr.ht/~sircmpwn/aerc/models" +) + +// A Message is an individual email inside of a maildir.Dir. +type Message struct { + dir maildir.Dir + uid uint32 + key string +} + +// NewReader reads a message into memory and returns an io.Reader for it. +func (m Message) NewReader() (io.Reader, error) { + f, err := m.dir.Open(m.key) + if err != nil { + return nil, err + } + defer f.Close() + b, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +// Flags fetches the set of flags currently applied to the message. +func (m Message) Flags() ([]maildir.Flag, error) { + return m.dir.Flags(m.key) +} + +// SetFlags replaces the message's flags with a new set. +func (m Message) SetFlags(flags []maildir.Flag) error { + return m.dir.SetFlags(m.key, flags) +} + +// MarkRead either adds or removes the maildir.FlagSeen flag from the message. +func (m Message) MarkRead(seen bool) error { + flags, err := m.Flags() + if err != nil { + return fmt.Errorf("could not read previous flags: %v", err) + } + if seen { + flags = append(flags, maildir.FlagSeen) + return m.SetFlags(flags) + } + var newFlags []maildir.Flag + for _, flag := range flags { + if flag != maildir.FlagSeen { + newFlags = append(newFlags, flag) + } + } + return m.SetFlags(newFlags) +} + +// Remove deletes the email immediately. +func (m Message) Remove() error { + return m.dir.Remove(m.key) +} + +// MessageInfo populates a models.MessageInfo struct for the message. +func (m Message) MessageInfo() (*models.MessageInfo, error) { + f, err := m.dir.Open(m.key) + if err != nil { + return nil, err + } + defer f.Close() + msg, err := message.Read(f) + if err != nil { + return nil, fmt.Errorf("could not read message: %v", err) + } + bs, err := parseEntityStructure(msg) + if err != nil { + return nil, fmt.Errorf("could not get structure: %v", err) + } + env, err := parseEnvelope(&msg.Header) + if err != nil { + return nil, fmt.Errorf("could not get envelope: %v", err) + } + flags, err := m.Flags() + if err != nil { + return nil, fmt.Errorf("could not read flags: %v", err) + } + return &models.MessageInfo{ + BodyStructure: bs, + Envelope: env, + Flags: translateFlags(flags), + InternalDate: env.Date, + RFC822Headers: &mail.Header{msg.Header}, + Size: 0, + Uid: m.uid, + }, nil +} + +// NewBodyPartReader creates a new io.Reader for the requested body part(s) of +// the message. +func (m Message) NewBodyPartReader(requestedParts []int) (io.Reader, error) { + f, err := m.dir.Open(m.key) + if err != nil { + return nil, err + } + defer f.Close() + msg, err := message.Read(f) + if err != nil { + return nil, fmt.Errorf("could not read message: %v", err) + } + return fetchEntityPartReader(msg, requestedParts) +} + +func fetchEntityPartReader(e *message.Entity, index []int) (io.Reader, error) { + if len(index) < 1 { + return nil, fmt.Errorf("no part to read") + } + if mpr := e.MultipartReader(); mpr != nil { + idx := 0 + for { + idx++ + part, err := mpr.NextPart() + if err != nil { + return nil, err + } + if idx == index[0] { + rest := index[1:] + if len(rest) < 1 { + return fetchEntityReader(part) + } + return fetchEntityPartReader(part, index[1:]) + } + } + } + if index[0] != 1 { + return nil, fmt.Errorf("cannont return non-first part of non-multipart") + } + return fetchEntityReader(e) +} + +// fetchEntityReader makes an io.Reader for the given entity. Since the +// go-message package decodes the body for us, and the UI expects to deal with +// a reader whose bytes are encoded with the part's encoding, we are in the +// interesting position of needing to re-encode the reader before sending it +// off to the UI layer. +// +// TODO: probably change the UI to expect an already-decoded reader and decode +// in the IMAP worker. +func fetchEntityReader(e *message.Entity) (io.Reader, error) { + enc := e.Header.Get("content-transfer-encoding") + var buf bytes.Buffer + + // base64 + if strings.EqualFold(enc, "base64") { + wc := base64.NewEncoder(base64.StdEncoding, &buf) + defer wc.Close() + if _, err := io.Copy(wc, e.Body); err != nil { + return nil, fmt.Errorf("could not base64 encode: %v", err) + } + return &buf, nil + } + + // quoted-printable + if strings.EqualFold(enc, "quoted-printable") { + wc := quotedprintable.NewWriter(&buf) + defer wc.Close() + if _, err := io.Copy(wc, e.Body); err != nil { + return nil, fmt.Errorf("could not quoted-printable encode: %v", err) + } + return &buf, nil + } + + // other general encoding + if _, err := io.Copy(&buf, e.Body); err != nil { + return nil, err + } + + return &buf, nil +} + +// split a MIME type into its major and minor parts +func splitMIME(m string) (string, string) { + parts := strings.Split(m, "/") + if len(parts) != 2 { + return parts[0], "" + } + return parts[0], parts[1] +} + +func parseEntityStructure(e *message.Entity) (*models.BodyStructure, error) { + var body models.BodyStructure + contentType, ctParams, err := e.Header.ContentType() + if err != nil { + return nil, fmt.Errorf("could not parse content type: %v", err) + } + mimeType, mimeSubType := splitMIME(contentType) + body.MIMEType = mimeType + body.MIMESubType = mimeSubType + body.Params = ctParams + body.Description = e.Header.Get("content-description") + body.Encoding = e.Header.Get("content-transfer-encoding") + if cd := e.Header.Get("content-disposition"); cd != "" { + contentDisposition, cdParams, err := e.Header.ContentDisposition() + if err != nil { + return nil, fmt.Errorf("could not parse content disposition: %v", err) + } + body.Disposition = contentDisposition + body.DispositionParams = cdParams + } + body.Parts = []*models.BodyStructure{} + if mpr := e.MultipartReader(); mpr != nil { + for { + part, err := mpr.NextPart() + if err == io.EOF { + return &body, nil + } else if err != nil { + return nil, err + } + ps, err := parseEntityStructure(part) + if err != nil { + return nil, fmt.Errorf("could not parse child entity structure: %v", err) + } + body.Parts = append(body.Parts, ps) + } + } + return &body, nil +} + +func parseEnvelope(h *message.Header) (*models.Envelope, error) { + date, err := gomail.ParseDate(h.Get("date")) + if err != nil { + return nil, fmt.Errorf("could not parse date header: %v", err) + } + from, err := parseAddressList(h, "from") + if err != nil { + return nil, fmt.Errorf("could not read from address: %v", err) + } + to, err := parseAddressList(h, "to") + if err != nil { + return nil, fmt.Errorf("could not read to address: %v", err) + } + cc, err := parseAddressList(h, "cc") + if err != nil { + return nil, fmt.Errorf("could not read cc address: %v", err) + } + bcc, err := parseAddressList(h, "bcc") + if err != nil { + return nil, fmt.Errorf("could not read bcc address: %v", err) + } + return &models.Envelope{ + Date: date, + Subject: h.Get("subject"), + MessageId: h.Get("message-id"), + From: from, + To: to, + Cc: cc, + Bcc: bcc, + }, nil +} + +func parseAddressList(h *message.Header, key string) ([]*models.Address, error) { + var converted []*models.Address + hdr := h.Get(key) + if strings.TrimSpace(hdr) == "" { + return converted, nil + } + addrs, err := gomail.ParseAddressList(hdr) + if err != nil { + if strings.Index(hdr, "@") < 0 { + return []*models.Address{&models.Address{ + Name: hdr, + }}, nil + } + return nil, err + } + for _, addr := range addrs { + parts := strings.Split(addr.Address, "@") + var mbox, host string + if len(parts) > 1 { + mbox = strings.Join(parts[0:len(parts)-1], "@") + host = parts[len(parts)-1] + } else { + mbox = addr.Address + } + converted = append(converted, &models.Address{ + Name: addr.Name, + Mailbox: mbox, + Host: host, + }) + } + return converted, nil +} + +var flagMap = map[maildir.Flag]models.Flag{ + maildir.FlagReplied: models.AnsweredFlag, + maildir.FlagSeen: models.SeenFlag, + maildir.FlagTrashed: models.DeletedFlag, + maildir.FlagFlagged: models.FlaggedFlag, + // maildir.FlagDraft Flag = 'D' + // maildir.FlagPassed Flag = 'P' +} + +func translateFlags(maildirFlags []maildir.Flag) []models.Flag { + var flags []models.Flag + for _, maildirFlag := range maildirFlags { + if flag, ok := flagMap[maildirFlag]; ok { + flags = append(flags, flag) + } + } + return flags +} diff --git a/worker/maildir/worker.go b/worker/maildir/worker.go new file mode 100644 index 0000000..f0c92ed --- /dev/null +++ b/worker/maildir/worker.go @@ -0,0 +1,353 @@ +package maildir + +import ( + "fmt" + "io" + "net/url" + "os" + "path/filepath" + + "github.com/emersion/go-maildir" + + "git.sr.ht/~sircmpwn/aerc/models" + "git.sr.ht/~sircmpwn/aerc/worker/types" +) + +var errUnsupported = fmt.Errorf("unsupported command") + +// A Worker handles interfacing between aerc's UI and a group of maildirs. +type Worker struct { + c *Container + selected *maildir.Dir + worker *types.Worker +} + +// NewWorker creates a new maildir worker with the provided worker. +func NewWorker(worker *types.Worker) *Worker { + return &Worker{worker: worker} +} + +// Run starts the worker's message handling loop. +func (w *Worker) Run() { + for { + action := <-w.worker.Actions + msg := w.worker.ProcessAction(action) + if err := w.handleMessage(msg); err == errUnsupported { + w.worker.PostMessage(&types.Unsupported{ + Message: types.RespondTo(msg), + }, nil) + } else if err != nil { + w.worker.PostMessage(&types.Error{ + Message: types.RespondTo(msg), + Error: err, + }, nil) + } + } +} + +func (w *Worker) done(msg types.WorkerMessage) { + w.worker.PostMessage(&types.Done{types.RespondTo(msg)}, nil) +} + +func (w *Worker) err(msg types.WorkerMessage, err error) { + w.worker.PostMessage(&types.Error{ + Message: types.RespondTo(msg), + Error: err, + }, nil) +} + +func (w *Worker) handleMessage(msg types.WorkerMessage) error { + switch msg := msg.(type) { + case *types.Unsupported: + // No-op + case *types.Configure: + return w.handleConfigure(msg) + case *types.Connect: + return w.handleConnect(msg) + case *types.ListDirectories: + return w.handleListDirectories(msg) + case *types.OpenDirectory: + return w.handleOpenDirectory(msg) + case *types.FetchDirectoryContents: + return w.handleFetchDirectoryContents(msg) + case *types.CreateDirectory: + return w.handleCreateDirectory(msg) + case *types.FetchMessageHeaders: + return w.handleFetchMessageHeaders(msg) + case *types.FetchMessageBodyPart: + return w.handleFetchMessageBodyPart(msg) + case *types.FetchFullMessages: + return w.handleFetchFullMessages(msg) + case *types.DeleteMessages: + return w.handleDeleteMessages(msg) + case *types.ReadMessages: + return w.handleReadMessages(msg) + case *types.CopyMessages: + return w.handleCopyMessages(msg) + case *types.AppendMessage: + return w.handleAppendMessage(msg) + case *types.SearchDirectory: + return w.handleSearchDirectory(msg) + } + return errUnsupported +} + +func (w *Worker) handleConfigure(msg *types.Configure) error { + defer w.done(msg) + u, err := url.Parse(msg.Config.Source) + if err != nil { + w.worker.Logger.Printf("error configuring maildir worker: %v", err) + return err + } + dir := u.Path + if u.Host == "~" { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("could not resolve home directory: %v", err) + } + dir = filepath.Join(home, u.Path) + } + w.c = NewContainer(dir, w.worker.Logger) + w.worker.Logger.Printf("configured base maildir: %s", dir) + return nil +} + +func (w *Worker) handleConnect(msg *types.Connect) error { + w.done(msg) + return nil +} + +func (w *Worker) handleListDirectories(msg *types.ListDirectories) error { + defer w.done(msg) + dirs, err := w.c.ListFolders() + if err != nil { + w.worker.Logger.Printf("error listing directories: %v", err) + return err + } + for _, name := range dirs { + w.worker.PostMessage(&types.Directory{ + Message: types.RespondTo(msg), + Dir: &models.Directory{ + Name: name, + Attributes: []string{}, + }, + }, nil) + } + return nil +} + +func (w *Worker) handleOpenDirectory(msg *types.OpenDirectory) error { + defer w.done(msg) + w.worker.Logger.Printf("opening %s", msg.Directory) + dir, err := w.c.OpenDirectory(msg.Directory) + if err != nil { + return err + } + w.selected = &dir + // TODO: why does this need to be sent twice?? + info := &types.DirectoryInfo{ + Info: &models.DirectoryInfo{ + Name: msg.Directory, + Flags: []string{}, + ReadOnly: false, + // total messages + Exists: 0, + // new messages since mailbox was last opened + Recent: 0, + // total unread + Unseen: 0, + }, + } + w.worker.PostMessage(info, nil) + w.worker.PostMessage(info, nil) + return nil +} + +func (w *Worker) handleFetchDirectoryContents( + msg *types.FetchDirectoryContents) error { + defer w.done(msg) + uids, err := w.c.UIDs(*w.selected) + if err != nil { + w.worker.Logger.Printf("error scanning uids: %v", err) + return err + } + w.worker.PostMessage(&types.DirectoryContents{ + Message: types.RespondTo(msg), + Uids: uids, + }, nil) + return nil +} + +func (w *Worker) handleCreateDirectory(msg *types.CreateDirectory) error { + dir := w.c.Dir(msg.Directory) + defer w.done(msg) + if err := dir.Create(); err != nil { + w.worker.Logger.Printf("could not create directory %s: %v", + msg.Directory, err) + return err + } + return nil +} + +func (w *Worker) handleFetchMessageHeaders( + msg *types.FetchMessageHeaders) error { + defer w.done(msg) + for _, uid := range msg.Uids { + m, err := w.c.Message(*w.selected, uid) + if err != nil { + w.worker.Logger.Printf("could not get message: %v", err) + w.err(msg, err) + continue + } + info, err := m.MessageInfo() + if err != nil { + w.worker.Logger.Printf("could not get message info: %v", err) + w.err(msg, err) + continue + } + w.worker.PostMessage(&types.MessageInfo{ + Message: types.RespondTo(msg), + Info: info, + }, nil) + } + return nil +} + +func (w *Worker) handleFetchMessageBodyPart( + msg *types.FetchMessageBodyPart) error { + defer w.done(msg) + + // get reader + m, err := w.c.Message(*w.selected, msg.Uid) + if err != nil { + w.worker.Logger.Printf("could not get message %d: %v", msg.Uid, err) + return err + } + r, err := m.NewBodyPartReader(msg.Part) + if err != nil { + w.worker.Logger.Printf( + "could not get body part reader for message=%d, parts=%#v: %v", + msg.Uid, msg.Part, err) + return err + } + w.worker.PostMessage(&types.MessageBodyPart{ + Message: types.RespondTo(msg), + Part: &models.MessageBodyPart{ + Reader: r, + Uid: msg.Uid, + }, + }, nil) + + // mark message as read + if err := m.MarkRead(true); err != nil { + w.worker.Logger.Printf("could not mark message as read: %v", err) + return err + } + + // send updated flags to ui + info, err := m.MessageInfo() + if err != nil { + w.worker.Logger.Printf("could not fetch message info: %v", err) + return err + } + w.worker.PostMessage(&types.MessageInfo{ + Message: types.RespondTo(msg), + Info: info, + }, nil) + + return nil +} + +func (w *Worker) handleFetchFullMessages(msg *types.FetchFullMessages) error { + defer w.done(msg) + for _, uid := range msg.Uids { + m, err := w.c.Message(*w.selected, uid) + if err != nil { + w.worker.Logger.Printf("could not get message %d: %v", uid, err) + return err + } + r, err := m.NewReader() + if err != nil { + w.worker.Logger.Printf("could not get message reader: %v", err) + return err + } + w.worker.PostMessage(&types.FullMessage{ + Message: types.RespondTo(msg), + Content: &models.FullMessage{ + Uid: uid, + Reader: r, + }, + }, nil) + } + return nil +} + +func (w *Worker) handleDeleteMessages(msg *types.DeleteMessages) error { + defer w.done(msg) + deleted, err := w.c.DeleteAll(*w.selected, msg.Uids) + if len(deleted) > 0 { + w.worker.PostMessage(&types.MessagesDeleted{ + Message: types.RespondTo(msg), + Uids: deleted, + }, nil) + } + if err != nil { + w.worker.Logger.Printf("error removing some messages: %v", err) + return err + } + return nil +} + +func (w *Worker) handleReadMessages(msg *types.ReadMessages) error { + defer w.done(msg) + for _, uid := range msg.Uids { + m, err := w.c.Message(*w.selected, uid) + if err != nil { + w.worker.Logger.Printf("could not get message: %v", err) + w.err(msg, err) + continue + } + if err := m.MarkRead(msg.Read); err != nil { + w.worker.Logger.Printf("could not mark message as read: %v", err) + w.err(msg, err) + continue + } + info, err := m.MessageInfo() + if err != nil { + w.worker.Logger.Printf("could not get message info: %v", err) + w.err(msg, err) + continue + } + w.worker.PostMessage(&types.MessageInfo{ + Message: types.RespondTo(msg), + Info: info, + }, nil) + } + return nil +} + +func (w *Worker) handleCopyMessages(msg *types.CopyMessages) error { + // TODO: this. + return nil +} + +func (w *Worker) handleAppendMessage(msg *types.AppendMessage) error { + defer w.done(msg) + dest := w.c.Dir(msg.Destination) + delivery, err := dest.NewDelivery() + if err != nil { + w.worker.Logger.Printf("could not deliver message to %s: %v", + msg.Destination, err) + return err + } + defer delivery.Close() + if _, err := io.Copy(delivery, msg.Reader); err != nil { + w.worker.Logger.Printf("could not write message to destination: %v", err) + return err + } + return nil +} + +func (w *Worker) handleSearchDirectory(msg *types.SearchDirectory) error { + return errUnsupported +} -- cgit v1.2.3