From dea6ee40f067e0a3ef81972cfcfac1761eeb27d4 Mon Sep 17 00:00:00 2001 From: Ben Burwell Date: Tue, 24 Sep 2019 14:06:09 -0400 Subject: Implement monitor package --- main.go | 100 +++++++++++++++++---------------------- monitor/database.go | 56 ++++++++++++++++++++++ monitor/monitor.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++ monitor/security_error.go | 7 +++ sumdb/client.go | 90 ++++++++++++++++++++++------------- 5 files changed, 280 insertions(+), 90 deletions(-) create mode 100644 monitor/database.go create mode 100644 monitor/monitor.go create mode 100644 monitor/security_error.go diff --git a/main.go b/main.go index f1d200b..f60372f 100644 --- a/main.go +++ b/main.go @@ -2,79 +2,63 @@ package main import ( "log" - "sync" "time" - "git.sr.ht/~benburwell/gosumdbaudit/sumdb" + "git.sr.ht/~benburwell/gosumdbaudit/monitor" + // "git.sr.ht/~benburwell/gosumdbaudit/sumdb" ) func main() { - dbs := []*database{ - &database{ - host: "sum.golang.org", + mon := monitor.NewMonitor(&monitor.Database{ + URL: "sum.golang.org", + Key: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", + }) - key: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", - // key: "sum.golang.org+033de0ae+BADBADBADBADBADBADBADBADBADBADBADBADBADBADBA", - - pollInterval: 10 * time.Second, - }, - } - var wg sync.WaitGroup - wg.Add(len(dbs)) - for _, db := range dbs { - go func(db *database) { - defer wg.Done() - if err := monitor(db); err != nil { - log.Printf("AUDIT FAILED: %s", err.Error()) - return - } - }(db) + if err := mon.Watch(10 * time.Second); err != nil { + log.Printf("AUDIT FAILED: %v", err) } - wg.Wait() } -func monitor(db *database) error { - log.Printf("starting monitor for %s", db.host) +// func monitor(db *database) error { +// log.Printf("starting monitor for %s", db.host) - client := sumdb.NewClient(db) +// client := sumdb.NewClient(db) - lines, err := client.Lookup("golang.org/x/text", "v0.3.0") - if err != nil { - return err - } - log.Printf("got lines: %s", lines) +// tree, err := client.FetchLatest() +// if err != nil { +// return err +// } +// log.Printf("got latest: N=%d, Hash=%s", tree.N, tree.Hash) - // fetch & verify current STH - // latest, err := client.Latest() - // if err != nil { - // return err - // } +// if err := client.FetchTreeProof(tree); err != nil { +// return err +// } - // fetch all entries in the tree according to the STH - // entries := client.Entries(nil, latest) +// // fetch all entries in the tree according to the STH +// // entries := client.Entries(nil, latest) - // confirm the tree made from the entries produces the same hash as the STH - // IF NOT: the server has signed invalid data +// // confirm the tree made from the entries produces the same hash as the STH +// // IF NOT: the server has signed invalid data - // prev := latest - for { - // await a new STH - // prev = latest - time.Sleep(db.pollInterval) - log.Printf("checking %s for new STH...", db.host) - // awaitNewSTH() +// // prev := latest +// for { +// // await a new STH +// // prev = latest +// time.Sleep(db.pollInterval) +// log.Printf("checking %s for new STH...", db.host) +// // awaitNewSTH() - // latest, err := client.Latest() - // if err != nil { - // return err - // } +// // latest, err := client.Latest() +// // if err != nil { +// // return err +// // } - // fetch all NEW entries between prev and latest - // if unavailable for an extended period, this should be viewed as misbehavior - // entries := client.Entries(prev, latest) +// // fetch all NEW entries between prev and latest +// // if unavailable for an extended period, this should be viewed as misbehavior +// // entries := client.Entries(prev, latest) - // fetch a consistency proof for the new STH with the previous STH - // verify consistency proof - // verify the new entries generate the corresponding elements in the consistency proof - } -} +// // fetch a consistency proof for the new STH with the previous STH +// // verify consistency proof +// // verify the new entries generate the corresponding elements in the consistency proof +// } +// } diff --git a/monitor/database.go b/monitor/database.go new file mode 100644 index 0000000..7a0ead1 --- /dev/null +++ b/monitor/database.go @@ -0,0 +1,56 @@ +package monitor + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" +) + +type Database struct { + URL string + Key string +} + +func (db *Database) readRemote(path string) ([]byte, error) { + log.Printf("GET %s", path) + resp, err := http.Get("https://" + db.URL + path) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("get %s: expected HTTP OK but got %s", path, resp.Status) + } + if resp.ContentLength > 1e6 { + return nil, fmt.Errorf("get %s: body too large (%d bytes)", path, resp.ContentLength) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("get %s: %v", err) + } + return body, nil +} + +func (db *Database) VerifyNote(msg []byte) (*note.Note, error) { + verifier, err := note.NewVerifier(db.Key) + if err != nil { + return nil, err + } + return note.Open(msg, note.VerifierList(verifier)) +} + +func (db *Database) FetchSTH() (tlog.Tree, error) { + resp, err := db.readRemote("/latest") + if err != nil { + return tlog.Tree{}, err + } + note, err := db.VerifyNote(resp) + if err != nil { + return tlog.Tree{}, err + } + return tlog.ParseTree([]byte(note.Text)) +} diff --git a/monitor/monitor.go b/monitor/monitor.go new file mode 100644 index 0000000..e291a63 --- /dev/null +++ b/monitor/monitor.go @@ -0,0 +1,117 @@ +package monitor + +import ( + "fmt" + "log" + "sync" + "time" + + "golang.org/x/mod/sumdb/tlog" +) + +type Monitor struct { + db *Database + + // the latest verified tree + latest tlog.Tree +} + +func NewMonitor(db *Database) *Monitor { + return &Monitor{db: db} +} + +// Watch polls the database to check for a new Signed Tree Head every interval. +// It runs indefinitely; only returning an error if an inconsistency is found. +func (m *Monitor) Watch(interval time.Duration) error { + for { + log.Printf("checking for new STH") + t, err := m.db.FetchSTH() + if err != nil { + return err + } + log.Printf("got STH: %#v", t) + + if m.latest.N == t.N { + log.Printf("current STH matches latest proved") + } else { + if err := m.TreeProof(m.latest, t); err != nil { + return err + } + log.Printf("proof succeeded") + m.latest = t + } + + time.Sleep(interval) + } + return nil +} + +// TreeProof proves that tree older is a prefix of tree newer, returning an +// error if the proof fails. +func (m *Monitor) TreeProof(older, newer tlog.Tree) error { + log.Printf("proving that N=%d is a prefix of N'=%d", older.N, newer.N) + if older.N == 0 { + log.Printf("N=0, so using bootstrap proof") + return m.BootstrapProof(newer) + } + thr := tlog.TileHashReader(newer, m) + p, err := tlog.ProveTree(newer.N, older.N, thr) + if err != nil { + return err + } + log.Printf("Proof: %#v", p) + return nil +} + +func (m *Monitor) BootstrapProof(t tlog.Tree) error { + thr := tlog.TileHashReader(t, m) + h, err := tlog.TreeHash(t.N, thr) + if err != nil { + return err + } + log.Printf("computed hash: %v", h) + log.Printf("expected hash: %v", t.Hash) + if h != t.Hash { + return fmt.Errorf("computed hash %v does not match expected: %v", h, t.Hash) + } + return nil +} + +func (m *Monitor) Height() int { + return 8 +} + +// ReadTiles reads and returns the requested tiles, +// either from the on-disk cache or the server. +func (m *Monitor) ReadTiles(tiles []tlog.Tile) ([][]byte, error) { + log.Printf("reading %d tiles", len(tiles)) + // Read all the tiles in parallel. + data := make([][]byte, len(tiles)) + errs := make([]error, len(tiles)) + var wg sync.WaitGroup + for i, tile := range tiles { + wg.Add(1) + go func(i int, tile tlog.Tile) { + defer wg.Done() + log.Printf("reading tile %v", tile) + data[i], errs[i] = m.readTile(tile) + }(i, tile) + } + wg.Wait() + + for _, err := range errs { + if err != nil { + return nil, err + } + } + + return data, nil +} + +func (m *Monitor) readTile(tile tlog.Tile) ([]byte, error) { + return m.db.readRemote("/" + tile.Path()) +} + +func (m *Monitor) SaveTiles(tiles []tlog.Tile, data [][]byte) { + // noop +} diff --git a/monitor/security_error.go b/monitor/security_error.go new file mode 100644 index 0000000..aa036eb --- /dev/null +++ b/monitor/security_error.go @@ -0,0 +1,7 @@ +package monitor + +type SecurityError string + +func (e SecurityError) Error() string { + return string(e) +} diff --git a/sumdb/client.go b/sumdb/client.go index 8943eb3..136f611 100644 --- a/sumdb/client.go +++ b/sumdb/client.go @@ -8,9 +8,9 @@ import ( "bytes" "errors" "fmt" + "log" "strings" "sync" - "sync/atomic" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/note" @@ -69,8 +69,6 @@ var ErrSecurity = errors.New("security error: misbehaving server") type Client struct { ops ClientOps // access to operations in the external world - didLookup uint32 - // one-time initialized data initOnce sync.Once initErr error // init error, if any @@ -107,16 +105,16 @@ func (c *Client) init() error { // initWork does the actual initialization work. func (c *Client) initWork() { + log.Printf("initializing sumdb client") defer func() { + log.Printf("done init") if c.initErr != nil { c.initErr = fmt.Errorf("initializing sumdb.Client: %v", c.initErr) } }() c.tileReader.c = c - if c.tileHeight == 0 { - c.tileHeight = 8 - } + c.tileHeight = 8 c.tileSaved = make(map[tlog.Tile]bool) vkey, err := c.ops.ReadConfig("key") @@ -132,41 +130,24 @@ func (c *Client) initWork() { c.verifiers = note.VerifierList(verifier) c.name = verifier.Name() + log.Printf("init: reading latest") data, err := c.ops.ReadConfig(c.name + "/latest") if err != nil { c.initErr = err return } + + log.Printf("init: merging latest") if err := c.mergeLatest(data); err != nil { c.initErr = err return } } -// SetTileHeight sets the tile height for the Client. -// Any call to SetTileHeight must happen before the first call to Lookup. -// If SetTileHeight is not called, the Client defaults to tile height 8. -// SetTileHeight can be called at most once, -// and if so it must be called before the first call to Lookup. -func (c *Client) SetTileHeight(height int) { - if atomic.LoadUint32(&c.didLookup) != 0 { - panic("SetTileHeight used after Lookup") - } - if height <= 0 { - panic("invalid call to SetTileHeight") - } - if c.tileHeight != 0 { - panic("multiple calls to SetTileHeight") - } - c.tileHeight = height -} - // Lookup returns the go.sum lines for the given module path and version. // The version may end in a /go.mod suffix, in which case Lookup returns // the go.sum lines for the module's go.mod-only hash. func (c *Client) Lookup(path, vers string) (lines []string, err error) { - atomic.StoreUint32(&c.didLookup, 1) - defer func() { if err != nil { err = fmt.Errorf("%s@%s: %v", path, vers, err) @@ -199,7 +180,6 @@ func (c *Client) Lookup(path, vers string) (lines []string, err error) { err error } result := c.record.Do(file, func() interface{} { - // Try the on-disk cache, or else get from web. data, err := c.ops.ReadRemote(remotePath) if err != nil { return cached{nil, err} @@ -235,6 +215,37 @@ func (c *Client) Lookup(path, vers string) (lines []string, err error) { return hashes, nil } +func (c *Client) FetchLatest() (*tlog.Tree, error) { + if err := c.init(); err != nil { + return nil, err + } + + msg, err := c.ops.ReadRemote("/latest") + if err != nil { + return nil, err + } + note, err := note.Open(msg, c.verifiers) + if err != nil { + return nil, fmt.Errorf("reading latest note: %v\nnote:\n%s", err, msg) + } + tree, err := tlog.ParseTree([]byte(note.Text)) + if err != nil { + return nil, fmt.Errorf("reading tree: %v\ntree:\n%s", err, note.Text) + } + return &tree, nil +} + +func (c *Client) FetchTreeProof(t *tlog.Tree) error { + log.Printf("fetching proof for tree %v", t) + + if err := c.checkTrees(tlog.Tree{}, nil, *t, nil); err != nil { + log.Printf("check tree error: %v", err) + return err + } + + return nil +} + // mergeLatest merges the tree head in msg // with the Client's current latest tree head, // ensuring the result is a consistent timeline. @@ -245,14 +256,17 @@ func (c *Client) Lookup(path, vers string) (lines []string, err error) { // mergeLatest updates the underlying configuration file as well, // taking care to merge any independent updates to that configuration. func (c *Client) mergeLatest(msg []byte) error { + log.Printf("merge latest: start") // Merge msg into our in-memory copy of the latest tree head. when, err := c.mergeLatestMem(msg) if err != nil { return err } + log.Printf("merge latest: when=%d", when) if when != msgFuture { // msg matched our present or was in the past. // No change to our present, so no update of config file. + log.Printf("merge latest: future; done") return nil } @@ -261,6 +275,7 @@ func (c *Client) mergeLatest(msg []byte) error { // we need to merge any updates made there as well. // Note that writeConfig is an atomic compare-and-swap. for { + log.Printf("merge latest: reading latest...") msg, err := c.ops.ReadConfig(c.name + "/latest") if err != nil { return err @@ -301,6 +316,7 @@ const ( // msgNow means msg was exactly c.latest, and // msgFuture means msg was from after c.latest, which has now been updated. func (c *Client) mergeLatestMem(msg []byte) (when int, err error) { + log.Printf("mergeLatestMem: start with msg=%v", msg) if len(msg) == 0 { // Accept empty msg as the unsigned, empty timeline. c.latestMu.Lock() @@ -312,6 +328,7 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) { return msgPast, nil } + log.Printf("mergeMemLatest: reading tree note: %s", msg) note, err := note.Open(msg, c.verifiers) if err != nil { return 0, fmt.Errorf("reading tree note: %v\nnote:\n%s", err, msg) @@ -332,6 +349,7 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) { for { // If the tree head looks old, check that it is on our timeline. if tree.N <= latest.N { + log.Printf("mergeLatestMem: looks old") if err := c.checkTrees(tree, msg, latest, latestMsg); err != nil { return 0, err } @@ -343,9 +361,12 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) { // The tree head looks new. Check that we are on its timeline and try to move our timeline forward. if err := c.checkTrees(latest, latestMsg, tree, msg); err != nil { + log.Printf("mergeLatestMem: looks new") return 0, err } + log.Printf("mergeLatestMem: install msg if possible") + // Install our msg if possible. // Otherwise we will go around again. c.latestMu.Lock() @@ -371,6 +392,7 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) { // If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed // message and calls log.Fatal. func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error { + log.Printf("checking trees: older=%v, newer=%v", older, newer) thr := tlog.TileHashReader(newer, &c.tileReader) h, err := tlog.TreeHash(older.N, thr) if err != nil { @@ -379,9 +401,12 @@ func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, } return fmt.Errorf("checking tree#%d against tree#%d: %v", older.N, newer.N, err) } + log.Printf("computed tree hash %v", h) if h == older.Hash { + log.Printf("hashes match: %v", older.Hash) return nil } + log.Printf("HASHES DO NOT MATCH h=%v older=%v", h, older.Hash) // Detected a fork in the tree timeline. // Start by reporting the inconsistent signed tree notes. @@ -420,6 +445,7 @@ func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, // checkRecord checks that record #id's hash matches data. func (c *Client) checkRecord(id int64, data []byte) error { + log.Printf("checking record %d with %s", id, data) c.latestMu.Lock() latest := c.latest c.latestMu.Unlock() @@ -431,7 +457,9 @@ func (c *Client) checkRecord(id int64, data []byte) error { if err != nil { return err } + log.Printf("checkRecord: got hashes %v", hashes) if hashes[0] == tlog.RecordHash(data) { + log.Printf("tile hash %s matches record hash %s", hashes[0], data) return nil } return fmt.Errorf("cannot authenticate record data in server response") @@ -445,12 +473,14 @@ type tileReader struct { } func (r *tileReader) Height() int { + log.Printf("checking height") return r.c.tileHeight } // ReadTiles reads and returns the requested tiles, // either from the on-disk cache or the server. func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) { + log.Printf("reading %d tiles", len(tiles)) // Read all the tiles in parallel. data := make([][]byte, len(tiles)) errs := make([]error, len(tiles)) @@ -459,6 +489,7 @@ func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) { wg.Add(1) go func(i int, tile tlog.Tile) { defer wg.Done() + log.Printf("reading tile %v", tile) data[i], errs[i] = r.c.readTile(tile) }(i, tile) } @@ -473,11 +504,6 @@ func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) { return data, nil } -// tileCacheKey returns the cache key for the tile. -func (c *Client) tileCacheKey(tile tlog.Tile) string { - return c.name + "/" + tile.Path() -} - // tileRemotePath returns the remote path for the tile. func (c *Client) tileRemotePath(tile tlog.Tile) string { return "/" + tile.Path() -- cgit v1.2.3