Implement monitor package

5 files changed, 280 insertions, 90 deletions

diff --git a/main.go b/main.go
index f1d200b..f60372f 100644
--- a/main.go
+++ b/main.go
@@ -2,79 +2,63 @@ package main
 
 import (
 	"log"
-	"sync"
 	"time"
 
-	"git.sr.ht/~benburwell/gosumdbaudit/sumdb"
+	"git.sr.ht/~benburwell/gosumdbaudit/monitor"
+	// "git.sr.ht/~benburwell/gosumdbaudit/sumdb"
 )
 
 func main() {
-	dbs := []*database{
-		&database{
-			host: "sum.golang.org",
+	mon := monitor.NewMonitor(&monitor.Database{
+		URL: "sum.golang.org",
+		Key: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8",
+	})
 
-			key: "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8",
-			// key: "sum.golang.org+033de0ae+BADBADBADBADBADBADBADBADBADBADBADBADBADBADBA",
-
-			pollInterval: 10 * time.Second,
-		},
-	}
-	var wg sync.WaitGroup
-	wg.Add(len(dbs))
-	for _, db := range dbs {
-		go func(db *database) {
-			defer wg.Done()
-			if err := monitor(db); err != nil {
-				log.Printf("AUDIT FAILED: %s", err.Error())
-				return
-			}
-		}(db)
+	if err := mon.Watch(10 * time.Second); err != nil {
+		log.Printf("AUDIT FAILED: %v", err)
 	}
-	wg.Wait()
 }
 
-func monitor(db *database) error {
-	log.Printf("starting monitor for %s", db.host)
+// func monitor(db *database) error {
+// 	log.Printf("starting monitor for %s", db.host)
 
-	client := sumdb.NewClient(db)
+// 	client := sumdb.NewClient(db)
 
-	lines, err := client.Lookup("golang.org/x/text", "v0.3.0")
-	if err != nil {
-		return err
-	}
-	log.Printf("got lines: %s", lines)
+// 	tree, err := client.FetchLatest()
+// 	if err != nil {
+// 		return err
+// 	}
+// 	log.Printf("got latest: N=%d, Hash=%s", tree.N, tree.Hash)
 
-	// fetch & verify current STH
-	// latest, err := client.Latest()
-	// if err != nil {
-	// 	return err
-	// }
+// 	if err := client.FetchTreeProof(tree); err != nil {
+// 		return err
+// 	}
 
-	// fetch all entries in the tree according to the STH
-	// entries := client.Entries(nil, latest)
+// 	// fetch all entries in the tree according to the STH
+// 	// entries := client.Entries(nil, latest)
 
-	// confirm the tree made from the entries produces the same hash as the STH
-	// IF NOT: the server has signed invalid data
+// 	// confirm the tree made from the entries produces the same hash as the STH
+// 	// IF NOT: the server has signed invalid data
 
-	// prev := latest
-	for {
-		// await a new STH
-		// prev = latest
-		time.Sleep(db.pollInterval)
-		log.Printf("checking %s for new STH...", db.host)
-		// awaitNewSTH()
+// 	// prev := latest
+// 	for {
+// 		// await a new STH
+// 		// prev = latest
+// 		time.Sleep(db.pollInterval)
+// 		log.Printf("checking %s for new STH...", db.host)
+// 		// awaitNewSTH()
 
-		// latest, err := client.Latest()
-		// if err != nil {
-		// 	return err
-		// }
+// 		// latest, err := client.Latest()
+// 		// if err != nil {
+// 		// 	return err
+// 		// }
 
-		// fetch all NEW entries between prev and latest
-		// if unavailable for an extended period, this should be viewed as misbehavior
-		// entries := client.Entries(prev, latest)
+// 		// fetch all NEW entries between prev and latest
+// 		// if unavailable for an extended period, this should be viewed as misbehavior
+// 		// entries := client.Entries(prev, latest)
 
-		// fetch a consistency proof for the new STH with the previous STH
-		// verify consistency proof
-		// verify the new entries generate the corresponding elements in the consistency proof
-	}
-}
+// 		// fetch a consistency proof for the new STH with the previous STH
+// 		// verify consistency proof
+// 		// verify the new entries generate the corresponding elements in the consistency proof
+// 	}
+// }
diff --git a/monitor/database.go b/monitor/database.go
new file mode 100644
index 0000000..7a0ead1
--- /dev/null
+++ b/monitor/database.go
@@ -0,0 +1,56 @@
+package monitor
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"golang.org/x/mod/sumdb/note"
+	"golang.org/x/mod/sumdb/tlog"
+)
+
+type Database struct {
+	URL string
+	Key string
+}
+
+func (db *Database) readRemote(path string) ([]byte, error) {
+	log.Printf("GET %s", path)
+	resp, err := http.Get("https://" + db.URL + path)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("get %s: expected HTTP OK but got %s", path, resp.Status)
+	}
+	if resp.ContentLength > 1e6 {
+		return nil, fmt.Errorf("get %s: body too large (%d bytes)", path, resp.ContentLength)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("get %s: %v", err)
+	}
+	return body, nil
+}
+
+func (db *Database) VerifyNote(msg []byte) (*note.Note, error) {
+	verifier, err := note.NewVerifier(db.Key)
+	if err != nil {
+		return nil, err
+	}
+	return note.Open(msg, note.VerifierList(verifier))
+}
+
+func (db *Database) FetchSTH() (tlog.Tree, error) {
+	resp, err := db.readRemote("/latest")
+	if err != nil {
+		return tlog.Tree{}, err
+	}
+	note, err := db.VerifyNote(resp)
+	if err != nil {
+		return tlog.Tree{}, err
+	}
+	return tlog.ParseTree([]byte(note.Text))
+}
diff --git a/monitor/monitor.go b/monitor/monitor.go
new file mode 100644
index 0000000..e291a63
--- /dev/null
+++ b/monitor/monitor.go
@@ -0,0 +1,117 @@
+package monitor
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"golang.org/x/mod/sumdb/tlog"
+)
+
+type Monitor struct {
+	db *Database
+
+	// the latest verified tree
+	latest tlog.Tree
+}
+
+func NewMonitor(db *Database) *Monitor {
+	return &Monitor{db: db}
+}
+
+// Watch polls the database to check for a new Signed Tree Head every interval.
+// It runs indefinitely; only returning an error if an inconsistency is found.
+func (m *Monitor) Watch(interval time.Duration) error {
+	for {
+		log.Printf("checking for new STH")
+		t, err := m.db.FetchSTH()
+		if err != nil {
+			return err
+		}
+		log.Printf("got STH: %#v", t)
+
+		if m.latest.N == t.N {
+			log.Printf("current STH matches latest proved")
+		} else {
+			if err := m.TreeProof(m.latest, t); err != nil {
+				return err
+			}
+			log.Printf("proof succeeded")
+			m.latest = t
+		}
+
+		time.Sleep(interval)
+	}
+	return nil
+}
+
+// TreeProof proves that tree older is a prefix of tree newer, returning an
+// error if the proof fails.
+func (m *Monitor) TreeProof(older, newer tlog.Tree) error {
+	log.Printf("proving that N=%d is a prefix of N'=%d", older.N, newer.N)
+	if older.N == 0 {
+		log.Printf("N=0, so using bootstrap proof")
+		return m.BootstrapProof(newer)
+	}
+	thr := tlog.TileHashReader(newer, m)
+	p, err := tlog.ProveTree(newer.N, older.N, thr)
+	if err != nil {
+		return err
+	}
+	log.Printf("Proof: %#v", p)
+	return nil
+}
+
+func (m *Monitor) BootstrapProof(t tlog.Tree) error {
+	thr := tlog.TileHashReader(t, m)
+	h, err := tlog.TreeHash(t.N, thr)
+	if err != nil {
+		return err
+	}
+	log.Printf("computed hash: %v", h)
+	log.Printf("expected hash: %v", t.Hash)
+	if h != t.Hash {
+		return fmt.Errorf("computed hash %v does not match expected: %v", h, t.Hash)
+	}
+	return nil
+}
+
+func (m *Monitor) Height() int {
+	return 8
+}
+
+// ReadTiles reads and returns the requested tiles,
+// either from the on-disk cache or the server.
+func (m *Monitor) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
+	log.Printf("reading %d tiles", len(tiles))
+	// Read all the tiles in parallel.
+	data := make([][]byte, len(tiles))
+	errs := make([]error, len(tiles))
+	var wg sync.WaitGroup
+	for i, tile := range tiles {
+		wg.Add(1)
+		go func(i int, tile tlog.Tile) {
+			defer wg.Done()
+			log.Printf("reading tile %v", tile)
+			data[i], errs[i] = m.readTile(tile)
+		}(i, tile)
+	}
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return data, nil
+}
+
+func (m *Monitor) readTile(tile tlog.Tile) ([]byte, error) {
+	return m.db.readRemote("/" + tile.Path())
+}
+
+func (m *Monitor) SaveTiles(tiles []tlog.Tile, data [][]byte) {
+	// noop
+}
diff --git a/monitor/security_error.go b/monitor/security_error.go
new file mode 100644
index 0000000..aa036eb
--- /dev/null
+++ b/monitor/security_error.go
@@ -0,0 +1,7 @@
+package monitor
+
+type SecurityError string
+
+func (e SecurityError) Error() string {
+	return string(e)
+}
diff --git a/sumdb/client.go b/sumdb/client.go
index 8943eb3..136f611 100644
--- a/sumdb/client.go
+++ b/sumdb/client.go
@@ -8,9 +8,9 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"log"
 	"strings"
 	"sync"
-	"sync/atomic"
 
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/sumdb/note"
@@ -69,8 +69,6 @@ var ErrSecurity = errors.New("security error: misbehaving server")
 type Client struct {
 	ops ClientOps // access to operations in the external world
 
-	didLookup uint32
-
 	// one-time initialized data
 	initOnce   sync.Once
 	initErr    error          // init error, if any
@@ -107,16 +105,16 @@ func (c *Client) init() error {
 
 // initWork does the actual initialization work.
 func (c *Client) initWork() {
+	log.Printf("initializing sumdb client")
 	defer func() {
+		log.Printf("done init")
 		if c.initErr != nil {
 			c.initErr = fmt.Errorf("initializing sumdb.Client: %v", c.initErr)
 		}
 	}()
 
 	c.tileReader.c = c
-	if c.tileHeight == 0 {
-		c.tileHeight = 8
-	}
+	c.tileHeight = 8
 	c.tileSaved = make(map[tlog.Tile]bool)
 
 	vkey, err := c.ops.ReadConfig("key")
@@ -132,41 +130,24 @@ func (c *Client) initWork() {
 	c.verifiers = note.VerifierList(verifier)
 	c.name = verifier.Name()
 
+	log.Printf("init: reading latest")
 	data, err := c.ops.ReadConfig(c.name + "/latest")
 	if err != nil {
 		c.initErr = err
 		return
 	}
+
+	log.Printf("init: merging latest")
 	if err := c.mergeLatest(data); err != nil {
 		c.initErr = err
 		return
 	}
 }
 
-// SetTileHeight sets the tile height for the Client.
-// Any call to SetTileHeight must happen before the first call to Lookup.
-// If SetTileHeight is not called, the Client defaults to tile height 8.
-// SetTileHeight can be called at most once,
-// and if so it must be called before the first call to Lookup.
-func (c *Client) SetTileHeight(height int) {
-	if atomic.LoadUint32(&c.didLookup) != 0 {
-		panic("SetTileHeight used after Lookup")
-	}
-	if height <= 0 {
-		panic("invalid call to SetTileHeight")
-	}
-	if c.tileHeight != 0 {
-		panic("multiple calls to SetTileHeight")
-	}
-	c.tileHeight = height
-}
-
 // Lookup returns the go.sum lines for the given module path and version.
 // The version may end in a /go.mod suffix, in which case Lookup returns
 // the go.sum lines for the module's go.mod-only hash.
 func (c *Client) Lookup(path, vers string) (lines []string, err error) {
-	atomic.StoreUint32(&c.didLookup, 1)
-
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("%s@%s: %v", path, vers, err)
@@ -199,7 +180,6 @@ func (c *Client) Lookup(path, vers string) (lines []string, err error) {
 		err  error
 	}
 	result := c.record.Do(file, func() interface{} {
-		// Try the on-disk cache, or else get from web.
 		data, err := c.ops.ReadRemote(remotePath)
 		if err != nil {
 			return cached{nil, err}
@@ -235,6 +215,37 @@ func (c *Client) Lookup(path, vers string) (lines []string, err error) {
 	return hashes, nil
 }
 
+func (c *Client) FetchLatest() (*tlog.Tree, error) {
+	if err := c.init(); err != nil {
+		return nil, err
+	}
+
+	msg, err := c.ops.ReadRemote("/latest")
+	if err != nil {
+		return nil, err
+	}
+	note, err := note.Open(msg, c.verifiers)
+	if err != nil {
+		return nil, fmt.Errorf("reading latest note: %v\nnote:\n%s", err, msg)
+	}
+	tree, err := tlog.ParseTree([]byte(note.Text))
+	if err != nil {
+		return nil, fmt.Errorf("reading tree: %v\ntree:\n%s", err, note.Text)
+	}
+	return &tree, nil
+}
+
+func (c *Client) FetchTreeProof(t *tlog.Tree) error {
+	log.Printf("fetching proof for tree %v", t)
+
+	if err := c.checkTrees(tlog.Tree{}, nil, *t, nil); err != nil {
+		log.Printf("check tree error: %v", err)
+		return err
+	}
+
+	return nil
+}
+
 // mergeLatest merges the tree head in msg
 // with the Client's current latest tree head,
 // ensuring the result is a consistent timeline.
@@ -245,14 +256,17 @@ func (c *Client) Lookup(path, vers string) (lines []string, err error) {
 // mergeLatest updates the underlying configuration file as well,
 // taking care to merge any independent updates to that configuration.
 func (c *Client) mergeLatest(msg []byte) error {
+	log.Printf("merge latest: start")
 	// Merge msg into our in-memory copy of the latest tree head.
 	when, err := c.mergeLatestMem(msg)
 	if err != nil {
 		return err
 	}
+	log.Printf("merge latest: when=%d", when)
 	if when != msgFuture {
 		// msg matched our present or was in the past.
 		// No change to our present, so no update of config file.
+		log.Printf("merge latest: future; done")
 		return nil
 	}
 
@@ -261,6 +275,7 @@ func (c *Client) mergeLatest(msg []byte) error {
 	// we need to merge any updates made there as well.
 	// Note that writeConfig is an atomic compare-and-swap.
 	for {
+		log.Printf("merge latest: reading latest...")
 		msg, err := c.ops.ReadConfig(c.name + "/latest")
 		if err != nil {
 			return err
@@ -301,6 +316,7 @@ const (
 // msgNow means msg was exactly c.latest, and
 // msgFuture means msg was from after c.latest, which has now been updated.
 func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
+	log.Printf("mergeLatestMem: start with msg=%v", msg)
 	if len(msg) == 0 {
 		// Accept empty msg as the unsigned, empty timeline.
 		c.latestMu.Lock()
@@ -312,6 +328,7 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
 		return msgPast, nil
 	}
 
+	log.Printf("mergeMemLatest: reading tree note: %s", msg)
 	note, err := note.Open(msg, c.verifiers)
 	if err != nil {
 		return 0, fmt.Errorf("reading tree note: %v\nnote:\n%s", err, msg)
@@ -332,6 +349,7 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
 	for {
 		// If the tree head looks old, check that it is on our timeline.
 		if tree.N <= latest.N {
+			log.Printf("mergeLatestMem: looks old")
 			if err := c.checkTrees(tree, msg, latest, latestMsg); err != nil {
 				return 0, err
 			}
@@ -343,9 +361,12 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
 
 		// The tree head looks new. Check that we are on its timeline and try to move our timeline forward.
 		if err := c.checkTrees(latest, latestMsg, tree, msg); err != nil {
+			log.Printf("mergeLatestMem: looks new")
 			return 0, err
 		}
 
+		log.Printf("mergeLatestMem: install msg if possible")
+
 		// Install our msg if possible.
 		// Otherwise we will go around again.
 		c.latestMu.Lock()
@@ -371,6 +392,7 @@ func (c *Client) mergeLatestMem(msg []byte) (when int, err error) {
 // If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed
 // message and calls log.Fatal.
 func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error {
+	log.Printf("checking trees: older=%v, newer=%v", older, newer)
 	thr := tlog.TileHashReader(newer, &c.tileReader)
 	h, err := tlog.TreeHash(older.N, thr)
 	if err != nil {
@@ -379,9 +401,12 @@ func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree,
 		}
 		return fmt.Errorf("checking tree#%d against tree#%d: %v", older.N, newer.N, err)
 	}
+	log.Printf("computed tree hash %v", h)
 	if h == older.Hash {
+		log.Printf("hashes match: %v", older.Hash)
 		return nil
 	}
+	log.Printf("HASHES DO NOT MATCH h=%v older=%v", h, older.Hash)
 
 	// Detected a fork in the tree timeline.
 	// Start by reporting the inconsistent signed tree notes.
@@ -420,6 +445,7 @@ func (c *Client) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree,
 
 // checkRecord checks that record #id's hash matches data.
 func (c *Client) checkRecord(id int64, data []byte) error {
+	log.Printf("checking record %d with %s", id, data)
 	c.latestMu.Lock()
 	latest := c.latest
 	c.latestMu.Unlock()
@@ -431,7 +457,9 @@ func (c *Client) checkRecord(id int64, data []byte) error {
 	if err != nil {
 		return err
 	}
+	log.Printf("checkRecord: got hashes %v", hashes)
 	if hashes[0] == tlog.RecordHash(data) {
+		log.Printf("tile hash %s matches record hash %s", hashes[0], data)
 		return nil
 	}
 	return fmt.Errorf("cannot authenticate record data in server response")
@@ -445,12 +473,14 @@ type tileReader struct {
 }
 
 func (r *tileReader) Height() int {
+	log.Printf("checking height")
 	return r.c.tileHeight
 }
 
 // ReadTiles reads and returns the requested tiles,
 // either from the on-disk cache or the server.
 func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
+	log.Printf("reading %d tiles", len(tiles))
 	// Read all the tiles in parallel.
 	data := make([][]byte, len(tiles))
 	errs := make([]error, len(tiles))
@@ -459,6 +489,7 @@ func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
 		wg.Add(1)
 		go func(i int, tile tlog.Tile) {
 			defer wg.Done()
+			log.Printf("reading tile %v", tile)
 			data[i], errs[i] = r.c.readTile(tile)
 		}(i, tile)
 	}
@@ -473,11 +504,6 @@ func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
 	return data, nil
 }
 
-// tileCacheKey returns the cache key for the tile.
-func (c *Client) tileCacheKey(tile tlog.Tile) string {
-	return c.name + "/" + tile.Path()
-}
-
 // tileRemotePath returns the remote path for the tile.
 func (c *Client) tileRemotePath(tile tlog.Tile) string {
 	return "/" + tile.Path()