ShutterTEE MS1+2: block verificaiton

rolling-shutter/.gitignore

@@ -9,3 +9,7 @@
 /godoc.idx
 /go.work
 /go.work.sum
+
+enclave.json
+private.pem
+public.pem

rolling-shutter/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/bitwurx/jrpc2 v0.0.0-20220302204700-52c6dbbeb536
 	github.com/deckarep/golang-set/v2 v2.6.0
 	github.com/deepmap/oapi-codegen v1.9.1
+	github.com/edgelesssys/ego v1.7.0
 	github.com/ethereum/go-ethereum v1.15.11
 	github.com/ferranbt/fastssz v0.1.3
 	github.com/getkin/kin-openapi v0.87.0
@@ -93,6 +94,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
 	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect

rolling-shutter/go.sum

@@ -195,6 +195,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/edgelesssys/ego v1.7.0 h1:NzCiKZKalHbeRgG7+11xgADgOPR8laSxxazOH303QVw=
+github.com/edgelesssys/ego v1.7.0/go.mod h1:xBU369lGvxWTuMLlJv3tT6bI66hrpkCFn/292zzl7U4=
 github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
 github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
@@ -250,6 +252,8 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
+github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
 github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4=
 github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=

rolling-shutter/keyper/database/extend.go

@@ -12,6 +12,7 @@ import (
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/p2pmsg"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/shmsg"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/tee"
 )
 
 func GetKeyperIndex(addr common.Address, keypers []string) (uint64, bool) {
@@ -31,10 +32,16 @@ func (bc *TendermintBatchConfig) KeyperIndex(addr common.Address) (uint64, bool)
 func (q *Queries) InsertDecryptionKeysMsg(ctx context.Context, msg *p2pmsg.DecryptionKeys) error {
 	for _, key := range msg.Keys {
 		identityPreimage := identitypreimage.IdentityPreimage(key.IdentityPreimage)
+
+		sealedKey, err := tee.SealSecret(key.Key)
+		if err != nil {
+			return err
+		}
+
 		tag, err := q.InsertDecryptionKey(ctx, InsertDecryptionKeyParams{
 			Eon:           int64(msg.Eon),
 			EpochID:       identityPreimage.Bytes(),
-			DecryptionKey: key.Key,
+			DecryptionKey: sealedKey,
 		})
 		if err != nil {
 			return errors.Wrapf(err, "failed to insert decryption key for identity %s", identityPreimage)
@@ -49,11 +56,16 @@ func (q *Queries) InsertDecryptionKeysMsg(ctx context.Context, msg *p2pmsg.Decry
 
 func (q *Queries) InsertDecryptionKeySharesMsg(ctx context.Context, msg *p2pmsg.DecryptionKeyShares) error {
 	for _, share := range msg.GetShares() {
-		err := q.InsertDecryptionKeyShare(ctx, InsertDecryptionKeyShareParams{
+		sealedShare, err := tee.SealSecret(share.Share)
+		if err != nil {
+			return err
+		}
+
+		err = q.InsertDecryptionKeyShare(ctx, InsertDecryptionKeyShareParams{
 			Eon:                int64(msg.Eon),
 			EpochID:            share.IdentityPreimage,
 			KeyperIndex:        int64(msg.KeyperIndex),
-			DecryptionKeyShare: share.Share,
+			DecryptionKeyShare: sealedShare,
 		})
 		if err != nil {
 			return errors.Wrapf(

rolling-shutter/keyper/smobserver/smstate.go

@@ -28,6 +28,7 @@ import (
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/shmsg"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/tee"
 )
 
 type Config interface {
@@ -216,7 +217,13 @@ func (st *ShuttermintState) sendPolyEvals(ctx context.Context, queries *database
 		if err != nil {
 			return fmt.Errorf("failed to decode encryption public key for %s from db: %w", eval.ReceiverAddress, err)
 		}
-		encrypted, err := ecies.Encrypt(rand.Reader, pubkey, eval.Eval, nil, nil)
+
+		plainEval, err := tee.UnsealSecret(eval.Eval)
+		if err != nil {
+			return err
+		}
+
+		encrypted, err := ecies.Encrypt(rand.Reader, pubkey, plainEval, nil, nil)
 		if err != nil {
 			return err
 		}
@@ -390,10 +397,15 @@ func (st *ShuttermintState) startPhase1Dealing(
 	).Inc()
 
 	for _, eval := range polyEvals {
+		sealedEval, err := tee.SealSecret(shdb.EncodeBigint(eval.Eval))
+		if err != nil {
+			return err
+		}
+
 		err = queries.InsertPolyEval(ctx, database.InsertPolyEvalParams{
 			Eon:             int64(eon),
 			ReceiverAddress: shdb.EncodeAddress(dkg.keypers[eval.Receiver]),
-			Eval:            shdb.EncodeBigint(eval.Eval),
+			Eval:            sealedEval,
 		})
 		if err != nil {
 			return err

rolling-shutter/medley/chainsync/syncer/unsafehead.go

@@ -3,6 +3,7 @@ package syncer
 import (
 	"context"
 	"errors"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -11,37 +12,138 @@ import (
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable/number"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/tee"
 )
 
 type UnsafeHeadSyncer struct {
+	//  Not used when eVerify is true
 	Client  client.Client
 	Log     log.Logger
 	Handler event.BlockHandler
 
+	// Not used when eVerify is true.
 	newLatestHeadCh chan *types.Header
 }
 
 func (s *UnsafeHeadSyncer) Start(ctx context.Context, runner service.Runner) error {
+	// Extended/External/Enclave verification
+	// This can be true when not running in SGX, too.
+	// We currently assume the runner (separate process that contains the enclave) is already running.
+	// It might be possible to start it from within the enclave, but I am not sure about that.
+	eVerify := true
+
 	if s.Handler == nil {
 		return errors.New("no handler registered")
 	}
-	s.newLatestHeadCh = make(chan *types.Header, 1)
 
-	subs, err := s.Client.SubscribeNewHead(ctx, s.newLatestHeadCh)
-	if err != nil {
-		return err
-	}
-	runner.Go(func() error {
-		err := s.watchLatestUnsafeHead(ctx, subs.Err())
+	// Difference between SubscribeNewHead and the ethereum-enclave:
+	// - SubscribeNewHead returns all headers (no gaps, potentially with duplicates)
+	// - On a reorg: SubscribeNeHead returns all headers (starting at the reorg)
+	// - ethereum-enclave (finalized) returns all headers if AFTER the start blocknum (no gaps)
+	// - ethereum-enclave (finalized) has gaps BEFORE the start blocknum
+	// - ethereum-enclave (finalized) will panic if there is a reorg (major consensus failure and slashing)
+	// - ethereum-enclave (optimistic) has gaps
+	if eVerify {
+		// We don't want to trust the geth client. This provides a channel containing blocks.
+		// The tee.Config allows configuring which external verifiers should be considered valid.
+		// It is only relevant when we're running in SGX, too. See comments.
+		// conn.Attestation() contains additional configuration like the hash the etheruem-enclave started at (genesis).
+		// It should be checked or forwarded in a remote attestation to make sure we're not following the wrong chain.
+		conn, err := tee.DialVerifiedChainDataChannel("127.0.0.1:8001", tee.Config{
+			// // Configuration options for SGX verification:
+			// SameSigner: false,		// Require that the etheruem-enclave is signed with the same key as we are
+			// SignerID:   []byte{},	// Alternative: Specify the SignerID
+			// MrEnclave:  []byte{},	// Hash of the code the ethereum-enclave is running
+			// ProductID:  new(uint16),	// Number by the signer to identify the ethereum-enclave to distinguish different products/binaries
+			// MinISVSVN:  0,			// Minimum (security) version number specified when signing the ethereum enclave
+
+			// Currently we do not use events verified by the ethereum-enclave, so we can just tell it to not extract them.
+			EventExtractionStartBlocknum: ^uint64(0),
+			Contracts:                    nil, // For now: We are not interested in events
+		})
+		if err != nil {
+			return err
+		}
+		runner.Go(func() error {
+			err := s.watchLatestUnsafeHeadEVerify(ctx, conn)
+			if err != nil {
+				s.Log.Error("error watching latest unsafe head with e-verify", err.Error())
+			}
+			conn.Close()
+			return err
+		})
+	} else {
+		s.newLatestHeadCh = make(chan *types.Header, 1)
+		subs, err := s.Client.SubscribeNewHead(ctx, s.newLatestHeadCh)
 		if err != nil {
-			s.Log.Error("error watching latest unsafe head", err.Error())
+			return err
 		}
-		subs.Unsubscribe()
-		return err
-	})
+		runner.Go(func() error {
+			err := s.watchLatestUnsafeHead(ctx, subs.Err())
+			if err != nil {
+				s.Log.Error("error watching latest unsafe head", err.Error())
+			}
+			subs.Unsubscribe()
+			return err
+		})
+	}
+
 	return nil
 }
 
+func (s *UnsafeHeadSyncer) watchLatestUnsafeHeadEVerify(ctx context.Context, conn *tee.Connection) error {
+	for {
+		select {
+		case verified, ok := <-conn.Headers():
+			// We have three options:
+			// - Fully trust the go-ethereum node (problematic/insecure)
+			// - Use the Finalized header (about 21 minutes delay but no reorgs unless there is a major chain failure)
+			// - Use the optimistic header
+			//
+			// I do not know for sure which one is best suited:
+			// - The existing code accounts for reorgs, so the optimistic header should be fine.
+			// - But on the other hand it might result in leaking/exposing/outputting key material too early (which cannot be undone).
+			header := verified.Optimistic
+
+			if !ok {
+				return nil
+			}
+			num := big.NewInt(0)
+			num.SetUint64(header.Number)
+
+			// The ethereum enclave (external verifier) primarily uses beacon-chain data. Thus it
+			// has the execution layer header data in a different format and not exactly the same data.
+			// However, shutter does not need the majority of that data.
+			// The fields below can be extended by other data from ExecutionPayloadHeader if needed.
+			// I could not find any place in this repo that uses the other fields.
+			ev := &event.LatestBlock{
+				Number:    number.BigToBlockNumber(num),
+				BlockHash: header.BlockHash,
+				Header: &types.Header{
+					ParentHash: header.ParentHash,
+					Number:     num,
+					Time:       header.Timestamp,
+				},
+			}
+			err := s.Handler(ctx, ev)
+			if err != nil {
+				s.Log.Error(
+					"handler for `NewLatestBlock` errored",
+					"error",
+					err.Error(),
+				)
+			}
+		case err := <-conn.Errors():
+			if err != nil {
+				s.Log.Error("subscription error for watchLatestUnsafeHead", err.Error())
+				return err
+			}
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+}
+
 func (s *UnsafeHeadSyncer) watchLatestUnsafeHead(ctx context.Context, subsErr <-chan error) error {
 	for {
 		select {

rolling-shutter/medley/encodeable/keys/ecdsa.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable/hex"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/tee"
 )
 
 func GenerateECDSAKey(src io.Reader) (*ECDSAPrivate, error) {
@@ -60,7 +61,7 @@ func (k *ECDSAPrivate) Equal(b *ECDSAPrivate) bool {
 }
 
 func (k *ECDSAPrivate) UnmarshalText(b []byte) error {
-	dec, err := hex.DecodeHex(b)
+	dec, err := tee.UnsealSecretFromHex(string(b))
 	if err != nil {
 		return err
 	}
@@ -73,7 +74,8 @@ func (k *ECDSAPrivate) UnmarshalText(b []byte) error {
 }
 
 func (k *ECDSAPrivate) MarshalText() ([]byte, error) {
-	return hex.EncodeHex(k.Bytes()), nil
+	str, err := tee.SealSecretAsHex(k.Bytes())
+	return []byte(str), err
 }
 
 func (k *ECDSAPrivate) String() string {

rolling-shutter/medley/encodeable/keys/ed25519.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable/hex"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/tee"
 )
 
 type nullReader struct{}
@@ -69,7 +70,7 @@ func (k *Ed25519Private) Equal(b *Ed25519Private) bool {
 }
 
 func (k *Ed25519Private) UnmarshalText(b []byte) error {
-	seed, err := hex.DecodeHex(b)
+	seed, err := tee.UnsealSecretFromHex(string(b))
 	if err != nil {
 		return err
 	}
@@ -85,7 +86,8 @@ func (k *Ed25519Private) UnmarshalText(b []byte) error {
 }
 
 func (k *Ed25519Private) MarshalText() ([]byte, error) {
-	return hex.EncodeHex(k.Key.Seed()), nil
+	str, err := tee.SealSecretAsHex(k.Key.Seed())
+	return []byte(str), err
 }
 
 func (k *Ed25519Private) String() string {

rolling-shutter/medley/encodeable/keys/libp2p.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/encodeable/address"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/tee"
 )
 
 func GenerateLibp2pPrivate(src io.Reader) (*Libp2pPrivate, error) {
@@ -103,7 +104,7 @@ func (k *Libp2pPrivate) Equal(b *Libp2pPrivate) bool {
 }
 
 func (k *Libp2pPrivate) UnmarshalText(b []byte) error {
-	dec, err := crypto.ConfigDecodeKey(string(b))
+	dec, err := tee.UnsealSecretFromCustomText(string(b), crypto.ConfigDecodeKey)
 	if err != nil {
 		return errors.Wrap(errFailedUnmarshalLibp2pPrivate, err.Error())
 	}
@@ -127,8 +128,8 @@ func (k *Libp2pPrivate) MarshalText() ([]byte, error) {
 	if err != nil {
 		return []byte{}, errors.Wrap(errFailedMarshalLibp2pPrivate, err.Error())
 	}
-	enc := crypto.ConfigEncodeKey(b)
-	return []byte(enc), nil
+	str, err := tee.SealSecretAsCustomText(b, crypto.ConfigEncodeKey)
+	return []byte(str), err
 }
 
 func (k *Libp2pPrivate) String() string {