From ffd276bd3eb1ca1d56e0cb7e60064d711e301bf4 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Sat, 5 Nov 2022 15:23:29 +0100 Subject: [PATCH] Refactor how nebula certs are signed and propagated I had previously made the mistake of thinking that the Curve25519 key which is generated for each host to use in nebula communication could also be used for signing. This is not the case, Ed25519 is used for signing and is different thant Curve25519. Rather than figuring out how to convert the Curve25519 key into an Ed25519 key, which there is no apparent support for in the standard library, I opted to instead ship a separate key just for signing with each host. Doing this required a bit of refactoring in order to keep all the different keys straight and ensure all data which needs a signature still has it. --- entrypoint/src/bootstrap/bootstrap.go | 4 +- .../src/bootstrap/garage_global_bucket.go | 43 +++++-- entrypoint/src/bootstrap/hosts.go | 51 ++++++++- entrypoint/src/cmd/entrypoint/admin.go | 26 ++++- entrypoint/src/cmd/entrypoint/nebula_util.go | 6 +- entrypoint/src/garage/admin_client.go | 3 + entrypoint/src/nebula/nebula.go | 108 ++++++++++-------- entrypoint/src/nebula/nebula_test.go | 28 ++--- 8 files changed, 186 insertions(+), 83 deletions(-) diff --git a/entrypoint/src/bootstrap/bootstrap.go b/entrypoint/src/bootstrap/bootstrap.go index 29ea5a2..1b70261 100644 --- a/entrypoint/src/bootstrap/bootstrap.go +++ b/entrypoint/src/bootstrap/bootstrap.go @@ -37,7 +37,9 @@ type Bootstrap struct { HostName string `yaml:"hostname"` Nebula struct { - HostCredentials nebula.HostCredentials `yaml:"host_credentials"` + CAPublicCredentials nebula.CAPublicCredentials `yaml:"ca_public_credentials"` + HostCredentials nebula.HostCredentials `yaml:"host_credentials"` + SignedPublicCredentials string `yaml:"signed_public_credentials"` } `yaml:"nebula"` Garage struct { diff --git a/entrypoint/src/bootstrap/garage_global_bucket.go b/entrypoint/src/bootstrap/garage_global_bucket.go index 9052c32..d3069b0 100644 --- a/entrypoint/src/bootstrap/garage_global_bucket.go +++ b/entrypoint/src/bootstrap/garage_global_bucket.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/minio/minio-go/v7" "gopkg.in/yaml.v3" @@ -26,6 +27,12 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { host := b.ThisHost() client := b.GlobalBucketS3APIClient() + // the base Bootstrap has the public credentials signed by the CA, but we + // need this to be presented in the data stored into garage, so other hosts + // can verify that the stored host object is signed by the host public key, + // and that the host public key is signed by the CA. + host.Nebula.SignedPublicCredentials = b.Nebula.SignedPublicCredentials + hostB, err := yaml.Marshal(host) if err != nil { return fmt.Errorf("yaml encoding host data: %w", err) @@ -33,7 +40,7 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { buf := new(bytes.Buffer) - err = nebula.SignAndWrap(buf, b.Nebula.HostCredentials.HostKeyPEM, hostB) + err = nebula.SignAndWrap(buf, b.Nebula.HostCredentials.SigningPrivateKeyPEM, hostB) if err != nil { return fmt.Errorf("signing encoded host data: %w", err) } @@ -82,7 +89,6 @@ func (b Bootstrap) GetGarageBootstrapHosts( map[string]Host, error, ) { - caCertPEM := b.Nebula.HostCredentials.CACertPEM client := b.GlobalBucketS3APIClient() hosts := map[string]Host{} @@ -109,7 +115,7 @@ func (b Bootstrap) GetGarageBootstrapHosts( return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err) } - hostB, sig, err := nebula.Unwrap(obj) + hostB, hostSig, err := nebula.Unwrap(obj) obj.Close() if err != nil { @@ -121,15 +127,36 @@ func (b Bootstrap) GetGarageBootstrapHosts( return nil, fmt.Errorf("yaml decoding object %q: %w", objInfo.Key, err) } - hostCertPEM := host.Nebula.CertPEM + hostPublicCredsB, hostPublicCredsSig, err := nebula.Unwrap( + strings.NewReader(host.Nebula.SignedPublicCredentials), + ) - if err := nebula.ValidateSignature(hostCertPEM, hostB, sig); err != nil { - fmt.Fprintf(os.Stderr, "invalid host data for %q: %v\n", objInfo.Key, err) + if err != nil { + fmt.Fprintf(os.Stderr, "unwrapping signed public creds for %q: %v\n", objInfo.Key, err) continue } - if err := nebula.ValidateHostCertPEM(caCertPEM, hostCertPEM); err != nil { - fmt.Fprintf(os.Stderr, "invalid nebula cert for %q: %v\n", objInfo.Key, err) + err = nebula.ValidateSignature( + b.Nebula.CAPublicCredentials.SigningKeyPEM, + hostPublicCredsB, + hostPublicCredsSig, + ) + + if err != nil { + fmt.Fprintf(os.Stderr, "invalid signed public creds for %q: %v\n", objInfo.Key, err) + continue + } + + var hostPublicCreds nebula.HostPublicCredentials + if err := yaml.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { + fmt.Fprintf(os.Stderr, "yaml unmarshaling signed public creds for %q: %v\n", objInfo.Key, err) + continue + } + + err = nebula.ValidateSignature(hostPublicCreds.SigningKeyPEM, hostB, hostSig) + + if err != nil { + fmt.Fprintf(os.Stderr, "invalid host data for %q: %v\n", objInfo.Key, err) continue } diff --git a/entrypoint/src/bootstrap/hosts.go b/entrypoint/src/bootstrap/hosts.go index 2439eab..badf38d 100644 --- a/entrypoint/src/bootstrap/hosts.go +++ b/entrypoint/src/bootstrap/hosts.go @@ -1,16 +1,45 @@ package bootstrap import ( + "bytes" "cryptic-net/nebula" "fmt" "net" + "strings" + + "gopkg.in/yaml.v3" ) // NebulaHost describes the nebula configuration of a Host which is relevant for // other hosts to know. type NebulaHost struct { - CertPEM string `yaml:"cert_pem"` - PublicAddr string `yaml:"public_addr,omitempty"` + SignedPublicCredentials string `yaml:"signed_public_credentials"` + PublicAddr string `yaml:"public_addr,omitempty"` +} + +// NewNebulaHostSignedPublicCredentials constructs the SignedPublicCredentials +// field of the NebulaHost struct, using the CACredentials to sign the +// HostPublicCredentials. +func NewNebulaHostSignedPublicCredentials( + caCreds nebula.CACredentials, + hostPublicCreds nebula.HostPublicCredentials, +) ( + string, error, +) { + + hostPublicCredsB, err := yaml.Marshal(hostPublicCreds) + if err != nil { + return "", fmt.Errorf("yaml marshaling host's public credentials: %w", err) + } + + buf := new(bytes.Buffer) + + err = nebula.SignAndWrap(buf, caCreds.SigningPrivateKeyPEM, hostPublicCredsB) + if err != nil { + return "", fmt.Errorf("signing host's public credentials: %w", err) + } + + return buf.String(), nil } // GarageHost describes a single garage instance in the GarageHost. @@ -36,9 +65,25 @@ type Host struct { // IP returns the IP address encoded in the Host's nebula certificate, or panics // if there is an error. +// +// This assumes that the Host and its data has already been verified against the +// CA signing key. func (h Host) IP() net.IP { - ip, err := nebula.IPFromHostCertPEM(h.Nebula.CertPEM) + hostPublicCredsB, _, err := nebula.Unwrap( + strings.NewReader(h.Nebula.SignedPublicCredentials), + ) + + if err != nil { + panic(fmt.Errorf("unwrapping host's signed public credentials: %w", err)) + } + + var hostPublicCreds nebula.HostPublicCredentials + if err := yaml.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { + panic(fmt.Errorf("yaml unmarshaling host's public credentials: %w", err)) + } + + ip, err := nebula.IPFromHostCertPEM(hostPublicCreds.CertPEM) if err != nil { panic(fmt.Errorf("could not parse IP out of cert for host %q: %w", h.Name, err)) } diff --git a/entrypoint/src/cmd/entrypoint/admin.go b/entrypoint/src/cmd/entrypoint/admin.go index 5218df9..46a3113 100644 --- a/entrypoint/src/cmd/entrypoint/admin.go +++ b/entrypoint/src/cmd/entrypoint/admin.go @@ -132,6 +132,15 @@ var subCmdAdminCreateNetwork = subCmd{ return fmt.Errorf("creating nebula cert for host: %w", err) } + nebulaHostSignedPublicCreds, err := bootstrap.NewNebulaHostSignedPublicCredentials( + nebulaCACreds, + nebulaHostCreds.Public, + ) + + if err != nil { + return fmt.Errorf("creating signed public credentials for host: %w", err) + } + adminCreationParams := admin.CreationParams{ ID: randStr(32), Name: *name, @@ -144,14 +153,17 @@ var subCmdAdminCreateNetwork = subCmd{ *hostName: bootstrap.Host{ Name: *hostName, Nebula: bootstrap.NebulaHost{ - CertPEM: nebulaHostCreds.HostCertPEM, + SignedPublicCredentials: nebulaHostSignedPublicCreds, }, }, }, HostName: *hostName, } + hostBootstrap.Nebula.CAPublicCredentials = nebulaCACreds.Public hostBootstrap.Nebula.HostCredentials = nebulaHostCreds + hostBootstrap.Nebula.SignedPublicCredentials = nebulaHostSignedPublicCreds + hostBootstrap.Garage.RPCSecret = randStr(32) hostBootstrap.Garage.AdminToken = randStr(32) hostBootstrap.Garage.GlobalBucketS3APICredentials = garage.NewS3APICredentials() @@ -289,6 +301,15 @@ var subCmdAdminMakeBootstrap = subCmd{ return fmt.Errorf("creating new nebula host key/cert: %w", err) } + nebulaHostSignedPublicCreds, err := bootstrap.NewNebulaHostSignedPublicCredentials( + adm.Nebula.CACredentials, + nebulaHostCreds.Public, + ) + + if err != nil { + return fmt.Errorf("creating signed public credentials for host: %w", err) + } + newHostBootstrap := bootstrap.Bootstrap{ AdminCreationParams: adm.CreationParams, @@ -296,7 +317,10 @@ var subCmdAdminMakeBootstrap = subCmd{ HostName: *hostName, } + newHostBootstrap.Nebula.CAPublicCredentials = adm.Nebula.CACredentials.Public newHostBootstrap.Nebula.HostCredentials = nebulaHostCreds + newHostBootstrap.Nebula.SignedPublicCredentials = nebulaHostSignedPublicCreds + newHostBootstrap.Garage.RPCSecret = adm.Garage.RPCSecret newHostBootstrap.Garage.AdminToken = randStr(32) newHostBootstrap.Garage.GlobalBucketS3APICredentials = adm.Garage.GlobalBucketS3APICredentials diff --git a/entrypoint/src/cmd/entrypoint/nebula_util.go b/entrypoint/src/cmd/entrypoint/nebula_util.go index 703c66b..d0f58bf 100644 --- a/entrypoint/src/cmd/entrypoint/nebula_util.go +++ b/entrypoint/src/cmd/entrypoint/nebula_util.go @@ -58,9 +58,9 @@ func nebulaPmuxProcConfig( config := map[string]interface{}{ "pki": map[string]string{ - "ca": hostBootstrap.Nebula.HostCredentials.CACertPEM, - "cert": hostBootstrap.Nebula.HostCredentials.HostCertPEM, - "key": hostBootstrap.Nebula.HostCredentials.HostKeyPEM, + "ca": hostBootstrap.Nebula.CAPublicCredentials.CertPEM, + "cert": hostBootstrap.Nebula.HostCredentials.Public.CertPEM, + "key": hostBootstrap.Nebula.HostCredentials.KeyPEM, }, "static_host_map": staticHostMap, "punchy": map[string]bool{ diff --git a/entrypoint/src/garage/admin_client.go b/entrypoint/src/garage/admin_client.go index b4e5bb5..53227fe 100644 --- a/entrypoint/src/garage/admin_client.go +++ b/entrypoint/src/garage/admin_client.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "time" ) // AdminClientError gets returned from AdminClient's Do method for non-200 @@ -130,5 +131,7 @@ func (c *AdminClient) Wait(ctx context.Context) error { if numUp >= ReplicationFactor-1 { return nil } + + time.Sleep(250 * time.Millisecond) } } diff --git a/entrypoint/src/nebula/nebula.go b/entrypoint/src/nebula/nebula.go index 56a80c1..eb99931 100644 --- a/entrypoint/src/nebula/nebula.go +++ b/entrypoint/src/nebula/nebula.go @@ -21,19 +21,34 @@ import ( // fails. var ErrInvalidSignature = errors.New("invalid signature") +// HostPublicCredentials contains certificate and signing public keys which are +// able to be broadcast publicly. +type HostPublicCredentials struct { + CertPEM string `yaml:"cert_pem"` + SigningKeyPEM string `yaml:"signing_key_pem"` +} + // HostCredentials contains the certificate and private key files which will // need to be present on a particular host. Each file is PEM encoded. type HostCredentials struct { - CACertPEM string `yaml:"ca_cert_pem"` - HostKeyPEM string `yaml:"host_key_pem"` - HostCertPEM string `yaml:"host_cert_pem"` + Public HostPublicCredentials `yaml:"public"` + KeyPEM string `yaml:"key_pem"` + SigningPrivateKeyPEM string `yaml:"signing_private_key_pem"` +} + +// CAPublicCredentials contains certificate and signing public keys which are +// able to be broadcast publicly. The signing public key is the same one which +// is embedded into the certificate. +type CAPublicCredentials struct { + CertPEM string `yaml:"cert_pem"` + SigningKeyPEM string `yaml:"signing_key_pem"` } // CACredentials contains the certificate and private files which can be used to // create and validate HostCredentials. Each file is PEM encoded. type CACredentials struct { - CACertPEM string `yaml:"ca_cert_pem"` - CAKeyPEM string `yaml:"ca_key_pem"` + Public CAPublicCredentials `yaml:"public"` + SigningPrivateKeyPEM string `yaml:"signing_private_key_pem"` } // NewHostCredentials generates a new key/cert for a nebula host using the CA @@ -47,12 +62,12 @@ func NewHostCredentials( // The logic here is largely based on // https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go - caKey, _, err := cert.UnmarshalEd25519PrivateKey([]byte(caCreds.CAKeyPEM)) + caSigningKey, _, err := cert.UnmarshalEd25519PrivateKey([]byte(caCreds.SigningPrivateKeyPEM)) if err != nil { return HostCredentials{}, fmt.Errorf("unmarshaling ca.key: %w", err) } - caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCreds.CACertPEM)) + caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCreds.Public.CertPEM)) if err != nil { return HostCredentials{}, fmt.Errorf("unmarshaling ca.crt: %w", err) } @@ -69,6 +84,14 @@ func NewHostCredentials( return HostCredentials{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet) } + signingPubKey, signingPrivKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + panic(fmt.Errorf("generating ed25519 key: %w", err)) + } + + signingPrivKeyPEM := cert.MarshalEd25519PrivateKey(signingPrivKey) + signingPubKeyPEM := cert.MarshalEd25519PublicKey(signingPubKey) + var hostPub, hostKey []byte { var pubkey, privkey [32]byte @@ -98,7 +121,7 @@ func NewHostCredentials( return HostCredentials{}, fmt.Errorf("validating certificate constraints: %w", err) } - if err := hostCert.Sign(caKey); err != nil { + if err := hostCert.Sign(caSigningKey); err != nil { return HostCredentials{}, fmt.Errorf("signing host cert with ca.key: %w", err) } @@ -110,9 +133,12 @@ func NewHostCredentials( } return HostCredentials{ - CACertPEM: caCreds.CACertPEM, - HostKeyPEM: string(hostKeyPEM), - HostCertPEM: string(hostCertPEM), + Public: HostPublicCredentials{ + CertPEM: string(hostCertPEM), + SigningKeyPEM: string(signingPubKeyPEM), + }, + KeyPEM: string(hostKeyPEM), + SigningPrivateKeyPEM: string(signingPrivKeyPEM), }, nil } @@ -120,7 +146,10 @@ func NewHostCredentials( // and is included in the signing certificate's Name field. func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) { - pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) + // The logic here is largely based on + // https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/ca.go + + signingPubKey, signingPrivKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { panic(fmt.Errorf("generating ed25519 key: %w", err)) } @@ -134,51 +163,32 @@ func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) { Subnets: []*net.IPNet{subnet}, NotBefore: now, NotAfter: expireAt, - PublicKey: pubKey, + PublicKey: signingPubKey, IsCA: true, }, } - if err := caCert.Sign(privKey); err != nil { + if err := caCert.Sign(signingPrivKey); err != nil { return CACredentials{}, fmt.Errorf("signing caCert: %w", err) } - caKeyPEM := cert.MarshalEd25519PrivateKey(privKey) + signingPrivKeyPEM := cert.MarshalEd25519PrivateKey(signingPrivKey) + signingPubKeyPEM := cert.MarshalEd25519PublicKey(signingPubKey) - caCertPEM, err := caCert.MarshalToPEM() + certPEM, err := caCert.MarshalToPEM() if err != nil { return CACredentials{}, fmt.Errorf("marshaling caCert: %w", err) } return CACredentials{ - CACertPEM: string(caCertPEM), - CAKeyPEM: string(caKeyPEM), + Public: CAPublicCredentials{ + CertPEM: string(certPEM), + SigningKeyPEM: string(signingPubKeyPEM), + }, + SigningPrivateKeyPEM: string(signingPrivKeyPEM), }, nil } -// ValidateHostCertPEM checks if the given host certificate was signed by the -// given CA certificate, and returns ErrInvalidSignature if validation fails. -func ValidateHostCertPEM(caCertPEM, hostCertPEM string) error { - - caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCertPEM)) - if err != nil { - return fmt.Errorf("unmarshaling CA certificate as PEM: %w", err) - } - - hostCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(hostCertPEM)) - if err != nil { - return fmt.Errorf("unmarshaling host certificate as PEM: %w", err) - } - - caPubKey := ed25519.PublicKey(caCert.Details.PublicKey) - - if !hostCert.CheckSignature(caPubKey) { - return ErrInvalidSignature - } - - return nil -} - // IPFromHostCertPEM is a convenience function for parsing the IP of a host out // of its nebula cert. func IPFromHostCertPEM(hostCertPEM string) (net.IP, error) { @@ -196,11 +206,11 @@ func IPFromHostCertPEM(hostCertPEM string) (net.IP, error) { return ips[0].IP, nil } -// SignAndWrap signs the given bytes using the keyPEM, and writes an +// SignAndWrap signs the given bytes using the host key, and writes an // encoded, versioned structure containing the signature and the given bytes. -func SignAndWrap(into io.Writer, keyPEM string, b []byte) error { +func SignAndWrap(into io.Writer, signingKeyPEM string, b []byte) error { - key, _, err := cert.UnmarshalEd25519PrivateKey([]byte(keyPEM)) + key, _, err := cert.UnmarshalEd25519PrivateKey([]byte(signingKeyPEM)) if err != nil { return fmt.Errorf("unmarshaling private key: %w", err) } @@ -215,7 +225,7 @@ func SignAndWrap(into io.Writer, keyPEM string, b []byte) error { } err = pem.Encode(into, &pem.Block{ - Type: "SIGNATURE", + Type: "NEBULA ED25519 SIGNATURE", Bytes: sig, }) @@ -231,7 +241,7 @@ func SignAndWrap(into io.Writer, keyPEM string, b []byte) error { } // Unwrap reads a stream of bytes which was produced by SignAndWrap, and returns -// the original inpute to SignAndWrap as well as the signature which was +// the original input to SignAndWrap as well as the signature which was // created. ValidateSignature can be used to validate the signature. func Unwrap(from io.Reader) (b, sig []byte, err error) { @@ -255,15 +265,13 @@ func Unwrap(from io.Reader) (b, sig []byte, err error) { } // ValidateSignature can be used to validate a signature produced by Unwrap. -func ValidateSignature(certPEM string, b, sig []byte) error { +func ValidateSignature(signingPubKeyPEM string, b, sig []byte) error { - cert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(certPEM)) + pubKey, _, err := cert.UnmarshalEd25519PublicKey([]byte(signingPubKeyPEM)) if err != nil { return fmt.Errorf("unmarshaling certificate as PEM: %w", err) } - pubKey := ed25519.PublicKey(cert.Details.PublicKey) - if !ed25519.Verify(pubKey, b, sig) { return ErrInvalidSignature } diff --git a/entrypoint/src/nebula/nebula_test.go b/entrypoint/src/nebula/nebula_test.go index c5622c5..654fedf 100644 --- a/entrypoint/src/nebula/nebula_test.go +++ b/entrypoint/src/nebula/nebula_test.go @@ -8,9 +8,10 @@ import ( ) var ( - ip net.IP - ipNet *net.IPNet - caCredsA, caCredsB CACredentials + ip net.IP + ipNet *net.IPNet + caCredsA, caCredsB CACredentials + hostCredsA, hostCredsB HostCredentials ) func init() { @@ -30,24 +31,17 @@ func init() { if err != nil { panic(err) } -} -func TestValidateHostCredentials(t *testing.T) { - - hostCreds, err := NewHostCredentials(caCredsA, "foo", ip) + hostCredsA, err = NewHostCredentials(caCredsA, "foo", ip) if err != nil { - t.Fatal(err) + panic(err) } - err = ValidateHostCertPEM(hostCreds.CACertPEM, hostCreds.HostCertPEM) + hostCredsB, err = NewHostCredentials(caCredsB, "bar", ip) if err != nil { - t.Fatal(err) + panic(err) } - err = ValidateHostCertPEM(caCredsB.CACertPEM, hostCreds.HostCertPEM) - if !errors.Is(err, ErrInvalidSignature) { - t.Fatalf("expected ErrInvalidSignature, got %v", err) - } } func TestSignAndWrap(t *testing.T) { @@ -55,7 +49,7 @@ func TestSignAndWrap(t *testing.T) { b := []byte("foo bar baz") buf := new(bytes.Buffer) - if err := SignAndWrap(buf, caCredsA.CAKeyPEM, b); err != nil { + if err := SignAndWrap(buf, hostCredsA.SigningPrivateKeyPEM, b); err != nil { t.Fatal(err) } @@ -67,11 +61,11 @@ func TestSignAndWrap(t *testing.T) { t.Fatalf("got %q but expected %q", gotB, b) } - if err := ValidateSignature(caCredsA.CACertPEM, b, gotSig); err != nil { + if err := ValidateSignature(hostCredsA.Public.SigningKeyPEM, b, gotSig); err != nil { t.Fatal(err) } - if err := ValidateSignature(caCredsB.CACertPEM, b, gotSig); !errors.Is(err, ErrInvalidSignature) { + if err := ValidateSignature(hostCredsB.Public.SigningKeyPEM, b, gotSig); !errors.Is(err, ErrInvalidSignature) { t.Fatalf("expected ErrInvalidSignature but got %v", err) } }