diff --git a/entrypoint/src/bootstrap/bootstrap.go b/entrypoint/src/bootstrap/bootstrap.go index 29ea5a2..1b70261 100644 --- a/entrypoint/src/bootstrap/bootstrap.go +++ b/entrypoint/src/bootstrap/bootstrap.go @@ -37,7 +37,9 @@ type Bootstrap struct { HostName string `yaml:"hostname"` Nebula struct { - HostCredentials nebula.HostCredentials `yaml:"host_credentials"` + CAPublicCredentials nebula.CAPublicCredentials `yaml:"ca_public_credentials"` + HostCredentials nebula.HostCredentials `yaml:"host_credentials"` + SignedPublicCredentials string `yaml:"signed_public_credentials"` } `yaml:"nebula"` Garage struct { diff --git a/entrypoint/src/bootstrap/garage_global_bucket.go b/entrypoint/src/bootstrap/garage_global_bucket.go index 9052c32..d3069b0 100644 --- a/entrypoint/src/bootstrap/garage_global_bucket.go +++ b/entrypoint/src/bootstrap/garage_global_bucket.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/minio/minio-go/v7" "gopkg.in/yaml.v3" @@ -26,6 +27,12 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { host := b.ThisHost() client := b.GlobalBucketS3APIClient() + // the base Bootstrap has the public credentials signed by the CA, but we + // need this to be presented in the data stored into garage, so other hosts + // can verify that the stored host object is signed by the host public key, + // and that the host public key is signed by the CA. + host.Nebula.SignedPublicCredentials = b.Nebula.SignedPublicCredentials + hostB, err := yaml.Marshal(host) if err != nil { return fmt.Errorf("yaml encoding host data: %w", err) @@ -33,7 +40,7 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { buf := new(bytes.Buffer) - err = nebula.SignAndWrap(buf, b.Nebula.HostCredentials.HostKeyPEM, hostB) + err = nebula.SignAndWrap(buf, b.Nebula.HostCredentials.SigningPrivateKeyPEM, hostB) if err != nil { return fmt.Errorf("signing encoded host data: %w", err) } @@ -82,7 +89,6 @@ func (b Bootstrap) GetGarageBootstrapHosts( map[string]Host, error, ) { - caCertPEM := b.Nebula.HostCredentials.CACertPEM client := b.GlobalBucketS3APIClient() hosts := map[string]Host{} @@ -109,7 +115,7 @@ func (b Bootstrap) GetGarageBootstrapHosts( return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err) } - hostB, sig, err := nebula.Unwrap(obj) + hostB, hostSig, err := nebula.Unwrap(obj) obj.Close() if err != nil { @@ -121,15 +127,36 @@ func (b Bootstrap) GetGarageBootstrapHosts( return nil, fmt.Errorf("yaml decoding object %q: %w", objInfo.Key, err) } - hostCertPEM := host.Nebula.CertPEM + hostPublicCredsB, hostPublicCredsSig, err := nebula.Unwrap( + strings.NewReader(host.Nebula.SignedPublicCredentials), + ) - if err := nebula.ValidateSignature(hostCertPEM, hostB, sig); err != nil { - fmt.Fprintf(os.Stderr, "invalid host data for %q: %v\n", objInfo.Key, err) + if err != nil { + fmt.Fprintf(os.Stderr, "unwrapping signed public creds for %q: %v\n", objInfo.Key, err) continue } - if err := nebula.ValidateHostCertPEM(caCertPEM, hostCertPEM); err != nil { - fmt.Fprintf(os.Stderr, "invalid nebula cert for %q: %v\n", objInfo.Key, err) + err = nebula.ValidateSignature( + b.Nebula.CAPublicCredentials.SigningKeyPEM, + hostPublicCredsB, + hostPublicCredsSig, + ) + + if err != nil { + fmt.Fprintf(os.Stderr, "invalid signed public creds for %q: %v\n", objInfo.Key, err) + continue + } + + var hostPublicCreds nebula.HostPublicCredentials + if err := yaml.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { + fmt.Fprintf(os.Stderr, "yaml unmarshaling signed public creds for %q: %v\n", objInfo.Key, err) + continue + } + + err = nebula.ValidateSignature(hostPublicCreds.SigningKeyPEM, hostB, hostSig) + + if err != nil { + fmt.Fprintf(os.Stderr, "invalid host data for %q: %v\n", objInfo.Key, err) continue } diff --git a/entrypoint/src/bootstrap/hosts.go b/entrypoint/src/bootstrap/hosts.go index 2439eab..badf38d 100644 --- a/entrypoint/src/bootstrap/hosts.go +++ b/entrypoint/src/bootstrap/hosts.go @@ -1,16 +1,45 @@ package bootstrap import ( + "bytes" "cryptic-net/nebula" "fmt" "net" + "strings" + + "gopkg.in/yaml.v3" ) // NebulaHost describes the nebula configuration of a Host which is relevant for // other hosts to know. type NebulaHost struct { - CertPEM string `yaml:"cert_pem"` - PublicAddr string `yaml:"public_addr,omitempty"` + SignedPublicCredentials string `yaml:"signed_public_credentials"` + PublicAddr string `yaml:"public_addr,omitempty"` +} + +// NewNebulaHostSignedPublicCredentials constructs the SignedPublicCredentials +// field of the NebulaHost struct, using the CACredentials to sign the +// HostPublicCredentials. +func NewNebulaHostSignedPublicCredentials( + caCreds nebula.CACredentials, + hostPublicCreds nebula.HostPublicCredentials, +) ( + string, error, +) { + + hostPublicCredsB, err := yaml.Marshal(hostPublicCreds) + if err != nil { + return "", fmt.Errorf("yaml marshaling host's public credentials: %w", err) + } + + buf := new(bytes.Buffer) + + err = nebula.SignAndWrap(buf, caCreds.SigningPrivateKeyPEM, hostPublicCredsB) + if err != nil { + return "", fmt.Errorf("signing host's public credentials: %w", err) + } + + return buf.String(), nil } // GarageHost describes a single garage instance in the GarageHost. @@ -36,9 +65,25 @@ type Host struct { // IP returns the IP address encoded in the Host's nebula certificate, or panics // if there is an error. +// +// This assumes that the Host and its data has already been verified against the +// CA signing key. func (h Host) IP() net.IP { - ip, err := nebula.IPFromHostCertPEM(h.Nebula.CertPEM) + hostPublicCredsB, _, err := nebula.Unwrap( + strings.NewReader(h.Nebula.SignedPublicCredentials), + ) + + if err != nil { + panic(fmt.Errorf("unwrapping host's signed public credentials: %w", err)) + } + + var hostPublicCreds nebula.HostPublicCredentials + if err := yaml.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { + panic(fmt.Errorf("yaml unmarshaling host's public credentials: %w", err)) + } + + ip, err := nebula.IPFromHostCertPEM(hostPublicCreds.CertPEM) if err != nil { panic(fmt.Errorf("could not parse IP out of cert for host %q: %w", h.Name, err)) } diff --git a/entrypoint/src/cmd/entrypoint/admin.go b/entrypoint/src/cmd/entrypoint/admin.go index 5218df9..46a3113 100644 --- a/entrypoint/src/cmd/entrypoint/admin.go +++ b/entrypoint/src/cmd/entrypoint/admin.go @@ -132,6 +132,15 @@ var subCmdAdminCreateNetwork = subCmd{ return fmt.Errorf("creating nebula cert for host: %w", err) } + nebulaHostSignedPublicCreds, err := bootstrap.NewNebulaHostSignedPublicCredentials( + nebulaCACreds, + nebulaHostCreds.Public, + ) + + if err != nil { + return fmt.Errorf("creating signed public credentials for host: %w", err) + } + adminCreationParams := admin.CreationParams{ ID: randStr(32), Name: *name, @@ -144,14 +153,17 @@ var subCmdAdminCreateNetwork = subCmd{ *hostName: bootstrap.Host{ Name: *hostName, Nebula: bootstrap.NebulaHost{ - CertPEM: nebulaHostCreds.HostCertPEM, + SignedPublicCredentials: nebulaHostSignedPublicCreds, }, }, }, HostName: *hostName, } + hostBootstrap.Nebula.CAPublicCredentials = nebulaCACreds.Public hostBootstrap.Nebula.HostCredentials = nebulaHostCreds + hostBootstrap.Nebula.SignedPublicCredentials = nebulaHostSignedPublicCreds + hostBootstrap.Garage.RPCSecret = randStr(32) hostBootstrap.Garage.AdminToken = randStr(32) hostBootstrap.Garage.GlobalBucketS3APICredentials = garage.NewS3APICredentials() @@ -289,6 +301,15 @@ var subCmdAdminMakeBootstrap = subCmd{ return fmt.Errorf("creating new nebula host key/cert: %w", err) } + nebulaHostSignedPublicCreds, err := bootstrap.NewNebulaHostSignedPublicCredentials( + adm.Nebula.CACredentials, + nebulaHostCreds.Public, + ) + + if err != nil { + return fmt.Errorf("creating signed public credentials for host: %w", err) + } + newHostBootstrap := bootstrap.Bootstrap{ AdminCreationParams: adm.CreationParams, @@ -296,7 +317,10 @@ var subCmdAdminMakeBootstrap = subCmd{ HostName: *hostName, } + newHostBootstrap.Nebula.CAPublicCredentials = adm.Nebula.CACredentials.Public newHostBootstrap.Nebula.HostCredentials = nebulaHostCreds + newHostBootstrap.Nebula.SignedPublicCredentials = nebulaHostSignedPublicCreds + newHostBootstrap.Garage.RPCSecret = adm.Garage.RPCSecret newHostBootstrap.Garage.AdminToken = randStr(32) newHostBootstrap.Garage.GlobalBucketS3APICredentials = adm.Garage.GlobalBucketS3APICredentials diff --git a/entrypoint/src/cmd/entrypoint/nebula_util.go b/entrypoint/src/cmd/entrypoint/nebula_util.go index 703c66b..d0f58bf 100644 --- a/entrypoint/src/cmd/entrypoint/nebula_util.go +++ b/entrypoint/src/cmd/entrypoint/nebula_util.go @@ -58,9 +58,9 @@ func nebulaPmuxProcConfig( config := map[string]interface{}{ "pki": map[string]string{ - "ca": hostBootstrap.Nebula.HostCredentials.CACertPEM, - "cert": hostBootstrap.Nebula.HostCredentials.HostCertPEM, - "key": hostBootstrap.Nebula.HostCredentials.HostKeyPEM, + "ca": hostBootstrap.Nebula.CAPublicCredentials.CertPEM, + "cert": hostBootstrap.Nebula.HostCredentials.Public.CertPEM, + "key": hostBootstrap.Nebula.HostCredentials.KeyPEM, }, "static_host_map": staticHostMap, "punchy": map[string]bool{ diff --git a/entrypoint/src/garage/admin_client.go b/entrypoint/src/garage/admin_client.go index b4e5bb5..53227fe 100644 --- a/entrypoint/src/garage/admin_client.go +++ b/entrypoint/src/garage/admin_client.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "time" ) // AdminClientError gets returned from AdminClient's Do method for non-200 @@ -130,5 +131,7 @@ func (c *AdminClient) Wait(ctx context.Context) error { if numUp >= ReplicationFactor-1 { return nil } + + time.Sleep(250 * time.Millisecond) } } diff --git a/entrypoint/src/nebula/nebula.go b/entrypoint/src/nebula/nebula.go index 56a80c1..eb99931 100644 --- a/entrypoint/src/nebula/nebula.go +++ b/entrypoint/src/nebula/nebula.go @@ -21,19 +21,34 @@ import ( // fails. var ErrInvalidSignature = errors.New("invalid signature") +// HostPublicCredentials contains certificate and signing public keys which are +// able to be broadcast publicly. +type HostPublicCredentials struct { + CertPEM string `yaml:"cert_pem"` + SigningKeyPEM string `yaml:"signing_key_pem"` +} + // HostCredentials contains the certificate and private key files which will // need to be present on a particular host. Each file is PEM encoded. type HostCredentials struct { - CACertPEM string `yaml:"ca_cert_pem"` - HostKeyPEM string `yaml:"host_key_pem"` - HostCertPEM string `yaml:"host_cert_pem"` + Public HostPublicCredentials `yaml:"public"` + KeyPEM string `yaml:"key_pem"` + SigningPrivateKeyPEM string `yaml:"signing_private_key_pem"` +} + +// CAPublicCredentials contains certificate and signing public keys which are +// able to be broadcast publicly. The signing public key is the same one which +// is embedded into the certificate. +type CAPublicCredentials struct { + CertPEM string `yaml:"cert_pem"` + SigningKeyPEM string `yaml:"signing_key_pem"` } // CACredentials contains the certificate and private files which can be used to // create and validate HostCredentials. Each file is PEM encoded. type CACredentials struct { - CACertPEM string `yaml:"ca_cert_pem"` - CAKeyPEM string `yaml:"ca_key_pem"` + Public CAPublicCredentials `yaml:"public"` + SigningPrivateKeyPEM string `yaml:"signing_private_key_pem"` } // NewHostCredentials generates a new key/cert for a nebula host using the CA @@ -47,12 +62,12 @@ func NewHostCredentials( // The logic here is largely based on // https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go - caKey, _, err := cert.UnmarshalEd25519PrivateKey([]byte(caCreds.CAKeyPEM)) + caSigningKey, _, err := cert.UnmarshalEd25519PrivateKey([]byte(caCreds.SigningPrivateKeyPEM)) if err != nil { return HostCredentials{}, fmt.Errorf("unmarshaling ca.key: %w", err) } - caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCreds.CACertPEM)) + caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCreds.Public.CertPEM)) if err != nil { return HostCredentials{}, fmt.Errorf("unmarshaling ca.crt: %w", err) } @@ -69,6 +84,14 @@ func NewHostCredentials( return HostCredentials{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet) } + signingPubKey, signingPrivKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + panic(fmt.Errorf("generating ed25519 key: %w", err)) + } + + signingPrivKeyPEM := cert.MarshalEd25519PrivateKey(signingPrivKey) + signingPubKeyPEM := cert.MarshalEd25519PublicKey(signingPubKey) + var hostPub, hostKey []byte { var pubkey, privkey [32]byte @@ -98,7 +121,7 @@ func NewHostCredentials( return HostCredentials{}, fmt.Errorf("validating certificate constraints: %w", err) } - if err := hostCert.Sign(caKey); err != nil { + if err := hostCert.Sign(caSigningKey); err != nil { return HostCredentials{}, fmt.Errorf("signing host cert with ca.key: %w", err) } @@ -110,9 +133,12 @@ func NewHostCredentials( } return HostCredentials{ - CACertPEM: caCreds.CACertPEM, - HostKeyPEM: string(hostKeyPEM), - HostCertPEM: string(hostCertPEM), + Public: HostPublicCredentials{ + CertPEM: string(hostCertPEM), + SigningKeyPEM: string(signingPubKeyPEM), + }, + KeyPEM: string(hostKeyPEM), + SigningPrivateKeyPEM: string(signingPrivKeyPEM), }, nil } @@ -120,7 +146,10 @@ func NewHostCredentials( // and is included in the signing certificate's Name field. func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) { - pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) + // The logic here is largely based on + // https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/ca.go + + signingPubKey, signingPrivKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { panic(fmt.Errorf("generating ed25519 key: %w", err)) } @@ -134,51 +163,32 @@ func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) { Subnets: []*net.IPNet{subnet}, NotBefore: now, NotAfter: expireAt, - PublicKey: pubKey, + PublicKey: signingPubKey, IsCA: true, }, } - if err := caCert.Sign(privKey); err != nil { + if err := caCert.Sign(signingPrivKey); err != nil { return CACredentials{}, fmt.Errorf("signing caCert: %w", err) } - caKeyPEM := cert.MarshalEd25519PrivateKey(privKey) + signingPrivKeyPEM := cert.MarshalEd25519PrivateKey(signingPrivKey) + signingPubKeyPEM := cert.MarshalEd25519PublicKey(signingPubKey) - caCertPEM, err := caCert.MarshalToPEM() + certPEM, err := caCert.MarshalToPEM() if err != nil { return CACredentials{}, fmt.Errorf("marshaling caCert: %w", err) } return CACredentials{ - CACertPEM: string(caCertPEM), - CAKeyPEM: string(caKeyPEM), + Public: CAPublicCredentials{ + CertPEM: string(certPEM), + SigningKeyPEM: string(signingPubKeyPEM), + }, + SigningPrivateKeyPEM: string(signingPrivKeyPEM), }, nil } -// ValidateHostCertPEM checks if the given host certificate was signed by the -// given CA certificate, and returns ErrInvalidSignature if validation fails. -func ValidateHostCertPEM(caCertPEM, hostCertPEM string) error { - - caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCertPEM)) - if err != nil { - return fmt.Errorf("unmarshaling CA certificate as PEM: %w", err) - } - - hostCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(hostCertPEM)) - if err != nil { - return fmt.Errorf("unmarshaling host certificate as PEM: %w", err) - } - - caPubKey := ed25519.PublicKey(caCert.Details.PublicKey) - - if !hostCert.CheckSignature(caPubKey) { - return ErrInvalidSignature - } - - return nil -} - // IPFromHostCertPEM is a convenience function for parsing the IP of a host out // of its nebula cert. func IPFromHostCertPEM(hostCertPEM string) (net.IP, error) { @@ -196,11 +206,11 @@ func IPFromHostCertPEM(hostCertPEM string) (net.IP, error) { return ips[0].IP, nil } -// SignAndWrap signs the given bytes using the keyPEM, and writes an +// SignAndWrap signs the given bytes using the host key, and writes an // encoded, versioned structure containing the signature and the given bytes. -func SignAndWrap(into io.Writer, keyPEM string, b []byte) error { +func SignAndWrap(into io.Writer, signingKeyPEM string, b []byte) error { - key, _, err := cert.UnmarshalEd25519PrivateKey([]byte(keyPEM)) + key, _, err := cert.UnmarshalEd25519PrivateKey([]byte(signingKeyPEM)) if err != nil { return fmt.Errorf("unmarshaling private key: %w", err) } @@ -215,7 +225,7 @@ func SignAndWrap(into io.Writer, keyPEM string, b []byte) error { } err = pem.Encode(into, &pem.Block{ - Type: "SIGNATURE", + Type: "NEBULA ED25519 SIGNATURE", Bytes: sig, }) @@ -231,7 +241,7 @@ func SignAndWrap(into io.Writer, keyPEM string, b []byte) error { } // Unwrap reads a stream of bytes which was produced by SignAndWrap, and returns -// the original inpute to SignAndWrap as well as the signature which was +// the original input to SignAndWrap as well as the signature which was // created. ValidateSignature can be used to validate the signature. func Unwrap(from io.Reader) (b, sig []byte, err error) { @@ -255,15 +265,13 @@ func Unwrap(from io.Reader) (b, sig []byte, err error) { } // ValidateSignature can be used to validate a signature produced by Unwrap. -func ValidateSignature(certPEM string, b, sig []byte) error { +func ValidateSignature(signingPubKeyPEM string, b, sig []byte) error { - cert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(certPEM)) + pubKey, _, err := cert.UnmarshalEd25519PublicKey([]byte(signingPubKeyPEM)) if err != nil { return fmt.Errorf("unmarshaling certificate as PEM: %w", err) } - pubKey := ed25519.PublicKey(cert.Details.PublicKey) - if !ed25519.Verify(pubKey, b, sig) { return ErrInvalidSignature } diff --git a/entrypoint/src/nebula/nebula_test.go b/entrypoint/src/nebula/nebula_test.go index c5622c5..654fedf 100644 --- a/entrypoint/src/nebula/nebula_test.go +++ b/entrypoint/src/nebula/nebula_test.go @@ -8,9 +8,10 @@ import ( ) var ( - ip net.IP - ipNet *net.IPNet - caCredsA, caCredsB CACredentials + ip net.IP + ipNet *net.IPNet + caCredsA, caCredsB CACredentials + hostCredsA, hostCredsB HostCredentials ) func init() { @@ -30,24 +31,17 @@ func init() { if err != nil { panic(err) } -} -func TestValidateHostCredentials(t *testing.T) { - - hostCreds, err := NewHostCredentials(caCredsA, "foo", ip) + hostCredsA, err = NewHostCredentials(caCredsA, "foo", ip) if err != nil { - t.Fatal(err) + panic(err) } - err = ValidateHostCertPEM(hostCreds.CACertPEM, hostCreds.HostCertPEM) + hostCredsB, err = NewHostCredentials(caCredsB, "bar", ip) if err != nil { - t.Fatal(err) + panic(err) } - err = ValidateHostCertPEM(caCredsB.CACertPEM, hostCreds.HostCertPEM) - if !errors.Is(err, ErrInvalidSignature) { - t.Fatalf("expected ErrInvalidSignature, got %v", err) - } } func TestSignAndWrap(t *testing.T) { @@ -55,7 +49,7 @@ func TestSignAndWrap(t *testing.T) { b := []byte("foo bar baz") buf := new(bytes.Buffer) - if err := SignAndWrap(buf, caCredsA.CAKeyPEM, b); err != nil { + if err := SignAndWrap(buf, hostCredsA.SigningPrivateKeyPEM, b); err != nil { t.Fatal(err) } @@ -67,11 +61,11 @@ func TestSignAndWrap(t *testing.T) { t.Fatalf("got %q but expected %q", gotB, b) } - if err := ValidateSignature(caCredsA.CACertPEM, b, gotSig); err != nil { + if err := ValidateSignature(hostCredsA.Public.SigningKeyPEM, b, gotSig); err != nil { t.Fatal(err) } - if err := ValidateSignature(caCredsB.CACertPEM, b, gotSig); !errors.Is(err, ErrInvalidSignature) { + if err := ValidateSignature(hostCredsB.Public.SigningKeyPEM, b, gotSig); !errors.Is(err, ErrInvalidSignature) { t.Fatalf("expected ErrInvalidSignature but got %v", err) } }