Implement admin create-network command

This required a lot of re-implementation of how garage gets interacted
with, including updating cluster layout using the admin API and
initialization of the global bucket key.

go-workspace/src/admin/admin.go

@@ -27,8 +27,8 @@ const (
 // CreationParams are general parameters used when creating a new network. These
 // are available to all hosts within the network via their bootstrap files.
 type CreationParams struct {
-	Domain string   `yaml:"domain"`
+	ID     string `yaml:"id"`
-	CIDRs  []string `yaml:"cidrs"`
+	Domain string `yaml:"domain"`
 }
 
 // Admin is used for accessing all information contained within an admin.tgz.

go-workspace/src/cmd/entrypoint/admin.go

@@ -1,14 +1,22 @@
 package entrypoint
 
 import (
+	"context"
 	"cryptic-net/admin"
 	"cryptic-net/bootstrap"
+	"cryptic-net/garage"
 	"cryptic-net/nebula"
 	"crypto/rand"
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"log"
+	"net"
 	"os"
+	"strconv"
+	"strings"
+
+	"github.com/cryptic-io/pmux/pmuxlib"
 )
 
 func randStr(l int) string {
@@ -40,6 +48,294 @@ func readAdmin(path string) (admin.Admin, error) {
 	return admin.FromReader(f)
 }
 
+func garageInitializeGlobalBucket(
+	ctx context.Context,
+	adminClient *garage.AdminClient,
+	globalBucketCreds garage.S3APICredentials,
+) error {
+
+	// first attempt to import the key
+	err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
+		"accessKeyId":     globalBucketCreds.ID,
+		"secretAccessKey": globalBucketCreds.Secret,
+		"name":            "shared-global-bucket-key",
+	})
+
+	if err != nil {
+		return fmt.Errorf("importing global bucket key into garage: %w", err)
+	}
+
+	// create global bucket
+	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
+		"globalAlias": garage.GlobalBucket,
+	})
+
+	if err != nil {
+		return fmt.Errorf("creating global bucket: %w", err)
+	}
+
+	// retrieve newly created bucket's id
+	var getBucketRes struct {
+		ID string `json:"id"`
+	}
+
+	err = adminClient.Do(
+		ctx, &getBucketRes,
+		"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
+	)
+
+	if err != nil {
+		return fmt.Errorf("fetching global bucket id: %w", err)
+	}
+
+	// allow shared global bucket key to perform all operations
+	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
+		"bucketId":    getBucketRes.ID,
+		"accessKeyId": globalBucketCreds.ID,
+		"permissions": map[string]bool{
+			"read":  true,
+			"write": true,
+		},
+	})
+
+	if err != nil {
+		return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
+	}
+
+	return nil
+}
+
+var subCmdAdminCreateNetwork = subCmd{
+	name:  "create-network",
+	descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
+	do: func(subCmdCtx subCmdCtx) error {
+
+		flags := subCmdCtx.flagSet(false)
+
+		daemonYmlPath := flags.StringP(
+			"config-path", "c", "",
+			"Optional path to a daemon.yml file to load configuration from.",
+		)
+
+		dumpConfig := flags.Bool(
+			"dump-config", false,
+			"Write the default configuration file to stdout and exit.",
+		)
+
+		domain := flags.StringP(
+			"domain", "d", "",
+			"Domain name that should be used as the root domain in the network.",
+		)
+
+		subnetStr := flags.StringP(
+			"subnet", "s", "",
+			"CIDR which denotes the subnet that IPs hosts on the network can be assigned.",
+		)
+
+		hostName := flags.StringP(
+			"name", "n", "",
+			"Name of the host which will be the first host in the network",
+		)
+
+		if err := flags.Parse(subCmdCtx.args); err != nil {
+			return fmt.Errorf("parsing flags: %w", err)
+		}
+
+		env := subCmdCtx.env
+
+		if *dumpConfig {
+			return writeBuiltinDaemonYml(env, os.Stdout)
+		}
+
+		if *domain == "" || *subnetStr == "" || *hostName == "" {
+			return errors.New("--domain, --subnet, and --name are required")
+		}
+
+		*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
+
+		ip, subnet, err := net.ParseCIDR(*subnetStr)
+		if err != nil {
+			return fmt.Errorf("parsing %q as a CIDR: %w", *subnetStr, err)
+		}
+
+		if err := validateHostName(*hostName); err != nil {
+			return fmt.Errorf("invalid hostname %q: %w", *hostName, err)
+		}
+
+		adminCreationParams := admin.CreationParams{
+			ID:     randStr(32),
+			Domain: *domain,
+		}
+
+		{
+			runtimeDirPath := env.RuntimeDirPath
+
+			fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
+
+			if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
+				return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
+			}
+
+			defer func() {
+				fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
+				if err := os.RemoveAll(runtimeDirPath); err != nil {
+					fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
+				}
+			}()
+		}
+
+		if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
+			return fmt.Errorf("merging and writing daemon.yml file: %w", err)
+		}
+
+		daemon := env.ThisDaemon()
+
+		if len(daemon.Storage.Allocations) < 3 {
+			return fmt.Errorf("daemon.yml with at least 3 allocations was not provided")
+		}
+
+		nebulaCACert, err := nebula.NewCACert(*domain, subnet)
+		if err != nil {
+			return fmt.Errorf("creating nebula CA cert: %w", err)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(nebulaCACert, *hostName, ip)
+		if err != nil {
+			return fmt.Errorf("creating nebula cert for host: %w", err)
+		}
+
+		env.Bootstrap = bootstrap.Bootstrap{
+			AdminCreationParams: adminCreationParams,
+			Hosts: map[string]bootstrap.Host{
+				*hostName: bootstrap.Host{
+					Name: *hostName,
+					Nebula: bootstrap.NebulaHost{
+						IP: ip.String(),
+					},
+				},
+			},
+			HostName:                           *hostName,
+			NebulaHostCert:                     nebulaHostCert,
+			GarageRPCSecret:                    randStr(32),
+			GarageGlobalBucketS3APICredentials: garage.NewS3APICredentials(),
+		}
+
+		// this will also write the bootstrap file
+		if err := mergeDaemonIntoBootstrap(env); err != nil {
+			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
+		}
+
+		// TODO this can be gotten rid of once nebula-entrypoint is rolled into
+		// daemon itself
+		for key, val := range env.ToMap() {
+			if err := os.Setenv(key, val); err != nil {
+				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
+			}
+		}
+
+		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
+		if err != nil {
+			return fmt.Errorf("generating garage children configs: %w", err)
+		}
+
+		pmuxConfig := pmuxlib.Config{
+			Processes: append(
+				[]pmuxlib.ProcessConfig{
+					nebulaEntrypointPmuxProcConfig(),
+				},
+				garageChildrenPmuxProcConfigs...,
+			),
+		}
+
+		ctx, cancel := context.WithCancel(env.Context)
+		pmuxDoneCh := make(chan struct{})
+
+		log.Printf("starting child processes")
+		go func() {
+			pmuxlib.Run(ctx, pmuxConfig)
+			close(pmuxDoneCh)
+		}()
+
+		defer func() {
+			cancel()
+			log.Printf("waiting for child processes to exit")
+			<-pmuxDoneCh
+		}()
+
+		var garageAdminClient *garage.AdminClient
+		garageAdminClients := map[string]*garage.AdminClient{}
+
+		for _, alloc := range daemon.Storage.Allocations {
+
+			garageAdminAddr := net.JoinHostPort(ip.String(), strconv.Itoa(alloc.AdminPort))
+
+			garageAdminClient = garage.NewAdminClient(
+				garageAdminAddr,
+				env.Bootstrap.GarageAdminToken,
+			)
+
+			garageAdminClients[garageAdminAddr] = garageAdminClient
+		}
+
+		log.Printf("waiting for garage instances to come online")
+		for garageAdminAddr, garageAdminClient := range garageAdminClients {
+			if err := garageAdminClient.Wait(ctx); err != nil {
+				return fmt.Errorf("waiting for garage instance %q to start up: %w", garageAdminAddr, err)
+			}
+		}
+
+		log.Printf("applying initial garage layout")
+		err = garageApplyLayout(
+			ctx,
+			garageAdminClient,
+			*hostName, ip.String(),
+			daemon.Storage.Allocations,
+		)
+
+		if err != nil {
+			return fmt.Errorf("applying initial garage layout: %w", err)
+		}
+
+		log.Printf("initializing garage shared global bucket")
+		err = garageInitializeGlobalBucket(
+			ctx,
+			garageAdminClient,
+			env.Bootstrap.GarageGlobalBucketS3APICredentials,
+		)
+
+		if err != nil {
+			return fmt.Errorf("initializing garage shared global bucket: %w", err)
+		}
+
+		garageS3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
+		if err != nil {
+			return fmt.Errorf("initializing garage shared global bucket client: %w", err)
+		}
+
+		log.Printf("writing data for this host into garage")
+		err = bootstrap.PutGarageBoostrapHost(ctx, garageS3Client, env.Bootstrap.ThisHost())
+		if err != nil {
+			return fmt.Errorf("putting host data into garage: %w", err)
+		}
+
+		log.Printf("cluster initialized successfully, writing admin.tgz to stdout")
+
+		err = admin.Admin{
+			CreationParams:                     adminCreationParams,
+			NebulaCACert:                       nebulaCACert,
+			GarageRPCSecret:                    env.Bootstrap.GarageRPCSecret,
+			GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials,
+			GarageAdminBucketS3APICredentials:  garage.NewS3APICredentials(),
+		}.WriteTo(os.Stdout)
+
+		if err != nil {
+			return fmt.Errorf("writing admin.tgz to stdout")
+		}
+
+		return nil
+	},
+}
+
 var subCmdAdminMakeBootstrap = subCmd{
 	name:      "make-bootstrap",
 	descr:     "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
@@ -92,7 +388,12 @@ var subCmdAdminMakeBootstrap = subCmd{
 			return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
 		}
 
-		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
+		ip := net.ParseIP(host.Nebula.IP)
+		if ip == nil {
+			return fmt.Errorf("invalid IP stored with host %q: %q", *name, host.Nebula.IP)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, ip)
 		if err != nil {
 			return fmt.Errorf("creating new nebula host key/cert: %w", err)
 		}

go-workspace/src/cmd/entrypoint/daemon_util.go

@@ -4,14 +4,10 @@ import (
 	"bytes"
 	crypticnet "cryptic-net"
 	"cryptic-net/bootstrap"
-	"cryptic-net/garage"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"path/filepath"
-	"strconv"
-	"time"
 
 	"github.com/cryptic-io/pmux/pmuxlib"
 )
@@ -83,29 +79,6 @@ func waitForNebulaArgs(env *crypticnet.Env, args ...string) []string {
 	return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
 }
 
-func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
-
-	thisHost := env.Bootstrap.ThisHost()
-	allocs := env.ThisDaemon().Storage.Allocations
-
-	if len(allocs) == 0 {
-		return waitForNebulaArgs(env, args...)
-	}
-
-	var preArgs []string
-
-	for _, alloc := range allocs {
-		preArgs = append(
-			preArgs,
-			"wait-for",
-			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-			"--",
-		)
-	}
-
-	return append(preArgs, args...)
-}
-
 func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
 	return pmuxlib.ProcessConfig{
 		Name: "nebula",
@@ -115,91 +88,3 @@ func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
 		},
 	}
 }
-
-func garageWriteChildConf(
-	env *crypticnet.Env,
-	alloc crypticnet.DaemonYmlStorageAllocation,
-) (
-	string, error,
-) {
-
-	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
-		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
-	}
-
-	thisHost := env.Bootstrap.ThisHost()
-
-	peer := garage.Peer{
-		IP:        thisHost.Nebula.IP,
-		RPCPort:   alloc.RPCPort,
-		S3APIPort: alloc.S3APIPort,
-	}
-
-	pubKey, privKey := peer.RPCPeerKey()
-
-	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
-	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
-
-	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
-		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
-
-	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
-		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
-	}
-
-	garageTomlPath := filepath.Join(
-		env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
-	)
-
-	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
-		MetaPath: alloc.MetaPath,
-		DataPath: alloc.DataPath,
-
-		RPCSecret:  env.Bootstrap.GarageRPCSecret,
-		AdminToken: env.Bootstrap.GarageAdminToken,
-
-		RPCAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-		APIAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
-		AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
-
-		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
-	})
-
-	if err != nil {
-		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
-	}
-
-	return garageTomlPath, nil
-}
-
-func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
-
-	var pmuxProcConfigs []pmuxlib.ProcessConfig
-
-	for _, alloc := range env.ThisDaemon().Storage.Allocations {
-
-		childConfPath, err := garageWriteChildConf(env, alloc)
-
-		if err != nil {
-			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
-		}
-
-		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
-			Name:        fmt.Sprintf("garage-%d", alloc.RPCPort),
-			Cmd:         "garage",
-			Args:        []string{"-c", childConfPath, "server"},
-			SigKillWait: 1 * time.Minute,
-		})
-	}
-
-	return pmuxProcConfigs, nil
-}
-
-func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
-	return pmuxlib.ProcessConfig{
-		Name:        "garage-apply-layout-diff",
-		Cmd:         "bash",
-		Args:        waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
-		NoRestartOn: []int{0},
-	}
-}

go-workspace/src/cmd/entrypoint/garage_util.go

@@ -0,0 +1,189 @@
+package entrypoint
+
+import (
+	"context"
+	crypticnet "cryptic-net"
+	"cryptic-net/garage"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/cryptic-io/pmux/pmuxlib"
+)
+
+func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
+
+	thisHost := env.Bootstrap.ThisHost()
+	allocs := env.ThisDaemon().Storage.Allocations
+
+	if len(allocs) == 0 {
+		return waitForNebulaArgs(env, args...)
+	}
+
+	var preArgs []string
+
+	for _, alloc := range allocs {
+		preArgs = append(
+			preArgs,
+			"wait-for",
+			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+			"--",
+		)
+	}
+
+	return append(preArgs, args...)
+}
+
+func garageWriteChildConf(
+	env *crypticnet.Env,
+	alloc crypticnet.DaemonYmlStorageAllocation,
+) (
+	string, error,
+) {
+
+	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
+		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
+	}
+
+	thisHost := env.Bootstrap.ThisHost()
+
+	peer := garage.Peer{
+		IP:        thisHost.Nebula.IP,
+		RPCPort:   alloc.RPCPort,
+		S3APIPort: alloc.S3APIPort,
+	}
+
+	pubKey, privKey := peer.RPCPeerKey()
+
+	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
+	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
+
+	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
+		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
+
+	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
+		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
+	}
+
+	garageTomlPath := filepath.Join(
+		env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
+	)
+
+	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
+		MetaPath: alloc.MetaPath,
+		DataPath: alloc.DataPath,
+
+		RPCSecret:  env.Bootstrap.GarageRPCSecret,
+		AdminToken: env.Bootstrap.GarageAdminToken,
+
+		RPCAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+		APIAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
+		AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
+
+		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
+	}
+
+	return garageTomlPath, nil
+}
+
+func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
+
+	var pmuxProcConfigs []pmuxlib.ProcessConfig
+
+	for _, alloc := range env.ThisDaemon().Storage.Allocations {
+
+		childConfPath, err := garageWriteChildConf(env, alloc)
+
+		if err != nil {
+			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
+		}
+
+		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
+			Name:        fmt.Sprintf("garage-%d", alloc.RPCPort),
+			Cmd:         "garage",
+			Args:        []string{"-c", childConfPath, "server"},
+			SigKillWait: 1 * time.Minute,
+		})
+	}
+
+	return pmuxProcConfigs, nil
+}
+
+func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
+	return pmuxlib.ProcessConfig{
+		Name:        "garage-apply-layout-diff",
+		Cmd:         "bash",
+		Args:        waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
+		NoRestartOn: []int{0},
+	}
+}
+
+func garageApplyLayout(
+	ctx context.Context,
+	adminClient *garage.AdminClient,
+	hostName, ipStr string,
+	allocs []crypticnet.DaemonYmlStorageAllocation,
+) error {
+
+	type peerLayout struct {
+		Capacity int      `json:"capacity"`
+		Zone     string   `json:"zone"`
+		Tags     []string `json:"tags"`
+	}
+
+	{
+		clusterLayout := map[string]peerLayout{}
+
+		for _, alloc := range allocs {
+
+			peer := garage.Peer{
+				IP:        ipStr,
+				RPCPort:   alloc.RPCPort,
+				S3APIPort: alloc.S3APIPort,
+			}
+
+			clusterLayout[peer.RPCPeerID()] = peerLayout{
+				Capacity: alloc.Capacity / 100,
+				Zone:     hostName,
+			}
+		}
+
+		err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
+		if err != nil {
+			return fmt.Errorf("staging layout changes: %w", err)
+		}
+	}
+
+	var clusterLayout struct {
+		Version           int                   `json:"version"`
+		StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
+	}
+
+	if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
+		return fmt.Errorf("retrieving staged layout change: %w", err)
+	}
+
+	if len(clusterLayout.StagedRoleChanges) == 0 {
+		return nil
+	}
+
+	applyClusterLayout := struct {
+		Version int `json:"version"`
+	}{
+		Version: clusterLayout.Version + 1,
+	}
+
+	err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
+	if err != nil {
+		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
+	}
+
+	return nil
+}

go-workspace/src/garage/admin_client.go

@@ -82,3 +82,38 @@ func (c *AdminClient) Do(
 
 	return nil
 }
+
+// Wait will block until the instance connected to can see at least
+// ReplicationFactor-1 other garage instances. If the context is canceled it
+// will return the context error.
+func (c *AdminClient) Wait(ctx context.Context) error {
+	for {
+
+		var clusterStatus struct {
+			KnownNodes map[string]struct {
+				IsUp bool `json:"is_up"`
+			} `json:"knownNodes"`
+		}
+
+		err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
+
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return ctxErr
+
+		} else if err != nil {
+			continue
+		}
+
+		var numUp int
+
+		for _, knownNode := range clusterStatus.KnownNodes {
+			if knownNode.IsUp {
+				numUp++
+			}
+		}
+
+		if numUp >= ReplicationFactor-1 {
+			return nil
+		}
+	}
+}

go-workspace/src/garage/client.go

@@ -1,12 +1,22 @@
 package garage
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
+func randStr(l int) string {
+	b := make([]byte, l)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(b)
+}
+
 // IsKeyNotFound returns true if the given error is the result of a key not
 // being found in a bucket.
 func IsKeyNotFound(err error) bool {
@@ -24,6 +34,14 @@ type S3APICredentials struct {
 	Secret string `yaml:"secret"`
 }
 
+// NewS3APICredentials returns a new usable instance of S3APICredentials.
+func NewS3APICredentials() S3APICredentials {
+	return S3APICredentials{
+		ID:     randStr(8),
+		Secret: randStr(32),
+	}
+}
+
 // NewS3APIClient returns a minio client configured to use the given garage S3 API
 // endpoint.
 func NewS3APIClient(addr string, creds S3APICredentials) (S3APIClient, error) {

go-workspace/src/garage/garage.go

@@ -10,4 +10,8 @@ const (
 	// GlobalBucket is the name of the global garage bucket which is
 	// accessible to all hosts in the network.
 	GlobalBucket = "cryptic-net-global"
+
+	// ReplicationFactor indicates the replication factor set on the garage
+	// cluster. We currently only support a factor of 3.
+	ReplicationFactor = 3
 )

go-workspace/src/nebula/nebula.go

@@ -14,15 +14,6 @@ import (
 	"golang.org/x/crypto/curve25519"
 )
 
-// TODO this should one day not be hardcoded
-var ipCIDRMask = func() net.IPMask {
-	_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
-	if err != nil {
-		panic(err)
-	}
-	return ipNet.Mask
-}()
-
 // HostCert contains the certificate and private key files which will need to
 // be present on a particular host. Each file is PEM encoded.
 type HostCert struct {
@@ -41,7 +32,7 @@ type CACert struct {
 // NewHostCert generates a new key/cert for a nebula host using the CA key
 // which will be found in the adminFS.
 func NewHostCert(
-	caCert CACert, hostName, hostIP string,
+	caCert CACert, hostName string, ip net.IP,
 ) (
 	HostCert, error,
 ) {
@@ -66,14 +57,9 @@ func NewHostCert(
 
 	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
 
-	ip := net.ParseIP(hostIP)
+	subnet := caCrt.Details.Subnets[0]
-	if ip == nil {
+	if !subnet.Contains(ip) {
-		return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
+		return HostCert{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
-	}
-
-	ipNet := &net.IPNet{
-		IP:   ip,
-		Mask: ipCIDRMask,
 	}
 
 	var hostPub, hostKey []byte
@@ -88,8 +74,11 @@ func NewHostCert(
 
 	hostCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
-			Name:      hostName,
+			Name: hostName,
-			Ips:       []*net.IPNet{ipNet},
+			Ips: []*net.IPNet{{
+				IP:   ip,
+				Mask: subnet.Mask,
+			}},
 			NotBefore: time.Now(),
 			NotAfter:  expireAt,
 			PublicKey: hostPub,
@@ -122,7 +111,7 @@ func NewHostCert(
 
 // NewCACert generates a CACert. The domain should be the network's root domain,
 // and is included in the signing certificate's Name field.
-func NewCACert(domain string) (CACert, error) {
+func NewCACert(domain string, subnet *net.IPNet) (CACert, error) {
 
 	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
 	if err != nil {
@@ -135,6 +124,7 @@ func NewCACert(domain string) (CACert, error) {
 	caCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
 			Name:      fmt.Sprintf("%s cryptic-net root cert", domain),
+			Subnets:   []*net.IPNet{subnet},
 			NotBefore: now,
 			NotAfter:  expireAt,
 			PublicKey: pubKey,