WIP

The daemon entrypoint now starts the garage child processes directly,
without the extra step of indirection

AppDir/etc/daemon.yml

@@ -64,7 +64,7 @@ storage:
   # Capacity declares how many gigabytes can be stored in each allocation, and
   # is required. It must be a multiple of 100.
   #
-  # The ports are all required and must all be unique within and across
+  # The various ports are all required and must all be unique within and across
   # allocations.
   allocations:
 
@@ -73,4 +73,4 @@ storage:
     #  capacity: 1200
     #  api_port: 3900
     #  rpc_port: 3901
-    #  admin_port: 3902
+    #  web_port: 3902

docs/operator/contributing-storage.md

@@ -36,6 +36,7 @@ storage:
       capacity: 1200
       api_port: 3900
       rpc_port: 3901
+      web_port: 3902
 
     # 100 GB (the minimum) are being shared from drive2
     - data_path: /mnt/drive2/cryptic-net/data
@@ -43,6 +44,7 @@ storage:
       capacity: 100
       api_port: 3910
       rpc_port: 3911
+      web_port: 3912
 ```
 
 ## Setup Firewall

go-workspace/src/bootstrap/bootstrap.go

@@ -36,7 +36,6 @@ type Bootstrap struct {
 	NebulaHostCert nebula.HostCert
 
 	GarageRPCSecret                    string
-	GarageAdminToken                   string
 	GarageGlobalBucketS3APICredentials garage.S3APICredentials
 }
 
@@ -76,7 +75,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 		{&b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
 		{&b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
 		{&b.GarageRPCSecret, garageRPCSecretPath},
-		{&b.GarageAdminToken, garageAdminTokenPath},
 	}
 
 	for _, f := range filesToLoadAsString {
@@ -157,7 +155,6 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
 		{b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
 		{b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
 		{b.GarageRPCSecret, garageRPCSecretPath},
-		{b.GarageAdminToken, garageAdminTokenPath},
 	}
 
 	for _, f := range filesToWriteAsString {

go-workspace/src/bootstrap/garage.go

@@ -7,9 +7,8 @@ import (
 
 // Paths within the bootstrap FS related to garage.
 const (
-	garageRPCSecretPath          = "garage/rpc-secret.txt"
-	garageAdminTokenPath         = "garage/admin-token.txt"
 	garageGlobalBucketKeyYmlPath = "garage/cryptic-net-global-bucket-key.yml"
+	garageRPCSecretPath          = "garage/rpc-secret.txt"
 )
 
 // GaragePeers returns a Peer for each known garage instance in the network.

go-workspace/src/bootstrap/hosts.go

@@ -24,6 +24,7 @@ type NebulaHost struct {
 type GarageHostInstance struct {
 	RPCPort   int `yaml:"rpc_port"`
 	S3APIPort int `yaml:"s3_api_port"`
+	WebPort   int `yaml:"web_port"`
 }
 
 // GarageHost describes the garage configuration of a Host which is relevant for

go-workspace/src/cmd/entrypoint/admin.go

@@ -2,10 +2,8 @@ package entrypoint
 
 import (
 	"context"
-	crypticnet "cryptic-net"
 	"cryptic-net/admin"
 	"cryptic-net/bootstrap"
-	"cryptic-net/garage"
 	"cryptic-net/nebula"
 	"crypto/rand"
 	"encoding/hex"
@@ -13,7 +11,6 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"strconv"
 	"strings"
 
 	"github.com/cryptic-io/pmux/pmuxlib"
@@ -48,73 +45,6 @@ func readAdmin(path string) (admin.Admin, error) {
 	return admin.FromReader(f)
 }
 
-func garageInitializeGlobalBucket(
-	env *crypticnet.Env, globalBucketCreds garage.S3APICredentials,
-) error {
-
-	var (
-		ctx        = env.Context
-		thisHost   = env.Bootstrap.ThisHost()
-		thisDaemon = env.ThisDaemon()
-		allocs     = thisDaemon.Storage.Allocations
-	)
-
-	adminClient := garage.NewAdminClient(
-		net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(allocs[0].AdminPort)),
-		env.Bootstrap.GarageAdminToken,
-	)
-
-	// first attempt to import the key
-	err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
-		"accessKeyId":     globalBucketCreds.ID,
-		"secretAccessKey": globalBucketCreds.Secret,
-		"name":            "shared-global-bucket-key",
-	})
-
-	if err != nil {
-		return fmt.Errorf("importing global bucket key into garage: %w", err)
-	}
-
-	// create global bucket
-	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
-		"globalAlias": garage.GlobalBucket,
-	})
-
-	if err != nil {
-		return fmt.Errorf("creating global bucket: %w", err)
-	}
-
-	// retrieve newly created bucket's id
-	var getBucketRes struct {
-		ID string `json:"id"`
-	}
-
-	err = adminClient.Do(
-		ctx, &getBucketRes,
-		"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
-	)
-
-	if err != nil {
-		return fmt.Errorf("fetching global bucket id: %w", err)
-	}
-
-	// allow shared global bucket key to perform all operations
-	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
-		"bucketId":    getBucketRes.ID,
-		"accessKeyId": globalBucketCreds.ID,
-		"permissions": map[string]bool{
-			"read":  true,
-			"write": true,
-		},
-	})
-
-	if err != nil {
-		return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
-	}
-
-	return nil
-}
-
 var subCmdAdminCreateNetwork = subCmd{
 	name:  "create-network",
 	descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
@@ -253,6 +183,7 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		ctx, cancel := context.WithCancel(env.Context)
+		defer cancel()
 		pmuxDoneCh := make(chan struct{})
 
 		go func() {
@@ -260,21 +191,6 @@ var subCmdAdminCreateNetwork = subCmd{
 			close(pmuxDoneCh)
 		}()
 
-		defer func() {
-			cancel()
-			<-pmuxDoneCh
-		}()
-
-		globalBucketCreds := garage.S3APICredentials{} // TODO
-
-		// TODO wait for garage to be confirmed as booted up
-		// TODO apply layout
-
-		if err := garageInitializeGlobalBucket(env, globalBucketCreds); err != nil {
-			return fmt.Errorf("initializing shared global bucket: %w", err)
-		}
-
-		panic("TODO: create and output admin.tgz")
 	},
 }
 
@@ -330,12 +246,7 @@ var subCmdAdminMakeBootstrap = subCmd{
 			return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
 		}
 
-		ip := net.ParseIP(host.Nebula.IP)
-		if ip == nil {
-			return fmt.Errorf("invalid IP stored with host %q: %q", *name, host.Nebula.IP)
-		}
-
-		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, ip)
+		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
 		if err != nil {
 			return fmt.Errorf("creating new nebula host key/cert: %w", err)
 		}
@@ -349,7 +260,6 @@ var subCmdAdminMakeBootstrap = subCmd{
 			NebulaHostCert: nebulaHostCert,
 
 			GarageRPCSecret:                    adm.GarageRPCSecret,
-			GarageAdminToken:                   randStr(32),
 			GarageGlobalBucketS3APICredentials: adm.GarageGlobalBucketS3APICredentials,
 		}
 

go-workspace/src/cmd/entrypoint/daemon_util.go

@@ -60,6 +60,7 @@ func mergeDaemonIntoBootstrap(env *crypticnet.Env) error {
 			host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
 				RPCPort:   alloc.RPCPort,
 				S3APIPort: alloc.S3APIPort,
+				WebPort:   alloc.WebPort,
 			})
 		}
 	}
@@ -155,12 +156,11 @@ func garageWriteChildConf(
 		MetaPath: alloc.MetaPath,
 		DataPath: alloc.DataPath,
 
-		RPCSecret:  env.Bootstrap.GarageRPCSecret,
-		AdminToken: env.Bootstrap.GarageAdminToken,
+		RPCSecret: env.Bootstrap.GarageRPCSecret,
 
-		RPCAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-		APIAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
-		AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
+		RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+		APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
+		WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
 
 		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
 	})

go-workspace/src/cmd/nebula-entrypoint/main.go

@@ -102,6 +102,11 @@ func Main() {
 				Proto: "tcp",
 				Host:  "any",
 			},
+			crypticnet.ConfigFirewallRule{
+				Port:  strconv.Itoa(alloc.WebPort),
+				Proto: "tcp",
+				Host:  "any",
+			},
 		)
 	}
 

go-workspace/src/daemon_yml.go

@@ -31,9 +31,9 @@ type DaemonYmlStorageAllocation struct {
 	DataPath  string `yaml:"data_path"`
 	MetaPath  string `yaml:"meta_path"`
 	Capacity  int    `yaml:"capacity"`
-	S3APIPort int    `yaml:"s3_api_port"`
+	S3APIPort int    `yaml:"api_port"` // TODO fix field name here
 	RPCPort   int    `yaml:"rpc_port"`
-	AdminPort int    `yaml:"admin_port"`
+	WebPort   int    `yaml:"web_port"`
 }
 
 // DaemonYml describes the structure of the daemon.yml file.

go-workspace/src/garage/admin_client.go

@@ -1,84 +0,0 @@
-package garage
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-)
-
-// AdminClient is a helper type for performing actions against the garage admin
-// interface.
-type AdminClient struct {
-	c          *http.Client
-	addr       string
-	adminToken string
-}
-
-// NewAdminClient initializes and returns an AdminClient which will use the
-// given address and adminToken for all requests made.
-func NewAdminClient(addr, adminToken string) *AdminClient {
-	return &AdminClient{
-		c: &http.Client{
-			Transport: http.DefaultTransport.(*http.Transport).Clone(),
-		},
-		addr:       addr,
-		adminToken: adminToken,
-	}
-}
-
-// Do performs an HTTP request with the given method (GET, POST) and path, and
-// using the json marshaling of the given body as the request body (unless body
-// is nil). It will JSON unmarshal the response into rcv, unless rcv is nil.
-func (c *AdminClient) Do(
-	ctx context.Context, rcv interface{}, method, path string, body interface{},
-) error {
-
-	var bodyR io.Reader
-
-	if body != nil {
-		bodyBuf := new(bytes.Buffer)
-		bodyR = bodyBuf
-
-		if err := json.NewEncoder(bodyBuf).Encode(body); err != nil {
-			return fmt.Errorf("json marshaling body: %w", err)
-		}
-	}
-
-	urlStr := fmt.Sprintf("http://%s%s", c.addr, path)
-
-	req, err := http.NewRequestWithContext(ctx, method, urlStr, bodyR)
-	if err != nil {
-		return fmt.Errorf("initializing request: %w", err)
-	}
-
-	req.Header.Set("Authorization", "Bearer "+c.adminToken)
-
-	res, err := c.c.Do(req)
-	if err != nil {
-		return fmt.Errorf("performing http request: %w", err)
-	}
-
-	defer res.Body.Close()
-
-	if res.StatusCode != 200 {
-		return fmt.Errorf("unexpected %s response returned", res.Status)
-	}
-
-	if rcv == nil {
-
-		if _, err := io.Copy(io.Discard, res.Body); err != nil {
-			return fmt.Errorf("discarding response body: %w", err)
-		}
-
-		return nil
-	}
-
-	if err := json.NewDecoder(res.Body).Decode(rcv); err != nil {
-		return fmt.Errorf("decoding json response body: %w", err)
-	}
-
-	return nil
-}

go-workspace/src/garage/tpl.go

@@ -13,12 +13,11 @@ type GarageTomlData struct {
 	MetaPath string
 	DataPath string
 
-	RPCSecret  string
-	AdminToken string
+	RPCSecret string
 
-	RPCAddr   string
-	APIAddr   string
-	AdminAddr string
+	RPCAddr string
+	APIAddr string
+	WebAddr string
 
 	BootstrapPeers []string
 }
@@ -42,9 +41,9 @@ bootstrap_peers = [{{- range .BootstrapPeers }}
 api_bind_addr = "{{ .APIAddr }}"
 s3_region = "garage"
 
-[admin]
-api_bind_addr = "{{ .AdminAddr }}"
-admin_token = "{{ .AdminToken }}"
+[s3_web]
+bind_addr = "{{ .WebAddr }}"
+root_domain = ".example.com"
 
 `))