From f13a08abfb1d5862981a9cb328af0cb96aa7cb4c Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 10 Jun 2024 18:56:36 +0200 Subject: [PATCH] Use JSON instead of YAML for files which aren't intended for human editing --- default.nix | 3 +- docs/admin/adding-a-host-to-the-network.md | 42 +++++++------- docs/admin/creating-a-new-network.md | 14 ++--- docs/dev/daemon-process-tree.plantuml | 4 +- docs/dev/daemon-process-tree.svg | 58 +------------------ docs/user/getting-started.md | 6 +- go/.golangci.yml | 4 ++ go/admin/admin.go | 35 ++++++----- go/bootstrap/bootstrap.go | 58 +++++++++---------- go/bootstrap/garage_global_bucket.go | 26 ++++----- go/bootstrap/hosts.go | 31 +++++----- go/cmd/entrypoint/admin.go | 24 ++++---- go/cmd/entrypoint/daemon.go | 10 ++-- go/cmd/entrypoint/hosts.go | 22 +++---- go/cmd/entrypoint/nebula.go | 18 +++--- go/cmd/entrypoint/nebula_util.go | 2 +- go/daemon/config.go | 1 - go/garage/client.go | 4 +- go/go.mod | 4 +- go/go.sum | 4 ++ go/jsonutil/jsonutil.go | 58 +++++++++++++++++++ go/nebula/nebula.go | 18 +++--- go/yamlutil/yamlutil.go | 35 ++--------- tests/cases/admin/01-create-network.sh | 14 ++--- tests/cases/admin/02-create-bootstrap.sh | 12 ++-- .../utils/with-1-data-1-empty-node-cluster.sh | 6 +- 26 files changed, 247 insertions(+), 266 deletions(-) create mode 100644 go/.golangci.yml create mode 100644 go/jsonutil/jsonutil.go diff --git a/default.nix b/default.nix index 391f858..9821512 100644 --- a/default.nix +++ b/default.nix @@ -102,7 +102,7 @@ in rec { builder = builtins.toFile "builder.sh" '' source $stdenv/setup mkdir -p "$out"/share - cp "$src" "$out"/share/bootstrap.yml + cp "$src" "$out"/share/bootstrap.json ''; }; @@ -168,7 +168,6 @@ in rec { export PATH=${pkgs.lib.makeBinPath [ appImage pkgs.busybox - pkgs.yq-go pkgs.jq pkgs.dig ]} diff --git a/docs/admin/adding-a-host-to-the-network.md b/docs/admin/adding-a-host-to-the-network.md index 1deb720..552ff7c 100644 --- a/docs/admin/adding-a-host-to-the-network.md +++ b/docs/admin/adding-a-host-to-the-network.md @@ -7,12 +7,12 @@ wishes to add. There are two ways for a user to add a host to the isle network. - If the user is savy enough to obtain their own `isle` binary, they can - do so. The admin can then generate a `bootstrap.yml` file for their host, + do so. The admin can then generate a `bootstrap.json` file for their host, give that to the user, and the user can run `isle daemon` using that bootstrap file. - If the user is not so savy, the admin can generate a custom `isle` - binary with the `bootstrap.yml` embedded into it. The user can be given this + binary with the `bootstrap.json` embedded into it. The user can be given this binary and run `isle daemon` without any configuration on their end. From the admin's perspective the only difference between these cases is one @@ -35,57 +35,57 @@ The admin should choose an IP for the host. The IP you choose for the new host should be one which is not yet used by any other host, and which is in subnet which was configured when creating the network. -## Step 3: Create a `bootstrap.yml` File +## Step 3: Create a `bootstrap.json` File -Access to an `admin.yml` file is required for this step. +Access to an `admin.json` file is required for this step. -To create a `bootstrap.yml` file for the new host, the admin should perform the +To create a `bootstrap.json` file for the new host, the admin should perform the following command from their own host: ``` isle admin create-bootstrap \ --hostname \ --ip \ - --admin-path \ - > bootstrap.yml + --admin-path \ + > bootstrap.json ``` -The resulting `bootstrap.yml` file should be treated as a secret file that is -shared only with the user it was generated for. The `bootstrap.yml` file should +The resulting `bootstrap.json` file should be treated as a secret file that is +shared only with the user it was generated for. The `bootstrap.json` file should not be re-used between hosts either. If the user already has access to a `isle` binary then the new -`bootstrap.yml` file can be given to them as-is, and they can proceed with +`bootstrap.json` file can be given to them as-is, and they can proceed with running their host's `isle daemon`. -### Encrypted `admin.yml` +### Encrypted `admin.json` -If `admin.yml` is kept in an encrypted format on disk (it should be!) then the +If `admin.json` is kept in an encrypted format on disk (it should be!) then the decrypted form can be piped into `create-bootstrap` over stdin. For example, if -GPG is being used to secure `admin.yml` then the following could be used to -generate a `bootstrap.yml`: +GPG is being used to secure `admin.json` then the following could be used to +generate a `bootstrap.json`: ``` -gpg -d | isle admin create-bootstrap \ +gpg -d | isle admin create-bootstrap \ --hostname \ --ip \ --admin-path - \ - > bootstrap.yml + > bootstrap.json ``` -Note that the value of `--admin-path` is `-`, indicating that `admin.yml` should -be read from stdin. +Note that the value of `--admin-path` is `-`, indicating that `admin.json` +should be read from stdin. ## Step 4: Optionally, Build Binary -If you wish to embed the `bootstrap.yml` into a custom binary for the user (to +If you wish to embed the `bootstrap.json` into a custom binary for the user (to make installation _extremely_ easy for them) then you can run the following: ``` -nix-build --arg bootstrap -A appImage +nix-build --arg bootstrap -A appImage ``` The resulting binary can be found in the `result` directory which is created. -This binary should be treated like a `bootstrap.yml` in terms of its uniqueness +This binary should be treated like a `bootstrap.json` in terms of its uniqueness and sensitivity. diff --git a/docs/admin/creating-a-new-network.md b/docs/admin/creating-a-new-network.md index ab3e6f5..32f380d 100644 --- a/docs/admin/creating-a-new-network.md +++ b/docs/admin/creating-a-new-network.md @@ -83,9 +83,9 @@ be chosen with care. * IP: The IP of your host, which will be the first host in the network. This IP must be within the chosen subnet range. -## Step 3: Prepare to Encrypt `admin.yml` +## Step 3: Prepare to Encrypt `admin.json` -The `admin.yml` file (which will be created in the next step) is the most +The `admin.json` file (which will be created in the next step) is the most sensitive part of an isle network. If it falls into the wrong hands it can be used to completely compromise your network, impersonate hosts on the network, and will likely lead to someone stealing or deleting all of your data. @@ -97,9 +97,9 @@ This guide assumes that you have GPG already set up with your own secret key, and that you are familiar with how it works. There is no requirement to use GPG, if you care to use a different method. -## Step 4: Create the `admin.yml` File +## Step 4: Create the `admin.json` File -To create the `admin.yml` file, which effectively creates the network itself, +To create the `admin.json` file, which effectively creates the network itself, you can run: ``` @@ -110,7 +110,7 @@ sudo isle admin create-network \ --domain \ --hostname \ | gpg -e -r \ - > admin.yml.gpg + > admin.json.gpg ``` A couple of notes here: @@ -121,14 +121,14 @@ A couple of notes here: * Only one gpg recipient is specified. If you intend on including other users as network administrators you can add them to the recipients list at this step, - so they will be able to use the `admin.yml` file as well. You can also + so they will be able to use the `admin.json` file as well. You can also manually add them as recipients later. You will see a lot of output, as `create-network` starts up many child processes in order to set the network up. It should exit successfully on its own after a few seconds. -At this point you should have an `admin.yml.gpg` file in your current directory. +At this point you should have an `admin.json.gpg` file in your current directory. ## Step 5: Run the Daemon diff --git a/docs/dev/daemon-process-tree.plantuml b/docs/dev/daemon-process-tree.plantuml index 8b091c2..bac3f61 100644 --- a/docs/dev/daemon-process-tree.plantuml +++ b/docs/dev/daemon-process-tree.plantuml @@ -15,8 +15,8 @@ state AppDir { entrypoint : * Create runtime dir at $_RUNTIME_DIR_PATH entrypoint : * Lock runtime dir entrypoint : * Merge given and default daemon.yml files - entrypoint : * Copy bootstrap.yml into $_DATA_DIR_PATH, if it's not there - entrypoint : * Merge daemon.yml config into bootstrap.yml + entrypoint : * Copy bootstrap.json into $_DATA_DIR_PATH, if it's not there + entrypoint : * Merge daemon.yml config into bootstrap.json entrypoint : * Create $_RUNTIME_DIR_PATH/dnsmasq.conf entrypoint : * Create $_RUNTIME_DIR_PATH/nebula.yml entrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n (one per storage allocation) diff --git a/docs/dev/daemon-process-tree.svg b/docs/dev/daemon-process-tree.svg index 4164532..4304b0c 100644 --- a/docs/dev/daemon-process-tree.svg +++ b/docs/dev/daemon-process-tree.svg @@ -1,57 +1 @@ -AppDirAll relative paths are relative to the root of the AppDir./bin/dnsmasq -d -C $_RUNTIME_DIR_PATH/dnsmasq.conf./bin/nebula -config $_RUNTIME_DIR_PATH/nebula.yml./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server./AppRunSet PATH to APPDIR/bin./bin/entrypoint daemon -c ./daemon.ymlCreate runtime dir at $_RUNTIME_DIR_PATHLock runtime dirMerge given and default daemon.yml filesCopy bootstrap.yml into $_DATA_DIR_PATH, if it's not thereMerge daemon.yml config into bootstrap.ymlCreate $_RUNTIME_DIR_PATH/dnsmasq.confCreate $_RUNTIME_DIR_PATH/nebula.ymlCreate $_RUNTIME_DIR_PATH/garage-N.toml(one per storage allocation)Run child processes(in the background) Updates garage cluster layout(in the background) Stores host info in global bucket./isle daemon -c ./daemon.ymlexecexecchildchildchild (one per storage allocation) \ No newline at end of file +AppDirAll relative paths are relative to the root of the AppDir./AppRunSet PATH to APPDIR/bin./bin/entrypoint daemon -c ./daemon.ymlCreate runtime dir at $_RUNTIME_DIR_PATHLock runtime dirMerge given and default daemon.yml filesCopy bootstrap.json into $_DATA_DIR_PATH, if it's not thereMerge daemon.yml config into bootstrap.jsonCreate $_RUNTIME_DIR_PATH/dnsmasq.confCreate $_RUNTIME_DIR_PATH/nebula.ymlCreate $_RUNTIME_DIR_PATH/garage-N.toml(one per storage allocation)Run child processes(in the background) Updates garage cluster layout(in the background) Stores host info in global bucket./bin/dnsmasq -d -C $_RUNTIME_DIR_PATH/dnsmasq.conf./bin/nebula -config $_RUNTIME_DIR_PATH/nebula.yml./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server./isle daemon -c ./daemon.ymlexecexecchildchildchild (one per storage allocation) \ No newline at end of file diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index 2934ff9..26184fc 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -43,7 +43,7 @@ The resulting binary can be found in the `result` directory which is created. ## Obtaining Your Bootstrap File -The `bootstrap.yml` file contains all information required for your particular +The `bootstrap.json` file contains all information required for your particular host to join the network, and must be generated and provided to you by an admin for the network. @@ -54,11 +54,11 @@ sub-command as the root user. This can most easily be done using the `sudo` command, in a terminal: ``` -sudo /path/to/isle daemon --bootstrap-path /path/to/bootstrap.yml +sudo /path/to/isle daemon --bootstrap-path /path/to/bootstrap.json ``` This will start the daemon process, which will keep running until you kill it -with `ctrl-c`. The `--bootstrap-path /path/to/bootstrap.yml` argument is only +with `ctrl-c`. The `--bootstrap-path /path/to/bootstrap.json` argument is only required the first time the daemon is run, it will be ignored on subsequent runs. diff --git a/go/.golangci.yml b/go/.golangci.yml new file mode 100644 index 0000000..267cd9d --- /dev/null +++ b/go/.golangci.yml @@ -0,0 +1,4 @@ +# https://github.com/golangci/golangci-lint/issues/4733 +linters-settings: + errcheck: + ignore : "" diff --git a/go/admin/admin.go b/go/admin/admin.go index e215351..6bfe797 100644 --- a/go/admin/admin.go +++ b/go/admin/admin.go @@ -1,44 +1,43 @@ -// Package admin deals with the parsing and creation of admin.yml files. +// Package admin deals with the parsing and creation of admin.json files. package admin import ( + "encoding/json" + "io" "isle/garage" "isle/nebula" - "io" - - "gopkg.in/yaml.v3" ) // CreationParams are general parameters used when creating a new network. These // are available to all hosts within the network via their bootstrap files. type CreationParams struct { - ID string `yaml:"id"` - Name string `yaml:"name"` - Domain string `yaml:"domain"` + ID string + Name string + Domain string } -// Admin is used for accessing all information contained within an admin.yml. +// Admin is used for accessing all information contained within an admin.json. type Admin struct { - CreationParams CreationParams `yaml:"creation_params"` + CreationParams CreationParams Nebula struct { - CACredentials nebula.CACredentials `yaml:"ca_credentials"` - } `yaml:"nebula"` + CACredentials nebula.CACredentials + } Garage struct { - RPCSecret string `yaml:"rpc_secret"` - GlobalBucketS3APICredentials garage.S3APICredentials `yaml:"global_bucket_s3_api_credentials"` - } `yaml:"garage"` + RPCSecret string + GlobalBucketS3APICredentials garage.S3APICredentials + } } -// FromReader reads an admin.yml from the given io.Reader. +// FromReader reads an admin.json from the given io.Reader. func FromReader(r io.Reader) (Admin, error) { var a Admin - err := yaml.NewDecoder(r).Decode(&a) + err := json.NewDecoder(r).Decode(&a) return a, err } -// WriteTo writes the Admin as an admin.yml to the given io.Writer. +// WriteTo writes the Admin as an admin.json to the given io.Writer. func (a Admin) WriteTo(into io.Writer) error { - return yaml.NewEncoder(into).Encode(a) + return json.NewEncoder(into).Encode(a) } diff --git a/go/bootstrap/bootstrap.go b/go/bootstrap/bootstrap.go index 7c52e84..1ce8778 100644 --- a/go/bootstrap/bootstrap.go +++ b/go/bootstrap/bootstrap.go @@ -1,62 +1,61 @@ -// Package bootstrap deals with the parsing and creation of bootstrap.yml files. -// It also contains some helpers which rely on bootstrap data. +// Package bootstrap deals with the parsing and creation of bootstrap.json +// files. It also contains some helpers which rely on bootstrap data. package bootstrap import ( + "crypto/sha512" + "encoding/json" + "fmt" + "io" "isle/admin" "isle/garage" "isle/nebula" - "crypto/sha512" - "fmt" - "io" "os" "path/filepath" "sort" - - "gopkg.in/yaml.v3" ) // DataDirPath returns the path within the user's data directory where the // bootstrap file is stored. func DataDirPath(dataDirPath string) string { - return filepath.Join(dataDirPath, "bootstrap.yml") + return filepath.Join(dataDirPath, "bootstrap.json") } // AppDirPath returns the path within the AppDir where an embedded bootstrap // file might be found. func AppDirPath(appDirPath string) string { - return filepath.Join(appDirPath, "share/bootstrap.yml") + return filepath.Join(appDirPath, "share/bootstrap.json") } // Bootstrap is used for accessing all information contained within a -// bootstrap.yml file. +// bootstrap.json file. type Bootstrap struct { - AdminCreationParams admin.CreationParams `yaml:"admin_creation_params"` + AdminCreationParams admin.CreationParams - Hosts map[string]Host `yaml:"hosts"` - HostName string `yaml:"hostname"` + Hosts map[string]Host + HostName string Nebula struct { - CAPublicCredentials nebula.CAPublicCredentials `yaml:"ca_public_credentials"` - HostCredentials nebula.HostCredentials `yaml:"host_credentials"` - SignedPublicCredentials string `yaml:"signed_public_credentials"` - } `yaml:"nebula"` + CAPublicCredentials nebula.CAPublicCredentials + HostCredentials nebula.HostCredentials + SignedPublicCredentials string + } Garage struct { - RPCSecret string `yaml:"rpc_secret"` - AdminToken string `yaml:"admin_token"` - GlobalBucketS3APICredentials garage.S3APICredentials `yaml:"global_bucket_s3_api_credentials"` - } `yaml:"garage"` + RPCSecret string + AdminToken string + GlobalBucketS3APICredentials garage.S3APICredentials + } } -// FromReader reads a bootstrap.yml file from the given io.Reader. +// FromReader reads a bootstrap file from the given io.Reader. func FromReader(r io.Reader) (Bootstrap, error) { var b Bootstrap - err := yaml.NewDecoder(r).Decode(&b) + err := json.NewDecoder(r).Decode(&b) return b, err } -// FromFile reads a bootstrap.yml from a file at the given path. +// FromFile reads a bootstrap from a file at the given path. func FromFile(path string) (Bootstrap, error) { f, err := os.Open(path) @@ -68,9 +67,9 @@ func FromFile(path string) (Bootstrap, error) { return FromReader(f) } -// WriteTo writes the Bootstrap as a new bootstrap.yml to the given io.Writer. +// WriteTo writes the Bootstrap as a new bootstrap to the given io.Writer. func (b Bootstrap) WriteTo(into io.Writer) error { - return yaml.NewEncoder(into).Encode(b) + return json.NewEncoder(into).Encode(b) } // ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the @@ -93,11 +92,12 @@ func HostsHash(hostsMap map[string]Host) ([]byte, error) { hosts = append(hosts, host) } - sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name }) + sort.Slice(hosts, func(i, j int) bool { + return hosts[i].Name < hosts[j].Name + }) h := sha512.New() - - if err := yaml.NewEncoder(h).Encode(hosts); err != nil { + if err := json.NewEncoder(h).Encode(hosts); err != nil { return nil, err } diff --git a/go/bootstrap/garage_global_bucket.go b/go/bootstrap/garage_global_bucket.go index df6b719..caea77e 100644 --- a/go/bootstrap/garage_global_bucket.go +++ b/go/bootstrap/garage_global_bucket.go @@ -3,16 +3,16 @@ package bootstrap import ( "bytes" "context" + "encoding/json" + "fmt" "isle/garage" "isle/nebula" - "fmt" "path/filepath" "strings" "github.com/mediocregopher/mediocre-go-lib/v2/mctx" "github.com/mediocregopher/mediocre-go-lib/v2/mlog" "github.com/minio/minio-go/v7" - "gopkg.in/yaml.v3" ) // Paths within garage's global bucket @@ -20,7 +20,7 @@ const ( garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" ) -// PutGarageBoostrapHost places the .yml.signed file for this host +// PutGarageBoostrapHost places the .json.signed file for this host // into garage so that other hosts are able to see relevant configuration for // it. func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { @@ -34,9 +34,9 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { // and that the host public key is signed by the CA. host.Nebula.SignedPublicCredentials = b.Nebula.SignedPublicCredentials - hostB, err := yaml.Marshal(host) + hostB, err := json.Marshal(host) if err != nil { - return fmt.Errorf("yaml encoding host data: %w", err) + return fmt.Errorf("encoding host data: %w", err) } buf := new(bytes.Buffer) @@ -48,7 +48,7 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { filePath := filepath.Join( garageGlobalBucketBootstrapHostsDirPath, - host.Name+".yml.signed", + host.Name+".json.signed", ) _, err = client.PutObject( @@ -63,7 +63,7 @@ func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { return nil } -// RemoveGarageBootstrapHost removes the .yml.signed for the given +// RemoveGarageBootstrapHost removes the .json.signed for the given // host from garage. // // The given client should be for the global bucket. @@ -73,7 +73,7 @@ func RemoveGarageBootstrapHost( filePath := filepath.Join( garageGlobalBucketBootstrapHostsDirPath, - hostName+".yml.signed", + hostName+".json.signed", ) return client.RemoveObject( @@ -82,7 +82,7 @@ func RemoveGarageBootstrapHost( ) } -// GetGarageBootstrapHosts loads the .yml.signed file for all hosts +// GetGarageBootstrapHosts loads the .json.signed file for all hosts // stored in garage. func (b Bootstrap) GetGarageBootstrapHosts( ctx context.Context, @@ -127,8 +127,8 @@ func (b Bootstrap) GetGarageBootstrapHosts( } var host Host - if err = yaml.Unmarshal(hostB, &host); err != nil { - return nil, fmt.Errorf("yaml decoding object %q: %w", objInfo.Key, err) + if err = json.Unmarshal(hostB, &host); err != nil { + return nil, fmt.Errorf("decoding object %q: %w", objInfo.Key, err) } hostPublicCredsB, hostPublicCredsSig, err := nebula.Unwrap( @@ -152,8 +152,8 @@ func (b Bootstrap) GetGarageBootstrapHosts( } var hostPublicCreds nebula.HostPublicCredentials - if err := yaml.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { - logger.Warn(ctx, "yaml unmarshaling signed public creds", err) + if err := json.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { + logger.Warn(ctx, "unmarshaling signed public creds", err) continue } diff --git a/go/bootstrap/hosts.go b/go/bootstrap/hosts.go index 8c8395c..37dbcbb 100644 --- a/go/bootstrap/hosts.go +++ b/go/bootstrap/hosts.go @@ -2,19 +2,18 @@ package bootstrap import ( "bytes" - "isle/nebula" + "encoding/json" "fmt" + "isle/nebula" "net" "strings" - - "gopkg.in/yaml.v3" ) // NebulaHost describes the nebula configuration of a Host which is relevant for // other hosts to know. type NebulaHost struct { - SignedPublicCredentials string `yaml:"signed_public_credentials"` - PublicAddr string `yaml:"public_addr,omitempty"` + SignedPublicCredentials string + PublicAddr string } // NewNebulaHostSignedPublicCredentials constructs the SignedPublicCredentials @@ -27,9 +26,9 @@ func NewNebulaHostSignedPublicCredentials( string, error, ) { - hostPublicCredsB, err := yaml.Marshal(hostPublicCreds) + hostPublicCredsB, err := json.Marshal(hostPublicCreds) if err != nil { - return "", fmt.Errorf("yaml marshaling host's public credentials: %w", err) + return "", fmt.Errorf("marshaling host's public credentials: %w", err) } buf := new(bytes.Buffer) @@ -44,23 +43,23 @@ func NewNebulaHostSignedPublicCredentials( // GarageHost describes a single garage instance in the GarageHost. type GarageHostInstance struct { - ID string `yaml:"id"` - RPCPort int `yaml:"rpc_port"` - S3APIPort int `yaml:"s3_api_port"` + ID string + RPCPort int + S3APIPort int } // GarageHost describes the garage configuration of a Host which is relevant for // other hosts to know. type GarageHost struct { - Instances []GarageHostInstance `yaml:"instances"` + Instances []GarageHostInstance } // Host consolidates all information about a single host from the bootstrap // file. type Host struct { - Name string `yaml:"name"` - Nebula NebulaHost `yaml:"nebula"` - Garage GarageHost `yaml:"garage,omitempty"` + Name string + Nebula NebulaHost + Garage GarageHost } // IP returns the IP address encoded in the Host's nebula certificate, or panics @@ -79,8 +78,8 @@ func (h Host) IP() net.IP { } var hostPublicCreds nebula.HostPublicCredentials - if err := yaml.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { - panic(fmt.Errorf("yaml unmarshaling host's public credentials: %w", err)) + if err := json.Unmarshal(hostPublicCredsB, &hostPublicCreds); err != nil { + panic(fmt.Errorf("unmarshaling host's public credentials: %w", err)) } ip, err := nebula.IPFromHostCertPEM(hostPublicCreds.CertPEM) diff --git a/go/cmd/entrypoint/admin.go b/go/cmd/entrypoint/admin.go index dd8765f..25d64eb 100644 --- a/go/cmd/entrypoint/admin.go +++ b/go/cmd/entrypoint/admin.go @@ -33,7 +33,7 @@ func readAdmin(path string) (admin.Admin, error) { adm, err := admin.FromReader(os.Stdin) if err != nil { - return admin.Admin{}, fmt.Errorf("parsing admin.yml from stdin: %w", err) + return admin.Admin{}, fmt.Errorf("parsing admin.json from stdin: %w", err) } return adm, nil @@ -50,7 +50,7 @@ func readAdmin(path string) (admin.Admin, error) { var subCmdAdminCreateNetwork = subCmd{ name: "create-network", - descr: "Creates a new isle network, outputting the resulting admin.yml to stdout", + descr: "Creates a new isle network, outputting the resulting admin.json to stdout", do: func(subCmdCtx subCmdCtx) error { flags := subCmdCtx.flagSet(false) @@ -212,7 +212,7 @@ var subCmdAdminCreateNetwork = subCmd{ logger.Info(ctx, "starting child processes") go func() { // NOTE both stdout and stderr are sent to stderr, so that the user - // can pipe the resulting admin.yml to stdout. + // can pipe the resulting admin.json to stdout. pmuxlib.Run(ctx, os.Stderr, os.Stderr, pmuxConfig) close(pmuxDoneCh) }() @@ -243,7 +243,7 @@ var subCmdAdminCreateNetwork = subCmd{ return fmt.Errorf("initializing garage shared global bucket: %w", err) } - logger.Info(ctx, "cluster initialized successfully, writing admin.yml to stdout") + logger.Info(ctx, "cluster initialized successfully, writing admin.json to stdout") adm := admin.Admin{ CreationParams: adminCreationParams, @@ -253,7 +253,7 @@ var subCmdAdminCreateNetwork = subCmd{ adm.Garage.GlobalBucketS3APICredentials = hostBootstrap.Garage.GlobalBucketS3APICredentials if err := adm.WriteTo(os.Stdout); err != nil { - return fmt.Errorf("writing admin.yml to stdout") + return fmt.Errorf("writing admin.json to stdout") } return nil @@ -262,7 +262,7 @@ var subCmdAdminCreateNetwork = subCmd{ var subCmdAdminCreateBootstrap = subCmd{ name: "create-bootstrap", - descr: "Creates a new bootstrap.yml file for a particular host and writes it to stdout", + descr: "Creates a new bootstrap.json file for a particular host and writes it to stdout", checkLock: true, do: func(subCmdCtx subCmdCtx) error { @@ -270,7 +270,7 @@ var subCmdAdminCreateBootstrap = subCmd{ hostName := flags.StringP( "hostname", "h", "", - "Name of the host to generate bootstrap.yml for", + "Name of the host to generate bootstrap.json for", ) ipStr := flags.StringP( @@ -280,7 +280,7 @@ var subCmdAdminCreateBootstrap = subCmd{ adminPath := flags.StringP( "admin-path", "a", "", - `Path to admin.yml file. If the given path is "-" then stdin is used.`, + `Path to admin.json file. If the given path is "-" then stdin is used.`, ) if err := flags.Parse(subCmdCtx.args); err != nil { @@ -303,7 +303,7 @@ var subCmdAdminCreateBootstrap = subCmd{ adm, err := readAdmin(*adminPath) if err != nil { - return fmt.Errorf("reading admin.yml with --admin-path of %q: %w", *adminPath, err) + return fmt.Errorf("reading admin.json with --admin-path of %q: %w", *adminPath, err) } hostBootstrap, err := loadHostBootstrap() @@ -354,7 +354,7 @@ var subCmdAdminCreateNebulaCert = subCmd{ hostName := flags.StringP( "hostname", "h", "", - "Name of the host to generate bootstrap.yml for", + "Name of the host to generate bootstrap.json for", ) ipStr := flags.StringP( @@ -364,7 +364,7 @@ var subCmdAdminCreateNebulaCert = subCmd{ adminPath := flags.StringP( "admin-path", "a", "", - `Path to admin.yml file. If the given path is "-" then stdin is used.`, + `Path to admin.json file. If the given path is "-" then stdin is used.`, ) pubKeyPath := flags.StringP( @@ -392,7 +392,7 @@ var subCmdAdminCreateNebulaCert = subCmd{ adm, err := readAdmin(*adminPath) if err != nil { - return fmt.Errorf("reading admin.yml with --admin-path of %q: %w", *adminPath, err) + return fmt.Errorf("reading admin.json with --admin-path of %q: %w", *adminPath, err) } hostPubPEM, err := os.ReadFile(*pubKeyPath) diff --git a/go/cmd/entrypoint/daemon.go b/go/cmd/entrypoint/daemon.go index 5935fe2..3ef1b7b 100644 --- a/go/cmd/entrypoint/daemon.go +++ b/go/cmd/entrypoint/daemon.go @@ -213,7 +213,7 @@ var subCmdDaemon = subCmd{ bootstrapPath := flags.StringP( "bootstrap-path", "b", "", - `Path to a bootstrap.yml file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`, + `Path to a bootstrap.json file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`, ) logLevelStr := flags.StringP( @@ -263,7 +263,7 @@ var subCmdDaemon = subCmd{ return false } else if err != nil { - err = fmt.Errorf("parsing bootstrap.yml at %q: %w", path, err) + err = fmt.Errorf("parsing bootstrap.json at %q: %w", path, err) return false } @@ -281,9 +281,9 @@ var subCmdDaemon = subCmd{ case *bootstrapPath != "" && tryLoadBootstrap(*bootstrapPath): case tryLoadBootstrap(bootstrapAppDirPath): case err != nil: - return fmt.Errorf("attempting to load bootstrap.yml file: %w", err) + return fmt.Errorf("attempting to load bootstrap.json file: %w", err) default: - return errors.New("No bootstrap.yml file could be found, and one is not provided with --bootstrap-path") + return errors.New("No bootstrap.json file could be found, and one is not provided with --bootstrap-path") } if hostBootstrapPath != bootstrapDataDirPath { @@ -291,7 +291,7 @@ var subCmdDaemon = subCmd{ // If the bootstrap file is not being stored in the data dir, copy // it there, so it can be loaded from there next time. if err := writeBootstrapToDataDir(hostBootstrap); err != nil { - return fmt.Errorf("writing bootstrap.yml to data dir: %w", err) + return fmt.Errorf("writing bootstrap.json to data dir: %w", err) } } diff --git a/go/cmd/entrypoint/hosts.go b/go/cmd/entrypoint/hosts.go index a216f09..16207d6 100644 --- a/go/cmd/entrypoint/hosts.go +++ b/go/cmd/entrypoint/hosts.go @@ -1,15 +1,15 @@ package main import ( - "isle/bootstrap" "errors" "fmt" + "isle/bootstrap" + "isle/jsonutil" "os" "regexp" "sort" "github.com/mediocregopher/mediocre-go-lib/v2/mlog" - "gopkg.in/yaml.v3" ) var hostNameRegexp = regexp.MustCompile(`^[a-z][a-z0-9\-]*$`) @@ -60,29 +60,29 @@ var subCmdHostsList = subCmd{ } type host struct { - Name string `yaml:"name"` - Nebula struct { - IP string `yaml:"ip"` - } `yaml:"nebula"` - Garage bootstrap.GarageHost `yaml:"garage,omitempty"` + Name string + VPN struct { + IP string + } + Storage bootstrap.GarageHost `json:",omitempty"` } hosts := make([]host, 0, len(hostsMap)) for _, h := range hostsMap { host := host{ - Name: h.Name, - Garage: h.Garage, + Name: h.Name, + Storage: h.Garage, } - host.Nebula.IP = h.IP().String() + host.VPN.IP = h.IP().String() hosts = append(hosts, host) } sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name }) - return yaml.NewEncoder(os.Stdout).Encode(hosts) + return jsonutil.WriteIndented(os.Stdout, hosts) }, } diff --git a/go/cmd/entrypoint/nebula.go b/go/cmd/entrypoint/nebula.go index a7a7eb7..8f30e81 100644 --- a/go/cmd/entrypoint/nebula.go +++ b/go/cmd/entrypoint/nebula.go @@ -2,15 +2,15 @@ package main import ( "fmt" + "isle/jsonutil" "os" "github.com/slackhq/nebula/cert" - "gopkg.in/yaml.v3" ) var subCmdNebulaShow = subCmd{ name: "show", - descr: "Writes nebula network information to stdout in yaml format", + descr: "Writes nebula network information to stdout in JSON format", do: func(subCmdCtx subCmdCtx) error { flags := subCmdCtx.flagSet(false) @@ -39,14 +39,14 @@ var subCmdNebulaShow = subCmd{ subnet := caCert.Details.Subnets[0] type outLighthouse struct { - PublicAddr string `yaml:"public_addr"` - IP string `yaml:"ip"` + PublicAddr string + IP string } out := struct { - CACert string `yaml:"ca_cert_pem"` - SubnetCIDR string `yaml:"subnet_cidr"` - Lighthouses []outLighthouse `yaml:"lighthouses"` + CACert string + SubnetCIDR string + Lighthouses []outLighthouse }{ CACert: caPublicCreds.CertPEM, SubnetCIDR: subnet.String(), @@ -63,8 +63,8 @@ var subCmdNebulaShow = subCmd{ }) } - if err := yaml.NewEncoder(os.Stdout).Encode(out); err != nil { - return fmt.Errorf("yaml encoding to stdout: %w", err) + if err := jsonutil.WriteIndented(os.Stdout, out); err != nil { + return fmt.Errorf("encoding to stdout: %w", err) } return nil diff --git a/go/cmd/entrypoint/nebula_util.go b/go/cmd/entrypoint/nebula_util.go index 47f7557..a3fb2fa 100644 --- a/go/cmd/entrypoint/nebula_util.go +++ b/go/cmd/entrypoint/nebula_util.go @@ -105,7 +105,7 @@ func nebulaPmuxProcConfig( nebulaYmlPath := filepath.Join(envRuntimeDirPath, "nebula.yml") - if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil { + if err := yamlutil.WriteYamlFile(config, nebulaYmlPath, 0440); err != nil { return pmuxlib.ProcessConfig{}, fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err) } diff --git a/go/daemon/config.go b/go/daemon/config.go index 363b8bf..a021112 100644 --- a/go/daemon/config.go +++ b/go/daemon/config.go @@ -66,7 +66,6 @@ func (c *Config) fillDefaults() { var firewallGarageInbound []ConfigFirewallRule for i := range c.Storage.Allocations { - if c.Storage.Allocations[i].RPCPort == 0 { c.Storage.Allocations[i].RPCPort = 3900 + (i * 10) } diff --git a/go/garage/client.go b/go/garage/client.go index a925f25..bb355f2 100644 --- a/go/garage/client.go +++ b/go/garage/client.go @@ -31,8 +31,8 @@ type S3APIClient = *minio.Client // S3APICredentials describe data fields necessary for authenticating with a // garage S3 API endpoint. type S3APICredentials struct { - ID string `yaml:"id"` - Secret string `yaml:"secret"` + ID string + Secret string } // NewS3APICredentials returns a new usable instance of S3APICredentials. diff --git a/go/go.mod b/go/go.mod index ffa8785..c64d10e 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,8 +1,9 @@ module isle -go 1.17 +go 1.22 require ( + code.betamike.com/micropelago/pmux v0.0.0-20230706154656-fde8c2be7540 github.com/adrg/xdg v0.4.0 github.com/imdario/mergo v0.3.12 github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.1.0.20221113151154-07f3889a705b @@ -15,7 +16,6 @@ require ( ) require ( - code.betamike.com/micropelago/pmux v0.0.0-20230706154656-fde8c2be7540 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/go-cmp v0.5.6 // indirect diff --git a/go/go.sum b/go/go.sum index 6dc70f2..3d045de 100644 --- a/go/go.sum +++ b/go/go.sum @@ -62,6 +62,7 @@ github.com/slackhq/nebula v1.6.1/go.mod h1:UmkqnXe4O53QwToSl/gG7sM4BroQwAB7dd4hU github.com/smartystreets/assertions v1.13.0 h1:Dx1kYM01xsSqKPno3aqLnrwac2LetPvN23diwyr69Qs= github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrxfxaaSlJazyFLYVFx8= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -69,6 +70,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= @@ -89,6 +91,7 @@ golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2 h1:GLw7MR8AfAG2GmGcmVgObF golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -99,6 +102,7 @@ gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/jsonutil/jsonutil.go b/go/jsonutil/jsonutil.go new file mode 100644 index 0000000..1f93d92 --- /dev/null +++ b/go/jsonutil/jsonutil.go @@ -0,0 +1,58 @@ +// Package contains utility functions and types related to (un)marshaling JSON. +package jsonutil + +import ( + "encoding/json" + "fmt" + "io" + "os" +) + +// LoadFile reads the file at the given path and unmarshals it into the +// given pointer. +func LoadFile(into any, path string) error { + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening file: %w", err) + } + defer file.Close() + + if err = json.NewDecoder(file).Decode(into); err != nil { + return fmt.Errorf("decoding json: %w", err) + } + + return nil +} + +// WriteFile encodes the given data as a JSON document, and writes it to the +// given file path, overwriting any previous data. +func WriteFile(data any, path string, fileMode os.FileMode) error { + + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileMode) + if err != nil { + return fmt.Errorf("opening file: %w", err) + } + defer file.Close() + + if err = json.NewEncoder(file).Encode(data); err != nil { + return fmt.Errorf("writing/encoding file: %w", err) + } + + return nil +} + +// WriteIndented is a helper which calls json.MarshalIndent on the given +// value and writes the result to the given io.Writer. +func WriteIndented(into io.Writer, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return fmt.Errorf("json marshaling: %w", err) + } + + if _, err := os.Stdout.Write(b); err != nil { + return fmt.Errorf("writing: %w", err) + } + + return nil +} diff --git a/go/nebula/nebula.go b/go/nebula/nebula.go index 9ddbe08..f1bac81 100644 --- a/go/nebula/nebula.go +++ b/go/nebula/nebula.go @@ -24,31 +24,31 @@ var ErrInvalidSignature = errors.New("invalid signature") // HostPublicCredentials contains certificate and signing public keys which are // able to be broadcast publicly. type HostPublicCredentials struct { - CertPEM string `yaml:"cert_pem"` - SigningKeyPEM string `yaml:"signing_key_pem"` + CertPEM string + SigningKeyPEM string } // HostCredentials contains the certificate and private key files which will // need to be present on a particular host. Each file is PEM encoded. type HostCredentials struct { - Public HostPublicCredentials `yaml:"public"` - PrivateKeyPEM string `yaml:"key_pem"` // TODO should be private_key_pem - SigningPrivateKeyPEM string `yaml:"signing_private_key_pem"` + Public HostPublicCredentials + PrivateKeyPEM string + SigningPrivateKeyPEM string } // CAPublicCredentials contains certificate and signing public keys which are // able to be broadcast publicly. The signing public key is the same one which // is embedded into the certificate. type CAPublicCredentials struct { - CertPEM string `yaml:"cert_pem"` - SigningKeyPEM string `yaml:"signing_key_pem"` + CertPEM string + SigningKeyPEM string // TODO remove redundant field } // CACredentials contains the certificate and private files which can be used to // create and validate HostCredentials. Each file is PEM encoded. type CACredentials struct { - Public CAPublicCredentials `yaml:"public"` - SigningPrivateKeyPEM string `yaml:"signing_private_key_pem"` + Public CAPublicCredentials + SigningPrivateKeyPEM string } // NewHostCertPEM generates and signs a new host certificate containing the diff --git a/go/yamlutil/yamlutil.go b/go/yamlutil/yamlutil.go index f76fee5..06cf7d4 100644 --- a/go/yamlutil/yamlutil.go +++ b/go/yamlutil/yamlutil.go @@ -2,7 +2,6 @@ package yamlutil import ( "fmt" - "io/fs" "os" "gopkg.in/yaml.v3" @@ -10,14 +9,12 @@ import ( // LoadYamlFile reads the file at the given path and unmarshals it into the // given pointer. -func LoadYamlFile(into interface{}, path string) error { +func LoadYamlFile(into any, path string) error { file, err := os.Open(path) - if err != nil { return fmt.Errorf("opening file: %w", err) } - defer file.Close() if err = yaml.NewDecoder(file).Decode(into); err != nil { @@ -29,39 +26,17 @@ func LoadYamlFile(into interface{}, path string) error { // WriteYamlFile encodes the given data as a yaml document, and writes it to the // given file path, overwriting any previous data. -func WriteYamlFile(data interface{}, path string) error { - - file, err := os.OpenFile( - path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640, - ) +func WriteYamlFile(data any, path string, fileMode os.FileMode) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileMode) if err != nil { return fmt.Errorf("opening file: %w", err) } + defer file.Close() - err = yaml.NewEncoder(file).Encode(data) - - file.Close() - - if err != nil { + if err := yaml.NewEncoder(file).Encode(data); err != nil { return fmt.Errorf("writing/encoding file: %w", err) } return nil } - -// LoadYamlFSFile is like LoadYamlFile, but it will read the file from the given -// fs.FS instance. -func LoadYamlFSFile(into interface{}, f fs.FS, path string) error { - - body, err := fs.ReadFile(f, path) - if err != nil { - return fmt.Errorf("reading file from FS: %w", err) - } - - if err := yaml.Unmarshal(body, into); err != nil { - return fmt.Errorf("yaml unmarshaling: %w", err) - } - - return nil -} diff --git a/tests/cases/admin/01-create-network.sh b/tests/cases/admin/01-create-network.sh index de41177..4c8535b 100644 --- a/tests/cases/admin/01-create-network.sh +++ b/tests/cases/admin/01-create-network.sh @@ -5,12 +5,12 @@ source "$UTILS"/with-1-data-1-empty-node-cluster.sh [ "$(cat b/meta/isle/rpc_port)" = "3910" ] [ "$(cat c/meta/isle/rpc_port)" = "3920" ] -[ "$(yq admin.yml + > admin.json isle daemon --config-path daemon.yml >daemon.log 2>&1 & pid="$!" @@ -68,7 +68,7 @@ EOF echo "Creating secondus bootstrap" isle admin create-bootstrap \ - --admin-path admin.yml \ + --admin-path admin.json \ --hostname secondus \ --ip "$secondus_ip" \ > "$secondus_bootstrap"