Compare commits

..

8 Commits

Author SHA1 Message Date
Alex Auvolat
b1ee3e54ba
Merge branch 'main-0.8.x' into sync-08-09 2023-10-26 11:27:56 +02:00
Alex
4b3dee2ca3 Merge pull request 's3 api: add missing CORS headers to PostObject responses (fix #609)' (#656) from fix-cors-post-object into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/656
2023-10-26 09:17:14 +00:00
Alex Auvolat
ac04934dae s3 api: add missing CORS headers to PostObject responses (fix #609) 2023-10-20 10:37:48 +02:00
Alex
0215b11402 Merge pull request 'Add support for specifying rpc_secret_file, metrics_token_file and admin_token_file using environment variables' (#643) from networkException/garage:token-file-env into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/643
2023-10-19 09:33:12 +00:00
networkException
8599051c49
garage: support specifying token / secret as environment variables
this patch adds support for specifying the `rpc_secret_file`,
`metrics_token_file` and `admin_token_file` as environment variables.
2023-10-19 03:39:02 +02:00
networkException
4a19ee94bb
garage: fix admin-token description 2023-10-19 03:31:50 +02:00
networkException
c99cb58d71
util: move reading secret file into seperate helper
this patch moves the logic to read a secret file (and check for correct
permissions) from `secret_from_file` into a new `read_secret_file`
helper.
2023-10-19 03:29:48 +02:00
networkException
5feb6a1f64
docs: add documentation for specifying token / secret file as environment variables 2023-10-19 03:28:44 +02:00
34 changed files with 6907 additions and 2026 deletions

3
.cargo/config.toml Normal file
View File

@ -0,0 +1,3 @@
[target.x86_64-unknown-linux-gnu]
linker = "clang"
rustflags = ["-C", "link-arg=-fuse-ld=mold"]

4
.gitignore vendored
View File

@ -3,6 +3,4 @@
/pki
**/*.rs.bk
*.swp
/.direnv
/.cargo
/result
/.direnv

15
Cargo.lock generated
View File

@ -1271,6 +1271,7 @@ dependencies = [
"http-range",
"httpdate",
"hyper",
"hyperlocal",
"idna",
"md-5",
"multer",
@ -1464,6 +1465,7 @@ dependencies = [
"garage_util",
"http",
"hyper",
"hyperlocal",
"opentelemetry",
"percent-encoding",
"tokio",
@ -1776,6 +1778,19 @@ dependencies = [
"tokio-io-timeout",
]
[[package]]
name = "hyperlocal"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c"
dependencies = [
"futures-util",
"hex",
"hyper",
"pin-project",
"tokio",
]
[[package]]
name = "iana-time-zone"
version = "0.1.57"

6326
Cargo.nix Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,166 +1,56 @@
{
buildSystem ? builtins.currentSystem,
targetSystem ? buildSystem,
gitVersion ? null,
release ? false,
features ? null,
}:
{ system ? builtins.currentSystem, git_version ? null, }:
with import ./nix/common.nix;
let
pkgsSrc = import ./nix/pkgs.nix;
newBuildTarget = {
nixPkgsSystem,
rustTarget ? nixPkgsSystem,
nativeBuildInputs ? pkgsCross: [],
rustFlags ? pkgsCross: [],
}: {
inherit nixPkgsSystem rustTarget nativeBuildInputs rustFlags;
pkgs = import pkgsSrc { };
compile = import ./nix/compile.nix;
build_debug_and_release = (target: {
debug = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = false;
}).workspace.garage { compileMode = "build"; };
release = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = true;
}).workspace.garage { compileMode = "build"; };
});
test = (rustPkgs:
pkgs.symlinkJoin {
name = "garage-tests";
paths =
builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; })
(builtins.attrNames rustPkgs.workspace);
});
in {
pkgs = {
amd64 = build_debug_and_release "x86_64-unknown-linux-musl";
i386 = build_debug_and_release "i686-unknown-linux-musl";
arm64 = build_debug_and_release "aarch64-unknown-linux-musl";
arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
};
# centralize per-target configuration in a single place.
buildTargets = {
"x86_64-linux" = newBuildTarget {
nixPkgsSystem = "x86_64-unknown-linux-musl";
};
"i686-linux" = newBuildTarget {
nixPkgsSystem = "i686-unknown-linux-musl";
};
"aarch64-linux" = newBuildTarget {
nixPkgsSystem = "aarch64-unknown-linux-musl";
};
# Old Raspberry Pi's (not currently supported due to linking errors with
# libsqlite3 and libsodium
#"armv6l-linux" = newBuildTarget {
# nixPkgsSystem = "armv6l-unknown-linux-musleabihf";
# rustTarget = "arm-unknown-linux-musleabihf";
#};
"x86_64-windows" = newBuildTarget {
nixPkgsSystem = "x86_64-w64-mingw32";
rustTarget = "x86_64-pc-windows-gnu";
nativeBuildInputs = pkgsCross: [ pkgsCross.windows.pthreads ];
rustFlags = pkgsCross: [
"-C" "link-arg=-L${pkgsCross.windows.pthreads}/lib"
];
};
};
buildTarget = buildTargets.${targetSystem};
pkgs = import pkgsSrc { system = buildSystem; };
pkgsCross = import pkgsSrc {
system = buildSystem;
crossSystem.config = buildTarget.nixPkgsSystem;
};
rustTarget = buildTarget.rustTarget;
toolchain = let
fenix = import (pkgs.fetchFromGitHub {
owner = "nix-community";
repo = "fenix";
rev = "81ab0b4f7ae9ebb57daa0edf119c4891806e4d3a";
hash = "sha256-bZmI7ytPAYLpyFNgj5xirDkKuAniOkj1xHdv5aIJ5GM=";
}) {
system = buildSystem;
};
mkToolchain = fenixTarget: fenixTarget.toolchainOf {
channel = "1.68.2";
sha256 = "sha256-4vetmUhTUsew5FODnjlnQYInzyLNyDwocGa4IvMk3DM=";
};
in
fenix.combine [
(mkToolchain fenix).rustc
(mkToolchain fenix).rustfmt
(mkToolchain fenix).cargo
(mkToolchain fenix).clippy
(mkToolchain fenix.targets.${rustTarget}).rust-std
];
naersk = let
naerskSrc = pkgs.fetchFromGitHub {
owner = "nix-community";
repo = "naersk";
rev = "d9a33d69a9c421d64c8d925428864e93be895dcc";
hash = "sha256-e136hTT7LqQ2QjOTZQMW+jnsevWwBpMj78u6FRUsH9I=";
};
in
pkgs.callPackages naerskSrc {
cargo = toolchain;
rustc = toolchain;
};
builtFeatures = if features != null then
features
else (
[ "garage/bundled-libs" "garage/sled" "garage/lmdb" "garage/k2v" ] ++ (
if release then [
"garage/consul-discovery"
"garage/kubernetes-discovery"
"garage/metrics"
"garage/telemetry-otlp"
test = {
amd64 = test (compile {
inherit system git_version pkgsSrc cargo2nixOverlay;
target = "x86_64-unknown-linux-musl";
features = [
"garage/bundled-libs"
"garage/k2v"
"garage/sled"
"garage/lmdb"
"garage/sqlite"
] else [ ]
)
);
# For some reason the pkgsCross.pkgsStatic build of libsodium doesn't contain
# a `.a` file when compiled to a windows target, but rather contains
# a `.dll.a` file which libsodium-sys doesn't pick up on. Copying the one to
# the be the other seems to work.
libsodium = pkgs.runCommand "libsodium-wrapped" {
libsodium = pkgsCross.pkgsStatic.libsodium;
} ''
cp -rL "$libsodium" "$out"
chmod -R +w "$out"
if [ ! -e "$out"/lib/libsodium.a ] && [ -f "$out"/lib/libsodium.dll.a ]; then
cp "$out"/lib/libsodium.dll.a "$out"/lib/libsodium.a
fi
'';
in rec {
inherit pkgs pkgsCross;
# Exported separately so it can be used from shell.nix
buildEnv = rec {
nativeBuildInputs = (buildTarget.nativeBuildInputs pkgsCross) ++ [
toolchain
pkgs.protobuf
# Required for shell because of rust dependency build scripts which must
# run on the build system.
pkgs.stdenv.cc
];
SODIUM_LIB_DIR = "${libsodium}/lib";
# Required because ring crate is special. This also seems to have
# fixed some issues with the x86_64-windows cross-compile :shrug:
TARGET_CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
CARGO_BUILD_TARGET = rustTarget;
CARGO_BUILD_RUSTFLAGS = [
"-C" "target-feature=+crt-static"
"-C" "link-arg=-static"
# https://github.com/rust-lang/cargo/issues/4133
"-C" "linker=${TARGET_CC}"
] ++ (buildTarget.rustFlags pkgsCross);
];
});
};
clippy = {
amd64 = (compile {
inherit system git_version pkgsSrc cargo2nixOverlay;
target = "x86_64-unknown-linux-musl";
compiler = "clippy";
}).workspace.garage { compileMode = "build"; };
};
build = naersk.buildPackage (rec {
inherit release;
src = ./.;
strictDeps = true;
doCheck = false;
cargoBuildOptions = prev: prev++[
"--features=${builtins.concatStringsSep "," builtFeatures}"
];
} // buildEnv);
}

View File

@ -1,24 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>Garage Adminstration API v0</title>
<!-- needed for adaptive design -->
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="./css/redoc.css" rel="stylesheet">
<!--
Redoc doesn't change outer page styles
-->
<style>
body {
margin: 0;
padding: 0;
}
</style>
</head>
<body>
<redoc spec-url='./garage-admin-v1.yml'></redoc>
<script src="./redoc.standalone.js"> </script>
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@ -37,84 +37,30 @@ import (
"context"
"fmt"
"os"
"strings"
garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang"
)
func main() {
// Initialization
// Set Host and other parameters
configuration := garage.NewConfiguration()
configuration.Host = "127.0.0.1:3903"
// We can now generate a client
client := garage.NewAPIClient(configuration)
// Authentication is handled through the context pattern
ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t")
// Nodes
fmt.Println("--- nodes ---")
nodes, _, _ := client.NodesApi.GetNodes(ctx).Execute()
fmt.Fprintf(os.Stdout, "First hostname: %v\n", nodes.KnownNodes[0].Hostname)
capa := int64(1000000000)
change := []garage.NodeRoleChange{
garage.NodeRoleChange{NodeRoleUpdate: &garage.NodeRoleUpdate {
Id: *nodes.KnownNodes[0].Id,
Zone: "dc1",
Capacity: *garage.NewNullableInt64(&capa),
Tags: []string{ "fast", "amd64" },
}},
// Send a request
resp, r, err := client.NodesApi.GetNodes(ctx).Execute()
if err != nil {
fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err)
fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
}
staged, _, _ := client.LayoutApi.AddLayout(ctx).NodeRoleChange(change).Execute()
msg, _, _ := client.LayoutApi.ApplyLayout(ctx).LayoutVersion(*garage.NewLayoutVersion(staged.Version + 1)).Execute()
fmt.Printf(strings.Join(msg.Message, "\n")) // Layout configured
health, _, _ := client.NodesApi.GetHealth(ctx).Execute()
fmt.Printf("Status: %s, nodes: %v/%v, storage: %v/%v, partitions: %v/%v\n", health.Status, health.ConnectedNodes, health.KnownNodes, health.StorageNodesOk, health.StorageNodes, health.PartitionsAllOk, health.Partitions)
// Key
fmt.Println("\n--- key ---")
key := "openapi-key"
keyInfo, _, _ := client.KeyApi.AddKey(ctx).AddKeyRequest(garage.AddKeyRequest{Name: *garage.NewNullableString(&key) }).Execute()
defer client.KeyApi.DeleteKey(ctx).Id(*keyInfo.AccessKeyId).Execute()
fmt.Printf("AWS_ACCESS_KEY_ID=%s\nAWS_SECRET_ACCESS_KEY=%s\n", *keyInfo.AccessKeyId, *keyInfo.SecretAccessKey.Get())
id := *keyInfo.AccessKeyId
canCreateBucket := true
updateKeyRequest := *garage.NewUpdateKeyRequest()
updateKeyRequest.SetName("openapi-key-updated")
updateKeyRequest.SetAllow(garage.UpdateKeyRequestAllow { CreateBucket: &canCreateBucket })
update, _, _ := client.KeyApi.UpdateKey(ctx).Id(id).UpdateKeyRequest(updateKeyRequest).Execute()
fmt.Printf("Updated %v with key name %v\n", *update.AccessKeyId, *update.Name)
keyList, _, _ := client.KeyApi.ListKeys(ctx).Execute()
fmt.Printf("Keys count: %v\n", len(keyList))
// Bucket
fmt.Println("\n--- bucket ---")
global_name := "global-ns-openapi-bucket"
local_name := "local-ns-openapi-bucket"
bucketInfo, _, _ := client.BucketApi.CreateBucket(ctx).CreateBucketRequest(garage.CreateBucketRequest{
GlobalAlias: &global_name,
LocalAlias: &garage.CreateBucketRequestLocalAlias {
AccessKeyId: keyInfo.AccessKeyId,
Alias: &local_name,
},
}).Execute()
defer client.BucketApi.DeleteBucket(ctx).Id(*bucketInfo.Id).Execute()
fmt.Printf("Bucket id: %s\n", *bucketInfo.Id)
updateBucketRequest := *garage.NewUpdateBucketRequest()
website := garage.NewUpdateBucketRequestWebsiteAccess()
website.SetEnabled(true)
website.SetIndexDocument("index.html")
website.SetErrorDocument("errors/4xx.html")
updateBucketRequest.SetWebsiteAccess(*website)
quotas := garage.NewUpdateBucketRequestQuotas()
quotas.SetMaxSize(1000000000)
quotas.SetMaxObjects(999999999)
updateBucketRequest.SetQuotas(*quotas)
updatedBucket, _, _ := client.BucketApi.UpdateBucket(ctx).Id(*bucketInfo.Id).UpdateBucketRequest(updateBucketRequest).Execute()
fmt.Printf("Bucket %v website activation: %v\n", *updatedBucket.Id, *updatedBucket.WebsiteAccess)
bucketList, _, _ := client.BucketApi.ListBuckets(ctx).Execute()
fmt.Printf("Bucket count: %v\n", len(bucketList))
// Process the response
fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname)
}
```

View File

@ -31,9 +31,9 @@ npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js.
A short example:
```javascript
const garage = require('garage_administration_api_v1garage_v0_9_0');
const garage = require('garage_administration_api_v0garage_v0_8_0');
const api = new garage.ApiClient("http://127.0.0.1:3903/v1");
const api = new garage.ApiClient("http://127.0.0.1:3903/v0");
api.authentications['bearerAuth'].accessToken = "s3cr3t";
const [node, layout, key, bucket] = [

View File

@ -80,7 +80,7 @@ from garage_admin_sdk.apis import *
from garage_admin_sdk.models import *
configuration = garage_admin_sdk.Configuration(
host = "http://localhost:3903/v1",
host = "http://localhost:3903/v0",
access_token = "s3cr3t"
)
@ -94,14 +94,13 @@ print(f"running garage {status.garage_version}, node_id {status.node}")
# Change layout of this node
current = layout.get_layout()
layout.add_layout([
NodeRoleChange(
id = status.node,
layout.add_layout({
status.node: NodeClusterInfo(
zone = "dc1",
capacity = 1000000000,
capacity = 1,
tags = [ "dev" ],
)
])
})
layout.apply_layout(LayoutVersion(
version = current.version + 1
))

View File

@ -38,7 +38,7 @@ Our website serving logic is as follow:
Now we need to infer the URL of your website through your bucket name.
Let assume:
- we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#web_root_domain))
- we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#root_domain))
- our bucket name is `garagehq.deuxfleurs.fr`.
Our bucket will be served if the Host field matches one of these 2 values (the port is ignored):

View File

@ -12,7 +12,7 @@ An introduction to building cluster layouts can be found in the [production depl
In Garage, all of the data that can be stored in a given cluster is divided
into slices which we call *partitions*. Each partition is stored by
one or several nodes in the cluster
(see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication_mode)).
(see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication-mode)).
The layout determines the correspondence between these partition,
which exist on a logical level, and actual storage nodes.

View File

@ -13,11 +13,8 @@ We will bump the version numbers prefixed to each API endpoint at each time the
or semantics change, meaning that code that relies on these endpoint will break
when changes are introduced.
Versions:
- Before Garage 0.7.2 - no admin API
- Garage 0.7.2 - admin APIv0
- Garage 0.9.0 - admin APIv1, deprecate admin APIv0
The Garage administration API was introduced in version 0.7.2, this document
does not apply to older versions of Garage.
## Access control
@ -134,9 +131,7 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl
### Cluster operations
These endpoints have a dedicated OpenAPI spec.
- APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
These endpoints are defined on a dedicated [Redocly page](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html). You can also download its [OpenAPI specification](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml).
Requesting the API from the command line can be as simple as running:

View File

@ -8,8 +8,6 @@ weight = 20
Here is an example `garage.toml` configuration file that illustrates all of the possible options:
```toml
replication_mode = "3"
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
metadata_fsync = true
@ -23,6 +21,8 @@ sled_cache_capacity = "128MiB"
sled_flush_every_ms = 2000
lmdb_map_size = "1T"
replication_mode = "3"
compression_level = 1
rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
@ -77,64 +77,157 @@ The following gives details about each available configuration option.
## Available configuration options
### Index
### `metadata_dir`
Top-level configuration options:
[`block_size`](#block_size),
[`bootstrap_peers`](#bootstrap_peers),
[`compression_level`](#compression_level),
[`data_dir`](#metadata_dir),
[`data_fsync`](#data_fsync),
[`db_engine`](#db_engine),
[`lmdb_map_size`](#lmdb_map_size),
[`metadata_dir`](#metadata_dir),
[`metadata_fsync`](#metadata_fsync),
[`replication_mode`](#replication_mode),
[`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_public_addr`](#rpc_public_addr),
[`rpc_secret`](#rpc_secret),
[`rpc_secret_file`](#rpc_secret),
[`sled_cache_capacity`](#sled_cache_capacity),
[`sled_flush_every_ms`](#sled_flush_every_ms).
The directory in which Garage will store its metadata. This contains the node identifier,
the network configuration and the peer list, the list of buckets and keys as well
as the index of all objects, object version and object blocks.
The `[consul_discovery]` section:
[`api`](#consul_api),
[`ca_cert`](#consul_ca_cert),
[`client_cert`](#consul_client_cert),
[`client_key`](#consul_client_cert),
[`consul_http_addr`](#consul_http_addr),
[`meta`](#consul_tags),
[`service_name`](#consul_service_name),
[`tags`](#consul_tags),
[`tls_skip_verify`](#consul_tls_skip_verify),
[`token`](#consul_token).
Store this folder on a fast SSD drive if possible to maximize Garage's performance.
The `[kubernetes_discovery]` section:
[`namespace`](#kube_namespace),
[`service_name`](#kube_service_name),
[`skip_crd`](#kube_skip_crd).
### `data_dir`
The `[s3_api]` section:
[`api_bind_addr`](#s3_api_bind_addr),
[`root_domain`](#s3_root_domain),
[`s3_region`](#s3_region).
The directory in which Garage will store the data blocks of objects.
This folder can be placed on an HDD. The space available for `data_dir`
should be counted to determine a node's capacity
when [adding it to the cluster layout](@/documentation/cookbook/real-world.md).
The `[s3_web]` section:
[`bind_addr`](#web_bind_addr),
[`root_domain`](#web_root_domain).
Since `v0.9.0`, Garage supports multiple data directories with the following syntax:
The `[admin]` section:
[`api_bind_addr`](#admin_api_bind_addr),
[`metrics_token`](#admin_metrics_token),
[`metrics_token_file`](#admin_metrics_token),
[`admin_token`](#admin_token),
[`admin_token_file`](#admin_token),
[`trace_sink`](#admin_trace_sink),
```toml
data_dir = [
{ path = "/path/to/old_data", read_only = true },
{ path = "/path/to/new_hdd1", capacity = "2T" },
{ path = "/path/to/new_hdd2", capacity = "4T" },
]
```
See [the dedicated documentation page](@/documentation/operations/multi-hdd.md)
on how to operate Garage in such a setup.
### Top-level configuration options
### `db_engine` (since `v0.8.0`)
#### `replication_mode` {#replication_mode}
By default, Garage uses the Sled embedded database library
to store its metadata on-disk. Since `v0.8.0`, Garage can use alternative storage backends as follows:
| DB engine | `db_engine` value | Database path |
| --------- | ----------------- | ------------- |
| [Sled](https://sled.rs) | `"sled"` | `<metadata_dir>/db/` |
| [LMDB](https://www.lmdb.tech) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
Performance characteristics of the different DB engines are as follows:
- Sled: the default database engine, which tends to produce
large data files and also has performance issues, especially when the metadata folder
is on a traditional HDD and not on SSD.
- LMDB: the recommended alternative on 64-bit systems,
much more space-efficiant and slightly faster. Note that the data format of LMDB is not portable
between architectures, so for instance the Garage database of an x86-64
node cannot be moved to an ARM64 node. Also note that, while LMDB can technically be used on 32-bit systems,
this will limit your node to very small database sizes due to how LMDB works; it is therefore not recommended.
- Sqlite: Garage supports Sqlite as a storage backend for metadata,
however it may have issues and is also very slow in its current implementation,
so it is not recommended to be used for now.
It is possible to convert Garage's metadata directory from one format to another with a small utility named `convert_db`,
which can be downloaded at the following locations:
[for amd64](https://garagehq.deuxfleurs.fr/_releases/convert_db/amd64/convert_db),
[for i386](https://garagehq.deuxfleurs.fr/_releases/convert_db/i386/convert_db),
[for arm64](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm64/convert_db),
[for arm](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm/convert_db).
The `convert_db` utility is used as folows:
```
convert-db -a <input db engine> -i <input db path> \
-b <output db engine> -o <output db path>
```
Make sure to specify the full database path as presented in the table above,
and not just the path to the metadata directory.
### `metadata_fsync`
Whether to enable synchronous mode for the database engine or not.
This is disabled (`false`) by default.
This reduces the risk of metadata corruption in case of power failures,
at the cost of a significant drop in write performance,
as Garage will have to pause to sync data to disk much more often
(several times for API calls such as PutObject).
Using this option reduces the risk of simultaneous metadata corruption on several
cluster nodes, which could lead to data loss.
If multi-site replication is used, this option is most likely not necessary, as
it is extremely unlikely that two nodes in different locations will have a
power failure at the exact same time.
(Metadata corruption on a single node is not an issue, the corrupted data file
can always be deleted and reconstructed from the other nodes in the cluster.)
Here is how this option impacts the different database engines:
| Database | `metadata_fsync = false` (default) | `metadata_fsync = true` |
|----------|------------------------------------|-------------------------------|
| Sled | default options | *unsupported* |
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
### `data_fsync`
Whether to `fsync` data blocks and their containing directory after they are
saved to disk.
This is disabled (`false`) by default.
This might reduce the risk that a data block is lost in rare
situations such as simultaneous node losing power,
at the cost of a moderate drop in write performance.
Similarly to `metatada_fsync`, this is likely not necessary
if geographical replication is used.
### `block_size`
Garage splits stored objects in consecutive chunks of size `block_size`
(except the last one which might be smaller). The default size is 1MiB and
should work in most cases. We recommend increasing it to e.g. 10MiB if
you are using Garage to store large files and have fast network connections
between all nodes (e.g. 1gbps).
If you are interested in tuning this, feel free to do so (and remember to
report your findings to us!). When this value is changed for a running Garage
installation, only files newly uploaded will be affected. Previously uploaded
files will remain available. This however means that chunks from existing files
will not be deduplicated with chunks from newly uploaded files, meaning you
might use more storage space that is optimally possible.
### `sled_cache_capacity`
This parameter can be used to tune the capacity of the cache used by
[sled](https://sled.rs), the database Garage uses internally to store metadata.
Tune this to fit the RAM you wish to make available to your Garage instance.
This value has a conservative default (128MB) so that Garage doesn't use too much
RAM by default, but feel free to increase this for higher performance.
### `sled_flush_every_ms`
This parameters can be used to tune the flushing interval of sled.
Increase this if sled is thrashing your SSD, at the risk of losing more data in case
of a power outage (though this should not matter much as data is replicated on other
nodes). The default value, 2000ms, should be appropriate for most use cases.
### `lmdb_map_size`
This parameters can be used to set the map size used by LMDB,
which is the size of the virtual memory region used for mapping the database file.
The value of this parameter is the maximum size the metadata database can take.
This value is not bound by the physical RAM size of the machine running Garage.
If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines.
### `replication_mode`
Garage supports the following replication modes:
@ -217,160 +310,7 @@ to the cluster while rebalancing is in progress. In theory, no data should be
lost as rebalancing is a routine operation for Garage, although we cannot
guarantee you that everything will go right in such an extreme scenario.
#### `metadata_dir` {#metadata_dir}
The directory in which Garage will store its metadata. This contains the node identifier,
the network configuration and the peer list, the list of buckets and keys as well
as the index of all objects, object version and object blocks.
Store this folder on a fast SSD drive if possible to maximize Garage's performance.
#### `data_dir` {#data_dir}
The directory in which Garage will store the data blocks of objects.
This folder can be placed on an HDD. The space available for `data_dir`
should be counted to determine a node's capacity
when [adding it to the cluster layout](@/documentation/cookbook/real-world.md).
Since `v0.9.0`, Garage supports multiple data directories with the following syntax:
```toml
data_dir = [
{ path = "/path/to/old_data", read_only = true },
{ path = "/path/to/new_hdd1", capacity = "2T" },
{ path = "/path/to/new_hdd2", capacity = "4T" },
]
```
See [the dedicated documentation page](@/documentation/operations/multi-hdd.md)
on how to operate Garage in such a setup.
#### `db_engine` (since `v0.8.0`) {#db_engine}
Since `v0.8.0`, Garage can use alternative storage backends as follows:
| DB engine | `db_engine` value | Database path |
| --------- | ----------------- | ------------- |
| [LMDB](https://www.lmdb.tech) (default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sled](https://sled.rs) (default up to `v0.8.0`) | `"sled"` | `<metadata_dir>/db/` |
| [Sqlite](https://sqlite.org) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
Sled was the only database engine up to Garage v0.7.0. Performance issues and
API limitations of Sled prompted the addition of alternative engines in v0.8.0.
Since v0.9.0, LMDB is the default engine instead of Sled, and Sled is
deprecated. We plan to remove Sled in Garage v1.0.
Performance characteristics of the different DB engines are as follows:
- Sled: tends to produce large data files and also has performance issues,
especially when the metadata folder is on a traditional HDD and not on SSD.
- LMDB: the recommended database engine on 64-bit systems, much more
space-efficient and slightly faster. Note that the data format of LMDB is not
portable between architectures, so for instance the Garage database of an
x86-64 node cannot be moved to an ARM64 node. Also note that, while LMDB can
technically be used on 32-bit systems, this will limit your node to very
small database sizes due to how LMDB works; it is therefore not recommended.
- Sqlite: Garage supports Sqlite as an alternative storage backend for
metadata, and although it has not been tested as much, it is expected to work
satisfactorily. Since Garage v0.9.0, performance issues have largely been
fixed by allowing for a no-fsync mode (see `metadata_fsync`). Sqlite does not
have the database size limitation of LMDB on 32-bit systems.
It is possible to convert Garage's metadata directory from one format to another
using the `garage convert-db` command, which should be used as follows:
```
garage convert-db -a <input db engine> -i <input db path> \
-b <output db engine> -o <output db path>
```
Make sure to specify the full database path as presented in the table above
(third colummn), and not just the path to the metadata directory.
#### `metadata_fsync` {#metadata_fsync}
Whether to enable synchronous mode for the database engine or not.
This is disabled (`false`) by default.
This reduces the risk of metadata corruption in case of power failures,
at the cost of a significant drop in write performance,
as Garage will have to pause to sync data to disk much more often
(several times for API calls such as PutObject).
Using this option reduces the risk of simultaneous metadata corruption on several
cluster nodes, which could lead to data loss.
If multi-site replication is used, this option is most likely not necessary, as
it is extremely unlikely that two nodes in different locations will have a
power failure at the exact same time.
(Metadata corruption on a single node is not an issue, the corrupted data file
can always be deleted and reconstructed from the other nodes in the cluster.)
Here is how this option impacts the different database engines:
| Database | `metadata_fsync = false` (default) | `metadata_fsync = true` |
|----------|------------------------------------|-------------------------------|
| Sled | default options | *unsupported* |
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
#### `data_fsync` {#data_fsync}
Whether to `fsync` data blocks and their containing directory after they are
saved to disk.
This is disabled (`false`) by default.
This might reduce the risk that a data block is lost in rare
situations such as simultaneous node losing power,
at the cost of a moderate drop in write performance.
Similarly to `metatada_fsync`, this is likely not necessary
if geographical replication is used.
#### `block_size` {#block_size}
Garage splits stored objects in consecutive chunks of size `block_size`
(except the last one which might be smaller). The default size is 1MiB and
should work in most cases. We recommend increasing it to e.g. 10MiB if
you are using Garage to store large files and have fast network connections
between all nodes (e.g. 1gbps).
If you are interested in tuning this, feel free to do so (and remember to
report your findings to us!). When this value is changed for a running Garage
installation, only files newly uploaded will be affected. Previously uploaded
files will remain available. This however means that chunks from existing files
will not be deduplicated with chunks from newly uploaded files, meaning you
might use more storage space that is optimally possible.
#### `sled_cache_capacity` {#sled_cache_capacity}
This parameter can be used to tune the capacity of the cache used by
[sled](https://sled.rs), the database Garage uses internally to store metadata.
Tune this to fit the RAM you wish to make available to your Garage instance.
This value has a conservative default (128MB) so that Garage doesn't use too much
RAM by default, but feel free to increase this for higher performance.
#### `sled_flush_every_ms` {#sled_flush_every_ms}
This parameters can be used to tune the flushing interval of sled.
Increase this if sled is thrashing your SSD, at the risk of losing more data in case
of a power outage (though this should not matter much as data is replicated on other
nodes). The default value, 2000ms, should be appropriate for most use cases.
#### `lmdb_map_size` {#lmdb_map_size}
This parameters can be used to set the map size used by LMDB,
which is the size of the virtual memory region used for mapping the database file.
The value of this parameter is the maximum size the metadata database can take.
This value is not bound by the physical RAM size of the machine running Garage.
If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines.
#### `compression_level` {#compression_level}
### `compression_level`
Zstd compression level to use for storing blocks.
@ -394,7 +334,7 @@ Compression is done synchronously, setting a value too high will add latency to
This value can be different between nodes, compression is done by the node which receive the
API call.
#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) {#rpc_secret}
### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env)
Garage uses a secret key, called an RPC secret, that is shared between all
nodes of the cluster in order to identify these nodes and allow them to
@ -406,7 +346,10 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is
given in the configuration variable `rpc_secret_file`, or specified as an
environment variable `GARAGE_RPC_SECRET`.
#### `rpc_bind_addr` {#rpc_bind_addr}
Since Garage `v0.9.0`, you can also specify the path of a file storing the secret
as the `GARAGE_RPC_SECRET_FILE` environment variable.
### `rpc_bind_addr`
The address and port on which to bind for inter-cluster communcations
(reffered to as RPC for remote procedure calls).
@ -415,14 +358,14 @@ the node, even in the case of a NAT: the NAT should be configured to forward the
port number to the same internal port nubmer. This means that if you have several nodes running
behind a NAT, they should each use a different RPC port number.
#### `rpc_public_addr` {#rpc_public_addr}
### `rpc_public_addr`
The address and port that other nodes need to use to contact this node for
RPC calls. **This parameter is optional but recommended.** In case you have
a NAT that binds the RPC port to a port that is different on your public IP,
this field might help making it work.
#### `bootstrap_peers` {#bootstrap_peers}
### `bootstrap_peers`
A list of peer identifiers on which to contact other Garage peers of this cluster.
These peer identifiers have the following syntax:
@ -439,42 +382,42 @@ key will be returned by `garage node id` and you will have to add the IP
yourself.
### The `[consul_discovery]` section
## The `[consul_discovery]` section
Garage supports discovering other nodes of the cluster using Consul. For this
to work correctly, nodes need to know their IP address by which they can be
reached by other nodes of the cluster, which should be set in `rpc_public_addr`.
#### `consul_http_addr` {#consul_http_addr}
### `consul_http_addr` and `service_name`
The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server.
#### `api` {#consul_api}
### `api`
Two APIs for service registration are supported: `catalog` and `agent`. `catalog`, the default, will register a service using
the `/v1/catalog` endpoints, enabling mTLS if `client_cert` and `client_key` are provided. The `agent` API uses the
`v1/agent` endpoints instead, where an optional `token` may be provided.
#### `service_name` {#consul_service_name}
### `service_name`
`service_name` should be set to the service name under which Garage's
RPC ports are announced.
#### `client_cert`, `client_key` {#consul_client_cert}
### `client_cert`, `client_key`
TLS client certificate and client key to use when communicating with Consul over TLS. Both are mandatory when doing so.
Only available when `api = "catalog"`.
#### `ca_cert` {#consul_ca_cert}
### `ca_cert`
TLS CA certificate to use when communicating with Consul over TLS.
#### `tls_skip_verify` {#consul_tls_skip_verify}
### `tls_skip_verify`
Skip server hostname verification in TLS handshake.
`ca_cert` is ignored when this is set.
#### `token` {#consul_token}
### `token`
Uses the provided token for communication with Consul. Only available when `api = "agent"`.
The policy assigned to this token should at least have these rules:
@ -494,49 +437,49 @@ node_prefix "" {
}
```
#### `tags` and `meta` {#consul_tags}
### `tags` and `meta`
Additional list of tags and map of service meta to add during service registration.
### The `[kubernetes_discovery]` section
## The `[kubernetes_discovery]` section
Garage supports discovering other nodes of the cluster using kubernetes custom
resources. For this to work, a `[kubernetes_discovery]` section must be present
with at least the `namespace` and `service_name` parameters.
#### `namespace` {#kube_namespace}
### `namespace`
`namespace` sets the namespace in which the custom resources are
configured.
#### `service_name` {#kube_service_name}
### `service_name`
`service_name` is added as a label to the advertised resources to
filter them, to allow for multiple deployments in a single namespace.
#### `skip_crd` {#kube_skip_crd}
### `skip_crd`
`skip_crd` can be set to true to disable the automatic creation and
patching of the `garagenodes.deuxfleurs.fr` CRD. You will need to create the CRD
manually.
### The `[s3_api]` section
## The `[s3_api]` section
#### `api_bind_addr` {#s3_api_bind_addr}
### `api_bind_addr`
The IP and port on which to bind for accepting S3 API calls.
This endpoint does not suport TLS: a reverse proxy should be used to provide it.
Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode.
#### `s3_region` {#s3_region}
### `s3_region`
Garage will accept S3 API calls that are targetted to the S3 region defined here.
API calls targetted to other regions will fail with a AuthorizationHeaderMalformed error
message that redirects the client to the correct region.
#### `root_domain` {#s3_root_domain}
### `root_domain` {#root_domain}
The optional suffix to access bucket using vhost-style in addition to path-style request.
Note path-style requests are always enabled, whether or not vhost-style is configured.
@ -548,12 +491,12 @@ using the hostname `my-bucket.s3.garage.eu`.
### The `[s3_web]` section
## The `[s3_web]` section
Garage allows to publish content of buckets as websites. This section configures the
behaviour of this module.
#### `bind_addr` {#web_bind_addr}
### `bind_addr`
The IP and port on which to bind for accepting HTTP requests to buckets configured
for website access.
@ -561,7 +504,7 @@ This endpoint does not suport TLS: a reverse proxy should be used to provide it.
Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode.
#### `root_domain` {#web_root_domain}
### `root_domain`
The optional suffix appended to bucket names for the corresponding HTTP Host.
@ -570,11 +513,11 @@ will be accessible either with hostname `deuxfleurs.fr.web.garage.eu`
or with hostname `deuxfleurs.fr`.
### The `[admin]` section
## The `[admin]` section
Garage has a few administration capabilities, in particular to allow remote monitoring. These features are detailed below.
#### `api_bind_addr` {#admin_api_bind_addr}
### `api_bind_addr`
If specified, Garage will bind an HTTP server to this port and address, on
which it will listen to requests for administration features.
@ -583,7 +526,7 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md
Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) {#admin_metrics_token}
### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env)
The token for accessing the Metrics endpoint. If this token is not set, the
Metrics endpoint can be accessed without access control.
@ -593,8 +536,9 @@ You can use any random string for this value. We recommend generating a random t
`metrics_token` was introduced in Garage `v0.7.2`.
`metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`.
`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.9.0`.
#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) {#admin_token}
### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env)
The token for accessing all of the other administration endpoints. If this
token is not set, access to these endpoints is disabled entirely.
@ -604,8 +548,9 @@ You can use any random string for this value. We recommend generating a random t
`admin_token` was introduced in Garage `v0.7.2`.
`admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.9.0`.
#### `trace_sink` {#admin_trace_sink}
### `trace_sink`
Optionally, the address of an OpenTelemetry collector. If specified,
Garage will send traces in the OpenTelemetry format to this endpoint. These

View File

@ -52,7 +52,7 @@ This is particularly usefull when nodes are far from one another and talk to one
Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data,
and with various levels of consistency, in order to adapt to a variety of usage scenarios.
Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_mode)
Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication-mode)
to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want).
### Web server for static websites

View File

@ -1,5 +1,31 @@
{
"nodes": {
"cargo2nix": {
"inputs": {
"flake-compat": [
"flake-compat"
],
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
],
"rust-overlay": "rust-overlay"
},
"locked": {
"lastModified": 1666087781,
"narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
"owner": "Alexis211",
"repo": "cargo2nix",
"rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
"type": "github"
},
"original": {
"owner": "Alexis211",
"repo": "cargo2nix",
"rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
"type": "github"
}
},
"flake-compat": {
"locked": {
"lastModified": 1688025799,
@ -20,19 +46,54 @@
"systems": "systems"
},
"locked": {
"lastModified": 1701680307,
"narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"id": "flake-utils",
"type": "indirect"
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1682109806,
"narHash": "sha256-d9g7RKNShMLboTWwukM+RObDWWpHKaqTYXB48clBWXI=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "2362848adf8def2866fabbffc50462e929d7fffb",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1682423271,
"narHash": "sha256-WHhl1GiOij1ob4cTLL+yhqr+vFOUH8E5wAX8Ir8fvjE=",
@ -50,9 +111,32 @@
},
"root": {
"inputs": {
"cargo2nix": "cargo2nix",
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"flake-utils": [
"cargo2nix",
"flake-utils"
],
"nixpkgs": "nixpkgs_2"
}
},
"rust-overlay": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1682389182,
"narHash": "sha256-8t2nmFnH+8V48+IJsf8AK51ebXNlVbOSVYOpiqJKvJE=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "74f1a64dd28faeeb85ef081f32cad2989850322c",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
},
"systems": {
@ -69,6 +153,21 @@
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",

View File

@ -8,23 +8,50 @@
inputs.flake-compat.url = "github:nix-community/flake-compat";
outputs = { self, nixpkgs, flake-utils, ... }:
inputs.cargo2nix = {
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
# As of 2023-04-25:
# - my two patches were merged into unstable (one for clippy and one to "fix" feature detection)
# - rustc v1.66
# url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2";
# Rust overlay as of 2023-04-25
inputs.rust-overlay.url =
"github:oxalica/rust-overlay/74f1a64dd28faeeb85ef081f32cad2989850322c";
inputs.nixpkgs.follows = "nixpkgs";
inputs.flake-compat.follows = "flake-compat";
};
inputs.flake-utils.follows = "cargo2nix/flake-utils";
outputs = { self, nixpkgs, cargo2nix, flake-utils, ... }:
let
gitVersion = self.lastModifiedDate;
git_version = self.lastModifiedDate;
compile = import ./nix/compile.nix;
in
flake-utils.lib.eachDefaultSystem (system:
let pkgs = nixpkgs.legacyPackages.${system};
in {
packages = {
default = (import ./default.nix {
inherit gitVersion;
buildSystem = system;
default = (compile {
inherit system git_version;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
release = true;
}).build;
}).workspace.garage { compileMode = "build"; };
};
devShell = (import ./shell.nix {
buildSystem = system;
}).rust;
devShell = (compile {
inherit system git_version;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
release = false;
}).workspaceShell { packages = with pkgs; [
rustfmt
clang
mold
]; };
});
}

View File

@ -1,6 +1,6 @@
{ path ? "/../aws-list.txt", }:
with import ./pkgs.nix;
with import ./common.nix;
let
pkgs = import pkgsSrc { };
lib = pkgs.lib;

17
nix/common.nix Normal file
View File

@ -0,0 +1,17 @@
let
lock = builtins.fromJSON (builtins.readFile ../flake.lock);
inherit (lock.nodes.flake-compat.locked) owner repo rev narHash;
flake-compat = fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz";
sha256 = narHash;
};
flake = (import flake-compat { system = builtins.currentSystem; src = ../.; });
in
rec {
pkgsSrc = flake.defaultNix.inputs.nixpkgs;
cargo2nix = flake.defaultNix.inputs.cargo2nix;
cargo2nixOverlay = cargo2nix.overlays.default;
}

View File

@ -1,8 +0,0 @@
let
lock = builtins.fromJSON (builtins.readFile ../flake.lock);
inherit (lock.nodes.nixpkgs.locked) owner repo rev narHash;
in
fetchTarball {
url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz";
sha256 = narHash;
}

View File

@ -1,12 +1,12 @@
{
buildSystem ? builtins.currentSystem,
targetSystem ? buildSystem,
}:
{ system ? builtins.currentSystem, }:
with import ./nix/pkgs.nix;
with import ./nix/common.nix;
let
inherit (import ./default.nix { inherit buildSystem targetSystem; }) pkgs pkgsCross buildEnv;
pkgs = import pkgsSrc {
inherit system;
overlays = [ cargo2nixOverlay ];
};
kaniko = (import ./nix/kaniko.nix) pkgs;
manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
winscp = (import ./nix/winscp.nix) pkgs;
@ -14,13 +14,22 @@ let
in {
# --- Rust Shell ---
# Use it to compile Garage
rust = pkgsCross.mkShell (buildEnv // {
inputsFrom = [
kaniko
manifest-tool
winscp
rust = pkgs.mkShell {
nativeBuildInputs = with pkgs; [
#rustPlatform.rust.rustc
rustPlatform.rust.cargo
clang
mold
#clippy
rustfmt
#perl
#protobuf
#pkg-config
#openssl
file
#cargo2nix.packages.x86_64-linux.cargo2nix
];
});
};
# --- Integration shell ---
# Use it to test Garage with common S3 clients

View File

@ -45,7 +45,7 @@ http = "0.2"
httpdate = "1.0"
http-range = "0.1"
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
#hyperlocal = { version = "0.8.0", default-features = false, features = ["server"] }
hyperlocal = { version = "0.8.0", default-features = false, features = ["server"] }
multer = "2.0"
percent-encoding = "2.1.0"
roxmltree = "0.18"

View File

@ -182,7 +182,7 @@ impl AdminApiServer {
),
};
let status_str = format!(
"{}\nConsult the full health check API endpoint at /v1/health for more details\n",
"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
status_str
);

View File

@ -1,4 +1,5 @@
//use std::fs::{self, Permissions};
use std::fs::{self, Permissions};
use std::os::unix::fs::PermissionsExt;
use std::sync::Arc;
use async_trait::async_trait;
@ -11,9 +12,9 @@ use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Request, Response, Server};
use hyper::{HeaderMap, StatusCode};
//use hyperlocal::UnixServerExt;
use hyperlocal::UnixServerExt;
//use tokio::net::UnixStream;
use tokio::net::UnixStream;
use opentelemetry::{
global,
@ -113,18 +114,18 @@ impl<A: ApiHandler> ApiServer<A> {
}
});
//let unix_service = make_service_fn(|_: &UnixStream| {
// let this = self.clone();
let unix_service = make_service_fn(|_: &UnixStream| {
let this = self.clone();
// let path = bind_addr.to_string();
// async move {
// Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
// let this = this.clone();
let path = bind_addr.to_string();
async move {
Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
let this = this.clone();
// this.handler(req, path.clone())
// }))
// }
//});
this.handler(req, path.clone())
}))
}
});
info!(
"{} API server listening on {}",
@ -139,24 +140,23 @@ impl<A: ApiHandler> ApiServer<A> {
.with_graceful_shutdown(shutdown_signal)
.await?
}
UnixOrTCPSocketAddress::UnixSocket(_path) => {
panic!("Unix sockets are not supported in this fork") // TODO(mediocregopher)
} //UnixOrTCPSocketAddress::UnixSocket(ref path) => {
// use std::os::unix::fs::PermissionsExt;
// remove_unix_socket_if_present(path).await?;
UnixOrTCPSocketAddress::UnixSocket(ref path) => {
if path.exists() {
fs::remove_file(path)?
}
// let bound = Server::bind_unix(path)?;
let bound = Server::bind_unix(path)?;
// fs::set_permissions(
// path,
// Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)),
// )?;
fs::set_permissions(
path,
Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)),
)?;
// bound
// .serve(unix_service)
// .with_graceful_shutdown(shutdown_signal)
// .await?;
//}
bound
.serve(unix_service)
.with_graceful_shutdown(shutdown_signal)
.await?;
}
};
Ok(())

View File

@ -344,7 +344,7 @@ impl ApiHandler for S3ApiServer {
bucket_id,
key,
upload_id,
part_number_marker: part_number_marker.map(|p| p.min(10000)),
part_number_marker: part_number_marker.map(|p| p.clamp(1, 10000)),
max_parts: max_parts.unwrap_or(1000).clamp(1, 1000),
},
)

View File

@ -15,6 +15,7 @@ use serde::Deserialize;
use garage_model::garage::Garage;
use crate::s3::cors::*;
use crate::s3::error::*;
use crate::s3::put::{get_headers, save_stream};
use crate::s3::xml as s3_xml;
@ -242,7 +243,7 @@ pub async fn handle_post_object(
let etag = format!("\"{}\"", md5);
let resp = if let Some(mut target) = params
let mut resp = if let Some(mut target) = params
.get("success_action_redirect")
.and_then(|h| h.to_str().ok())
.and_then(|u| url::Url::parse(u).ok())
@ -262,8 +263,7 @@ pub async fn handle_post_object(
} else {
let path = head
.uri
.into_parts()
.path_and_query
.path_and_query()
.map(|paq| paq.path().to_string())
.unwrap_or_else(|| "/".to_string());
let authority = head
@ -308,6 +308,13 @@ pub async fn handle_post_object(
}
};
let matching_cors_rule =
find_matching_cors_rule(&bucket, &Request::from_parts(head, Body::empty()))?;
if let Some(rule) = matching_cors_rule {
add_cors_headers(&mut resp, rule)
.ok_or_internal_error("Invalid bucket CORS configuration")?;
}
Ok(resp)
}

View File

@ -771,7 +771,11 @@ impl BlockManagerLocked {
// Now, we do an fsync on the containing directory, to ensure that the rename
// is persisted properly. See:
// http://thedjbway.b0llix.net/qmail/syncdir.html
let dir = fs::OpenOptions::new().read(true).open(directory).await?;
let dir = fs::OpenOptions::new()
.read(true)
.mode(0)
.open(directory)
.await?;
dir.sync_all().await?;
drop(dir);
}

View File

@ -28,7 +28,7 @@ use structopt::StructOpt;
use netapp::util::parse_and_resolve_peer_addr;
use netapp::NetworkKey;
use garage_util::config::Config;
use garage_util::config::{read_secret_file, Config};
use garage_util::error::*;
use garage_rpc::system::*;
@ -73,15 +73,30 @@ pub struct Secrets {
#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
pub rpc_secret: Option<String>,
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
/// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
/// when running the daemon or doing admin operations
#[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
pub rpc_secret_file: Option<String>,
/// Admin API authentication token, replaces admin.admin_token in config.toml when
/// running the Garage daemon
#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
pub admin_token: Option<String>,
/// Admin API authentication token file path, replaces admin.admin_token in config.toml
/// and admin-token when running the Garage daemon
#[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
pub admin_token_file: Option<String>,
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
/// running the Garage daemon
#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
pub metrics_token: Option<String>,
/// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
/// and metrics-token when running the Garage daemon
#[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
pub metrics_token_file: Option<String>,
}
#[tokio::main]
@ -262,15 +277,24 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
}
}
fn fill_secrets(mut config: Config, secrets: Secrets) -> Config {
fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
if secrets.rpc_secret.is_some() {
config.rpc_secret = secrets.rpc_secret;
} else if secrets.rpc_secret_file.is_some() {
config.rpc_secret = Some(read_secret_file(&secrets.rpc_secret_file.unwrap())?);
}
if secrets.admin_token.is_some() {
config.admin.admin_token = secrets.admin_token;
} else if secrets.admin_token_file.is_some() {
config.admin.admin_token = Some(read_secret_file(&secrets.admin_token_file.unwrap())?);
}
if secrets.metrics_token.is_some() {
config.admin.metrics_token = secrets.metrics_token;
} else if secrets.metrics_token_file.is_some() {
config.admin.metrics_token = Some(read_secret_file(&secrets.metrics_token_file.unwrap())?);
}
config
Ok(config)
}

View File

@ -20,7 +20,7 @@ pub async fn offline_repair(
}
info!("Loading configuration...");
let config = fill_secrets(read_config(config_file)?, secrets);
let config = fill_secrets(read_config(config_file)?, secrets)?;
info!("Initializing Garage main data store...");
let garage = Garage::new(config)?;

View File

@ -29,7 +29,7 @@ async fn wait_from(mut chan: watch::Receiver<bool>) {
pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> {
info!("Loading configuration...");
let config = fill_secrets(read_config(config_file)?, secrets);
let config = fill_secrets(read_config(config_file)?, secrets)?;
// ---- Initialize Garage internals ----

View File

@ -3,6 +3,7 @@ use std::collections::HashMap;
use std::io::{Read, Write};
use std::net::{IpAddr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering;
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};
@ -197,20 +198,6 @@ pub fn read_node_id(metadata_dir: &Path) -> Result<NodeID, Error> {
Ok(NodeID::from_slice(&key[..]).unwrap())
}
#[cfg(not(windows))]
fn set_private_key_perms(path: &Path) -> Result<(), Error> {
use std::os::unix::fs::PermissionsExt;
let perm = std::fs::Permissions::from_mode(0o600);
std::fs::set_permissions(path, perm)?;
Ok(())
}
#[cfg(windows)]
fn set_private_key_perms(_path: &Path) -> Result<(), Error> {
// TODO(mediocregopher) figure out how to do this, but it's not strictly necessary
Ok(())
}
pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
let mut key_file = metadata_dir.to_path_buf();
key_file.push("node_key");
@ -235,8 +222,11 @@ pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
let (pubkey, key) = ed25519::gen_keypair();
{
use std::os::unix::fs::PermissionsExt;
let mut f = std::fs::File::create(key_file.as_path())?;
set_private_key_perms(key_file.as_path())?;
let mut perm = f.metadata()?.permissions();
perm.set_mode(0o600);
std::fs::set_permissions(key_file.as_path(), perm)?;
f.write_all(&key[..])?;
}
@ -900,18 +890,6 @@ impl NodeStatus {
}
}
#[cfg(windows)]
fn update_disk_usage(
&mut self,
_meta_dir: &Path,
_data_dir: &DataDirEnum,
_metrics: &SystemMetrics,
) {
// TODO(mediocregopher) it'd be nice to have this for windows too, but it seems to only be
// used for OpenTelemetry so it's not a real requirement.
}
#[cfg(not(windows))]
fn update_disk_usage(
&mut self,
meta_dir: &Path,
@ -919,7 +897,6 @@ impl NodeStatus {
metrics: &SystemMetrics,
) {
use nix::sys::statvfs::statvfs;
use std::sync::atomic::Ordering;
let mount_avail = |path: &Path| match statvfs(path) {
Ok(x) => {
let avail = x.blocks_available() as u64 * x.fragment_size() as u64;
@ -978,7 +955,6 @@ impl NodeStatus {
}
}
#[cfg(not(windows))]
fn get_default_ip() -> Option<IpAddr> {
pnet_datalink::interfaces()
.iter()
@ -987,11 +963,6 @@ fn get_default_ip() -> Option<IpAddr> {
.map(|a| a.ip())
}
#[cfg(windows)]
fn get_default_ip() -> Option<IpAddr> {
None
}
async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> {
let mut ret = vec![];

View File

@ -265,6 +265,24 @@ pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
Ok(parsed_config)
}
pub fn read_secret_file(file_path: &String) -> Result<String, Error> {
#[cfg(unix)]
if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
use std::os::unix::fs::MetadataExt;
let metadata = std::fs::metadata(file_path)?;
if metadata.mode() & 0o077 != 0 {
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
}
}
let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?;
let mut secret_buf = String::new();
file.read_to_string(&mut secret_buf)?;
// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
// also editors sometimes add a trailing newline
Ok(String::from(secret_buf.trim_end()))
}
fn secret_from_file(
secret: &mut Option<String>,
secret_file: &Option<String>,
@ -277,22 +295,7 @@ fn secret_from_file(
(Some(_), Some(_)) => {
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
}
(None, Some(file_path)) => {
#[cfg(unix)]
if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
use std::os::unix::fs::MetadataExt;
let metadata = std::fs::metadata(file_path)?;
if metadata.mode() & 0o077 != 0 {
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
}
}
let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?;
let mut secret_buf = String::new();
file.read_to_string(&mut secret_buf)?;
// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
// also editors sometimes add a trailing newline
*secret = Some(String::from(secret_buf.trim_end()));
}
(None, Some(file_path)) => *secret = Some(read_secret_file(file_path)?),
}
Ok(())
}

View File

@ -27,7 +27,7 @@ futures = "0.3"
http = "0.2"
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
#hyperlocal = { version = "0.8.0", default-features = false, features = ["server"] }
hyperlocal = { version = "0.8.0", default-features = false, features = ["server"] }
tokio = { version = "1.0", default-features = false, features = ["net"] }

View File

@ -1,5 +1,5 @@
//use std::fs::{self, Permissions};
//use std::os::unix::prelude::PermissionsExt;
use std::fs::{self, Permissions};
use std::os::unix::prelude::PermissionsExt;
use std::{convert::Infallible, sync::Arc};
use futures::future::Future;
@ -11,9 +11,9 @@ use hyper::{
Body, Method, Request, Response, Server, StatusCode,
};
//use hyperlocal::UnixServerExt;
use hyperlocal::UnixServerExt;
//use tokio::net::UnixStream;
use tokio::net::UnixStream;
use opentelemetry::{
global,
@ -100,18 +100,18 @@ impl WebServer {
}
});
//let unix_service = make_service_fn(|_: &UnixStream| {
// let web_server = web_server.clone();
let unix_service = make_service_fn(|_: &UnixStream| {
let web_server = web_server.clone();
// let path = addr.to_string();
// async move {
// remove_unix_socket_if_present(&path).await.expect("could not remove existing unix socket");
// Ok::<_, Error>(service_fn(move |req: Request<Body>| {
// let web_server = web_server.clone();
// web_server.handle_request(req, path.clone())
// }))
// }
//});
let path = addr.to_string();
async move {
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
let web_server = web_server.clone();
web_server.handle_request(req, path.clone())
}))
}
});
info!("Web server listening on {}", addr);
@ -122,22 +122,20 @@ impl WebServer {
.with_graceful_shutdown(shutdown_signal)
.await?
}
UnixOrTCPSocketAddress::UnixSocket(_path) => {
panic!("Unix sockets are not supported in this fork") // TODO(mediocregopher)
} //UnixOrTCPSocketAddress::UnixSocket(ref path) => {
// if path.exists() {
// fs::remove_file(path)?
// }
UnixOrTCPSocketAddress::UnixSocket(ref path) => {
if path.exists() {
fs::remove_file(path)?
}
// let bound = Server::bind_unix(path)?;
let bound = Server::bind_unix(path)?;
// fs::set_permissions(path, Permissions::from_mode(0o222))?;
fs::set_permissions(path, Permissions::from_mode(0o222))?;
// bound
// .serve(unix_service)
// .with_graceful_shutdown(shutdown_signal)
// .await?;
//}
bound
.serve(unix_service)
.with_graceful_shutdown(shutdown_signal)
.await?;
}
};
Ok(())