Moved http proxy under the service module

.dev-config.yml

@@ -3,16 +3,6 @@ origin:
 domain:
   store_dir_path: /tmp/domani_dev_env/domain
   builtins:
-    foo:
-      kind: proxy
-      url: http://127.0.0.1:9000
-      request_http_headers:
-        - name: x-foo
-          value: BAR
-        - name: host
-          value: hi
-        - name: user-agent
-          value: ""
     bar:
       kind: git
       url: a
@@ -21,6 +11,16 @@ domain:
 service:
   http:
     form_method: GET
+    proxied_domains:
+      foo:
+        url: http://127.0.0.1:9000
+        request_headers:
+          - name: x-foo
+            value: BAR
+          - name: host
+            value: hi
+          - name: user-agent
+            value: ""
   passphrase: foobar
   dns_records:
     - kind: A

README.md

@@ -63,40 +63,10 @@ domain:
     #  kind: git
     #  url: "https://somewhere.com/some/repo.git"
     #  branch_name: main
+    #
+    #  # If true then the built-in will be included in the web interface's
+    #  # domain list, but will not be configurable in the web interface
     #  public: false
-    #
-    #  # Which protocols to serve the domain on. The given list overwrites the
-    #  # default, which is to serve on all available protocols.
-    #  #serve_protocols:
-    #  #- protocol: http
-    #  #- protocol: https
-
-    # An example built-in domain backed by a reverse-proxy to some other
-    # web-service. Requests to the backing service will automatically have
-    # X-Forwarded-For and (if HTTPS) X-Forwarded-Proto headers added to them.
-    #
-    # Proxies are currently limited in the following ways:
-    # * url must be to an http endpoint (not https)
-    # * dns.resolver_addr is ignored and the system-wide dns is used
-    #
-    #proxy.example.com:
-    #  kind: proxy
-    #  url: "http://some.other.service.com"
-    #
-    #  # Extra headers to add to requests being proxied
-    #  request_http_headers:
-    #    - name: Host
-    #      value: "yet.another.service.com"
-    #    - name: X-HEADER-TO-DELETE
-    #      value: ""
-    #
-    #  public: false
-    #
-    #  # Which protocols to serve the domain on. The given list overwrites the
-    #  # default, which is to serve on all available protocols.
-    #  #serve_protocols:
-    #  #- protocol: http
-    #  #- protocol: https
 
 service:
 
@@ -130,10 +100,32 @@ service:
 
     # The address to listen for HTTP requests on. This must use port 80 if
     # https_addr is set.
-    #http_addr: "[::]:3030"
+    #http_addr: "[::]:3080"
 
     # The address to listen for HTTPS requests on. This is optional.
     #https_addr: "[::]:443"
+
+    #proxied_domains:
+
+      # An example built-in domain backed by an HTTP reverse-proxy to some
+      # other web-service. Requests to the backing service will automatically
+      # have X-Forwarded-For and (if HTTPS) X-Forwarded-Proto headers added to
+      # them.
+      #
+      # Proxies are currently limited in the following ways:
+      # * url must be to an http endpoint (not https)
+      # * dns.resolver_addr is ignored and the system-wide dns is used
+      #
+      #proxy.example.com:
+      #  kind: http_proxy
+      #  url: "http://some.other.service.com"
+      #
+      #  # Extra headers to add to proxied requests
+      #  request_headers:
+      #    - name: Host
+      #      value: "yet.another.service.com"
+      #    - name: X-HEADER-TO-DELETE
+      #      value: ""
 ```
 
 The YAML config file can be passed to the Domani process via the `--config-path`

src/domain/manager.rs

@@ -248,10 +248,6 @@ impl Manager for ManagerImpl {
     ) -> Result<util::BoxByteStream, GetFileError> {
         let settings = self.domain_store.get(domain)?.settings;
 
-        if let origin::Descr::Proxy { .. } = settings.origin_descr {
-            return Err(unexpected::Error::from("origin is proxy, can't serve file").into());
-        }
-
         let path = settings.process_path(path);
 
         let f = self

src/domain/settings.rs

@@ -14,7 +14,6 @@ pub struct Settings {
     #[serde(flatten)]
     pub origin_descr: origin::Descr,
 
-    pub remove_path_prefix: Option<String>,
     pub add_path_prefix: Option<String>,
 }
 
@@ -25,31 +24,6 @@ impl Settings {
         Ok(h.finalize().encode_hex::<String>())
     }
 
-    fn remove_path_prefix<'path, 'prefix>(
-        path: borrow::Cow<'path, str>,
-        prefix: &'prefix str,
-    ) -> borrow::Cow<'path, str> {
-        if prefix.len() == 0 {
-            return path;
-        }
-
-        let mut prefix = prefix.trim_end_matches('/');
-        prefix = prefix.trim_start_matches('/');
-
-        let mut stripped_path = path.trim_start_matches('/');
-
-        if !stripped_path.starts_with(prefix) {
-            return path;
-        }
-
-        stripped_path = stripped_path.strip_prefix(prefix).unwrap();
-        if stripped_path.len() == 0 {
-            return borrow::Cow::Borrowed("/");
-        }
-
-        borrow::Cow::Owned(stripped_path.to_string())
-    }
-
     fn add_path_prefix<'path, 'prefix>(
         path: borrow::Cow<'path, str>,
         prefix: &'prefix str,
@@ -73,10 +47,6 @@ impl Settings {
     pub fn process_path<'a>(&self, path: &'a str) -> borrow::Cow<'a, str> {
         let mut path = borrow::Cow::Borrowed(path);
 
-        if let Some(ref prefix) = self.remove_path_prefix {
-            path = Self::remove_path_prefix(path, prefix);
-        }
-
         if let Some(ref prefix) = self.add_path_prefix {
             path = Self::add_path_prefix(path, prefix);
         }
@@ -90,26 +60,6 @@ mod tests {
     use super::*;
     use std::borrow;
 
-    #[test]
-    fn remove_path_prefix() {
-        let assert_remove = |want: &str, path: &str, prefix: &str| {
-            assert_eq!(
-                want,
-                Settings::remove_path_prefix(borrow::Cow::Borrowed(path), prefix).as_ref(),
-            )
-        };
-
-        assert_remove("/bar", "/foo/bar", "/foo");
-        assert_remove("/foo/bar", "/foo/bar", "/baz");
-        assert_remove("/bar", "/foo/bar", "/foo/");
-        assert_remove("/bar", "/foo/bar", "/foo///");
-        assert_remove("/", "/", "/");
-        assert_remove("/", "/foo/bar/", "/foo/bar");
-        assert_remove("/", "/foo/bar", "/foo/bar");
-        assert_remove("/", "/foo/bar", "/foo/bar///");
-        assert_remove("/bar", "/bar", "");
-    }
-
     #[test]
     fn add_path_prefix() {
         let assert_add = |want: &str, path: &str, prefix: &str| {

src/domain/store.rs

@@ -181,7 +181,6 @@ mod tests {
                 url: "bar".to_string(),
                 branch_name: "baz".to_string(),
             },
-            remove_path_prefix: None,
             add_path_prefix: None,
         };
 
@@ -205,7 +204,6 @@ mod tests {
                 url: "BAR".to_string(),
                 branch_name: "BAZ".to_string(),
             },
-            remove_path_prefix: None,
             add_path_prefix: None,
         };
 

src/main.rs

@@ -78,16 +78,6 @@ async fn main() {
             config.service.dns_records.push(primary_cname);
         }
 
-        for (domain, builtin_domain) in &config.domain.builtins {
-            if let domani::origin::Descr::Proxy { ref url, .. } =
-                builtin_domain.settings.origin_descr
-            {
-                if let Err(e) = domani::origin::proxy::validate_proxy_url(url) {
-                    panic!("invalid config for builtin domain {domain}: {e}");
-                }
-            }
-        }
-
         config
     };
 

src/origin.rs

@@ -2,7 +2,6 @@ mod config;
 mod descr;
 pub mod git;
 pub mod mux;
-pub mod proxy;
 
 pub use config::*;
 pub use descr::Descr;

src/origin/descr.rs

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct DescrProxyHttpHeader {
+pub struct DescrHttpProxyHeader {
     pub name: String,
     pub value: String,
 }
@@ -14,12 +14,6 @@ pub struct DescrProxyHttpHeader {
 pub enum Descr {
     #[serde(rename = "git")]
     Git { url: String, branch_name: String },
-
-    #[serde(rename = "proxy")]
-    Proxy {
-        url: String,
-        request_http_headers: Vec<DescrProxyHttpHeader>,
-    },
 }
 
 impl Descr {
@@ -38,18 +32,6 @@ impl Descr {
                 h_update(url);
                 h_update(branch_name);
             }
-            Descr::Proxy {
-                url,
-                request_http_headers,
-            } => {
-                h_update("proxy");
-                h_update(url);
-                for h in request_http_headers {
-                    h_update("header");
-                    h_update(&h.name);
-                    h_update(&h.value);
-                }
-            }
         }
 
         h.finalize().encode_hex::<String>()

src/origin/git.rs

@@ -57,15 +57,11 @@ impl FSStore {
     }
 
     fn deconstruct_descr(descr: &origin::Descr) -> (&str, &str) {
-        if let origin::Descr::Git {
+        let origin::Descr::Git {
             ref url,
             ref branch_name,
-        } = descr
+        } = descr;
-        {
+        (url, branch_name)
-            (url, branch_name)
-        } else {
-            panic!("non git descr passed in")
-        }
     }
 
     fn create_repo_snapshot(

src/origin/proxy.rs

@@ -1,78 +0,0 @@
-use crate::error::unexpected::{self, Mappable};
-use crate::{domain, origin};
-use http::header::{HeaderName, HeaderValue};
-use std::{net, str::FromStr};
-
-// proxy is a special case because it is so tied to the underlying protocol that a request is
-// being served on, it can't be abstracted out into a simple "get_file" operation like other
-// origins.
-
-pub fn validate_proxy_url(proxy_url: &str) -> unexpected::Result<()> {
-    let parsed_proxy_url =
-        http::Uri::from_str(proxy_url).or_unexpected_while("parsing proxy url {proxy_url}")?;
-
-    let scheme = parsed_proxy_url.scheme().map_unexpected_while(|| {
-        format!("expected a scheme of http in the proxy url {proxy_url}")
-    })?;
-
-    if scheme != "http" {
-        return Err(unexpected::Error::from(
-            format!("scheme of proxy url {proxy_url} should be 'http'",).as_str(),
-        ));
-    }
-
-    Ok(())
-}
-
-pub async fn serve_http_request(
-    settings: &domain::Settings,
-    client_ip: net::IpAddr,
-    mut req: hyper::Request<hyper::Body>,
-    req_is_https: bool,
-) -> unexpected::Result<hyper::Response<hyper::Body>> {
-    let (url, request_http_headers) = if let origin::Descr::Proxy {
-        ref url,
-        ref request_http_headers,
-    } = settings.origin_descr
-    {
-        (url, request_http_headers)
-    } else {
-        panic!("non-proxy domain settings passed in: {settings:?}")
-    };
-
-    for header in request_http_headers {
-        let name: HeaderName = header
-            .name
-            .as_str()
-            .try_into()
-            .map_unexpected_while(|| format!("parsing header name {}", &header.name))?;
-
-        if header.value == "" {
-            req.headers_mut().remove(name);
-            continue;
-        }
-
-        let value = HeaderValue::from_str(&header.value).map_unexpected_while(|| {
-            format!(
-                "parsing {} as value for header {}",
-                &header.value, &header.name
-            )
-        })?;
-
-        req.headers_mut().insert(name, value);
-    }
-
-    if req_is_https {
-        req.headers_mut()
-            .insert("x-forwarded-proto", HeaderValue::from_static("https"));
-    }
-
-    match hyper_reverse_proxy::call(client_ip, url, req).await {
-        Ok(res) => Ok(res),
-        // ProxyError doesn't actually implement Error :facepalm: so we have to format the error
-        // manually
-        Err(e) => Err(unexpected::Error::from(
-            format!("error while proxying to {url}: {e:?}").as_str(),
-        )),
-    }
-}

src/service/http.rs

@@ -1,4 +1,5 @@
 mod config;
+mod proxy;
 mod tasks;
 mod tpl;
 
@@ -12,7 +13,7 @@ use std::str::FromStr;
 use std::{future, net, sync};
 
 use crate::error::unexpected;
-use crate::{domain, origin, service, util};
+use crate::{domain, service, util};
 
 pub struct Service {
     domain_manager: sync::Arc<dyn domain::manager::Manager>,
@@ -158,34 +159,7 @@ impl<'svc> Service {
         )
     }
 
-    async fn serve_origin(
+    async fn serve_origin(&self, domain: domain::Name, req: Request<Body>) -> Response<Body> {
-        &self,
-        client_ip: net::IpAddr,
-        domain: domain::Name,
-        req: Request<Body>,
-        req_is_https: bool,
-    ) -> Response<Body> {
-        let settings = match self.domain_manager.get_settings(&domain) {
-            Ok(domain::manager::GetSettingsResult { settings, .. }) => settings,
-            Err(domain::manager::GetSettingsError::NotFound) => {
-                return self.render_error_page(404, "Domain not found");
-            }
-            Err(domain::manager::GetSettingsError::Unexpected(e)) => {
-                return self.internal_error(
-                    format!("failed to fetch settings for domain {domain}: {e}").as_str(),
-                );
-            }
-        };
-
-        // if the domain is backed by a proxy then that is handled specially.
-        if let origin::Descr::Proxy { .. } = settings.origin_descr {
-            return origin::proxy::serve_http_request(&settings, client_ip, req, req_is_https)
-                .await
-                .unwrap_or_else(|e| {
-                    self.internal_error(format!("proxying {domain}: {e}").as_str())
-                });
-        };
-
         let mut path_owned;
         let path = req.uri().path();
 
@@ -463,11 +437,28 @@ impl<'svc> Service {
             }
         }
 
-        // If a managed domain was given then serve that from its origin, which is possibly a proxy
+        // If a managed domain was given then serve that from its origin or a proxy.
         if let Some(domain) = maybe_host {
-            return self
+            if let Some(proxied_domain_config) = self.config.http.proxied_domains.get(&domain) {
-                .serve_origin(client_ip, domain, req, req_is_https)
+                return service::http::proxy::serve_http_request(
-                .await;
+                    proxied_domain_config,
+                    client_ip,
+                    req,
+                    req_is_https,
+                )
+                .await
+                .unwrap_or_else(|e| {
+                    self.internal_error(
+                        format!(
+                            "serving {domain} via proxy {}: {e}",
+                            proxied_domain_config.url.as_ref()
+                        )
+                        .as_str(),
+                    )
+                });
+            }
+
+            return self.serve_origin(domain, req).await;
         }
 
         // Serve main domani site

src/service/http/config.rs

@@ -1,5 +1,10 @@
+use crate::domain;
+use crate::error::unexpected::{self, Mappable};
+
 use serde::{Deserialize, Serialize};
-use std::{net, str::FromStr};
+use serde_with::{serde_as, TryFromInto};
+
+use std::{collections, net, str::FromStr};
 
 fn default_http_addr() -> net::SocketAddr {
     net::SocketAddr::from_str("[::]:3030").unwrap()
@@ -35,6 +40,105 @@ impl AsRef<hyper::Method> for ConfigFormMethod {
     }
 }
 
+#[derive(Clone)]
+pub struct ConfigProxiedDomainUrl(String);
+
+impl AsRef<str> for ConfigProxiedDomainUrl {
+    fn as_ref(&self) -> &str {
+        return &self.0;
+    }
+}
+
+impl From<ConfigProxiedDomainUrl> for String {
+    fn from(url: ConfigProxiedDomainUrl) -> Self {
+        url.0
+    }
+}
+
+impl TryFrom<String> for ConfigProxiedDomainUrl {
+    type Error = unexpected::Error;
+
+    fn try_from(url: String) -> Result<Self, Self::Error> {
+        let parsed = http::Uri::from_str(url.as_str())
+            .or_unexpected_while("parsing proxy url {proxy_url}")?;
+
+        let scheme = parsed
+            .scheme()
+            .map_unexpected_while(|| format!("expected a scheme of http in the proxy url {url}"))?;
+
+        if scheme != "http" {
+            return Err(unexpected::Error::from(
+                format!("scheme of proxy url {url} should be 'http'",).as_str(),
+            ));
+        }
+
+        Ok(ConfigProxiedDomainUrl(url))
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct ConfigProxiedDomainRequestHeader {
+    pub name: String,
+    pub value: String,
+}
+
+#[derive(Clone)]
+pub struct ConfigProxiedDomainRequestHeaders(http::header::HeaderMap);
+
+impl AsRef<http::header::HeaderMap> for ConfigProxiedDomainRequestHeaders {
+    fn as_ref(&self) -> &http::header::HeaderMap {
+        &self.0
+    }
+}
+
+impl Default for ConfigProxiedDomainRequestHeaders {
+    fn default() -> Self {
+        ConfigProxiedDomainRequestHeaders(http::header::HeaderMap::default())
+    }
+}
+
+impl TryFrom<ConfigProxiedDomainRequestHeaders> for Vec<ConfigProxiedDomainRequestHeader> {
+    type Error = http::header::ToStrError;
+
+    fn try_from(h: ConfigProxiedDomainRequestHeaders) -> Result<Self, Self::Error> {
+        let mut v = vec![];
+        for (name, value) in &h.0 {
+            v.push(ConfigProxiedDomainRequestHeader {
+                name: name.to_string(),
+                value: value.to_str()?.to_string(),
+            })
+        }
+        Ok(v)
+    }
+}
+
+impl TryFrom<Vec<ConfigProxiedDomainRequestHeader>> for ConfigProxiedDomainRequestHeaders {
+    type Error = unexpected::Error;
+
+    fn try_from(v: Vec<ConfigProxiedDomainRequestHeader>) -> Result<Self, Self::Error> {
+        use http::header::{HeaderMap, HeaderName, HeaderValue};
+
+        let mut h = HeaderMap::new();
+        for pair in v {
+            let name: HeaderName = pair.name.parse().or_unexpected()?;
+            let value: HeaderValue = pair.value.parse().or_unexpected()?;
+            h.insert(name, value);
+        }
+        Ok(ConfigProxiedDomainRequestHeaders(h))
+    }
+}
+
+#[serde_as]
+#[derive(Deserialize, Serialize)]
+pub struct ConfigProxiedDomain {
+    #[serde_as(as = "TryFromInto<String>")]
+    pub url: ConfigProxiedDomainUrl,
+
+    #[serde(default)]
+    #[serde_as(as = "TryFromInto<Vec<ConfigProxiedDomainRequestHeader>>")]
+    pub request_headers: ConfigProxiedDomainRequestHeaders,
+}
+
 #[derive(Deserialize, Serialize)]
 pub struct Config {
     #[serde(default = "default_http_addr")]
@@ -43,6 +147,9 @@ pub struct Config {
 
     #[serde(default)]
     pub form_method: ConfigFormMethod,
+
+    #[serde(default)]
+    pub proxied_domains: collections::HashMap<domain::Name, ConfigProxiedDomain>,
 }
 
 impl Default for Config {
@@ -51,6 +158,7 @@ impl Default for Config {
             http_addr: default_http_addr(),
             https_addr: None,
             form_method: Default::default(),
+            proxied_domains: Default::default(),
         }
     }
 }

src/service/http/proxy.rs

@@ -0,0 +1,36 @@
+use crate::error::unexpected::{self};
+use crate::service;
+use http::header::HeaderValue;
+use std::net;
+
+pub async fn serve_http_request(
+    config: &service::http::ConfigProxiedDomain,
+    client_ip: net::IpAddr,
+    mut req: hyper::Request<hyper::Body>,
+    req_is_https: bool,
+) -> unexpected::Result<hyper::Response<hyper::Body>> {
+    for (name, value) in config.request_headers.as_ref() {
+        if value == "" {
+            req.headers_mut().remove(name);
+            continue;
+        }
+
+        req.headers_mut().insert(name, value.clone());
+    }
+
+    if req_is_https {
+        req.headers_mut()
+            .insert("x-forwarded-proto", HeaderValue::from_static("https"));
+    }
+
+    let url = config.url.as_ref();
+
+    match hyper_reverse_proxy::call(client_ip, url, req).await {
+        Ok(res) => Ok(res),
+        // ProxyError doesn't actually implement Error :facepalm: so we have to format the error
+        // manually
+        Err(e) => Err(unexpected::Error::from(
+            format!("error while proxying to {url}: {e:?}").as_str(),
+        )),
+    }
+}

src/service/http/tpl/domain.html

@@ -51,23 +51,13 @@ automatically updated too!</p>
 
   <fieldset>
     <legend>Advanced Settings</legend>
-    <p>
-    <label>
-    Prefix to remove from URL paths:
-    <input name="domain_setting_remove_path_prefix"
-           type="text"
-           placeholder="/foo/bar"
-           value="{{ data.settings.remove_path_prefix }}"
-           required />
-    </label>
-    </p>
 
     <p>
     <label>
-    Prefix to add to URL paths:
+    Directory or sub-directory to serve files from:
     <input name="domain_setting_add_path_prefix"
            type="text"
-           placeholder="/foo/bar"
+           placeholder="directory/sub-directory"
            value="{{ data.settings.add_path_prefix }}"
            required />
     </label>

src/service/util.rs

@@ -15,10 +15,6 @@ pub struct FlatDomainSettings {
 
     domain_setting_origin_descr_proxy_url: Option<String>,
 
-    #[serde(default)]
-    #[serde_as(as = "NoneAsEmptyString")]
-    domain_setting_remove_path_prefix: Option<String>,
-
     #[serde(default)]
     #[serde_as(as = "NoneAsEmptyString")]
     domain_setting_add_path_prefix: Option<String>,
@@ -43,7 +39,6 @@ impl TryFrom<FlatDomainSettings> for domain::Settings {
 
         Ok(Self {
             origin_descr,
-            remove_path_prefix: v.domain_setting_remove_path_prefix,
             add_path_prefix: v.domain_setting_add_path_prefix,
         })
     }
@@ -61,14 +56,8 @@ impl TryFrom<domain::Settings> for FlatDomainSettings {
                 res.domain_setting_origin_descr_git_url = Some(url);
                 res.domain_setting_origin_descr_git_branch_name = Some(branch_name);
             }
-            origin::Descr::Proxy { .. } => {
-                return Err(unexpected::Error::from(
-                    "proxy origins not supported for forms",
-                ));
-            }
         }
 
-        res.domain_setting_remove_path_prefix = v.remove_path_prefix;
         res.domain_setting_add_path_prefix = v.add_path_prefix;
 
         Ok(res)