Clean up & integrate service network configuration into foundation module

Signed-off-by: Mel <mel@rnrd.eu>

8 files changed, 589 insertions, 305 deletions

diff --git a/modules/foundation/default.nix b/modules/foundation/default.nix
index 81140b3..68e102a 100644
--- a/modules/foundation/default.nix
+++ b/modules/foundation/default.nix
@@ -2,9 +2,9 @@
 
 {
   imports = [
-    ./services.nix
     ./tailnet.nix
     ./wireguard.nix
+    ./services
     ./monitoring
     ./www
   ];
diff --git a/modules/foundation/services.nix b/modules/foundation/services.nix
deleted file mode 100644
index bedceb1..0000000
--- a/modules/foundation/services.nix
+++ /dev/null
@@ -1,302 +0,0 @@
-{
-  lib,
-  config,
-  pkgs,
-  ...
-}:
-
-let
-  inherit (lib) mkOption types;
-
-  serviceOptions = {
-    options = {
-      image = mkOption {
-        type = types.nullOr types.package;
-        default = null;
-      };
-
-      fullImage = mkOption {
-        type = with types; nullOr (submodule {
-          options = {
-            image = mkOption { type = str; };
-            imageFile = mkOption { type = package; };
-            base = mkOption { type = nullOr anything; };
-          };
-        });
-        default = null;
-      };
-
-      ports = mkOption {
-        type = with types; listOf (
-          oneOf [(listOf port) str port]
-        );
-        default = [ ];
-      };
-
-      volumes = mkOption {
-        type = with types; listOf (listOf str);
-        default = [ ];
-      };
-
-      devices = mkOption {
-        type = with types; listOf str;
-        default = [ ];
-      };
-
-      capabilities = mkOption {
-        type = with types; listOf str;
-        default = [ ];
-      };
-
-      entrypoint = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-      };
-
-      cmd = mkOption {
-        type = with types; listOf str;
-        default = [ ];
-      };
-
-      workdir = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-      };
-
-      environment = mkOption {
-        type = types.attrsOf types.str;
-        default = { };
-      };
-
-      environmentFiles = mkOption {
-        type = types.listOf types.path;
-        default = [ ];
-      };
-
-      customNetwork = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-      };
-    };
-  };
-
-  cfg = config.foundation;
-
-  mkName =
-    {
-      group,
-      name,
-      full ? false,
-    }:
-    let
-      isGroup = group != "";
-      isDefault = name == "default" || name == group;
-
-      shortName = if isGroup && isDefault then group else name;
-
-      fullName = if isGroup then (if isDefault then group else "${group}-${name}") else name;
-    in
-    assert name != "";
-    assert isDefault -> isGroup;
-    if full then fullName else shortName;
-
-  processServices =
-    group: serviceSet:
-    lib.mapAttrsToList (
-      name: c:
-      {
-        name = mkName { inherit name group; };
-        fullName = mkName {
-          inherit name group;
-          full = true;
-        };
-        inherit group;
-      }
-      // c
-    ) serviceSet;
-
-  singleServices = processServices "" cfg.services;
-
-  groupedServices = lib.mapAttrs (
-    group: groupServices: processServices group groupServices
-  ) cfg.service;
-
-  allServices =
-    let
-      allSingleServices = singleServices;
-      allGroupedServices = lib.flatten (lib.attrValues groupedServices);
-    in
-    allSingleServices ++ allGroupedServices;
-
-  groupStructure = lib.mapAttrs (
-    group: groupServices: lib.catAttrs "fullName" groupServices
-  ) groupedServices;
-in
-{
-  options.foundation = {
-    service = mkOption {
-      type = with types; attrsOf (attrsOf (submodule serviceOptions));
-      default = { };
-    };
-
-    services = mkOption {
-      type = with types; attrsOf (submodule serviceOptions);
-      default = { };
-    };
-  };
-
-  config = lib.mkIf (cfg.service != { } || cfg.services != { }) {
-    virtualisation.oci-containers.containers =
-      let
-        mkOciPort =
-          portSetting:
-          if builtins.isList portSetting then
-            let
-              host = builtins.elemAt portSetting 0;
-              container = builtins.elemAt portSetting 1;
-            in
-            "127.0.0.1:${toString host}:${toString container}"
-          else if builtins.isInt portSetting then
-            "127.0.0.1:${toString portSetting}:${toString portSetting}"
-          else
-            portSetting;
-
-        mkOciVolume =
-          volumeTuple:
-          let
-            hostPath = builtins.elemAt volumeTuple 0;
-            containerPath = builtins.elemAt volumeTuple 1;
-          in
-          "${hostPath}:${containerPath}";
-
-        mkImage =
-          {
-            oldImage,
-            imageStream,
-          }:
-          if oldImage != null then
-            {
-              inherit (oldImage) image imageFile;
-            }
-          else if imageStream != null then
-            {
-              inherit imageStream;
-              image = "${imageStream.imageName}:${imageStream.imageTag}";
-            }
-          else
-            throw "can't use both `fullImage` and `image` together.";
-
-        mkOciContainer =
-          {
-            name,
-            fullImage,
-            image,
-            ports,
-            volumes,
-            devices,
-            capabilities,
-            entrypoint ? null,
-            cmd ? null,
-            workdir ? null,
-            environment ? null,
-            environmentFiles ? null,
-            customNetwork ? null,
-
-            group,
-            ...
-          }:
-          {
-            inherit
-              entrypoint
-              cmd
-              workdir
-              environment
-              environmentFiles
-              ;
-            ports = map mkOciPort ports;
-            volumes = map mkOciVolume volumes;
-
-            extraOptions = let
-              mapOptions = optionName: values:
-                map (v: "--${optionName}=${v}") values;
-
-              networkOptions =
-                if customNetwork != null then [
-                  "--network=${customNetwork}"
-                ] else if group != "" then [
-                  "--network-alias=${name}"
-                  "--network=${group}"
-                ] else [];
-
-              capabilityOptions = mapOptions "cap-add" capabilities;
-
-              deviceOptions = mapOptions "device" devices;
-            in
-            networkOptions ++ capabilityOptions ++ deviceOptions;
-          }
-          // (mkImage {
-            oldImage = fullImage;
-            imageStream = image;
-          });
-      in
-      builtins.listToAttrs (map (v: lib.nameValuePair v.fullName (mkOciContainer v)) allServices);
-
-    systemd =
-      let
-        mkGroupRootTargetName = group: "docker-${group}-root";
-        mkServiceName = fullName: "docker-${fullName}";
-        mkNetworkServiceName = group: "docker-${group}-network";
-
-        genericContainerService =
-          group:
-          let
-            network = "${mkNetworkServiceName group}.service";
-            root = "${mkGroupRootTargetName group}.target";
-          in
-          {
-            serviceConfig = {
-              Restart = lib.mkOverride 90 "always";
-              RestartMaxDelaySec = lib.mkOverride 90 "1m";
-              RestartSec = lib.mkOverride 90 "100ms";
-              RestartSteps = lib.mkOverride 90 9;
-            };
-            after = [ network ];
-            requires = [ network ];
-            partOf = [ root ];
-            wantedBy = [ root ];
-          };
-
-        networkService =
-          group:
-          let
-            root = "${mkGroupRootTargetName group}.target";
-          in
-          {
-            path = [ pkgs.docker ];
-            serviceConfig = {
-              Type = "oneshot";
-              RemainAfterExit = true;
-              ExecStop = "docker network rm -f ${group}";
-            };
-            script = ''
-              docker network inspect ${group} || docker network create ${group} --driver=bridge
-            '';
-            partOf = [ root ];
-            wantedBy = [ root ];
-          };
-      in
-      {
-        services = lib.concatMapAttrs (
-          group: serviceNames:
-          {
-            ${mkNetworkServiceName group} = networkService group;
-          }
-          // lib.genAttrs (map mkServiceName serviceNames) (_v: genericContainerService group)
-        ) groupStructure;
-
-        targets = lib.mapAttrs' (
-          group: _v: lib.nameValuePair (mkGroupRootTargetName group) { wantedBy = [ "multi-user.target" ]; }
-        ) groupStructure;
-      };
-  };
-}
diff --git a/modules/foundation/services/default.nix b/modules/foundation/services/default.nix
new file mode 100644
index 0000000..25477b1
--- /dev/null
+++ b/modules/foundation/services/default.nix
@@ -0,0 +1,16 @@
+{ ... }:
+
+{
+  imports = [
+    ./services.nix
+    ./networks.nix
+  ];
+
+  foundation.networks.foundation-default = {
+    enable = true;
+    default = true;
+
+    subnet = "2001:d0c:1::/48";
+    driver = "bridge";
+  };
+}
diff --git a/modules/foundation/services/networks.nix b/modules/foundation/services/networks.nix
new file mode 100644
index 0000000..d1f1a92
--- /dev/null
+++ b/modules/foundation/services/networks.nix
@@ -0,0 +1,215 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+
+let
+  utils = import ./utils.nix { inherit lib; };
+
+  inherit (lib)
+    mkOption
+    types
+    assertMsg
+    optional
+    getExe
+    concatStringsSep
+    filterAttrs
+    listToAttrs
+    attrsToList
+    imap0
+    attrNames
+    nameValuePair
+    ;
+  inherit (utils) naming;
+
+  cfg = config.foundation;
+in
+{
+  options.foundation =
+    let
+      networkSubmodule =
+        with types;
+        submodule {
+          options = {
+            enable = mkOption {
+              type = types.bool;
+              default = true;
+              description = "Should this network be created?";
+            };
+
+            default = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Should this network be the default for all services
+                that don't have an explicit network defined?
+              '';
+            };
+
+            subnet = mkOption {
+              type = types.nullOr types.str;
+              description = ''
+                IPv6 subnet for this network in CIDR notation.
+                Don't set to get a random subnet assigned to you within
+                the subnet defined in `defaultSubnetPrefix`.
+              '';
+              example = "2001:d0c:123::/64";
+              default = null;
+            };
+
+            driver = mkOption {
+              type = types.str;
+              default = "bridge";
+              description = "Docker network driver to use";
+            };
+
+            options = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              description = "Additional options to pass to docker network create";
+            };
+
+            serviceGroup = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                The group of the network, if it belongs to one.
+                This makes the network service a part of the
+                group target.
+              '';
+            };
+          };
+        };
+    in
+    {
+      networks = mkOption {
+        type = types.attrsOf networkSubmodule;
+        description = "List of service networks to create";
+        default = { };
+      };
+
+      defaultNetwork = mkOption {
+        type = types.nullOr types.str;
+        description = ''
+          The default network for services.
+          If at least one network is marked as default, you can
+          find it's name here.
+        '';
+      };
+
+      defaultSubnetPrefix = mkOption {
+        type = types.str;
+        description = ''
+          Default network subnet assigned to networks without
+          a set subnet.
+          Prefix length defined by `defaultSubnetLength`.
+        '';
+        default = "2001:d0c";
+      };
+
+      defaultSubnetLength = mkOption {
+        type = types.int;
+        description = ''
+          Default network subnet length assigned to networks without
+          a set subnet.
+          This should always be the length of the `defaultSubnetPrefix` + 16.
+        '';
+        default = 48;
+      };
+    };
+
+  config = lib.mkIf (cfg.networks != { }) {
+    # other modules can use this to look up which network
+    # they can use as the default!
+    foundation.defaultNetwork =
+      let
+        networksLabeledDefault = attrNames (filterAttrs (name: net: net.default) cfg.networks);
+        defaultsCount = builtins.length networksLabeledDefault;
+      in
+      assert assertMsg (defaultsCount <= 1) "multiple service networks labeled default";
+      assert assertMsg (defaultsCount != 0) "no default service network";
+      builtins.head networksLabeledDefault;
+
+    # we need these settings to allow our networks to be IPv6-enabled.
+    # we also ignore the default bridge, because it's setup
+    # is a lot more complicated...
+    virtualisation.docker.daemon.settings = {
+      experimental = true;
+      ipv6 = true;
+      ip6tables = true;
+      fixed-cidr-v6 = "${cfg.defaultSubnetPrefix}:255::/${toString cfg.defaultSubnetLength}";
+    };
+
+    boot.kernel.sysctl = {
+      "net.ipv6.conf.all.forwarding" = 1;
+      "net.ipv6.conf.default.forwarding" = 1;
+    };
+
+    systemd.services =
+      let
+        subnetOffset = 100;
+        subnetByIndex =
+          i:
+          "${cfg.defaultSubnetPrefix}:${toString (subnetOffset + i)}::/${toString cfg.defaultSubnetLength}";
+
+        # this could be moved out into library functions, it's pretty useful.
+        # mapAttrsIndexed' :: (Int -> String -> AttrSet -> { name:: String; value :: Any; }) -> AttrSet -> AttrSet
+        mapAttrsIndexed' = f: attrs: listToAttrs (imap0 (i: v: f i v.name v.value) (attrsToList attrs));
+        # mapAttrsIndexed :: (Int -> String -> AttrSet -> Any) -> AttrSet -> AttrSet
+        mapAttrsIndexed =
+          f: attrs:
+          mapAttrsIndexed' (
+            i: n: v:
+            nameValuePair v.name (f i v.name v.value)
+          ) attrs;
+
+        networkService =
+          index: name: network:
+          let
+            docker = getExe pkgs.docker;
+            options = concatStringsSep " " network.options;
+            subnet = if network.subnet == null then subnetByIndex index else network.subnet;
+          in
+          {
+            description = "Docker service network '${name}'";
+            after = [ "docker.service" ];
+            requires = [ "docker.service" ];
+
+            wantedBy = [ "multi-user.target" ];
+            partOf = optional (
+              network.serviceGroup != null
+            ) "${naming.groupTarget network.serviceGroup}.target";
+
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+
+              ExecStart = pkgs.writeShellScript "create-docker-network-${name}" ''
+                if ${docker} network inspect ${name} >/dev/null 2>&1; then
+                  ${docker} network rm ${name}
+                fi
+
+                ${docker} network create \
+                  --ipv6 \
+                  --subnet=${subnet} \
+                  --driver=${network.driver} \
+                  ${options} \
+                  ${name}
+              '';
+
+              ExecStop = pkgs.writeShellScript "delete-docker-network-${name}" ''
+                ${docker} network rm -f ${name}
+              '';
+            };
+          };
+
+        mkNetwork = index: name: network: {
+          name = naming.networkService name;
+          value = networkService index name network;
+        };
+      in
+      mapAttrsIndexed' mkNetwork (filterAttrs (name: net: net.enable) cfg.networks);
+  };
+}
diff --git a/modules/foundation/services/services.nix b/modules/foundation/services/services.nix
new file mode 100644
index 0000000..d9489aa
--- /dev/null
+++ b/modules/foundation/services/services.nix
@@ -0,0 +1,326 @@
+{
+  lib,
+  config,
+  ...
+}:
+
+let
+  utils = import ./utils.nix { };
+
+  inherit (lib)
+    mkOption
+    mkIf
+    mkOverride
+    types
+    mapAttrs
+    mapAttrsToList
+    catAttrs
+    attrNames
+    attrValues
+    listToAttrs
+    nameValuePair
+    optional
+    flatten
+    ;
+  inherit (utils) naming;
+
+  cfg = config.foundation;
+
+in
+{
+  options.foundation =
+    let
+      serviceSubmodule = types.submodule {
+        options = {
+          image = mkOption {
+            type = types.nullOr types.package;
+            default = null;
+          };
+
+          fullImage = mkOption {
+            type =
+              with types;
+              nullOr (submodule {
+                options = {
+                  image = mkOption { type = str; };
+                  imageFile = mkOption { type = package; };
+                  base = mkOption { type = nullOr anything; };
+                };
+              });
+            default = null;
+          };
+
+          ports = mkOption {
+            type =
+              with types;
+              listOf (oneOf [
+                (listOf port)
+                str
+                port
+              ]);
+            default = [ ];
+          };
+
+          volumes = mkOption {
+            type = with types; listOf (listOf str);
+            default = [ ];
+          };
+
+          devices = mkOption {
+            type = with types; listOf str;
+            default = [ ];
+          };
+
+          capabilities = mkOption {
+            type = with types; listOf str;
+            default = [ ];
+          };
+
+          entrypoint = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+          };
+
+          cmd = mkOption {
+            type = with types; listOf str;
+            default = [ ];
+          };
+
+          workdir = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+          };
+
+          environment = mkOption {
+            type = types.attrsOf types.str;
+            default = { };
+          };
+
+          environmentFiles = mkOption {
+            type = types.listOf types.path;
+            default = [ ];
+          };
+
+          network = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            description = ''
+              Makes the container part of a network defined by `foundation.networks`.
+              If null, the container network will either be that of the group it belongs to,
+              or the default network as defined by `foundation.defaultNetwork`.
+              If you need to connect the container to networks outside of the configuration
+              defined ones, see: `customNetworkOption`.
+            '';
+          };
+
+          customNetworkOption = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            description = ''
+              Overrides the normal group or default network with a custom network option.
+              This makes the containers systemd service not depend on any networks defined
+              in `foundation.networks`, so you have to ensure correct start-up order yourself.
+            '';
+          };
+        };
+      };
+    in
+    {
+      service = mkOption {
+        type = types.attrsOf (types.attrsOf serviceSubmodule);
+        default = { };
+      };
+
+      services = mkOption {
+        type = types.attrsOf serviceSubmodule;
+        default = { };
+      };
+    };
+
+  config =
+    let
+      processServices =
+        group: services:
+        mapAttrsToList (
+          name: c:
+          c
+          // {
+            name = naming.service { inherit name group; };
+            fullName = naming.service {
+              inherit name group;
+              full = true;
+            };
+
+            network =
+              if c.customNetworkOption != null then
+                "" # overrides all network configuration
+              else if c.network != null then
+                c.network
+              else if group != "" then
+                group
+              else
+                cfg.defaultNetwork;
+            inherit group;
+          }
+        ) services;
+
+      singleServices = processServices "" cfg.services;
+      groupedServices = mapAttrs (group: groupServices: processServices group groupServices) cfg.service;
+
+      allServices =
+        let
+          allSingleServices = singleServices;
+          allGroupedServices = flatten (attrValues groupedServices);
+        in
+        allSingleServices ++ allGroupedServices;
+
+      groupToServices = mapAttrs (
+        group: groupServices: catAttrs "fullName" groupServices
+      ) groupedServices;
+    in
+    mkIf (cfg.service != { } || cfg.services != { }) {
+      virtualisation.oci-containers.containers =
+        let
+          mkOciPort =
+            portSetting:
+            if builtins.isList portSetting then
+              let
+                host = builtins.elemAt portSetting 0;
+                container = builtins.elemAt portSetting 1;
+              in
+              "127.0.0.1:${toString host}:${toString container}"
+            else if builtins.isInt portSetting then
+              "127.0.0.1:${toString portSetting}:${toString portSetting}"
+            else
+              portSetting;
+
+          mkOciVolume =
+            volumeTuple:
+            let
+              hostPath = builtins.elemAt volumeTuple 0;
+              containerPath = builtins.elemAt volumeTuple 1;
+            in
+            "${hostPath}:${containerPath}";
+
+          mkImage =
+            {
+              oldImage,
+              imageStream,
+            }:
+            if oldImage != null then
+              {
+                inherit (oldImage) image imageFile;
+              }
+            else if imageStream != null then
+              {
+                inherit imageStream;
+                image = "${imageStream.imageName}:${imageStream.imageTag}";
+              }
+            else
+              throw "can't use both `fullImage` and `image` together.";
+
+          mkOciContainer =
+            {
+              name,
+              fullImage,
+              image,
+              ports,
+              volumes,
+              devices,
+              capabilities,
+              network,
+              customNetworkOption ? null,
+              entrypoint ? null,
+              cmd ? null,
+              workdir ? null,
+              environment ? null,
+              environmentFiles ? null,
+
+              group,
+              ...
+            }:
+            {
+              inherit
+                entrypoint
+                cmd
+                workdir
+                environment
+                environmentFiles
+                ;
+
+              ports = map mkOciPort ports;
+              volumes = map mkOciVolume volumes;
+
+              extraOptions =
+                let
+                  mapOptions = optionName: values: map (v: "--${optionName}=${v}") values;
+
+                  networkOptions =
+                    (optional (customNetworkOption != null) "--network=${customNetworkOption}")
+                    ++ (optional (network != "") "--network=${network}")
+                    ++ (optional (group != "" && customNetworkOption == null) "--network-alias=${name}"); # aliases not supported
+                  capabilityOptions = mapOptions "cap-add" capabilities;
+                  deviceOptions = mapOptions "device" devices;
+                in
+                networkOptions ++ capabilityOptions ++ deviceOptions;
+            }
+            // (mkImage {
+              oldImage = fullImage;
+              imageStream = image;
+            });
+        in
+        builtins.listToAttrs (map (v: nameValuePair v.fullName (mkOciContainer v)) allServices);
+
+      # define networks for all groups
+      foundation.networks = listToAttrs (
+        map (
+          group:
+          nameValuePair group {
+            enable = true;
+            serviceGroup = group;
+          }
+        ) (lib.attrNames groupToServices)
+      );
+
+      systemd =
+        let
+          containerService =
+            service:
+            let
+              grouped = service.group != "";
+              networked = service.network != "";
+
+              network = "${naming.networkService service.network}.service";
+              group = "${naming.groupTarget service.group}.target";
+            in
+            {
+              serviceConfig = {
+                Restart = mkOverride 90 "always";
+                RestartMaxDelaySec = mkOverride 90 "1m";
+                RestartSec = mkOverride 90 "100ms";
+                RestartSteps = mkOverride 90 9;
+              };
+
+              after = optional networked network;
+              requires = optional networked network;
+              partOf = optional grouped group;
+              wantedBy = optional grouped group;
+            };
+
+          groupTarget = {
+            wantedBy = [ "multi-user.target" ];
+          };
+        in
+        {
+          services = listToAttrs (
+            map (
+              service: nameValuePair (naming.serviceService service.fullName) (containerService service)
+            ) allServices
+          );
+
+          targets = listToAttrs (
+            map (group: nameValuePair (naming.groupTarget group) groupTarget) (attrNames groupToServices)
+          );
+        };
+    };
+}
diff --git a/modules/foundation/services/utils.nix b/modules/foundation/services/utils.nix
new file mode 100644
index 0000000..c7bbcf7
--- /dev/null
+++ b/modules/foundation/services/utils.nix
@@ -0,0 +1,29 @@
+{ ... }:
+
+{
+  naming = {
+    networkService = name: "docker-${name}-network";
+
+    groupTarget = group: "docker-${group}-group-root";
+
+    serviceService = fullName: "docker-${fullName}";
+
+    service =
+      {
+        group,
+        name,
+        full ? false,
+      }:
+      let
+        isGroup = group != "";
+        isDefault = name == "default" || name == group;
+
+        shortName = if isGroup && isDefault then group else name;
+
+        fullName = if isGroup then (if isDefault then group else "${group}-${name}") else name;
+      in
+      assert name != "";
+      assert isDefault -> isGroup;
+      if full then fullName else shortName;
+  };
+}
diff --git a/services/home-assistant.nix b/services/home-assistant.nix
index db49388..e356c46 100644
--- a/services/home-assistant.nix
+++ b/services/home-assistant.nix
@@ -21,7 +21,7 @@ in
       fullImage = homeImage;
       # give home-assistant control over the device network
       # stack to auto-discover devices on the network.
-      customNetwork = "host";
+      customNetworkOption = "host";
       # allow home-assistant to access zigbee/matter+thread
       # dongle.
       devices = [ "/dev/serial/by-id/usb-SMLIGHT_SMLIGHT_SLZB-07_6e29216e5272ef119d2f43848fcc3fa0-if00-port0" ];
diff --git a/services/transmission.nix b/services/transmission.nix
index ec98177..01bd1f6 100644
--- a/services/transmission.nix
+++ b/services/transmission.nix
@@ -57,7 +57,7 @@ in
         "--config-dir" "/var/lib/transmission/config"
       ];
 
-      customNetwork = "container:vpn";
+      customNetworkOption = "container:vpn";
     };
 
     vpn = {