Openstack production setup

flake.nix

@@ -65,6 +65,7 @@
             python3Packages
             writeText
             lib
+            libredirect
             ;
         };
 

modules/compute/compute.nix

@@ -2,6 +2,7 @@
 { ... }:
 {
   imports = [
+    ../generic/global-options.nix
     ../generic/controller-host-entry.nix
     (import ./neutron.nix { inherit neutron; })
     (import ./nova.nix { inherit nova; })

modules/compute/neutron.nix

@@ -25,7 +25,7 @@ let
     debug = false
     # File name for the paste.deploy config for api service (string value)
     api_paste_config = ${neutron}/etc/neutron/api-paste.ini
-    transport_url = rabbit://openstack:openstack@controller
+    transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname}
     log_dir = /var/log/neutron
 
     [agent]
@@ -50,6 +50,30 @@ let
     filterPath = "/etc/neutron/rootwrap.d";
     inherit utils_env;
   };
+
+  openvswitchPrepareScript = pkgs.writeShellScript "openvswitch-setup.sh" ''
+    export PATH=${
+      lib.makeBinPath [
+        pkgs.openvswitch
+      ]
+    }:$PATH
+
+    ovs-vsctl br-exists br-provider >/dev/null
+    status=$?
+
+    if [[ $status -eq 0 ]]; then
+      echo "br-provider already exists. No further setup tasks."
+    else
+      echo "br-provider didn't exist already. Proceed with basic setup."
+      set -euxo pipefail
+      ovs-vsctl add-br br-provider
+    fi
+    ovs-vsctl --may-exist add-port br-provider ${cfg.providerInterface}
+
+    # enable uplink provider interface
+    ip link set dev ${cfg.providerInterface} up
+  '';
+
 in
 {
   options.neutron = {
@@ -68,6 +92,13 @@ in
         The Neutron OpenVSwitch config.
       '';
     };
+    openvswitchPrepare = mkOption {
+      default = openvswitchPrepareScript;
+      description = ''
+        Default OpenVswitch prepare script of systemd unit: neutron-openvswitch-agent.service
+        This prepare script creates the basic openvswitch setup.
+      '';
+    };
     providerInterface = mkOption {
       default = "eth2";
       type = types.str;
@@ -78,7 +109,7 @@ in
     };
   };
 
-  config = mkIf cfg.enable {
+  config = {
     users.extraUsers.neutron = {
       group = "neutron";
       isSystemUser = true;
@@ -136,6 +167,7 @@ in
       after = [
         "network.target"
         "ovsdb.service"
+        "systemd-networkd.service"
       ];
       wantedBy = [ "multi-user.target" ];
       path = with pkgs; [
@@ -150,14 +182,12 @@ in
         conntrack-tools
       ];
       serviceConfig = {
-        ExecStartPre = pkgs.writeShellScript "neutron-openvswitch-agent-pre.sh" ''
-          ${pkgs.openvswitch}/bin/ovs-vsctl add-br br-provider
-          ${pkgs.openvswitch}/bin/ovs-vsctl add-port br-provider ${cfg.providerInterface}
-        '';
+        ExecStartPre = "${cfg.openvswitchPrepare}";
         ExecStart = pkgs.writeShellScript "neutron-openvswitch-agent.sh" ''
           ${neutron}/bin/neutron-openvswitch-agent --config-file=${cfg.config} --config-file=${cfg.openvswitchConfig}
         '';
       };
+      enable = cfg.enable;
     };
   };
 }

modules/compute/nova.nix

@@ -25,8 +25,8 @@ let
     state_path = /var/lib/nova
     rootwrap_config = ${rootwrapConf}
     compute_driver = libvirt.LibvirtDriver
-    my_ip = 10.0.0.39
-    transport_url = rabbit://openstack:openstack@controller
+    my_ip = ${config.openstack.myIp}
+    transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname}
 
     [api]
     auth_strategy = keystone
@@ -38,12 +38,12 @@ let
     connection = sqlite:////var/lib/nova/nova.sqlite
 
     [glance]
-    api_servers = http://controller:9292
+    api_servers = http://${config.openstack.controllerHostname}:9292
 
     [keystone_authtoken]
-    www_authenticate_uri = http://controller:5000/
-    auth_url = http://controller:5000/
-    memcached_servers = controller:11211
+    www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3
+    auth_url = http://${config.openstack.controllerHostname}:5000/v3
+    memcached_servers = ${config.openstack.controllerHostname}:11211
     auth_type = password
     project_domain_name = Default
     user_domain_name = Default
@@ -55,7 +55,7 @@ let
     virt_type = kvm
 
     [neutron]
-    auth_url = http://controller:5000
+    auth_url = http://${config.openstack.controllerHostname}:5000/v3
     auth_type = password
     project_domain_name = Default
     user_domain_name = Default
@@ -76,13 +76,13 @@ let
     project_name = service
     auth_type = password
     user_domain_name = Default
-    auth_url = http://controller:5000/v3
+    auth_url = http://${config.openstack.controllerHostname}:5000/v3
     username = placement
     password = placement
 
     [service_user]
     send_service_user_token = true
-    auth_url = http://controller:5000/
+    auth_url = http://${config.openstack.controllerHostname}:5000/v3
     auth_strategy = keystone
     auth_type = password
     project_domain_name = Default
@@ -146,7 +146,7 @@ in
     };
   };
 
-  config = mkIf cfg.enable {
+  config = {
     users.extraUsers.nova = {
       group = "nova";
       isSystemUser = true;
@@ -162,28 +162,28 @@ in
     systemd.tmpfiles.settings = {
       "10-nova" = {
         "/var/log/nova" = {
-          D = {
+          d = {
             group = "nova";
             mode = "0755";
             user = "nova";
           };
         };
         "/var/lock/nova" = {
-          D = {
+          d = {
             group = "nova";
             mode = "0755";
             user = "nova";
           };
         };
         "/var/lib/nova" = {
-          D = {
+          d = {
             group = "nova";
             mode = "0755";
             user = "nova";
           };
         };
         "/var/lib/nova/instances" = {
-          D = {
+          d = {
             group = "nova";
             mode = "0755";
             user = "nova";
@@ -194,7 +194,7 @@ in
     };
 
     services.openiscsi = {
-      enable = true;
+      enable = cfg.enable;
       name = "iqn.iscsi.${config.networking.hostName}";
     };
 
@@ -220,6 +220,7 @@ in
           lvm2
           openiscsi
           nfs-utils
+          e2fsprogs
         ]
         ++ cfg.extraPkgs;
       environment.PYTHONPATH = "${nova_env}/${pkgs.python3.sitePackages}";
@@ -228,6 +229,7 @@ in
           ${cfg.novaPackage}/bin/nova-compute --config-file=${cfg.config}
         '';
       };
+      enable = cfg.enable;
     };
   };
 }

modules/controller/cinder.nix

@@ -11,20 +11,20 @@ with lib;
 let
   cfg = config.cinder;
 
-  cinderConf = pkgs.writeText "cinder-api.conf" ''
+  cinderConf = pkgs.writeText "cinder.conf" ''
     [DEFAULT]
-    transport_url = rabbit://openstack:openstack@controller
+    transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname}
     auth_strategy = keystone
-    my_ip = controller
+    my_ip = ${config.openstack.controllerIP}
     verify_glance_signatures = disabled
 
     [database]
-    connection = mysql+pymysql://cinder:cinder@controller/cinder
+    connection = mysql+pymysql://cinder:cinder@${config.openstack.controllerHostname}/cinder
 
     [keystone_authtoken]
-    www_authenticate_uri = http://controller:5000
-    auth_url = http://controller:5000
-    memcached_servers = controller:11211
+    www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3
+    auth_url = http://${config.openstack.controllerHostname}:5000/v3
+    memcached_servers = ${config.openstack.controllerHostname}:11211
     auth_type = password
     project_domain_name = default
     user_domain_name = default
@@ -56,8 +56,26 @@ in
         The OpenStack Cinder package to use.
       '';
     };
+    envCinderApi = mkOption {
+      type = types.listOf types.str;
+      default = [
+        "PYTHONWARNINGS=ignore::DeprecationWarning"
+        "PATH=$PATH:/run/current-system/sw/bin"
+      ];
+      description = ''
+        Environment variables passed to the cinder-api uWSGI vassal.
+      '';
+    };
+    envCinderScheduler = mkOption {
+      default = {
+        PYTHONWARNINGS = "ignore::DeprecationWarning";
+      };
+      description = ''
+        Environment variables passed to the cinder-scheduler systemd unit.
+      '';
+    };
   };
-  config = mkIf cfg.enable {
+  config = {
 
     users.extraUsers.cinder = {
       group = "cinder";
@@ -71,21 +89,21 @@ in
     systemd.tmpfiles.settings = {
       "10-cinder" = {
         "/var/lib/cinder/" = {
-          D = {
+          d = {
             user = "cinder";
             group = "cinder";
             mode = "0755";
           };
         };
         "/var/lib/cinder/volumes" = {
-          D = {
+          d = {
             user = "cinder";
             group = "cinder";
             mode = "0755";
           };
         };
         "/var/log/cinder/" = {
-          D = {
+          d = {
             user = "cinder";
             group = "cinder";
             mode = "0755";
@@ -98,13 +116,14 @@ in
         };
         "/etc/cinder/cinder.conf" = {
           L = {
-            argument = "${cinderConf}";
+            argument = "${cfg.config}";
           };
         };
       };
     };
 
-    systemd.services.cinder-api = {
+    # create systemd service only if running in non production mode (CI/CD setup)
+    systemd.services.cinder-api = lib.mkIf (!config.openstack.production_setup) {
       description = "OpenStack Cinder API Daemon";
       after = [
         "cinder.service"
@@ -121,6 +140,31 @@ in
           .cinder-wsgi-wrapped --port 8776
         '';
       };
+      enable = cfg.enable;
+    };
+
+    # create uwsgi vassal configuration only in production setup
+    services.uwsgi = lib.mkIf (config.openstack.production_setup) {
+      instance.vassals.cinder-api = mkIf cfg.enable {
+        type = "normal";
+        http-socket = "0.0.0.0:8776";
+        wsgi-file = "${cinder}/bin/.cinder-wsgi-wrapped";
+        pyargv = "--config-file ${cfg.config}";
+        env = cfg.envCinderApi;
+
+        master = true;
+        processes = 4;
+        enable-threads = true;
+        thunder-lock = true;
+        lazy-apps = true;
+        die-on-term = true;
+        vacuum = true;
+        need-app = true;
+        buffer-size = 65535;
+
+        immediate-uid = "cinder";
+        immediate-gid = "cinder";
+      };
     };
 
     systemd.services.cinder-scheduler = {
@@ -140,6 +184,8 @@ in
           .cinder-scheduler-wrapped
         '';
       };
+      enable = cfg.enable;
+      environment = cfg.envCinderScheduler;
     };
   };
 }