initial commit of hypervisor configs

20 files changed, 444 insertions, 0 deletions

diff --git a/scripts/common/10-root-user b/scripts/common/10-root-user
new file mode 100644
index 0000000..9a9f5e6
--- /dev/null
+++ b/scripts/common/10-root-user
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Add root SSH pubkeys.
+set_authorized_keys root "$root_authorized_keys"
+
+# Set root password.
+set_password root "$root_password"
diff --git a/scripts/common/20-dns b/scripts/common/20-dns
new file mode 100644
index 0000000..e2d5ad6
--- /dev/null
+++ b/scripts/common/20-dns
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# For IDM servers, the resolver is localhost. In that case, we delay copying
+# this file until the IDM stack is fully up and running.
+if [ "$BOXCONF_HOSTCLASS" = idm_server ]; then
+  return
+fi
+
+install_template -m 0644 /etc/resolv.conf
diff --git a/scripts/hostclass/freebsd_hypervisor b/scripts/hostclass/freebsd_hypervisor
new file mode 100644
index 0000000..bdaa3c0
--- /dev/null
+++ b/scripts/hostclass/freebsd_hypervisor
@@ -0,0 +1,80 @@
+#!/bin/sh
+
+: ${hypervisor_trunk_interface:='lagg0'}
+: ${hypervisor_default_vlan:='1'}
+: ${hypervisor_default_prefix:='24'}
+: ${hypervisor_default_os_quota:='24G'}
+: ${hypervisor_default_data_quota:='8G'}
+
+: ${hypervisor_vm_home:='/usr/local/bhyve'}
+: ${hypervisor_vm_dataset:='tank/bhyve'}
+: ${hypervisor_vm_default_cpus:='2'}
+: ${hypervisor_vm_default_mem:='4G'}
+: ${hypervisor_vm_template_size:='10G'}
+: ${hypervisor_vm_default_autostart_delay:='2'}
+: ${hypervisor_vm_default_zfs_opts:='-o primarycache=metadata -o compress=off'}
+: ${hypervisor_vm_zfs_volblocksize:='64k'}
+
+: ${hypervisor_jail_home:='/usr/local/jails'}
+: ${hypervisor_jail_dataset:='tank/jails'}
+: ${hypervisor_jail_default_zfs_opts:='-o compress=lz4'}
+
+hypervisor_jail_bpf_ruleset=1000
+
+# Required for vnet jails.
+set_sysctl net.link.tap.up_on_open=1
+
+# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=262189
+set_sysctl vfs.zfs.vol.mode=2
+
+# Load required kernel modules.
+load_kernel_module vmm nmdm linux linux64
+set_loader_conf \
+  vmm_load=YES \
+  nmdm_load=YES \
+  linux_load=YES \
+  linux64_load=YES \
+  kern.racct.enable=1
+
+# Install vm/jail management dependencies.
+pkg install -y \
+  bhyve-firmware \
+  cdrkit-genisoimage \
+  qemu-tools
+
+# Create bhyve VM dataset.
+create_dataset -o "mountpoint=${hypervisor_vm_home}" "$hypervisor_vm_dataset"
+
+# Create dataset for bhyve templates.
+create_dataset -o volmode=none -o mountpoint=none "${hypervisor_vm_dataset}/templates"
+
+# Create jails dataset.
+create_dataset -o mountpoint="${hypervisor_jail_home}" "$hypervisor_jail_dataset"
+
+# Create dataset for jail templates.
+create_dataset -o mountpoint="${hypervisor_jail_home}/templates" "${hypervisor_jail_dataset}/templates"
+
+# Lock down permissions on the VM and jail directories.
+chmod 700 "$hypervisor_vm_home" "$hypervisor_jail_home"
+
+# Create directory for VM ISO files.
+install_directory -m 0755 "${hypervisor_vm_home}/isos"
+
+# Copy jail/bhyve management scripts.
+install_directory -m 0755 /usr/local/etc/rc.d
+
+install_file -m 0555 \
+  /usr/local/sbin/jailctl \
+  /usr/local/sbin/vmctl \
+  /usr/local/etc/rc.d/vmctl
+
+install_template -m 0644 \
+  /usr/local/etc/jailctl.conf \
+  /usr/local/etc/vmctl.conf
+
+install_template -m 0644 /etc/devfs.rules
+
+# Enable jails/bhyve to start on boot.
+sysrc -v \
+  vmctl_enable=YES \
+  jail_enable=YES
diff --git a/scripts/os/freebsd/10-bootloader b/scripts/os/freebsd/10-bootloader
new file mode 100644
index 0000000..0506606
--- /dev/null
+++ b/scripts/os/freebsd/10-bootloader
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Skip this file if running in a jail - jails don't have a bootloader.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" = jail ]; then
+  return
+fi
+
+# Configure serial console.
+install_file -m 0644 /boot.config
+install_file -m 0644 /etc/ttys
+kill -HUP 1
+
+set_loader_conf \
+  autoboot_delay=1 \
+  beastie_disable=YES \
+  boot_multicons=YES \
+  boot_serial=YES \
+  cc_htcp_load=YES \
+  console=comconsole,efi \
+  comconsole_speed=115200 \
+  kern.geom.label.disk_ident.enable=0 \
+  kern.geom.label.gptid.enable=0 \
+  net.inet.tcp.soreceive_stream=1 \
+  net.inet6.ip6.auto_linklocal=0 \
+  net.isr.defaultqlimit=2048 \
+  net.link.ifqmaxlen=2048 \
+  pf_load=YES \
+  pflog_load=YES \
+  security.bsd.allow_destructive_dtrace=0
diff --git a/scripts/os/freebsd/10-cpu b/scripts/os/freebsd/10-cpu
new file mode 100644
index 0000000..adc27d4
--- /dev/null
+++ b/scripts/os/freebsd/10-cpu
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Only run this file on baremetal hosts.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" != none ]; then
+  return
+fi
+
+# Allow lower C-states. As of FreeBSD 13, the default is to only allow C1.
+# My Xeon processor supports C2, and enabling that resulted in 15 watts of
+# power savings.
+#
+# Note that if your CPU supports *very* low C-states (likely for commodity
+# desktop and laptop hardware), you may not want them enabled, as transitioning
+# from a very low C-state can cause rather severe latency spikes.
+#
+# Experiment with your hardware and set $cx_lowest accordingly.
+sysrc -v \
+  microcode_update_enable=YES \
+  performance_cx_lowest="$cx_lowest" \
+  economy_cx_lowest="$cx_lowest"
+
+# Set energy/performance preference for Intel P-states.
+# 0 = most performance, 100 = most power savings
+if sysctl -n dev.hwpstate_intel.0.epp >/dev/null 2>&1; then
+  for n in $(seq 0 $(($(sysctl -n hw.ncpu)-1))); do
+    set_sysctl "dev.hwpstate_intel.${n}.epp=${intel_epp}"
+  done
+fi
diff --git a/scripts/os/freebsd/10-periodic b/scripts/os/freebsd/10-periodic
new file mode 100644
index 0000000..36ddd95
--- /dev/null
+++ b/scripts/os/freebsd/10-periodic
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# Disable periodic(8) reports, as well as tasks that generate lots of I/O.
+sysrc -v -f /etc/periodic.conf \
+  daily_show_success=NO \
+  daily_show_info=NO \
+  daily_clean_disks_verbose=NO \
+  daily_clean_tmps_verbose=NO \
+  daily_clean_preserve_verbose=NO \
+  daily_clean_rwho_verbose=NO \
+  daily_backup_passwd_enable=NO \
+  daily_backup_aliases_enable=NO \
+  daily_backup_gpart_enable=NO \
+  daily_status_disks_enable=NO \
+  daily_status_zfs_zpool_list_enable=NO \
+  daily_status_network_enable=NO \
+  daily_status_uptime_enable=NO \
+  daily_status_mailq_enable=NO \
+  daily_status_security_enable=NO \
+  daily_status_mail_rejects_enable=NO \
+  daily_status_world_kernel=NO \
+  weekly_show_success=NO \
+  weekly_show_info=NO \
+  weekly_locate_enable=NO \
+  weekly_whatis_enable=NO \
+  weekly_status_security_enable=NO \
+  monthly_show_success=NO \
+  monthly_show_info=NO \
+  monthly_accounting_enable=NO \
+  monthly_status_security_enable=NO \
+  security_show_success=NO \
+  security_show_info=NO \
+  security_status_chksetuid_enable=NO \
+  security_status_neggrpperm_enable=NO \
+  security_status_chkmounts_enable=NO \
+  security_status_chkuid0_enable=NO \
+  security_status_passwdless_enable=NO \
+  security_status_logincheck_enable=NO \
+  security_status_ipfwdenied_enable=NO \
+  security_status_ipfdenied_enable=NO \
+  security_status_pfdenied_enable=NO \
+  security_status_ipfwlimit_enable=NO \
+  security_status_ipf6denied_enable=NO \
+  security_status_kernelmsg_enable=NO \
+  security_status_loginfail_enable=NO \
+  security_status_tcpwrap_enable=NO
+
+# Sendmail-specific stuff
+sysrc -v -f /etc/periodic.conf \
+  daily_clean_hoststat_enable=NO \
+  daily_status_mail_rejects_enable=NO \
+  daily_status_include_submit_mailq=NO \
+  daily_submit_queuerun=NO
diff --git a/scripts/os/freebsd/10-rc-conf b/scripts/os/freebsd/10-rc-conf
new file mode 100644
index 0000000..a8a3d22
--- /dev/null
+++ b/scripts/os/freebsd/10-rc-conf
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+sysrc -v \
+  clear_tmp_enable=YES \
+  dumpdev=NO \
+  ipv6_activate_all_interfaces=NO \
+  syslogd_flags=-ss
diff --git a/scripts/os/freebsd/10-sysctls b/scripts/os/freebsd/10-sysctls
new file mode 100644
index 0000000..a59d54f
--- /dev/null
+++ b/scripts/os/freebsd/10-sysctls
@@ -0,0 +1,80 @@
+#!/bin/sh
+
+case $BOXCONF_OS_VERSION in
+  13.*)
+    set_sysctl \
+      net.inet.ip.check_interface=1 \
+      net.inet.tcp.rfc6675_pipe=1
+    ;;
+  *)
+    set_sysctl \
+      net.inet.ip.rfc1122_strong_es=1
+    ;;
+esac
+
+load_kernel_module cc_htcp
+
+set_sysctl \
+  net.inet.icmp.drop_redirect=1 \
+  net.inet.ip.process_options=0 \
+  net.inet.ip.random_id=1 \
+  net.inet.ip.redirect=0 \
+  net.inet.tcp.abc_l_var=44 \
+  net.inet.tcp.always_keepalive=0 \
+  net.inet.tcp.cc.abe=1 \
+  net.inet.tcp.cc.algorithm=htcp \
+  net.inet.tcp.cc.htcp.adaptive_backoff=1 \
+  net.inet.tcp.cc.htcp.rtt_scaling=1 \
+  net.inet.tcp.drop_synfin=1 \
+  net.inet.tcp.ecn.enable=1 \
+  net.inet.tcp.fastopen.server_enable=1 \
+  net.inet.tcp.icmp_may_rst=0 \
+  net.inet.tcp.initcwnd_segments=44 \
+  net.inet.tcp.minmss=536 \
+  net.inet.tcp.msl=2500 \
+  net.inet.tcp.mssdflt=1448 \
+  net.inet.tcp.nolocaltimewait=1 \
+  net.inet.tcp.path_mtu_discovery=0 \
+  net.inet.tcp.recvbuf_max="$tcp_buffer_size" \
+  net.inet.tcp.recvspace=65536 \
+  net.inet.tcp.sendbuf_inc=65536 \
+  net.inet.tcp.sendbuf_max="$tcp_buffer_size" \
+  net.inet.tcp.sendspace=65536 \
+  net.inet.tcp.syncookies=0 \
+  net.inet6.ip6.redirect=0 \
+  security.bsd.unprivileged_proc_debug="$allow_proc_debug"
+
+# Some sysctls cannot be set within jails.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" != jail ]; then
+  set_sysctl \
+    hw.kbd.keymap_restrict_change=4 \
+    kern.coredump=0 \
+    kern.elf32.allow_wx="$allow_wx" \
+    kern.elf32.aslr.pie_enable=1 \
+    kern.elf64.allow_wx="$allow_wx" \
+    kern.ipc.maxsockbuf="$tcp_buffer_size" \
+    kern.ipc.shm_use_phys=1 \
+    kern.ipc.soacceptqueue=1024 \
+    kern.ipc.somaxconn=1024 \
+    kern.random.fortuna.minpoolsize=128 \
+    kern.randompid=1 \
+    net.inet.tcp.fast_finwait2_recycle=1 \
+    net.inet.tcp.finwait2_timeout=5000 \
+    net.inet.tcp.keepcnt=2 \
+    net.inet.tcp.keepidle=62000 \
+    net.inet.tcp.keepinit=5000 \
+    net.inet.tcp.minmss=536 \
+    net.inet.tcp.minmss=536 \
+    security.bsd.hardlink_check_gid=0 \
+    security.bsd.hardlink_check_uid=0 \
+    security.bsd.see_other_gids=0 \
+    security.bsd.see_other_uids=0 \
+    security.bsd.unprivileged_read_msgbuf=0 \
+    vfs.zfs.min_auto_ashift=12
+
+  # FreeBSD automatically scales kern.maxfilesperproc with the amount of memory.
+  # On systems with large amounts of RAM, this can cause strange lags with some 
+  # applications that attempt to close every possible file descriptor.
+  # Therefore, we arbitrarily cap this value at 65535.
+  [ "$(sysctl -n kern.maxfilesperproc)" -le 65535 ] || set_sysctl kern.maxfilesperproc=65535
+fi
diff --git a/scripts/os/freebsd/20-hostname b/scripts/os/freebsd/20-hostname
new file mode 100644
index 0000000..bf7eeeb
--- /dev/null
+++ b/scripts/os/freebsd/20-hostname
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Set the fully qualified hostname.
+sysrc -v hostname="${BOXCONF_HOSTNAME}.${domain}"
+hostname "${BOXCONF_HOSTNAME}.${domain}"
+install_template -m 0644 /etc/hosts
diff --git a/scripts/os/freebsd/20-locale b/scripts/os/freebsd/20-locale
new file mode 100644
index 0000000..cf72d8d
--- /dev/null
+++ b/scripts/os/freebsd/20-locale
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Set the default system locale.
+install_template -m 0644   \
+  /etc/profile.d/locale.sh \
+  /etc/login.conf
+
+cap_mkdb /etc/login.conf
diff --git a/scripts/os/freebsd/20-motd b/scripts/os/freebsd/20-motd
new file mode 100644
index 0000000..9b1eadb
--- /dev/null
+++ b/scripts/os/freebsd/20-motd
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Disable motd.
+sysrc -v update_motd=NO
+rm -f /var/run/motd
diff --git a/scripts/os/freebsd/20-ntp b/scripts/os/freebsd/20-ntp
new file mode 100644
index 0000000..888bab4
--- /dev/null
+++ b/scripts/os/freebsd/20-ntp
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Jails don't need NTP.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" = jail ]; then
+  return
+fi
+
+install_template -m 0644 /etc/ntp.conf
+
+sysrc -v \
+  ntpd_enable=YES \
+  ntpd_sync_on_start=YES
+
+service ntpd restart
diff --git a/scripts/os/freebsd/20-root-ca b/scripts/os/freebsd/20-root-ca
new file mode 100644
index 0000000..1f88c69
--- /dev/null
+++ b/scripts/os/freebsd/20-root-ca
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Create local CA certificates directory.
+install_directory -m 0755  \
+  /usr/local/etc           \
+  /usr/local/etc/ssl       \
+  /usr/local/etc/ssl/certs
+
+# Install our root CA.
+install_ca_certificate "$site_cacert_path"
+
+certctl rehash
diff --git a/scripts/os/freebsd/20-timezone b/scripts/os/freebsd/20-timezone
new file mode 100644
index 0000000..22a3729
--- /dev/null
+++ b/scripts/os/freebsd/20-timezone
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Set the system timezone.
+cp -v "/usr/share/zoneinfo/${timezone}" /etc/localtime
diff --git a/scripts/os/freebsd/20-zfs b/scripts/os/freebsd/20-zfs
new file mode 100644
index 0000000..aa37c0a
--- /dev/null
+++ b/scripts/os/freebsd/20-zfs
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Every host should have a "state" dataset, which is a ZFS dataset which
+# persists across OS rebuilds.
+[ -n "${state_dataset:-}" ] || die 'state_dataset not defined!'
+create_dataset "$state_dataset"
+
+# If this is baremetal host or a VM, trim the zpools periodically.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" != jail ]; then
+  install_file -m 0644 /etc/cron.d/zfs-trim
+fi
diff --git a/scripts/os/freebsd/30-mail b/scripts/os/freebsd/30-mail
new file mode 100644
index 0000000..511ce69
--- /dev/null
+++ b/scripts/os/freebsd/30-mail
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ "$BOXCONF_HOSTCLASS" = smtp_server ]; then
+  return
+fi
+
+# Configure local mail agent.
+install_template -m 0644 /etc/dma/dma.conf
+install_template -m 0644 /etc/aliases
diff --git a/scripts/os/freebsd/30-ssh b/scripts/os/freebsd/30-ssh
new file mode 100644
index 0000000..91b1991
--- /dev/null
+++ b/scripts/os/freebsd/30-ssh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Create state dataset to persist SSH host keys across OS rebuilds.
+create_dataset -o "mountpoint=${ssh_host_key_dir}" "${state_dataset}/ssh"
+
+# If the state dataset contains existing host keys, symlink them into
+# /etc/ssh.
+#
+# If not, this is the first time we are building this box, so copy the
+# autogenerated host keys to the state partition.
+for key in  \
+  ssh_host_ecdsa_key \
+  ssh_host_ed25519_key \
+  ssh_host_rsa_key
+do
+  [ -f "${ssh_host_key_dir}/${key}" ] || \
+    mv -v "/etc/ssh/${key}" "/etc/ssh/${key}.pub" "$ssh_host_key_dir"
+
+  ln -snvf "${ssh_host_key_dir}/${key}"     "/etc/ssh/${key}"
+  ln -snvf "${ssh_host_key_dir}/${key}.pub" "/etc/ssh/${key}.pub"
+done
+
+# Copy SSH configs.
+install_directory -m 0755 /etc/ssh/sshd_config.d
+
+install_template -m 0644 \
+  /etc/ssh/sshd_config \
+  /etc/ssh/ssh_config
+
+# Restart sshd.
+service sshd restart
diff --git a/scripts/os/freebsd/30-syslog b/scripts/os/freebsd/30-syslog
new file mode 100644
index 0000000..6f3dc8c
--- /dev/null
+++ b/scripts/os/freebsd/30-syslog
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Copy syslog configuration.
+install_file -m 0644 /etc/syslog.conf
+
+# Restart syslogd.
+service syslogd restart
diff --git a/scripts/os/freebsd/40-pkg b/scripts/os/freebsd/40-pkg
new file mode 100644
index 0000000..7c1c828
--- /dev/null
+++ b/scripts/os/freebsd/40-pkg
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+case $BOXCONF_HOSTCLASS in
+  pkg_repository)
+    return # Do nothing.
+    ;;
+  freebsd_hypervisor)
+    ;; # Keep default FreeBSD pkg repository.
+  *)
+    # Configure on-prem pkg repository.
+    install_directory -m 0755 \
+      /usr/local/etc/pkg \
+      /usr/local/etc/pkg/repos
+
+    install_file -m 0644 \
+      /usr/local/etc/ssl/repo.crt \
+      /usr/local/etc/pkg/repos/FreeBSD.conf
+
+    install_template -m 0644 /usr/local/etc/pkg/repos/onprem.conf
+    ;;
+esac
+
+# Update packages.
+pkg update -f
+
+# Install default packages.
+if [ -n "${install_packages:-}" ]; then
+  pkg install -y $install_packages
+fi
diff --git a/scripts/os/freebsd/70-pf b/scripts/os/freebsd/70-pf
new file mode 100644
index 0000000..9ec9961
--- /dev/null
+++ b/scripts/os/freebsd/70-pf
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+if [ "$enable_pf" != true ]; then
+  return
+fi
+
+# Enable pf.
+sysrc -v pf_enable=YES
+
+# Copy pf configuration.
+install_template -m 0600 /etc/pf.conf
+
+# Start (or reload) pf.
+service pf status > /dev/null || service pf start
+service pf reload