From 241833b7f320e7fca84ba226f1ecbb0c963534f7 Mon Sep 17 00:00:00 2001
From: Cullum Smith <cullum@sacredheartsc.com>
Date: Fri, 12 Jul 2024 15:20:54 -0400
Subject: initial commit of hypervisor configs
---
README.md | 7 +-
files/boot.config.freebsd | 1 +
files/etc/aliases.freebsd | 38 +
files/etc/cron.d/zfs-trim.freebsd | 3 +
files/etc/devfs.rules.freebsd_hypervisor | 4 +
files/etc/dma/dma.conf.freebsd | 5 +
files/etc/hosts.freebsd | 4 +
files/etc/login.conf.freebsd | 64 ++
files/etc/ntp.conf.freebsd | 18 +
files/etc/pf.conf.freebsd | 37 +
files/etc/profile.d/locale.sh.freebsd | 2 +
files/etc/resolv.conf.common | 3 +
files/etc/ssh/ssh_config.freebsd_hypervisor | 1 +
files/etc/ssh/ssh_config.no_idm | 1 +
files/etc/ssh/sshd_config.freebsd_hypervisor | 1 +
files/etc/ssh/sshd_config.no_idm | 10 +
files/etc/syslog.conf.freebsd | 12 +
files/etc/ttys.freebsd | 24 +
.../usr/local/etc/jailctl.conf.freebsd_hypervisor | 29 +
files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor | 24 +
files/usr/local/etc/vmctl.conf.freebsd_hypervisor | 18 +
files/usr/local/sbin/jailctl.freebsd_hypervisor | 1098 ++++++++++++++++++
files/usr/local/sbin/vmctl.freebsd_hypervisor | 1198 ++++++++++++++++++++
lib/10-core | 9 +-
lib/40-user | 26 +
scripts/common/10-root-user | 7 +
scripts/common/20-dns | 9 +
scripts/hostclass/freebsd_hypervisor | 80 ++
scripts/os/freebsd/10-bootloader | 29 +
scripts/os/freebsd/10-cpu | 28 +
scripts/os/freebsd/10-periodic | 53 +
scripts/os/freebsd/10-rc-conf | 7 +
scripts/os/freebsd/10-sysctls | 80 ++
scripts/os/freebsd/20-hostname | 6 +
scripts/os/freebsd/20-locale | 8 +
scripts/os/freebsd/20-motd | 5 +
scripts/os/freebsd/20-ntp | 14 +
scripts/os/freebsd/20-root-ca | 12 +
scripts/os/freebsd/20-timezone | 4 +
scripts/os/freebsd/20-zfs | 11 +
scripts/os/freebsd/30-mail | 9 +
scripts/os/freebsd/30-ssh | 31 +
scripts/os/freebsd/30-syslog | 7 +
scripts/os/freebsd/40-pkg | 29 +
scripts/os/freebsd/70-pf | 15 +
vars/common | 18 +
vars/hostclass/freebsd_hypervisor | 5 +
vars/os/freebsd | 34 +
48 files changed, 3131 insertions(+), 7 deletions(-)
create mode 100644 files/boot.config.freebsd
create mode 100644 files/etc/aliases.freebsd
create mode 100644 files/etc/cron.d/zfs-trim.freebsd
create mode 100644 files/etc/devfs.rules.freebsd_hypervisor
create mode 100644 files/etc/dma/dma.conf.freebsd
create mode 100644 files/etc/hosts.freebsd
create mode 100644 files/etc/login.conf.freebsd
create mode 100644 files/etc/ntp.conf.freebsd
create mode 100644 files/etc/pf.conf.freebsd
create mode 100644 files/etc/profile.d/locale.sh.freebsd
create mode 100644 files/etc/resolv.conf.common
create mode 120000 files/etc/ssh/ssh_config.freebsd_hypervisor
create mode 100644 files/etc/ssh/ssh_config.no_idm
create mode 120000 files/etc/ssh/sshd_config.freebsd_hypervisor
create mode 100644 files/etc/ssh/sshd_config.no_idm
create mode 100644 files/etc/syslog.conf.freebsd
create mode 100644 files/etc/ttys.freebsd
create mode 100644 files/usr/local/etc/jailctl.conf.freebsd_hypervisor
create mode 100644 files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor
create mode 100644 files/usr/local/etc/vmctl.conf.freebsd_hypervisor
create mode 100644 files/usr/local/sbin/jailctl.freebsd_hypervisor
create mode 100644 files/usr/local/sbin/vmctl.freebsd_hypervisor
create mode 100644 lib/40-user
create mode 100644 scripts/common/10-root-user
create mode 100644 scripts/common/20-dns
create mode 100644 scripts/hostclass/freebsd_hypervisor
create mode 100644 scripts/os/freebsd/10-bootloader
create mode 100644 scripts/os/freebsd/10-cpu
create mode 100644 scripts/os/freebsd/10-periodic
create mode 100644 scripts/os/freebsd/10-rc-conf
create mode 100644 scripts/os/freebsd/10-sysctls
create mode 100644 scripts/os/freebsd/20-hostname
create mode 100644 scripts/os/freebsd/20-locale
create mode 100644 scripts/os/freebsd/20-motd
create mode 100644 scripts/os/freebsd/20-ntp
create mode 100644 scripts/os/freebsd/20-root-ca
create mode 100644 scripts/os/freebsd/20-timezone
create mode 100644 scripts/os/freebsd/20-zfs
create mode 100644 scripts/os/freebsd/30-mail
create mode 100644 scripts/os/freebsd/30-ssh
create mode 100644 scripts/os/freebsd/30-syslog
create mode 100644 scripts/os/freebsd/40-pkg
create mode 100644 scripts/os/freebsd/70-pf
create mode 100644 vars/common
create mode 100644 vars/hostclass/freebsd_hypervisor
create mode 100644 vars/os/freebsd
diff --git a/README.md b/README.md
index 8f21338..73d83aa 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ frustration with Ansible.
To execute boxconf on a target host, just run the following:
- ./boxconf $TARGET_HOSTNAME
+ ./boxconf $HOSTNAME
A deployment tarball will be generated and SCP'd to the remote box, where `boxconf`
will re-exec itself. After gathering some information about the target system (such
@@ -48,9 +48,8 @@ The `site/` directory does not exist in this repo. Its purpose is to hold person
site-specific variables and scripts that you would rather not share in a public git repo.
Ideally, you would use git submodules for this.
-The `hostname` value is taken from the short hostname of the remote system.
-If the remote hostname is incorrect (or unset), you can override the hostname
-detection by passing the `-o $HOSTNAME` flag to boxconf.
+If the hostname does not exist in DNS, you can manually specify the SSH
+target by passing the `-s $IP_ADDRESS` option to `boxconf`.
The `hostclass` value is matched based on the regular expressions listed in
the [hostclasses](./hostclasses) file.
diff --git a/files/boot.config.freebsd b/files/boot.config.freebsd
new file mode 100644
index 0000000..73d9cb4
--- /dev/null
+++ b/files/boot.config.freebsd
@@ -0,0 +1 @@
+-D -S115200
diff --git a/files/etc/aliases.freebsd b/files/etc/aliases.freebsd
new file mode 100644
index 0000000..b0aeb2d
--- /dev/null
+++ b/files/etc/aliases.freebsd
@@ -0,0 +1,38 @@
+# All local mail should end up forwarded to this address:
+root: ${root_mail_alias}
+
+# Basic system aliases -- these MUST be present
+MAILER-DAEMON: postmaster
+postmaster: root
+
+_dhcp: root
+_pflogd: root
+auditdistd: root
+bin: root
+bind: root
+daemon: root
+games: root
+hast: root
+kmem: root
+mailnull: postmaster
+man: root
+news: root
+nobody: root
+operator: root
+pop: root
+proxy: root
+smmsp: postmaster
+sshd: root
+system: root
+toor: root
+tty: root
+usenet: news
+uucp: root
+manager: root
+dumper:root
+
+# NETWORK OPERATIONS MAILBOX NAMES
+abuse: root
+security: root
+ftp: root
+ftp-bugs: ftp
diff --git a/files/etc/cron.d/zfs-trim.freebsd b/files/etc/cron.d/zfs-trim.freebsd
new file mode 100644
index 0000000..64b07b9
--- /dev/null
+++ b/files/etc/cron.d/zfs-trim.freebsd
@@ -0,0 +1,3 @@
+SHELL=/bin/sh
+PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
+@weekly root zfs list -Ho name | xargs -r -n1 zpool trim
diff --git a/files/etc/devfs.rules.freebsd_hypervisor b/files/etc/devfs.rules.freebsd_hypervisor
new file mode 100644
index 0000000..fe40b9c
--- /dev/null
+++ b/files/etc/devfs.rules.freebsd_hypervisor
@@ -0,0 +1,4 @@
+# Allow jails to access bpf device for DHCP.
+[devfsrules_jail_vnet_bpf=${hypervisor_jail_bpf_ruleset}]
+add include \$devfsrules_jail_vnet
+add path 'bpf*' unhide
diff --git a/files/etc/dma/dma.conf.freebsd b/files/etc/dma/dma.conf.freebsd
new file mode 100644
index 0000000..ff8aae0
--- /dev/null
+++ b/files/etc/dma/dma.conf.freebsd
@@ -0,0 +1,5 @@
+SMARTHOST ${smtp_host}
+SECURETRANSFER
+STARTTLS
+OPPORTUNISTIC_TLS
+MAILNAME ${email_domain}
diff --git a/files/etc/hosts.freebsd b/files/etc/hosts.freebsd
new file mode 100644
index 0000000..5551ff0
--- /dev/null
+++ b/files/etc/hosts.freebsd
@@ -0,0 +1,4 @@
+::1 localhost localhost.${domain}
+127.0.0.1 localhost localhost.${domain}
+
+${BOXCONF_DEFAULT_IPV4} ${BOXCONF_HOSTNAME}.${domain} ${BOXCONF_HOSTNAME}
diff --git a/files/etc/login.conf.freebsd b/files/etc/login.conf.freebsd
new file mode 100644
index 0000000..b7def42
--- /dev/null
+++ b/files/etc/login.conf.freebsd
@@ -0,0 +1,64 @@
+default:\\
+ :passwd_format=sha512:\\
+ :copyright=/etc/COPYRIGHT:\\
+ :welcome=/var/run/motd:\\
+ :setenv=BLOCKSIZE=K:\\
+ :mail=/var/mail/$:\\
+ :path=/sbin /bin /usr/sbin /usr/bin /usr/local/sbin /usr/local/bin ~/bin:\\
+ :nologin=/var/run/nologin:\\
+ :cputime=unlimited:\\
+ :datasize=unlimited:\\
+ :stacksize=unlimited:\\
+ :memorylocked=64M:\\
+ :memoryuse=unlimited:\\
+ :filesize=unlimited:\\
+ :coredumpsize=unlimited:\\
+ :openfiles=unlimited:\\
+ :maxproc=unlimited:\\
+ :sbsize=unlimited:\\
+ :vmemoryuse=unlimited:\\
+ :swapuse=unlimited:\\
+ :pseudoterminals=unlimited:\\
+ :kqueues=unlimited:\\
+ :umtxp=unlimited:\\
+ :priority=0:\\
+ :ignoretime@:\\
+ :umask=022:\\
+ :charset=UTF-8:\\
+ :lang=${locale}:
+
+#
+# A collection of common class names - forward them all to 'default'
+# (login would normally do this anyway, but having a class name
+# here suppresses the diagnostic)
+#
+standard:\\
+ :tc=default:
+xuser:\\
+ :tc=default:
+staff:\\
+ :tc=default:
+
+# This PATH may be clobbered by individual applications. Notably, by default,
+# rc(8), service(8), and cron(8) will all override it with a default PATH that
+# may not include /usr/local/sbin and /usr/local/bin when starting services or
+# jobs.
+daemon:\\
+ :path=/sbin /bin /usr/sbin /usr/bin /usr/local/sbin /usr/local/bin:\\
+ :mail@:\\
+ :memorylocked=128M:\\
+ :tc=default:
+news:\\
+ :tc=default:
+dialer:\\
+ :tc=default:
+
+#
+# Root can always login
+#
+# N.B. login_getpwclass(3) will use this entry for the root account,
+# in preference to 'default'.
+root:\\
+ :ignorenologin:\\
+ :memorylocked=unlimited:\\
+ :tc=default:
diff --git a/files/etc/ntp.conf.freebsd b/files/etc/ntp.conf.freebsd
new file mode 100644
index 0000000..d4b1a03
--- /dev/null
+++ b/files/etc/ntp.conf.freebsd
@@ -0,0 +1,18 @@
+interface ignore wildcard
+interface listen ${BOXCONF_DEFAULT_IPV4}
+
+tos minclock 3 maxclock 6
+
+$(if [ -n "${ntp_servers:-}" ]; then
+ printf 'server %s iburst\n' $ntp_servers
+ elif [ -n "${ntp_pools:-}" ]; then
+ printf 'pool %s iburst\n' $ntp_pools
+ fi)
+
+restrict default limited kod nomodify notrap noquery nopeer
+restrict source limited kod nomodify notrap noquery
+
+restrict 127.0.0.1
+restrict ::1
+
+leapfile "/var/db/ntpd.leap-seconds.list"
diff --git a/files/etc/pf.conf.freebsd b/files/etc/pf.conf.freebsd
new file mode 100644
index 0000000..633f3ef
--- /dev/null
+++ b/files/etc/pf.conf.freebsd
@@ -0,0 +1,37 @@
+egress = "${BOXCONF_DEFAULT_INTERFACE}"
+allowed_tcp_ports = "{ $(join ', ' ${allowed_tcp_ports:-}) }"
+allowed_udp_ports = "{ $(join ', ' ${allowed_udp_ports:-}) }"
+acme_standalone_port = ${acme_standalone_port}
+acme_standalone_user = ${acme_uid}
+nfscbd_port = ${nfscbd_port}
+
+set block-policy return
+set skip on lo
+scrub in on \$egress all fragment reassemble no-df
+
+$([ "${acme_standalone:-}" = true ] && echo \
+ 'rdr on $egress inet proto tcp to port http -> ($egress) port $acme_standalone_port'
+
+[ -n "${redirect_tcp_ports:-}" ] && printf \
+ 'rdr on $egress inet proto tcp to port %s -> ($egress) port %s\n' $redirect_tcp_ports
+
+[ -n "${redirect_udp_ports:-}" ] && printf \
+ 'rdr on $egress inet proto udp to port %s -> ($egress) port %s\n' $redirect_udp_ports)
+
+antispoof quick for \$egress
+
+block all
+pass out quick on \$egress inet
+pass in quick on \$egress inet proto icmp all icmp-type { echoreq, unreach }
+
+$([ "${acme_standalone:-}" = true ] && echo \
+ 'pass in quick on $egress inet proto tcp to port $acme_standalone_port user $acme_standalone_user'
+
+[ -n "${allowed_tcp_ports:-}" ] && echo \
+ 'pass in quick on $egress inet proto tcp to port $allowed_tcp_ports'
+
+[ -n "${allowed_udp_ports:-}" ] && echo \
+ 'pass in quick on $egress inet proto udp to port $allowed_udp_ports'
+
+[ "$BOXCONF_VIRTUALIZATION_TYPE" == jail ] || echo \
+ 'pass in quick on $egress inet proto { tcp, udp } to port $nfscbd_port')
diff --git a/files/etc/profile.d/locale.sh.freebsd b/files/etc/profile.d/locale.sh.freebsd
new file mode 100644
index 0000000..093e6d1
--- /dev/null
+++ b/files/etc/profile.d/locale.sh.freebsd
@@ -0,0 +1,2 @@
+export LANG=${locale}
+export CHARSET=UTF-8
diff --git a/files/etc/resolv.conf.common b/files/etc/resolv.conf.common
new file mode 100644
index 0000000..24c2044
--- /dev/null
+++ b/files/etc/resolv.conf.common
@@ -0,0 +1,3 @@
+$(printf 'nameserver %s\n' $resolvers)
+domain ${domain}
+options timeout:1
diff --git a/files/etc/ssh/ssh_config.freebsd_hypervisor b/files/etc/ssh/ssh_config.freebsd_hypervisor
new file mode 120000
index 0000000..338cdba
--- /dev/null
+++ b/files/etc/ssh/ssh_config.freebsd_hypervisor
@@ -0,0 +1 @@
+ssh_config.no_idm
\ No newline at end of file
diff --git a/files/etc/ssh/ssh_config.no_idm b/files/etc/ssh/ssh_config.no_idm
new file mode 100644
index 0000000..97f3ba8
--- /dev/null
+++ b/files/etc/ssh/ssh_config.no_idm
@@ -0,0 +1 @@
+# Intentionally empty.
diff --git a/files/etc/ssh/sshd_config.freebsd_hypervisor b/files/etc/ssh/sshd_config.freebsd_hypervisor
new file mode 120000
index 0000000..355377d
--- /dev/null
+++ b/files/etc/ssh/sshd_config.freebsd_hypervisor
@@ -0,0 +1 @@
+sshd_config.no_idm
\ No newline at end of file
diff --git a/files/etc/ssh/sshd_config.no_idm b/files/etc/ssh/sshd_config.no_idm
new file mode 100644
index 0000000..f38720c
--- /dev/null
+++ b/files/etc/ssh/sshd_config.no_idm
@@ -0,0 +1,10 @@
+PermitRootLogin prohibit-password
+AuthorizedKeysFile .ssh/authorized_keys
+
+KbdInteractiveAuthentication no
+PasswordAuthentication yes
+
+UsePAM yes
+UseDNS no
+
+Subsystem sftp /usr/libexec/sftp-server
diff --git a/files/etc/syslog.conf.freebsd b/files/etc/syslog.conf.freebsd
new file mode 100644
index 0000000..dda6710
--- /dev/null
+++ b/files/etc/syslog.conf.freebsd
@@ -0,0 +1,12 @@
+*.err;kern.warning;auth.notice;mail.crit /dev/console
+*.info;authpriv.none;auth.none;cron.none;kern.debug;mail.crit;news.err /var/log/messages
+security.* /var/log/security
+auth.info;authpriv.info /var/log/auth.log
+mail.info /var/log/maillog
+cron.* /var/log/cron
+!-devd
+*.=debug /var/log/debug.log
+*.emerg *
+!*
+include /etc/syslog.d
+include /usr/local/etc/syslog.d
diff --git a/files/etc/ttys.freebsd b/files/etc/ttys.freebsd
new file mode 100644
index 0000000..3ebdfe3
--- /dev/null
+++ b/files/etc/ttys.freebsd
@@ -0,0 +1,24 @@
+console none unknown off insecure
+#
+ttyv0 "/usr/libexec/getty Pc" xterm onifexists secure
+# Virtual terminals
+ttyv1 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv2 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv3 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv4 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv5 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv6 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv7 "/usr/libexec/getty Pc" xterm onifexists secure
+ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure
+# Serial terminals
+# The 'dialup' keyword identifies dialin lines to login, fingerd etc.
+ttyu0 "/usr/libexec/getty 3wire.115200" vt100 onifexists secure
+ttyu1 "/usr/libexec/getty 3wire" vt100 onifconsole secure
+ttyu2 "/usr/libexec/getty 3wire" vt100 onifconsole secure
+ttyu3 "/usr/libexec/getty 3wire" vt100 onifconsole secure
+# Dumb console
+dcons "/usr/libexec/getty std.9600" vt100 off secure
+# Xen Virtual console
+xc0 "/usr/libexec/getty Pc" xterm onifconsole secure
+# RISC-V HTIF console
+rcons "/usr/libexec/getty std.9600" vt100 onifconsole secure
diff --git a/files/usr/local/etc/jailctl.conf.freebsd_hypervisor b/files/usr/local/etc/jailctl.conf.freebsd_hypervisor
new file mode 100644
index 0000000..02b6065
--- /dev/null
+++ b/files/usr/local/etc/jailctl.conf.freebsd_hypervisor
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+JAIL_HOME='${hypervisor_jail_home}'
+JAIL_DATASET='${hypervisor_jail_dataset}'
+TRUNK_INTERFACE='${hypervisor_trunk_interface}'
+
+DEFAULT_DOMAIN='${domain}'
+DEFAULT_VLAN='${hypervisor_default_vlan}'
+DEFAULT_NETMASK='$(prefix2netmask "$hypervisor_default_prefix")'
+DEFAULT_OS_QUOTA='${hypervisor_default_os_quota}'
+DEFAULT_DATA_QUOTA='${hypervisor_default_data_quota}'
+
+ZFS_OPTS='${hypervisor_jail_default_zfs_opts}'
+
+DEFAULT_DEVFS_RULESET='5'
+BPF_ENABLED_DEVFS_RULESET='${hypervisor_jail_bpf_ruleset}'
+
+DEFAULT_PF_CONF='egress = "jail0"
+
+set block-policy return
+set skip on lo
+scrub in on \$egress all fragment reassemble no-df
+
+antispoof quick for \$egress
+
+block all
+pass out quick on \$egress inet
+pass in quick on \$egress inet proto icmp all icmp-type { echoreq, unreach }
+pass in quick on \$egress inet proto tcp to port ssh'
diff --git a/files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor b/files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor
new file mode 100644
index 0000000..5f1a84b
--- /dev/null
+++ b/files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# $FreeBSD$
+
+# PROVIDE: vmctl
+# REQUIRE: NETWORKING SERVERS dmesg
+# BEFORE: ipfw pf
+# KEYWORD: shutdown nojail
+
+. /etc/rc.subr
+
+name="vmctl"
+desc="Start and stop bhyve virtual machines"
+rcvar="vmctl_enable"
+
+: ${vmctl_enable:="NO"}
+
+command="/usr/local/sbin/${name}"
+start_cmd="${command} _start-all"
+stop_cmd="${command} _stop-all"
+status_cmd="${command} list"
+
+load_rc_config $name
+run_rc_command "$1"
diff --git a/files/usr/local/etc/vmctl.conf.freebsd_hypervisor b/files/usr/local/etc/vmctl.conf.freebsd_hypervisor
new file mode 100644
index 0000000..7bef759
--- /dev/null
+++ b/files/usr/local/etc/vmctl.conf.freebsd_hypervisor
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+VM_HOME='${hypervisor_vm_home}'
+VM_DATASET='${hypervisor_vm_dataset}'
+TRUNK_INTERFACE='${hypervisor_trunk_interface}'
+
+DEFAULT_DOMAIN='${domain}'
+DEFAULT_CPUS='${hypervisor_vm_default_cpus}'
+DEFAULT_MEMORY='${hypervisor_vm_default_mem}'
+DEFAULT_OS_SIZE='${hypervisor_default_os_quota}'
+DEFAULT_DATA_SIZE='${hypervisor_default_data_quota}'
+DEFAULT_PREFIXLEN='${hypervisor_default_prefix}'
+DEFAULT_VLAN='${hypervisor_default_vlan}'
+TEMPLATE_ZVOL_SIZE='${hypervisor_vm_template_size}'
+DEFAULT_AUTOSTART_DELAY='${hypervisor_vm_default_autostart_delay}'
+
+ZFS_OPTS='${hypervisor_vm_default_zfs_opts}'
+ZFS_VOLBLOCKSIZE='${hypervisor_vm_zfs_volblocksize}'
diff --git a/files/usr/local/sbin/jailctl.freebsd_hypervisor b/files/usr/local/sbin/jailctl.freebsd_hypervisor
new file mode 100644
index 0000000..05c0158
--- /dev/null
+++ b/files/usr/local/sbin/jailctl.freebsd_hypervisor
@@ -0,0 +1,1098 @@
+#!/bin/sh
+#
+# Jail management utility.
+
+set -eu -o pipefail
+
+. /usr/local/etc/jailctl.conf
+
+cmd::main(){
+ local usage="COMMAND [ARGS]...
+FreeBSD jail management utility.
+Commands:
+ create Create a new jail
+ create-snapshot Take a snapshot of a jail
+ create-template Create a template from a jail
+ destroy-snapshot Delete a jail snapshot
+ destroy-template Delete a template
+ destroy Delete a jail and its dataset
+ download-release Download and create a FreeBSD release template
+ edit Edit a jail's configuration
+ exec Run a command within the jail
+ list-snapshots List jail snapshots
+ list-templates List available templates
+ list List configured jails
+ reprovision Wipe and reprovision an OS dataset from template
+ restart Restart a jail
+ rollback Rollback a jail to a given snapshot
+ shell Run a shell within the jail
+ show Show jail configuration
+ start Start a jail
+ status Show running jail status
+ stop Stop a jail
+ update-release Update a FreeBSD release template"
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -ge 1 ] || cmd::usage 'no comand specified'
+ local cmd=$1; shift
+
+ case $cmd in
+ create) cmd::create "$@" ;;
+ create-snapshot|snapshot|snap) cmd::create_snapshot "$@" ;;
+ create-template) cmd::create_template "$@" ;;
+ destroy-snapshot|rms) cmd::destroy_snapshot "$@" ;;
+ destroy-template|rmt) cmd::destroy_template "$@" ;;
+ destroy|rm) cmd::destroy "$@" ;;
+ download-release) cmd::download_release "$@" ;;
+ edit) cmd::edit "$@" ;;
+ exec) cmd::exec "$@" ;;
+ list-snapshots|lss) cmd::list_snapshots "$@" ;;
+ list-templates|lst) cmd::list_templates "$@" ;;
+ list|ls) cmd::list "$@" ;;
+ reprovision) cmd::reprovision "$@" ;;
+ restart) cmd::restart "$@" ;;
+ rollback) cmd::rollback "$@" ;;
+ shell|sh) cmd::shell "$@" ;;
+ show) cmd::show "$@" ;;
+ start) cmd::start "$@" ;;
+ status) cmd::status "$@" ;;
+ stop) cmd::stop "$@" ;;
+ update-release) cmd::update_release "$@" ;;
+ # The following commands are internal to jailctl. Don't run them manually.
+ _create-epair) cmd::_create_epair "$@" ;;
+ _destroy-epair) cmd::_destroy_epair "$@" ;;
+ *) cmd::usage "unknown command: ${cmd}" ;;
+ esac
+}
+
+################################################################################
+# Standard helper functions.
+################################################################################
+die(){
+ printf '%s: %s\n' jailctl "$*" 1>&2
+ exit 1
+}
+
+warn(){
+ printf '%s\n' "$*" 1>&2
+}
+
+
+################################################################################
+# CLI-related functions.
+################################################################################
+cmd::help(){
+ printf 'Usage: %s %s\n' jailctl "$usage"
+ [ -n "${help:-}" ] && printf '%s\n' "$help"
+ exit 0
+}
+
+cmd::usage(){
+ [ $# -gt 0 ] && printf '%s: %s\n' "jailctl" "$1" 1>&2
+ printf 'Usage: %s %s\n' jailctl "${usage}" 1>&2
+ exit 2
+}
+
+cmd::getopt_help(){
+ local opt
+ while getopts :h opt; do
+ case $opt in
+ h) cmd::help ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+}
+
+cmd::create(){
+ local usage='create [-a IP] [-b] [-c CPUSET] [-d DOMAIN] [-g GATEWAY] [-k SSHKEY]
+ [-m MEMLIMIT] [-n NETMASK] [-q QUOTA] [-Q OS_QUOTA] [-r NAMESERVER]
+ [-s SEARCHDOMAIN] [-v VLANID] NAME TEMPLATE'
+ local help="Create a new jail.
+Options:
+ -a IP IPv4 address
+ -b Enable BPF device (allows DHCP, tcpdump, etc)
+ -c CPUSET CPU list for cpuset(1)
+ -d DOMAIN Host domain name
+ -g GATEWAY Default IPv4 gateway
+ -k SSHKEY Path to SSH pubkey for root's authorized_keys
+ -m MEMLIMIT Virtual memory limit
+ -n NETMASK IPv4 netmask
+ -q QUOTA Quota for delegated dataset
+ -Q OS_QUOTA Quota for root filesystem
+ -r NAMESERVER DNS resolver
+ -s SEARCHDOMAIN DNS search domain
+ -v VLANID VLAN ID number"
+
+ local \
+ bpf_enabled=false \
+ cpuset \
+ data_quota \
+ data_quota=$DEFAULT_DATA_QUOTA \
+ devfs_ruleset=$DEFAULT_DEVFS_RULESET \
+ domain=$DEFAULT_DOMAIN \
+ gateway \
+ ip \
+ memlimit \
+ nameservers \
+ netmask=$DEFAULT_NETMASK \
+ os_quota=$DEFAULT_OS_QUOTA \
+ searchdomains \
+ snapshot \
+ sshkey \
+ vlan=$DEFAULT_VLAN \
+ opt
+
+ while getopts :a:bc:d:g:hk:m:n:q:Q:r:s:v: opt; do
+ case $opt in
+ a) ip=$OPTARG ;;
+ b) bpf_enabled=true ;;
+ c) cpuset=$OPTARG ;;
+ d) domain=$OPTARG ;;
+ g) gateway=$OPTARG ;;
+ h) cmd::help ;;
+ k) sshkey=$OPTARG ;;
+ m) memlimit=$OPTARG ;;
+ n) netmask=$OPTARG ;;
+ q) data_quota=$OPTARG ;;
+ Q) os_quota=$OPTARG ;;
+ r) nameservers="${nameservers:-} ${OPTARG}" ;;
+ s) searchdomains="${searchdomains:-} ${OPTARG}" ;;
+ v) vlan=$OPTARG ;;
+ :) cmd::usage "missing option value: -${OPTARG}" ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'NAME not specified'
+ [ $# -lt 2 ] && cmd::usage 'TEMPLATE not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local name=$1 template=$2
+
+ jail::exists "$name" && die "jail name already in use: ${name}"
+ template::exists "$template" || die "no such template: ${template}"
+
+ interface::exists "bridge${vlan}" || interface::add_vlan "$vlan"
+
+ if [ -n "${sshkey:-}" ]; then
+ [ -f "$sshkey" ] || die "ssh key ${sshkey}: file not found"
+ fi
+
+ zfs::ensure_snapshot snapshot "${JAIL_DATASET}/templates/${template}"
+
+ # Clone template into new 'os' dataset.
+ zfs create -v "${JAIL_DATASET}/${name}"
+ zfs clone \
+ $ZFS_OPTS \
+ -o quota="$os_quota" \
+ "$snapshot" "${JAIL_DATASET}/${name}/os"
+
+ # Create delegated 'data' dataset.
+ zfs create -v \
+ $ZFS_OPTS \
+ -o mountpoint=none \
+ -o quota="$data_quota" \
+ "${JAIL_DATASET}/${name}/data"
+
+ # Copy timezone configuration from host.
+ cp -v /etc/localtime "${JAIL_HOME}/${name}/os/etc/localtime"
+
+ # Generate /etc/resolv.conf in the jail (word-splitting intentional).
+ [ -n "${nameservers:-}" ] && printf 'nameserver %s\n' $nameservers >> "${JAIL_HOME}/${name}/os/etc/resolv.conf"
+ [ -n "${searchdomains:-}" ] && printf 'search %s\n' "$searchdomains" >> "${JAIL_HOME}/${name}/os/etc/resolv.conf"
+
+ # Get the jail's virtual interface name.
+ local epair_name
+ epair_name=$(interface::epair::derive_name "$name")
+
+ # Set /etc/rc.conf values within the jail.
+ sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" \
+ "hostname=${name}.${domain}" \
+ "ifconfig_ej_${epair_name}_name=jail0" \
+ 'ipv6_activate_all_interfaces=NO' \
+ 'syslogd_flags=-ss' \
+ 'sendmail_enable=NONE' \
+ 'dumpdev=NO' \
+ 'pf_enable=YES'
+
+ echo "$DEFAULT_PF_CONF" > "${JAIL_HOME}/${name}/os/etc/pf.conf"
+
+ if [ -n "${ip:-}" ]; then
+ # If $ip set, configure /etc/rc.conf for static IP.
+ sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" \
+ "ifconfig_jail0=inet ${ip} netmask ${netmask}" \
+ "defaultrouter=${gateway}"
+ else
+ # Othersie, configure /etc/rc.conf for DHCP.
+ sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" "ifconfig_jail0=SYNCDHCP"
+ devfs_ruleset=$BPF_ENABLED_DEVFS_RULESET
+ fi
+
+ if [ -n "${sshkey:-}" ]; then
+ # If $sshkey set, enable sshd and root login.
+ echo "PermitRootLogin prohibit-password" | tee -a "${JAIL_HOME}/${name}/os/etc/ssh/sshd_config"
+ sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" 'sshd_enable=YES'
+ # Copy the ssh key into root's authorized_keys within the jail.
+ install -v -d -m 0700 "${JAIL_HOME}/${name}/os/root/.ssh"
+ install -v -m 0600 "$sshkey" "${JAIL_HOME}/${name}/os/root/.ssh/authorized_keys"
+ fi
+
+ # If BPF requested, use our custom bpf ruleset.
+ [ "${bpf_enabled}" = true ] && devfs_ruleset=$BPF_ENABLED_DEVFS_RULESET
+
+ # Generate jail config file.
+ local jailcfg="${JAIL_HOME}/${name}/jail.conf"
+ cat <<EOF > "$jailcfg"
+${name} {
+ path = "${JAIL_HOME}/\$name/os";
+ host.hostname = "\$name.${domain}";
+
+ exec.prestart = "jailctl _create-epair \$name vlan${vlan} bridge${vlan}";
+ exec.created = "zfs set jailed=on ${JAIL_DATASET}/\$name/data";
+ exec.created += "zfs jail \$name ${JAIL_DATASET}/\$name/data";
+ exec.start = "zfs mount -a";
+ exec.start += "/bin/sh /etc/rc";
+ exec.stop = "/bin/sh /etc/rc.shutdown";
+ exec.stop += "zfs list -Ho name,jailed,mounted | awk '\$2 == \\"on\\" && \$3 == \\"yes\\" {print \$1}' | xargs -rtn1 zfs unmount";
+ exec.poststop = "jailctl _destroy-epair \$name";
+ exec.poststop += "rctl -r jail:\$name:";
+ exec.clean;
+
+ exec.system_user = "root";
+ exec.jail_user = "root";
+
+ mount.devfs;
+ devfs_ruleset = "${devfs_ruleset}";
+
+ mount = "tmpfs \$path/tmp tmpfs rw,size=1G 0 0";
+ allow.mount = true;
+ allow.mount.zfs = true;
+ enforce_statfs = 1;
+
+ vnet;
+ vnet.interface = "ej_${epair_name}";
+EOF
+
+ ln -sv "$jailcfg" "/etc/jail.conf.d/${name}.conf"
+
+ # Configure resource limits.
+ [ -n "${cpuset:-}" ] && \
+ echo " exec.created += \"cpuset -j \$name -cl ${cpuset}\";" >> "$jailcfg"
+ [ -n "${memlimit:-}" ] && \
+ echo " exec.prestart += \"rctl -a jail:\$name:memoryuse:deny=${memlimit}\";" >> "$jailcfg"
+
+ # End jail config file.
+ echo '}' >> "$jailcfg"
+
+ # Update host's /etc/rc.conf to start the jail on boot.
+ sysrc -v jail_list+="$name"
+
+ # Start the jail.
+ jail::start "$name"
+}
+
+cmd::create_snapshot(){
+ local usage='create-snapshot [-d] [-o] JAIL [SNAPNAME]'
+ local help='Create a jail snapshot.
+Options:
+ -d Only snapshot the data dataset
+ -o Only snapshot the OS dataset'
+
+ local opt both=true os_only=false data_only=false snapname
+ while getopts :dho opt; do
+ case $opt in
+ d) data_only=true; both=false ;;
+ h) cmd::help ;;
+ o) os_only=true; both=false ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ jail=$1 snapname=${2:-}
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ [ -n "$snapname" ] || snapname=$(date +%Y-%m-%dT%H:%M:%S)
+
+ # Snapshot the OS dataset.
+ if [ "$both" = true ] || [ "$os_only" = true ]; then
+ zfs snapshot "${JAIL_DATASET}/${jail}/os@${snapname}"
+ fi
+
+ # Snapshot the data dataset.
+ if [ "$both" = true ] || [ "$data_only" = true ]; then
+ zfs snapshot "${JAIL_DATASET}/${jail}/data@${snapname}"
+ fi
+}
+
+cmd::create_template(){
+ local usage='create-template NAME JAIL'
+ local help='Create a template from a jail.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'NAME not specified'
+ [ $# -lt 2 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local name=$1 jail=$2
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ template::exists "$name" && die "template already exists: $name"
+ jail::running "$jail" && die "refusing to create template while jail is running: ${jail}"
+
+ local snapname
+ snapname=$(date +%Y-%m-%dT%H:%M:%S)
+
+ zfs snapshot "${JAIL_DATASET}/${jail}/os@${snapname}"
+ zfs send "${JAIL_DATASET}/${jail}/os@${snapname}" | zfs receive -v "${JAIL_DATASET}/templates/${name}"
+ zfs destroy "${JAIL_DATASET}/${jail}/os@${snapname}"
+}
+
+cmd::destroy_snapshot(){
+ local usage='destroy-snapshot [-y] JAIL SNAPSHOT'
+ local help="Delete a jail snapshot
+Options:
+ -y Don't prompt for confirmation"
+
+ local noconfirm=false answer opt
+
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local jail=$1 snapshot=$2
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ local datasets=''
+
+ if [ "${snapshot#*@}" != "$snapshot" ]; then
+ # If the snapshot name contains '@', then we have something like 'os@snapname'.
+ zfs::dataset_exists "${JAIL_DATASET}/${jail}/${snapshot}" && \
+ datasets="${JAIL_DATASET}/${jail}/${snapshot}"
+ else
+ # Otherwise, check if either os or data dataset contains a matching snapshot.
+ zfs::dataset_exists "${JAIL_DATASET}/${jail}/os@${snapshot}" && \
+ datasets="${JAIL_DATASET}/${jail}/os@${snapshot}"
+
+ zfs::dataset_exists "${JAIL_DATASET}/${jail}/data@${snapshot}" && \
+ datasets="${datasets} ${JAIL_DATASET}/${jail}/data@${snapshot}"
+ fi
+
+ [ -n "$datasets" ] || die "no such snapshot for jail ${jail}: ${snapshot}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really destroy ${jail} snapshot ${snapshot}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ local dataset
+ for dataset in $datasets; do
+ zfs destroy -v "$dataset"
+ done
+}
+
+cmd::destroy_template(){
+ local usage='destroy-template [-y] TEMPLATE'
+ local help="Delete a jail template.
+Options:
+ -y Don't prompt for confirmation
+Notes:
+ A template cannot be deleted while its clones still exist."
+
+ local noconfirm=false answer opt
+
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'TEMPLATE not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ template::exists "$jail" || die "no such template: ${jail}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really destroy template ${jail}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ zfs destroy -v -r "${JAIL_DATASET}/templates/${jail}"
+}
+
+cmd::destroy(){
+ local usage='destroy [-y] JAIL'
+ local help="Delete a jail and its dataset.
+Options:
+ -y Don't prompt for confirmation"
+
+ local noconfirm=false answer opt
+
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really destroy jail ${jail}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ jail::running "$jail" && jail::stop "$jail"
+
+ # Delete jail config file.
+ rm "/etc/jail.conf.d/${jail}.conf" "${JAIL_HOME}/${jail}/jail.conf"
+
+ # Remove the jail from the autostart list.
+ sysrc -v jail_list-="$jail" ||:
+
+ # Destroy the jail's dataset.
+ zfs destroy -v -f -r "${JAIL_DATASET}/${jail}"
+}
+
+cmd::download_release() {
+ local usage='download-release RELEASE'
+ local help='Download and create a FreeBSD release template.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'RELEASE not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local release=$1
+
+ local template
+ template::release2name template "$release"
+
+ template::exists "$template" && die "template already exists: ${template}"
+ template::download_release "$release"
+}
+
+cmd::edit(){
+ local usage='edit JAIL'
+ local help='Edit jail configuration.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ "$EDITOR" "${JAIL_HOME}/${jail}/jail.conf"
+}
+
+cmd::exec(){
+ local usage='exec JAIL COMMAND...'
+ local help='Run a command within the jail.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -lt 2 ] && cmd::usage 'COMMAND not specified'
+ jail=$1; shift
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ jail::exec "$jail" "$@"
+}
+
+cmd::list(){
+ local usage='list [-t]'
+ local help='List configured jails.
+Options:
+ -t Use terse output (jail names only)'
+
+ local file name status opt terse=false
+
+ while getopts :th opt; do
+ case $opt in
+ t) terse=true ;;
+ h) cmd::help ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -eq 0 ] || cmd::usage 'too many arguments'
+
+ { [ $terse = true ] || echo 'JAIL STATUS'
+ for file in "$JAIL_HOME"/*/jail.conf; do
+ [ -e "$file" ] || continue
+
+ name=$(basename "$(dirname "$file")")
+
+ if [ $terse = true ]; then
+ printf '%s\n' "$name"
+ else
+ if jail::running "$name"; then
+ status=running
+ else
+ status=stopped
+ fi
+ printf '%s %s\n' "$name" "$status"
+ fi
+ done
+ } | column -t
+}
+
+cmd::list_templates(){
+ local usage='list-templates'
+ local help='list template datasets.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -eq 0 ] || cmd::usage 'too many arguments'
+
+ ls -1 "${JAIL_HOME}/templates"
+}
+
+cmd::list_snapshots(){
+ local usage='list-snapshots JAIL'
+ local help='List jail snapshots
+Options:
+ -d List snapshots from data dataset only
+ -o List snapshots from OS dataset only
+ -t Use terse output'
+
+ local opt terse=false dataset=''
+ while getopts :dhot opt; do
+ case $opt in
+ d) dataset=/data ;;
+ h) cmd::help ;;
+ o) dataset=/os ;;
+ t) terse=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ if [ "$terse" = true ]; then
+ zfs list -r -t snapshot -H -o name -s creation "${JAIL_DATASET}/${jail}${dataset}" 2>/dev/null \
+ | sed 's/^.*\///' \
+ | sort -k1,1 -t@ -s
+ else
+ { echo 'DATASET SNAPSHOT USED REFER'
+ zfs list -r -t snapshot -H -o name,used,refer -s creation "${JAIL_DATASET}/${jail}${dataset}" 2>/dev/null
+ } | sed 's/^.*\///' \
+ | sort -k1,1 -t@ -s \
+ | tr '@' ' ' \
+ | column -t
+ fi
+}
+
+cmd::rollback(){
+ local usage='rollback JAIL SNAPSHOT'
+ local help='Rollback a jail to a given snapshot.
+Options:
+ -d Rollback the data dataset only
+ -f Attempt rollback even if the jail is running
+ -o Rollback the OS dataset only'
+
+ local opt force=false both=true os_only=false data_only=false
+ while getopts :dfho opt; do
+ case $opt in
+ d) data_only=true; both=false ;;
+ f) force=true ;;
+ h) cmd::help ;;
+ o) os_only=true; both=false ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local jail=$1 snapshot=$2
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ # Ensure OS snapshot exists, if requested.
+ if [ "$both" = true ] || [ "$os_only" = true ]; then
+ zfs::dataset_exists "${JAIL_DATASET}/${jail}/os@${snapshot}" \
+ || die "no such snapshot for ${jail}/os: ${snapshot}"
+ fi
+
+ # Ensure data snapshot exists, if requested.
+ if [ "$both" = true ] || [ "$data_only" = true ]; then
+ zfs::dataset_exists "${JAIL_DATASET}/${jail}/data@${snapshot}" \
+ || die "no such snapshot for ${jail}/data: ${snapshot}"
+ fi
+
+ # Don't rollback while jail is running.
+ if jail::running "$jail" && [ "$force" != true ]; then
+ die "jail ${jail} is running, refusing to rollback (-f to override)"
+ fi
+
+ # Rollback the OS snapshot, if requested.
+ if [ "$both" = true ] || [ "$os_only" = true ]; then
+ zfs rollback -r "${JAIL_DATASET}/${jail}/os@${snapshot}"
+ fi
+
+ # Rollback the data snapshot, if requested.
+ if [ "$both" = true ] || [ "$data_only" = true ]; then
+ zfs rollback -r "${JAIL_DATASET}/${jail}/data@${snapshot}"
+ fi
+}
+
+cmd::start(){
+ local usage='start JAIL'
+ local help='Start a jail.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ jail::running "$jail" && die "jail already running: ${jail}"
+
+ jail::start "$jail"
+}
+
+cmd::stop(){
+ local usage='stop JAIL'
+ local help='Stop a jail.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ jail::running "$jail" || die "jail not running: ${jail}"
+
+ jail::stop "$jail"
+}
+
+cmd::reprovision(){
+ local usage='reprovision JAIL TEMPLATE'
+ local help="Wipe and reprovision a jail's OS dataset from a template."
+
+ local opt noconfirm=false running=false
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -lt 2 ] && cmd::usage 'TEMPLATE not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local jail=$1 template=$2
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ template::exists "$template" || die "no such template: ${template}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really reprovision ${jail}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ # If the jail is running, stop it.
+ if jail::running "$jail"; then
+ running=true
+ jail::stop "$jail"
+ fi
+
+ local snapshot old_quota old_ifconfig old_defaultrouter old_hostname old_resolvconf
+
+ # Get the latest snapshot for the template (if not specified).
+ zfs::ensure_snapshot snapshot "${JAIL_DATASET}/templates/${template}"
+
+ # Stash old configuration data.
+ old_quota=$(zfs get -Hp -o value quota "${JAIL_DATASET}/${jail}/os")
+ old_hostname=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn hostname)
+ old_ifconfig=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn ifconfig_jail0)
+ old_defaultrouter=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn defaultrouter) ||:
+ old_pfenable=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn pf_enable) ||:
+ old_sshkey=$(cat "${JAIL_HOME}/${jail}/os/root/.ssh/authorized_keys") ||:
+ old_resolvconf=$(cat "${JAIL_HOME}/${jail}/os/etc/resolv.conf") ||:
+
+ # Reprovision OS dataset from template.
+ zfs destroy -v -f -r "${JAIL_DATASET}/${jail}/os"
+ zfs clone \
+ $ZFS_OPTS \
+ -o quota="$old_quota" \
+ "$snapshot" "${JAIL_DATASET}/${jail}/os"
+
+ # Copy timezone configuration from host.
+ cp -v /etc/localtime "${JAIL_HOME}/${jail}/os/etc/localtime"
+
+ # Restore stashed configuration data.
+ sysrc -v -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" \
+ "hostname=${old_hostname}" \
+ "ifconfig_ej_$(interface::epair::derive_name "$jail")_name=jail0" \
+ "ifconfig_jail0=${old_ifconfig}" \
+ 'ipv6_activate_all_interfaces=NO' \
+ 'syslogd_flags=-ss' \
+ 'sendmail_enable=NONE' \
+ 'dumpdev=NO' \
+ "pf_enable=YES"
+
+ [ -n "$old_defaultrouter" ] && sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" "defaultrouter=${old_defaultrouter}"
+
+ [ -n "$old_resolvconf" ] && printf '%s\n' "$old_resolvconf" | tee "${JAIL_HOME}/${jail}/os/etc/resolv.conf"
+
+ echo "$DEFAULT_PF_CONF" > "${JAIL_HOME}/${jail}/os/etc/pf.conf"
+
+ if [ -n "$old_sshkey" ]; then
+ echo "PermitRootLogin prohibit-password" | tee -a "${JAIL_HOME}/${jail}/os/etc/ssh/sshd_config"
+ sysrc -v -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" 'sshd_enable=YES'
+ install -v -d -m 0700 "${JAIL_HOME}/${jail}/os/root/.ssh"
+ install -v -m 0600 /dev/null "${JAIL_HOME}/${jail}/os/root/.ssh/authorized_keys"
+ printf '%s\n' "$old_sshkey" | tee "${JAIL_HOME}/${jail}/os/root/.ssh/authorized_keys"
+ fi
+
+ # If the jail was running, restart it.
+ if [ "$running" = true ]; then
+ jail::start "$jail"
+ fi
+}
+
+cmd::restart(){
+ local usage='restart JAIL'
+ local help='Restart a jail.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'no jail specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ jail::running "$jail" || die "jail not running: ${jail}"
+
+ jail::restart "$jail"
+}
+
+cmd::shell(){
+ local usage='shell JAIL'
+ local help='Run a shell within the jail.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ jail::running "$jail" || die "jail not running: ${jail}"
+
+ jail::exec "$jail" /bin/csh
+}
+
+cmd::show(){
+ local usage='show JAIL'
+ local help='Show jail configuration.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+
+ printf -- '------------------------- JAIL CONFIGURATION -------------------------\n'
+ cat "${JAIL_HOME}/${jail}/jail.conf"
+ printf -- '\n---------------------------- ZFS DATASET -----------------------------\n'
+ zfs list -o name,quota,used,avail,mountpoint -S name \
+ "${JAIL_DATASET}/${jail}/os" \
+ "${JAIL_DATASET}/${jail}/data"
+}
+
+cmd::status(){
+ local usage='status JAIL'
+ local help='Show running jail status.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ jail::exists "$jail" || die "no such jail: ${jail}"
+ jail::running "$jail" || die "jail not running: ${jail}"
+
+ printf -- '---------------------------- JAIL STATUS -----------------------------\n'
+ jls -j "$jail" -h jid name path osrelease host.hostname 2>/dev/null | column -t
+ printf -- '\n---------------------------- ZFS DATASET -----------------------------\n'
+ zfs list -o name,quota,used,avail,mountpoint -S name \
+ "${JAIL_DATASET}/${jail}/os" \
+ "${JAIL_DATASET}/${jail}/data" \
+ | sed "s|^${JAIL_DATASET}/${jail}/||" \
+ | column -t
+ printf -- '\n--------------------------- RESOURCE USAGE ---------------------------\n'
+ rctl -h -u "jail:${jail}:" \
+ | grep -E '^(maxproc|memoryuse|openfiles|pcpu|memoryuse|swapuse|readbps|writebps|readiops|writeiops)=' \
+ | rs -c= -C' ' -T \
+ | column -t
+ printf -- '\n----------------------------- PROCESSES ------------------------------\n'
+ ps -auxdr -J "$jail"
+}
+
+cmd::update_release(){
+ local usage='update-release TEMPLATE'
+ local help='Update a FreeBSD release template.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'TEMPLATE not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local jail=$1
+
+ template::exists "$jail" || die "no such template: ${jail}"
+ template::update_release "$jail"
+}
+
+cmd::_create_epair(){
+ local usage='_create-epair JAIL INTERFACE BRIDGE'
+ local help="Create an epair for a VNET jail.
+DO NOT RUN THIS COMMAND MANUALLY. It is invoked by each jail's exec.prestart."
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -lt 2 ] && cmd::usage 'INTERFACE not specified'
+ [ $# -lt 3 ] && cmd::usage 'BRIDGE not specified'
+ [ $# -gt 3 ] && cmd::usage 'too many arguments'
+ local jail=$1 interface=$2 bridge=$3 \
+ new_epair epair_name epair_bridge_mac epair_jail_mac
+
+ interface::exists "$interface" || die "no such interface: ${interface}"
+ interface::exists "$bridge" || die "no such bridge: ${bridge}"
+
+ epair_name=$(interface::epair::derive_name "$jail")
+ interface::epair::derive_mac "$interface" "$jail" epair_bridge_mac epair_jail_mac
+
+ new_epair=$(ifconfig epair create)
+ ifconfig "$bridge" addm "$new_epair"
+
+ ifconfig "$new_epair" name "eb_${epair_name}" descr "jail/${jail}"
+ ifconfig "eb_${epair_name}" up
+ ifconfig "eb_${epair_name}" ether "$epair_bridge_mac"
+
+ ifconfig "${new_epair%a}b" name "ej_${epair_name}"
+ ifconfig "ej_${epair_name}" up
+ ifconfig "ej_${epair_name}" ether "$epair_jail_mac"
+}
+
+cmd::_destroy_epair(){
+ local usage='_destroy-epair JAIL'
+ local help="Destroy an epair for a VNET jail.
+DO NOT RUN THIS COMMAND MANUALLY. It is invoked by each jail's exec.poststop."
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'JAIL not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local jail=$1 epair_name
+
+ epair_name=$(interface::epair::derive_name "$jail")
+ ifconfig "eb_${epair_name}" destroy
+}
+
+
+################################################################################
+# "Library" functions.
+################################################################################
+interface::add_vlan(){
+ local id=$1
+ ifconfig "vlan${id}" create
+ ifconfig "vlan${id}" vlan "$id" vlandev "$TRUNK_INTERFACE" up
+ ifconfig "bridge${id}" create
+ ifconfig "bridge${id}" addm "vlan${id}" up
+ sysrc -v \
+ "cloned_interfaces+=vlan${id} bridge${id}" \
+ "ifconfig_vlan${id}=vlan ${id} vlandev ${TRUNK_INTERFACE} up" \
+ "ifconfig_bridge${id}=addm vlan${id} up"
+}
+
+interface::exists(){
+ ifconfig "$1" > /dev/null 2>&1
+}
+
+interface::epair::derive_mac(){
+ # Derive unique mac addresses for an epair interface, based on the physical
+ # interface address + jail name.
+ local iface=$1 name=$2 upvar_a=$3 upvar_b=$4
+ local hwaddr checksum epair_mac_a epair_mac_b
+
+ # Similar to /usr/share/examples/jails/jib
+ hwaddr=$(ifconfig "$iface" ether | awk '/ether/,$0=$2')
+ # ??:??:??:II:II:II
+ epair_mac_a=${hwaddr#??:??:??} # => :II:II:II
+ # => :SS:SS:II:II:II
+ epair_mac_a=":$(printf '%s' "$name" | md5 | sed 's/^\(..\)\(..\).*/\1:\2/')${epair_mac_a}"
+ # => NP:SS:SS:II:II:II
+ case $hwaddr in
+ ?2:*) epair_mac_a="0a${epair_mac_a}" epair_mac_b="0e${epair_mac_a}" ;;
+ ?[Ee]:*) epair_mac_a="02${epair_mac_a}" epair_mac_b="06${epair_mac_a}" ;;
+ *) epair_mac_a="02${epair_mac_a}" epair_mac_b="0e${epair_mac_a}" ;;
+ esac
+
+ # Sanity check for duplicate MAC
+ for mac in $epair_mac_a $epair_mac_b; do
+ ifconfig | grep -qE "(ether|hwaddr) ${mac}" && die "MAC collision when configuring epair interface!"
+ done
+
+ setvar "$upvar_a" "$epair_mac_a"
+ setvar "$upvar_b" "$epair_mac_b"
+}
+
+interface::epair::derive_name(){
+ # Generate an epair(4) interface suffix, based on the jail name.
+ #
+ # The maximum length of a network interface name on FreeBSD is 15 characters.
+ # We use a prefix of 'eb_' for the bridge side of the interface, and 'ej_'
+ # for the jail side. This leaves 12 characters for a unique suffix for each
+ # jail.
+ #
+ # If the sanitized jail name is less than 12 characters, we'll simply use it
+ # for the suffix. Otherwise, we'll use the last 12 characters of the jail
+ # name's SHA-1 hash.
+ local name=$1 sanitized
+ sanitized=$(printf '%s' "$name" | tr -dC '[:alnum:]_')
+
+ if [ "${#sanitized}" -le 12 ]; then
+ printf '%s' "$sanitized"
+ else
+ printf '%s' "$name" | sha1 | tail -c 12
+ fi
+}
+
+jail::exec() {
+ local jail=$1; shift
+ jexec -l "$jail" "$@"
+}
+
+jail::exists(){
+ test -f "${JAIL_HOME}/${1}/jail.conf"
+}
+
+jail::restart(){
+ jail -v -f "${JAIL_HOME}/${1}/jail.conf" -rc "$1"
+}
+
+jail::running(){
+ jls -j "$1" > /dev/null 2>&1
+}
+
+jail::start(){
+ jail -v -f "${JAIL_HOME}/${1}/jail.conf" -c "$1"
+}
+
+jail::stop(){
+ jail -v -f "${JAIL_HOME}/${1}/jail.conf" -r "$1"
+}
+
+template::exists(){
+ zfs list -H "${JAIL_DATASET}/templates/${1}" > /dev/null 2>&1
+}
+
+template::download_release(){
+ # Download a given FreeBSD release and create a template jail.
+ local release=$1 arch base_tarball template
+
+ arch=$(uname -p)
+ base_tarball="https://download.freebsd.org/releases/${arch}/${release}/base.txz"
+
+ template::release2name template "$release"
+
+ zfs create -v -p $ZFS_OPTS "${JAIL_DATASET}/templates/${template}"
+
+ if ! fetch "$base_tarball" -o - | tar xzf - -C "${JAIL_HOME}/templates/${template}"; then
+ zfs destroy -v "${JAIL_DATASET}/templates/${template}"
+ die "failed to extract base tarball for ${release}"
+ fi
+
+ template::update_release "$template"
+}
+
+template::release2name(){
+ # Convert a FreeBSD release version to a template name.
+ # e.g. for "13.2-RELEASE", return "freebsd13".
+ setvar "$1" "freebsd${2%.*}"
+}
+
+template::update_release(){
+ # Run freebsd-update within a the given template and take a fresh snapshot.
+ local template=$1 snapshot
+
+ PAGER=/bin/cat freebsd-update -b "${JAIL_HOME}/templates/${template}" --not-running-from-cron fetch install
+
+ snapshot=$("${JAIL_HOME}/templates/${template}/bin/freebsd-version")
+ template::exists "${template}@${snapshot}" || zfs snapshot "${JAIL_DATASET}/templates/${template}@${snapshot}"
+}
+
+zfs::ensure_snapshot(){
+ # If the given zfs dataset is a snapshot, return as-is.
+ # Otherwise, get the latest snapshot from the given dataset and return that.
+ local upvar=$1 dataset=$2 latest
+
+ if [ "${dataset#*@}" = "$dataset" ]; then
+ latest=$(zfs list -t snapshot -o name -s creation -H "$dataset" | tail -1)
+ setvar "$upvar" "$latest"
+ else
+ zfs list -t snapshot -H "$dataset" > /dev/null 2>&1 || die "no such snapshot: ${dataset}"
+ setvar "$upvar" "$dataset"
+ fi
+}
+
+zfs::dataset_exists(){
+ zfs list "$1" > /dev/null 2>&1
+}
+
+
+cmd::main "$@"
diff --git a/files/usr/local/sbin/vmctl.freebsd_hypervisor b/files/usr/local/sbin/vmctl.freebsd_hypervisor
new file mode 100644
index 0000000..b5bc9cf
--- /dev/null
+++ b/files/usr/local/sbin/vmctl.freebsd_hypervisor
@@ -0,0 +1,1198 @@
+#!/bin/sh
+#
+# Bhyve management utility.
+
+set -eu -o pipefail
+
+. /usr/local/etc/vmctl.conf
+
+cmd::main(){
+ local usage="COMMAND [ARGS]...
+FreeBSD bhyve management utility.
+Commands:
+ create Create a new VM
+ create-snapshot Take a snapshot of a VM
+ create-template Create a template from a VM disk image
+ console Connect to a VM's serial console
+ destroy Destroy a VM and its dataset
+ destroy-iso Delete an ISO
+ destroy-snapshot Delete a VM snapshot
+ destroy-template Delete a VM template
+ download-iso Download an ISO
+ download-template Download a disk image
+ edit Edit a VM's configuration
+ list List VMs
+ list-isos List available ISOs
+ list-snapshots List VM snapshots
+ list-templates List VM templates
+ reprovision Wipe and reprovision an OS disk from a template
+ restart Restart a VM
+ rollback Rollback a VM to a given snapshot
+ show Show VM configuration
+ start Start a VM
+ status Show VM status
+ stop Stop a VM"
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -ge 1 ] || cmd::usage 'no comand specified'
+ local cmd=$1; shift
+
+ case $cmd in
+ console) cmd::console "$@" ;;
+ create) cmd::create "$@" ;;
+ create-snapshot|snapshot|snap) cmd::create_snapshot "$@" ;;
+ create-template) cmd::create_template "$@" ;;
+ destroy|rm) cmd::destroy "$@" ;;
+ destroy-iso|rmi) cmd::destroy_iso "$@" ;;
+ destroy-snapshot|rms) cmd::destroy_snapshot "$@" ;;
+ destroy-template|rmt) cmd::destroy_template "$@" ;;
+ download-iso|dli) cmd::download_iso "$@" ;;
+ download-template|dlt) cmd::download_template "$@" ;;
+ edit) cmd::edit "$@" ;;
+ list|ls) cmd::list "$@" ;;
+ list-isos|lsi) cmd::list_isos "$@" ;;
+ list-snapshots|lss) cmd::list_snapshots "$@" ;;
+ list-templates|lst) cmd::list_templates "$@" ;;
+ reprovision) cmd::reprovision "$@" ;;
+ restart|reboot) cmd::restart "$@" ;;
+ rollback) cmd::rollback "$@" ;;
+ show) cmd::show "$@" ;;
+ start|boot) cmd::start "$@" ;;
+ status) cmd::status "$@" ;;
+ stop|shutdown) cmd::stop "$@" ;;
+ # The following commands are internal and should not be invoked manually.
+ _start-all) cmd::_start_all "$@" ;;
+ _stop-all) cmd::_stop_all "$@" ;;
+ *) cmd::usage "unknown command: ${cmd}" ;;
+ esac
+}
+
+
+################################################################################
+# Standard helper functions.
+################################################################################
+die(){
+ printf '%s: %s\n' vmctl "$*" 1>&2
+ exit 1
+}
+
+
+################################################################################
+# CLI-related functions.
+################################################################################
+cmd::help(){
+ printf 'Usage: %s %s\n' vmctl "$usage"
+ [ -n "${help:-}" ] && printf '%s\n' "$help"
+ exit 0
+}
+
+cmd::usage(){
+ [ $# -gt 0 ] && printf '%s: %s\n' "vmctl" "$1" 1>&2
+ printf 'Usage: %s %s\n' vmctl "${usage}" 1>&2
+ exit 2
+}
+
+cmd::getopt_help(){
+ local opt
+ while getopts :h opt; do
+ case $opt in
+ h) cmd::help ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+}
+
+cmd::console(){
+ local usage='console VM'
+ local help='Connect to serial console.
+Notes:
+ Type ~. to return to your shell.
+ See `man 1 cu` for additional escape codes.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ vm::running "$vm" || die "vm not running: ${vm}"
+
+ exec cu -l "/dev/nmdm_${vm}B" -s 9600
+}
+
+cmd::create(){
+ local usage='create [-A] [-a IP] [-c CORES] [-d DOMAIN] [-g GATEWAY] [-i ISO] [-k SSHKEY]
+ [-m MEM] [-p PREFIXLEN] [-q DATA_SIZE] [-Q OS_SIZE] [-r NAMESERVER]
+ [-s SEARCHDOMAIN] [-v VLANID] NAME [TEMPLATE]'
+ local help="Create a new VM.
+Options:
+ -A Don't autostart on boot
+ -a IP IPv4 address (cloud-init)
+ -c CORES Number of CPU cores
+ -d DOMAIN Host domain name (cloud-init)
+ -g GATEWAY Default IPv4 gateway (cloud-init)
+ -i ISO Boot from an ISO
+ -k SSHKEY Path to SSH pubkey for root's authorized_keys (cloud-init)
+ -m MEM Size of memory (RAM)
+ -p PREFIXLEN IPv4 network prefix length (cloud-init)
+ -q DATA_SIZE Size of data disk
+ -Q OS_SIZE Size of OS disk
+ -r NAMESERVER DNS resolver (cloud-init)
+ -s SEARCHDOMAIN DNS search domain (cloud-init)
+ -v VLANID VLAN ID number"
+
+ local \
+ autostart=true \
+ cpus=$DEFAULT_CPUS \
+ data_size=$DEFAULT_DATA_SIZE \
+ domain=$DEFAULT_DOMAIN \
+ gateway \
+ ip \
+ memory=$DEFAULT_MEMORY \
+ nameservers \
+ prefixlen=$DEFAULT_PREFIXLEN \
+ os_size=$DEFAULT_OS_SIZE \
+ searchdomains \
+ vlan=$DEFAULT_VLAN \
+ ssh_key_files \
+ ssh_keys \
+ iso \
+ opt
+
+ while getopts :Aa:c:d:g:hi:k:m:p:q:Q:r:s:v: opt; do
+ case $opt in
+ A) autostart=false ;;
+ a) ip=${OPTARG} ;;
+ c) cpus=$OPTARG ;;
+ d) domain=$OPTARG ;;
+ g) gateway=$OPTARG ;;
+ h) cmd::help ;;
+ i) iso=$OPTARG ;;
+ k) ssh_key_files="${ssh_key_files:-} ${OPTARG}" ;;
+ m) memory=$OPTARG ;;
+ p) prefixlen=$OPTARG ;;
+ q) data_size=$OPTARG ;;
+ Q) os_size=$OPTARG ;;
+ r) nameservers="${nameservers:-}${OPTARG}," ;;
+ s) searchdomains="${searchdomains:-}${OPTARG}," ;;
+ v) vlan=$OPTARG ;;
+ :) cmd::usage "missing option value: -${OPTARG}" ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'NAME not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local name=$1 template=${2:-}
+
+ vm::exists "$name" && die "vm name already in use: ${name}"
+
+ if [ -n "$template" ]; then
+ template::exists "$template" || die "no such template: ${template}"
+ fi
+
+ if [ -n "${iso:-}" ]; then
+ iso::exists "$iso" || die "no such iso: ${iso}"
+ fi
+
+ local ssh_keys key
+ for key in ${ssh_key_files:-}; do
+ [ -f "${key}" ] || die "no such ssh key: ${key}"
+ ssh_keys="${ssh_keys:-} - $(cat "$key")
+"
+ done
+
+ interface::exists "bridge${vlan}" || interface::add_vlan "$vlan"
+
+ # Create the parent dataset.
+ zfs create -v "${VM_DATASET}/${name}"
+
+ if [ -n "$template" ]; then
+ # Clone the OS zvol from template.
+ zfs::ensure_snapshot snapshot "${VM_DATASET}/templates/${template}"
+ zfs clone $ZFS_OPTS -o volmode=dev "$snapshot" "${VM_DATASET}/${name}/os"
+ else
+ # Create a new OS zvol.
+ zfs create -v -o volblocksize="$ZFS_VOLBLOCKSIZE" $ZFS_OPTS -o volmode=dev -V "$os_size" "${VM_DATASET}/${name}/os"
+ fi
+
+ # Create a data zvol.
+ zfs create -v -o volblocksize="$ZFS_VOLBLOCKSIZE" $ZFS_OPTS -o volmode=dev -V "$data_size" "${VM_DATASET}/${name}/data"
+
+ local macaddr uuid
+ macaddr=$(interface::derive_mac "$TRUNK_INTERFACE" "$name")
+ uuid=$(uuidgen)
+
+ # Generate bhyve config file.
+ cat <<EOF > "${VM_HOME}/${name}/bhyve.conf"
+name=${name}
+uuid=${uuid}
+cpus=${cpus}
+memory.size=${memory}
+acpi_tables=true
+destroy_on_poweroff=true
+keyboard.layout=us_unix
+rtc.use_localtime=false
+x86.strictmsr=true
+x86.vmexit_on_hlt=true
+x86.vmexit_on_pause=true
+pci.0.0.0.device=hostbridge
+pci.0.1.0.device=lpc
+pci.0.2.0.device=virtio-net
+pci.0.2.0.backend=$(interface::tap::derive_name "$name")
+pci.0.2.0.mac=${macaddr}
+pci.0.3.0.device=nvme
+pci.0.3.0.path=/dev/zvol/${VM_DATASET}/%(name)/os
+pci.0.4.0.device=nvme
+pci.0.4.0.path=/dev/zvol/${VM_DATASET}/%(name)/data
+pci.0.5.0.device=ahci
+pci.0.5.0.port.0.type=cd
+pci.0.5.0.port.0.path=${VM_HOME}/%(name)/seed.iso
+pci.0.5.0.port.0.ro=true
+lpc.com1.path=/dev/nmdm_%(name)A
+lpc.bootrom=/usr/local/share/uefi-firmware/BHYVE_UEFI.fd
+lpc.bootvars=${VM_HOME}/${name}/uefi-vars.fd
+vmctl.vlan=${vlan}
+EOF
+
+ # Generate cloud-init network configuration.
+ if [ -z "${ip:-}" ]; then
+ cat <<EOF > "${VM_HOME}/${name}/network-config"
+version: 2
+ethernets:
+ id0:
+ match:
+ macaddress: '${macaddr}'
+ dhcp4: True
+ dhcp6: False
+EOF
+ else
+ cat <<EOF > "${VM_HOME}/${name}/network-config"
+version: 2
+ethernets:
+ id0:
+ match:
+ macaddress: '${macaddr}'
+ addresses: [${ip}/${prefixlen}]
+ routes:
+ - to: 0.0.0.0/0
+ via: ${gateway:-}
+ dhcp4: False
+ dhcp6: False
+ nameservers:
+ search: [${searchdomains:-}]
+ addresses: [${nameservers:-}]
+EOF
+ fi
+
+ # Generate cloud-init metadata.
+ cat <<EOF > "${VM_HOME}/${name}/meta-data"
+instance-id: ${uuid}
+local-hostname: ${name}
+EOF
+
+ # Generate cloud-init userdata.
+ cat <<EOF > "${VM_HOME}/${name}/user-data"
+#cloud-config
+disable_root: False
+ssh_pwauth: False
+resize_rootfs: True
+ssh_authorized_keys:
+${ssh_keys:-}
+users: []
+fqdn: ${name}.${domain}
+prefer_fqdn_over_hostname: True
+runcmd:
+ - sed -i.bak -E '/^#?PermitRootLogin/s/^.*$/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
+ - rm -f /etc/ssh/sshd_config.bak
+ - service ssh restart
+ - service sshd restart
+EOF
+
+ # Generate cloud-init ISO file.
+ genisoimage -output "${VM_HOME}/${name}/seed.iso" -volid cidata -joliet -input-charset utf-8 -rock \
+ "${VM_HOME}/${name}/network-config" \
+ "${VM_HOME}/${name}/meta-data" \
+ "${VM_HOME}/${name}/user-data"
+
+ # Copy initial UEFI vars.
+ cp /usr/local/share/uefi-firmware/BHYVE_UEFI_VARS.fd "${VM_HOME}/${name}/uefi-vars.fd"
+
+ # Enable autostart (if requested).
+ [ "$autostart" = true ] && sysrc "vmctl_list+=${name}"
+
+ # Start the VM.
+ vm::start "$name" "${iso:-}"
+}
+
+cmd::create_snapshot(){
+ local usage='create-snapshot [-d] [-o] VM [SNAPNAME]'
+ local help='Create a VM snapshot.
+Options:
+ -d Only snapshot the data dataset
+ -o Only snapshot the OS dataset'
+
+ local opt both=true os_only=false data_only=false snapname
+ while getopts :dho opt; do
+ case $opt in
+ d) data_only=true; both=false ;;
+ h) cmd::help ;;
+ o) os_only=true; both=false ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ vm=$1 snapname=${2:-}
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+
+ [ -n "$snapname" ] || snapname=$(date +%Y-%m-%dT%H:%M:%S)
+
+ # Snapshot the OS disk.
+ if [ "$both" = true ] || [ "$os_only" = true ]; then
+ zfs snapshot "${VM_DATASET}/${vm}/os@${snapname}"
+ fi
+
+ # Snapshot the data disk.
+ if [ "$both" = true ] || [ "$data_only" = true ]; then
+ zfs snapshot "${VM_DATASET}/${vm}/data@${snapname}"
+ fi
+}
+
+cmd::create_template(){
+ local usage='create-template NAME VM'
+ local help='Create a template from a VM.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'NAME not specified'
+ [ $# -lt 2 ] && cmd::usage 'VM not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local name=$1 vm=$2
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ template::exists "$name" && die "template already exists: $name"
+ vm::running "$vm" && die "refusing to create template while vm is running: ${vm}"
+
+ local snapname
+ snapname=$(date +%Y-%m-%dT%H:%M:%S)
+ zfs snapshot "${VM_DATASET}/${vm}/os@${snapname}"
+ zfs send "${VM_DATASET}/${vm}/os@${snapname}" | zfs receive -v $ZFS_OPTS -o volmode=dev "${VM_DATASET}/templates/${name}"
+ zfs destroy "${VM_DATASET}/${vm}/os@${snapname}"
+}
+
+cmd::destroy_iso(){
+ local usage='destroy-iso ISO'
+ local help='Delete an ISO.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'ISO not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ iso=$1
+
+ iso::exists "$iso" || die "no such iso: ${iso}"
+ rm "${VM_HOME}/isos/${iso}.iso"
+}
+
+cmd::destroy(){
+ local usage='destroy [-y] VM'
+ local help="Delete a VM and its dataset.
+Options:
+ -y Don't prompt for confirmation"
+
+ local noconfirm=false answer opt
+
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really destroy vm ${vm}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ # If the VM is running, stop it.
+ vm::running "$1" && vm::stop "$1" KILL
+
+ # Destroy the VM's dataset.
+ zfs destroy -v -r "${VM_DATASET}/${vm}"
+
+ # Remove VM from autostart list.
+ sysrc -v "vmctl_list-=${vm}"
+}
+
+cmd::destroy_snapshot(){
+ local usage='destroy-snapshot [-y] VM SNAPSHOT'
+ local help="Delete a VM snapshot.
+Options:
+ -y Don't prompt for confirmation"
+
+ local noconfirm=false answer opt
+
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local vm=$1 snapshot=$2
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+
+ local datasets=''
+
+ if [ "${snapshot#*@}" != "$snapshot" ]; then
+ # If the snapshot name contains '@', then we have something like 'os@snapname'.
+ zfs::dataset_exists "${VM_DATASET}/${vm}/${snapshot}" && \
+ datasets="${VM_DATASET}/${vm}/${snapshot}"
+ else
+ # Otherwise, check if either os or data dataset contains a matching snapshot.
+ zfs::dataset_exists "${VM_DATASET}/${vm}/os@${snapshot}" && \
+ datasets="${VM_DATASET}/${vm}/os@${snapshot}"
+
+ zfs::dataset_exists "${VM_DATASET}/${vm}/data@${snapshot}" && \
+ datasets="${datasets} ${VM_DATASET}/${vm}/data@${snapshot}"
+ fi
+
+ [ -n "$datasets" ] || die "no such snapshot for vm ${vm}: ${snapshot}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really destroy ${vm} snapshot ${snapshot}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ local dataset
+ for dataset in $datasets; do
+ zfs destroy -v "$dataset"
+ done
+}
+
+cmd::destroy_template(){
+ local usage='destroy-template [-y] TEMPLATE'
+ local help="Delete a VM template image.
+Options:
+ -y Don't prompt for confirmation
+Notes:
+ A template cannot be deleted while its clones still exist."
+
+ local noconfirm=false answer opt
+
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'TEMPLATE not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local template=$1
+
+ template::exists "$template" || die "no such template: ${template}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really destroy template ${template}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ zfs destroy -v -r "${VM_DATASET}/templates/${template}"
+}
+
+cmd::download_iso(){
+ local usage='download-iso URL [NAME]'
+ local help='Download an ISO.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'URL not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local url=$1 name=${2:-}
+
+ [ -n "$name" ] || name=$(basename "$url" .iso)
+
+ iso::exists "$name" && die "iso already exists: ${name}"
+
+ fetch -o "${VM_HOME}/isos/${name}.iso" "$url"
+}
+
+cmd::download_template(){
+ local usage='download-template URL [NAME]'
+ local help='Download a disk image.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'URL not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local url=$1 name=${2:-}
+
+ local ext image_ok=false
+ for ext in .raw .raw.xz .qcow2; do
+ if [ -z "${url%%*"$ext"}" ]; then
+ image_ok=true
+ [ -n "$name" ] || name=$(basename "$url" "$ext")
+ break
+ fi
+ done
+
+ [ "$image_ok" = true ] || die "unknown image type: $(basename "$url")"
+ template::exists "$name" && die "template already exists: ${name}"
+
+ local dataset="${VM_DATASET}/templates/${name}" \
+ zvol="/dev/zvol/${VM_DATASET}/templates/${name}"
+
+ # Create a ZFS dataset for the template.
+ zfs create -v -o volblocksize="$ZFS_VOLBLOCKSIZE" $ZFS_OPTS -o volmode=dev -V "$TEMPLATE_ZVOL_SIZE" "$dataset"
+
+ # Extract the template into the zvol, based on its file type.
+ case $url in
+ http://*.raw|https://*.raw)
+ fetch "$url" -o "$zvol" || zfs::cleanup_and_die "$dataset"
+ ;;
+ *.raw)
+ dd if="$url" of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset"
+ ;;
+ http://*.raw.xz|https://*.raw.xz)
+ fetch "$url" -o - | xz -d | dd of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset"
+ ;;
+ *.raw.xz)
+ xz -cd "$url" | dd of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset"
+ ;;
+ http://*.qcow2|https://*.qcow2)
+ fetch "$url" -o "/root/${name}.qcow2" || zfs::cleanup_and_die "$dataset"
+ qemu-img dd -O raw if="/root/${name}.qcow2" of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset"
+ rm "/root/${name}.qcow2"
+ ;;
+ *.qcow2)
+ qemu-img dd -O raw if="$url" of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset"
+ ;;
+ *) # NOTREACHED
+ ;;
+ esac
+
+ # Snapshot the template so it can be cloned.
+ zfs snapshot "${VM_DATASET}/templates/${name}@$(date +%Y-%m-%dT%H:%M:%S)"
+}
+
+cmd::edit(){
+ local usage='edit VM'
+ local help='Edit VM configuration.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ "$EDITOR" "${VM_HOME}/${vm}/bhyve.conf"
+}
+
+cmd::list(){
+ local usage='list [-t]'
+ local help='List configured VMs.'
+
+ local opt terse=false
+ while getopts :th opt; do
+ case $opt in
+ t) terse=true ;;
+ h) cmd::help ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -eq 0 ] || cmd::usage 'too many arguments'
+
+ local vm status
+ { [ $terse = true ] || echo 'VM STATUS'
+ for vm in $(vm::list); do
+ if [ $terse = true ]; then
+ printf '%s\n' "$vm"
+ else
+ if vm::running "${vm}"; then
+ status=running
+ else
+ status=stopped
+ fi
+ printf '%s %s\n' "$vm" "$status"
+ fi
+ done
+ } | column -t
+}
+
+cmd::list_isos(){
+ local usage='list-isos'
+ local help='List available ISOs.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+ [ $# -gt 0 ] && cmd::usage 'too many arguments'
+
+ local file
+ for file in "${VM_HOME}"/isos/*.iso; do
+ [ -e "$file" ] || continue
+ name=${file##*/}
+ name=${name%.iso}
+ printf '%s\n' "$name"
+ done
+}
+
+cmd::list_snapshots(){
+ local usage='list-snapshots VM'
+ local help='List VM snapshots.
+Options:
+ -d List snapshots from data dataset only
+ -o List snapshots from OS dataset only
+ -t Use terse output'
+
+ local opt terse=false dataset=''
+ while getopts :dhot opt; do
+ case $opt in
+ d) dataset=/data ;;
+ h) cmd::help ;;
+ o) dataset=/os ;;
+ t) terse=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+
+ if [ "$terse" = true ]; then
+ zfs list -r -t snapshot -H -o name -s creation "${VM_DATASET}/${vm}${dataset}" 2>/dev/null \
+ | sed 's/^.*\///' \
+ | sort -k1,1 -t@ -s
+ else
+ { echo 'DATASET SNAPSHOT USED REFER'
+ zfs list -r -t snapshot -H -o name,used,refer -s creation "${VM_DATASET}/${vm}${dataset}" 2>/dev/null
+ } | sed 's/^.*\///' \
+ | sort -k1,1 -t@ -s \
+ | tr '@' ' ' \
+ | column -t
+ fi
+}
+
+cmd::list_templates(){
+ local usage='list-templates'
+ local help='list template images.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -eq 0 ] || cmd::usage 'too many arguments'
+
+ ls -1 /dev/zvol/${VM_DATASET}/templates 2>/dev/null
+}
+
+cmd::reprovision(){
+ local usage='reprovision VM TEMPLATE'
+ local help="Wipe and reprovision a VM's OS dataset from a template."
+
+ local opt noconfirm=false running=false
+ while getopts :hy opt; do
+ case $opt in
+ h) cmd::help ;;
+ y) noconfirm=true ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -lt 2 ] && cmd::usage 'TEMPLATE not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local vm=$1 template=$2
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ template::exists "$template" || die "no such template: ${template}"
+
+ if [ "$noconfirm" != true ]; then
+ read -rp "Really reprovision ${vm}? (y/N) " answer
+ case $answer in
+ [yY]|[yY][eE][sS]) : ;;
+ *) die 'operation cancelled' ;;
+ esac
+ fi
+
+ # If the vm is running, stop it.
+ if vm::running "$vm"; then
+ running=true
+ vm::stop "$vm"
+ fi
+
+ local snapshot old_size
+
+ # Get the latest snapshot for the template (if not specified).
+ zfs::ensure_snapshot snapshot "${VM_DATASET}/templates/${template}"
+
+ # Stash old disk size.
+ old_size=$(zfs get -Hp -o value volsize "${VM_DATASET}/${vm}/os")
+
+ # Reprovision OS dataset from template.
+ zfs destroy -v -r "${VM_DATASET}/${vm}/os"
+ zfs clone $ZFS_OPTS -o volmode=dev -o volsize="$old_size" "$snapshot" "${VM_DATASET}/${vm}/os"
+
+ # If the jail was running, restart it.
+ if [ "$running" = true ]; then
+ vm::start "$vm"
+ fi
+}
+
+cmd::restart(){
+ local usage='restart VM'
+ local help='Restart a VM.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+
+ vm::exists "$1" || die "no such vm: ${1}"
+ vm::running "$1" || die "vm not running: ${1}"
+
+ vm::stop "$1"
+ vm::start "$1"
+}
+
+cmd::show(){
+ local usage='show VM'
+ local help='Show VM configuration.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+
+ printf -- '------------------------- BHYVE CONFIGURATION -------------------------\n'
+ cat "${VM_HOME}/${vm}/bhyve.conf"
+ printf -- '\n---------------------------- ZFS DATASET -----------------------------\n'
+ zfs list -o name,volsize,used,refer -S name \
+ "${VM_DATASET}/${vm}/os" \
+ "${VM_DATASET}/${vm}/data"
+}
+
+cmd::rollback(){
+ local usage='rollback VM SNAPSHOT'
+ local help='Rollback a VM to a given snapshot.
+Options:
+ -d Rollback the data zvol only
+ -f Attempt rollback even if the jail is running
+ -o Rollback the OS zvol only'
+
+ local opt force=false both=true os_only=false data_only=false
+ while getopts :dfho opt; do
+ case $opt in
+ d) data_only=true; both=false ;;
+ f) force=true ;;
+ h) cmd::help ;;
+ o) os_only=true; both=false ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified'
+ [ $# -gt 2 ] && cmd::usage 'too many arguments'
+ local vm=$1 snapshot=$2
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+
+ if [ "$both" = true ] || [ "$os_only" = true ]; then
+ zfs::dataset_exists "${VM_DATASET}/${vm}/os@${snapshot}" \
+ || die "no such snapshot for ${vm}/os: ${snapshot}"
+ fi
+ if [ "$both" = true ] || [ "$data_only" = true ]; then
+ zfs::dataset_exists "${VM_DATASET}/${vm}/data@${snapshot}" \
+ || die "no such snapshot for ${vm}/data: ${snapshot}"
+ fi
+
+ if vm::running "$vm" && [ "$force" != true ]; then
+ die "vm ${vm} is running, refusing to rollback (-f to override)"
+ fi
+
+ # Rollback the OS disk.
+ if [ "$both" = true ] || [ "$os_only" = true ]; then
+ zfs rollback -r "${VM_DATASET}/${vm}/os@${snapshot}"
+ fi
+
+ # Rollback the data disk.
+ if [ "$both" = true ] || [ "$data_only" = true ]; then
+ zfs rollback -r "${VM_DATASET}/${vm}/data@${snapshot}"
+ fi
+}
+
+cmd::start(){
+ local usage='start [-i ISO] VM'
+ local help='Start a VM.
+Options:
+ -i ISO Boot from an ISO'
+
+ local iso opt
+ while getopts :hi: opt; do
+ case $opt in
+ h) cmd::help ;;
+ i) iso=$OPTARG ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ vm::running "$vm" && die "vm already running: ${vm}"
+
+ if [ -n "${iso:-}" ]; then
+ iso::exists "$iso" || die "no such iso: ${iso}"
+ fi
+
+ vm::start "$vm" "${iso:-}"
+}
+
+cmd::_start_all(){
+ local usage='_start-all'
+ local help='Start all VMs.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+ [ $# -eq 0 ] || cmd::usage 'too many arguments'
+
+ rc::load_config
+
+ local vm
+ for vm in ${vmctl_list:-}; do
+ echo "vmctl: starting ${vm}"
+ vm::start "$vm"
+ sleep "${vmctl_delay:-$DEFAULT_AUTOSTART_DELAY}"
+ done
+}
+
+cmd::status(){
+ local usage='status VM'
+ local help='Show running VM status.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ vm::running "$vm" || die "vm not running: ${vm}"
+
+ local pid
+ pid=$(vm::pid "$vm")
+
+ printf -- '----------------------------- BHYVE PROCESS ------------------------------\n'
+ ps -auxdr -p "$pid"
+ printf -- '\n---------------------------- ZFS DATASET -----------------------------\n'
+ zfs list -o name,volsize,used,refer -S name \
+ "${VM_DATASET}/${vm}/os" \
+ "${VM_DATASET}/${vm}/data" \
+ | sed "s|^${VM_DATASET}/${vm}/||" \
+ | column -t
+ printf -- '\n--------------------------- RESOURCE USAGE ---------------------------\n'
+ rctl -h -u "process:${pid}" \
+ | grep -E '^(pcpu|memoryuse|vmemoryuse|swapuse|readbps|writebps|readiops|writeiops)=' \
+ | rs -c= -C' ' -T \
+ | column -t
+}
+
+cmd::stop(){
+ local usage='start VM'
+ local help='Shutdown a VM.
+Options:
+ -f Send SIGKILL'
+
+ local opt signal=TERM
+ while getopts :fh opt; do
+ case $opt in
+ f) signal=KILL ;;
+ h) cmd::help ;;
+ ?) cmd::usage "unknown option: -${OPTARG}" ;;
+ esac
+ done
+ shift $((OPTIND - 1))
+
+ [ $# -lt 1 ] && cmd::usage 'VM not specified'
+ [ $# -gt 1 ] && cmd::usage 'too many arguments'
+ local vm=$1
+
+ vm::exists "$vm" || die "no such vm: ${vm}"
+ vm::running "$vm" || die "vm not running: ${vm}"
+
+ vm::stop "$vm" "$signal"
+}
+
+cmd::_stop_all(){
+ local usage='_stop-all'
+ local help='Stop all VMs.'
+
+ cmd::getopt_help "$@"; shift $((OPTIND - 1))
+ [ $# -eq 0 ] || cmd::usage 'too many arguments'
+
+ rc::load_config
+
+ # Reverse the autostart list.
+ local vm vmctl_list_rev=''
+ for vm in ${vmctl_list:-}; do
+ vmctl_list_rev="${vm} ${vmctl_list_rev}"
+ done
+
+ # Stop the VMs in reverse order.
+ for vm in $vmctl_list_rev; do
+ if vm::running "$vm"; then
+ echo "vmctl: stopping ${vm}"
+ vm::stop "$vm"
+ sleep "${vmctl_delay:-$DEFAULT_AUTOSTART_DELAY}"
+ fi
+ done
+
+ # Stop any other VMs that might be running.
+ for vm in $(vm::list); do
+ if vm::running "$vm"; then
+ echo "vmctl: stopping ${vm}"
+ vm::stop "$vm"
+ fi
+ done
+}
+
+
+################################################################################
+# "Library" functions.
+################################################################################
+interface::add_vlan(){
+ local id=$1
+ ifconfig "vlan${id}" create
+ ifconfig "vlan${id}" vlan "$id" vlandev "$TRUNK_INTERFACE" up
+ ifconfig "bridge${id}" create
+ ifconfig "bridge${id}" addm "vlan${id}" up
+ sysrc -v \
+ "cloned_interfaces+=vlan${id} bridge${id}" \
+ "ifconfig_vlan${id}=vlan ${id} vlandev ${TRUNK_INTERFACE} up" \
+ "ifconfig_bridge${id}=addm vlan${id} up"
+}
+
+interface::derive_mac(){
+ # Derive unique mac addresses for a virtual NIC, based on the physical
+ # interface address + vm name.
+ local iface=$1 name=$2
+ local hwaddr checksum virtual_mac
+
+ # Similar to /usr/share/examples/jails/jib
+ hwaddr=$(ifconfig "$iface" ether | awk '/ether/,$0=$2')
+ # ??:??:??:II:II:II
+ virtual_mac=${hwaddr#??:??:??} # => :II:II:II
+ # => :SS:SS:II:II:II
+ virtual_mac=":$(printf '%s' "$name" | md5 | sed 's/^\(..\)\(..\).*/\1:\2/')${virtual_mac}"
+ # => NP:SS:SS:II:II:II
+ case $hwaddr in
+ ?2:*) virtual_mac="1e${virtual_mac}" ;;
+ ?[Ee]:*) virtual_mac="16${virtual_mac}" ;;
+ *) virtual_mac="1e${virtual_mac}" ;;
+ esac
+
+ # Sanity check for duplicate MAC
+ ifconfig | grep -qE "(ether|hwaddr) ${virtual_mac}" && die "MAC collision when configuring virutal interface!"
+
+ printf '%s' "${virtual_mac}"
+}
+
+interface::exists(){
+ ifconfig "$1" > /dev/null 2>&1
+}
+
+interface::tap::derive_name(){
+ # Generate an tap(4) interface name, based on the VM name.
+ #
+ # The maximum length of a network interface name on FreeBSD is 15 characters.
+ # We use a prefix of 'tap_', leaving 11 characters for a unique suffix for each
+ # VM.
+ #
+ # If the sanitized VM name is less than 11 characters, we'll simply use it
+ # for the suffix. Otherwise, we'll use the last 11 characters of the VM
+ # name's SHA-1 hash.
+ local name=$1 sanitized hash
+ sanitized=$(printf '%s' "$name" | tr -dC '[:alnum:]_')
+
+ if [ "${#sanitized}" -le 11 ]; then
+ printf 'tap_%s' "$sanitized"
+ else
+ hash=$(printf '%s' "$name" | sha1 | tail -c 11)
+ printf 'tap_%s' "$hash"
+ fi
+}
+
+iso::exists(){
+ test -f "${VM_HOME}/isos/${1}.iso"
+}
+
+rc::load_config(){
+ set +eu +o pipefail
+ . /etc/rc.subr
+ load_rc_config vmctl
+ set -eu -o pipefail
+}
+
+template::exists(){
+ zfs list -H "${VM_DATASET}/templates/${1}" > /dev/null 2>&1
+}
+
+vm::exists(){
+ test -f "${VM_HOME}/${1}/bhyve.conf"
+}
+
+vm::list(){
+ for file in "${VM_HOME}"/*/bhyve.conf; do
+ [ -e "$file" ] || continue
+ printf '%s\n' "$(basename "$(dirname "$file")")"
+ done
+}
+
+vm::pid(){
+ pgrep -fx "bhyve: ${1}"
+}
+
+vm::run(){
+ local vm=$1 iso=${2:-} bootdisk zvol iso_args bootcount=0 rc=0
+
+ zvol="/dev/zvol/${VM_DATASET}/${vm}/os"
+ bootdisk=$zvol
+ iso_args=''
+
+ # bhyve exit codes:
+ # 0: reboot
+ # 1: shutdown
+ # >1: error
+ # As long as rc=0, keep rebooting.
+
+ while [ "$rc" -eq 0 ]; do
+ if [ -n "$iso" ]; then
+ # If this is the first boot, or no bootsector exists, boot from ISO.
+ if [ "$bootcount" -eq 0 ] || ! file -bs "$zvol" | grep -q 'boot sector'; then
+ bootdisk="${VM_HOME}/isos/${iso}.iso"
+ iso_args="-s 31:0,ahci-cd,${bootdisk}"
+ else
+ # Otherwise, boot from disk.
+ bootdisk=$zvol
+ iso_args=''
+ fi
+ fi
+
+ bootcount=$(( bootcount + 1 ))
+
+ bhyve -k "${VM_HOME}/${vm}/bhyve.conf" $iso_args "$vm" && rc=$? || rc=$?
+ done
+
+ # If we reach this point, the bhyve process has terminated. Clean up the
+ # /dev/vmm device and tap interface.
+ bhyvectl --vm="${vm}" --destroy
+ ifconfig "$(interface::tap::derive_name "$vm")" destroy
+}
+
+vm::running(){
+ vm::pid "$1" > /dev/null
+}
+
+vm::stop(){
+ local vm=$1 signal=${2:-TERM} pid
+ pid=$(vm::pid "$vm")
+
+ kill -s "$signal" "$pid"
+
+ # Loop until the bhyve process terminates.
+ while kill -0 "$pid" 2>/dev/null; do
+ sleep 1
+ done
+}
+
+vm::start(){
+ local vm=$1 iso=${2:-} vlan newif
+
+ vlan=$(vm::config::get "$vm" vmctl.vlan)
+
+ # Create the tap interface.
+ newif=$(ifconfig tap create)
+ ifconfig "bridge${vlan}" addm "$newif"
+ ifconfig "$newif" name "$(interface::tap::derive_name "$vm")" descr "vm/${vm}"
+
+ # Start the bhyve process in the background.
+ vm::run "$vm" "$iso" > /dev/null 2>&1 &
+}
+
+vm::config::get(){
+ local vm=$1 key=$2
+ awk -F= -v "key=${key}" \
+ '$1 == key {rc=1; print $2} END {exit !rc}' \
+ "${VM_HOME}/${vm}/bhyve.conf" \
+ || die "config value not set for vm ${vm}: ${key}"
+}
+
+zfs::ensure_snapshot(){
+ # If the given zfs dataset is a snapshot, return as-is.
+ # Otherwise, get the latest snapshot from the given dataset and return that.
+ local upvar=$1 dataset=$2 latest
+
+ if [ "${dataset#*@}" = "$dataset" ]; then
+ latest=$(zfs list -t snapshot -o name -s creation -H "$dataset" | tail -1)
+ setvar "$upvar" "$latest"
+ else
+ zfs list -t snapshot -H "$dataset" > /dev/null 2>&1 || die "no such snapshot: ${dataset}"
+ setvar "$upvar" "$dataset"
+ fi
+}
+
+zfs::cleanup_and_die(){
+ zfs destroy -v -r "$1"
+ exit 1
+}
+
+zfs::dataset_exists(){
+ zfs list "$1" > /dev/null 2>&1
+}
+
+
+cmd::main "$@"
diff --git a/lib/10-core b/lib/10-core
index d7280d9..5fa2a16 100644
--- a/lib/10-core
+++ b/lib/10-core
@@ -98,7 +98,7 @@ _boxconf_include(){
elif [ -d "$1" ]; then
for _bci_file in "$1"/*; do
if [ -f "$_bci_file" ]; then
- log "sourcing ${1#${BOXCONF_ROOT}/}"
+ log "sourcing ${_bci_file#${BOXCONF_ROOT}/}"
BOXCONF_SOURCE=$_bci_file
. "$BOXCONF_SOURCE"
fi
@@ -141,8 +141,10 @@ _boxconf_stage(){
# Compex find expression to only copy files necessary for the target host.
# This avoids leaking site-wide secrets to hosts that don't require them.
- _bcs_relevant_files=$(find "${BOXCONF_ROOT}" -type f -and \( \
- -path "${BOXCONF_CA_DIR}/${_bcs_hostname}" \
+ set -f
+ _bcs_relevant_files=$(find -L "$BOXCONF_ROOT" -type f -and \( \
+ -path "${BOXCONF_CA_DIR}/ca.crt" \
+ -or -path "${BOXCONF_CA_DIR}/${_bcs_hostname}" \
-or -path "${BOXCONF_VAR_DIR}/common" \
-or -path "${BOXCONF_VAR_DIR}/common/*" \
-or -path "${BOXCONF_VAR_DIR}/os/*" \
@@ -194,6 +196,7 @@ _boxconf_stage(){
-or -path "${BOXCONF_SITE_FILE_DIR}/*.${BOXCONF_HOSTCLASS}" \
-or -path "${BOXCONF_SITE_FILE_DIR}/*.${_bcs_hostname}" \
\) )
+ set +f
OIFS=$IFS; IFS=$'\n'
set -- $_bcs_relevant_files
diff --git a/lib/40-user b/lib/40-user
new file mode 100644
index 0000000..42bbb82
--- /dev/null
+++ b/lib/40-user
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set_authorized_keys(){
+ # Add authorized_keys for a user.
+ # $1 = username
+ # $2 = newline-separated string of authorized keys
+ _sak_homedir=$(eval echo "~${1}")
+ _sak_group=$(getent passwd "$1" | awk -F: '{ print $4}')
+
+ # Create authorized keys file and set permissions.
+ install_directory -o "$1" -g "$_sak_group" -m 0700 "${_sak_homedir}/.ssh"
+ [ -f "${_sak_homedir}/.ssh/authorized_keys" ] || touch "${_sak_homedir}/.ssh/authorized_keys"
+ chown "$1" "${_sak_homedir}/.ssh/authorized_keys"
+ chgrp "$_sak_group" "${_sak_homedir}/.ssh/authorized_keys"
+ chmod 600 "${_sak_homedir}/.ssh/authorized_keys"
+
+ printf '%s\n' "${2}" > "${_sak_homedir}/.ssh/authorized_keys"
+ log "added authorized_keys for ${1}:"$'\n'"$2"
+}
+
+set_password(){
+ # Set password for a local user.
+ # $1 = username
+ # $2 = password
+ printf '%s\n%s\n' "$2" "$2" | passwd "$1" > /dev/null
+}
diff --git a/scripts/common/10-root-user b/scripts/common/10-root-user
new file mode 100644
index 0000000..9a9f5e6
--- /dev/null
+++ b/scripts/common/10-root-user
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Add root SSH pubkeys.
+set_authorized_keys root "$root_authorized_keys"
+
+# Set root password.
+set_password root "$root_password"
diff --git a/scripts/common/20-dns b/scripts/common/20-dns
new file mode 100644
index 0000000..e2d5ad6
--- /dev/null
+++ b/scripts/common/20-dns
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# For IDM servers, the resolver is localhost. In that case, we delay copying
+# this file until the IDM stack is fully up and running.
+if [ "$BOXCONF_HOSTCLASS" = idm_server ]; then
+ return
+fi
+
+install_template -m 0644 /etc/resolv.conf
diff --git a/scripts/hostclass/freebsd_hypervisor b/scripts/hostclass/freebsd_hypervisor
new file mode 100644
index 0000000..bdaa3c0
--- /dev/null
+++ b/scripts/hostclass/freebsd_hypervisor
@@ -0,0 +1,80 @@
+#!/bin/sh
+
+: ${hypervisor_trunk_interface:='lagg0'}
+: ${hypervisor_default_vlan:='1'}
+: ${hypervisor_default_prefix:='24'}
+: ${hypervisor_default_os_quota:='24G'}
+: ${hypervisor_default_data_quota:='8G'}
+
+: ${hypervisor_vm_home:='/usr/local/bhyve'}
+: ${hypervisor_vm_dataset:='tank/bhyve'}
+: ${hypervisor_vm_default_cpus:='2'}
+: ${hypervisor_vm_default_mem:='4G'}
+: ${hypervisor_vm_template_size:='10G'}
+: ${hypervisor_vm_default_autostart_delay:='2'}
+: ${hypervisor_vm_default_zfs_opts:='-o primarycache=metadata -o compress=off'}
+: ${hypervisor_vm_zfs_volblocksize:='64k'}
+
+: ${hypervisor_jail_home:='/usr/local/jails'}
+: ${hypervisor_jail_dataset:='tank/jails'}
+: ${hypervisor_jail_default_zfs_opts:='-o compress=lz4'}
+
+hypervisor_jail_bpf_ruleset=1000
+
+# Required for vnet jails.
+set_sysctl net.link.tap.up_on_open=1
+
+# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=262189
+set_sysctl vfs.zfs.vol.mode=2
+
+# Load required kernel modules.
+load_kernel_module vmm nmdm linux linux64
+set_loader_conf \
+ vmm_load=YES \
+ nmdm_load=YES \
+ linux_load=YES \
+ linux64_load=YES \
+ kern.racct.enable=1
+
+# Install vm/jail management dependencies.
+pkg install -y \
+ bhyve-firmware \
+ cdrkit-genisoimage \
+ qemu-tools
+
+# Create bhyve VM dataset.
+create_dataset -o "mountpoint=${hypervisor_vm_home}" "$hypervisor_vm_dataset"
+
+# Create dataset for bhyve templates.
+create_dataset -o volmode=none -o mountpoint=none "${hypervisor_vm_dataset}/templates"
+
+# Create jails dataset.
+create_dataset -o mountpoint="${hypervisor_jail_home}" "$hypervisor_jail_dataset"
+
+# Create dataset for jail templates.
+create_dataset -o mountpoint="${hypervisor_jail_home}/templates" "${hypervisor_jail_dataset}/templates"
+
+# Lock down permissions on the VM and jail directories.
+chmod 700 "$hypervisor_vm_home" "$hypervisor_jail_home"
+
+# Create directory for VM ISO files.
+install_directory -m 0755 "${hypervisor_vm_home}/isos"
+
+# Copy jail/bhyve management scripts.
+install_directory -m 0755 /usr/local/etc/rc.d
+
+install_file -m 0555 \
+ /usr/local/sbin/jailctl \
+ /usr/local/sbin/vmctl \
+ /usr/local/etc/rc.d/vmctl
+
+install_template -m 0644 \
+ /usr/local/etc/jailctl.conf \
+ /usr/local/etc/vmctl.conf
+
+install_template -m 0644 /etc/devfs.rules
+
+# Enable jails/bhyve to start on boot.
+sysrc -v \
+ vmctl_enable=YES \
+ jail_enable=YES
diff --git a/scripts/os/freebsd/10-bootloader b/scripts/os/freebsd/10-bootloader
new file mode 100644
index 0000000..0506606
--- /dev/null
+++ b/scripts/os/freebsd/10-bootloader
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Skip this file if running in a jail - jails don't have a bootloader.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" = jail ]; then
+ return
+fi
+
+# Configure serial console.
+install_file -m 0644 /boot.config
+install_file -m 0644 /etc/ttys
+kill -HUP 1
+
+set_loader_conf \
+ autoboot_delay=1 \
+ beastie_disable=YES \
+ boot_multicons=YES \
+ boot_serial=YES \
+ cc_htcp_load=YES \
+ console=comconsole,efi \
+ comconsole_speed=115200 \
+ kern.geom.label.disk_ident.enable=0 \
+ kern.geom.label.gptid.enable=0 \
+ net.inet.tcp.soreceive_stream=1 \
+ net.inet6.ip6.auto_linklocal=0 \
+ net.isr.defaultqlimit=2048 \
+ net.link.ifqmaxlen=2048 \
+ pf_load=YES \
+ pflog_load=YES \
+ security.bsd.allow_destructive_dtrace=0
diff --git a/scripts/os/freebsd/10-cpu b/scripts/os/freebsd/10-cpu
new file mode 100644
index 0000000..adc27d4
--- /dev/null
+++ b/scripts/os/freebsd/10-cpu
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Only run this file on baremetal hosts.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" != none ]; then
+ return
+fi
+
+# Allow lower C-states. As of FreeBSD 13, the default is to only allow C1.
+# My Xeon processor supports C2, and enabling that resulted in 15 watts of
+# power savings.
+#
+# Note that if your CPU supports *very* low C-states (likely for commodity
+# desktop and laptop hardware), you may not want them enabled, as transitioning
+# from a very low C-state can cause rather severe latency spikes.
+#
+# Experiment with your hardware and set $cx_lowest accordingly.
+sysrc -v \
+ microcode_update_enable=YES \
+ performance_cx_lowest="$cx_lowest" \
+ economy_cx_lowest="$cx_lowest"
+
+# Set energy/performance preference for Intel P-states.
+# 0 = most performance, 100 = most power savings
+if sysctl -n dev.hwpstate_intel.0.epp >/dev/null 2>&1; then
+ for n in $(seq 0 $(($(sysctl -n hw.ncpu)-1))); do
+ set_sysctl "dev.hwpstate_intel.${n}.epp=${intel_epp}"
+ done
+fi
diff --git a/scripts/os/freebsd/10-periodic b/scripts/os/freebsd/10-periodic
new file mode 100644
index 0000000..36ddd95
--- /dev/null
+++ b/scripts/os/freebsd/10-periodic
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# Disable periodic(8) reports, as well as tasks that generate lots of I/O.
+sysrc -v -f /etc/periodic.conf \
+ daily_show_success=NO \
+ daily_show_info=NO \
+ daily_clean_disks_verbose=NO \
+ daily_clean_tmps_verbose=NO \
+ daily_clean_preserve_verbose=NO \
+ daily_clean_rwho_verbose=NO \
+ daily_backup_passwd_enable=NO \
+ daily_backup_aliases_enable=NO \
+ daily_backup_gpart_enable=NO \
+ daily_status_disks_enable=NO \
+ daily_status_zfs_zpool_list_enable=NO \
+ daily_status_network_enable=NO \
+ daily_status_uptime_enable=NO \
+ daily_status_mailq_enable=NO \
+ daily_status_security_enable=NO \
+ daily_status_mail_rejects_enable=NO \
+ daily_status_world_kernel=NO \
+ weekly_show_success=NO \
+ weekly_show_info=NO \
+ weekly_locate_enable=NO \
+ weekly_whatis_enable=NO \
+ weekly_status_security_enable=NO \
+ monthly_show_success=NO \
+ monthly_show_info=NO \
+ monthly_accounting_enable=NO \
+ monthly_status_security_enable=NO \
+ security_show_success=NO \
+ security_show_info=NO \
+ security_status_chksetuid_enable=NO \
+ security_status_neggrpperm_enable=NO \
+ security_status_chkmounts_enable=NO \
+ security_status_chkuid0_enable=NO \
+ security_status_passwdless_enable=NO \
+ security_status_logincheck_enable=NO \
+ security_status_ipfwdenied_enable=NO \
+ security_status_ipfdenied_enable=NO \
+ security_status_pfdenied_enable=NO \
+ security_status_ipfwlimit_enable=NO \
+ security_status_ipf6denied_enable=NO \
+ security_status_kernelmsg_enable=NO \
+ security_status_loginfail_enable=NO \
+ security_status_tcpwrap_enable=NO
+
+# Sendmail-specific stuff
+sysrc -v -f /etc/periodic.conf \
+ daily_clean_hoststat_enable=NO \
+ daily_status_mail_rejects_enable=NO \
+ daily_status_include_submit_mailq=NO \
+ daily_submit_queuerun=NO
diff --git a/scripts/os/freebsd/10-rc-conf b/scripts/os/freebsd/10-rc-conf
new file mode 100644
index 0000000..a8a3d22
--- /dev/null
+++ b/scripts/os/freebsd/10-rc-conf
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+sysrc -v \
+ clear_tmp_enable=YES \
+ dumpdev=NO \
+ ipv6_activate_all_interfaces=NO \
+ syslogd_flags=-ss
diff --git a/scripts/os/freebsd/10-sysctls b/scripts/os/freebsd/10-sysctls
new file mode 100644
index 0000000..a59d54f
--- /dev/null
+++ b/scripts/os/freebsd/10-sysctls
@@ -0,0 +1,80 @@
+#!/bin/sh
+
+case $BOXCONF_OS_VERSION in
+ 13.*)
+ set_sysctl \
+ net.inet.ip.check_interface=1 \
+ net.inet.tcp.rfc6675_pipe=1
+ ;;
+ *)
+ set_sysctl \
+ net.inet.ip.rfc1122_strong_es=1
+ ;;
+esac
+
+load_kernel_module cc_htcp
+
+set_sysctl \
+ net.inet.icmp.drop_redirect=1 \
+ net.inet.ip.process_options=0 \
+ net.inet.ip.random_id=1 \
+ net.inet.ip.redirect=0 \
+ net.inet.tcp.abc_l_var=44 \
+ net.inet.tcp.always_keepalive=0 \
+ net.inet.tcp.cc.abe=1 \
+ net.inet.tcp.cc.algorithm=htcp \
+ net.inet.tcp.cc.htcp.adaptive_backoff=1 \
+ net.inet.tcp.cc.htcp.rtt_scaling=1 \
+ net.inet.tcp.drop_synfin=1 \
+ net.inet.tcp.ecn.enable=1 \
+ net.inet.tcp.fastopen.server_enable=1 \
+ net.inet.tcp.icmp_may_rst=0 \
+ net.inet.tcp.initcwnd_segments=44 \
+ net.inet.tcp.minmss=536 \
+ net.inet.tcp.msl=2500 \
+ net.inet.tcp.mssdflt=1448 \
+ net.inet.tcp.nolocaltimewait=1 \
+ net.inet.tcp.path_mtu_discovery=0 \
+ net.inet.tcp.recvbuf_max="$tcp_buffer_size" \
+ net.inet.tcp.recvspace=65536 \
+ net.inet.tcp.sendbuf_inc=65536 \
+ net.inet.tcp.sendbuf_max="$tcp_buffer_size" \
+ net.inet.tcp.sendspace=65536 \
+ net.inet.tcp.syncookies=0 \
+ net.inet6.ip6.redirect=0 \
+ security.bsd.unprivileged_proc_debug="$allow_proc_debug"
+
+# Some sysctls cannot be set within jails.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" != jail ]; then
+ set_sysctl \
+ hw.kbd.keymap_restrict_change=4 \
+ kern.coredump=0 \
+ kern.elf32.allow_wx="$allow_wx" \
+ kern.elf32.aslr.pie_enable=1 \
+ kern.elf64.allow_wx="$allow_wx" \
+ kern.ipc.maxsockbuf="$tcp_buffer_size" \
+ kern.ipc.shm_use_phys=1 \
+ kern.ipc.soacceptqueue=1024 \
+ kern.ipc.somaxconn=1024 \
+ kern.random.fortuna.minpoolsize=128 \
+ kern.randompid=1 \
+ net.inet.tcp.fast_finwait2_recycle=1 \
+ net.inet.tcp.finwait2_timeout=5000 \
+ net.inet.tcp.keepcnt=2 \
+ net.inet.tcp.keepidle=62000 \
+ net.inet.tcp.keepinit=5000 \
+ net.inet.tcp.minmss=536 \
+ net.inet.tcp.minmss=536 \
+ security.bsd.hardlink_check_gid=0 \
+ security.bsd.hardlink_check_uid=0 \
+ security.bsd.see_other_gids=0 \
+ security.bsd.see_other_uids=0 \
+ security.bsd.unprivileged_read_msgbuf=0 \
+ vfs.zfs.min_auto_ashift=12
+
+ # FreeBSD automatically scales kern.maxfilesperproc with the amount of memory.
+ # On systems with large amounts of RAM, this can cause strange lags with some
+ # applications that attempt to close every possible file descriptor.
+ # Therefore, we arbitrarily cap this value at 65535.
+ [ "$(sysctl -n kern.maxfilesperproc)" -le 65535 ] || set_sysctl kern.maxfilesperproc=65535
+fi
diff --git a/scripts/os/freebsd/20-hostname b/scripts/os/freebsd/20-hostname
new file mode 100644
index 0000000..bf7eeeb
--- /dev/null
+++ b/scripts/os/freebsd/20-hostname
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Set the fully qualified hostname.
+sysrc -v hostname="${BOXCONF_HOSTNAME}.${domain}"
+hostname "${BOXCONF_HOSTNAME}.${domain}"
+install_template -m 0644 /etc/hosts
diff --git a/scripts/os/freebsd/20-locale b/scripts/os/freebsd/20-locale
new file mode 100644
index 0000000..cf72d8d
--- /dev/null
+++ b/scripts/os/freebsd/20-locale
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Set the default system locale.
+install_template -m 0644 \
+ /etc/profile.d/locale.sh \
+ /etc/login.conf
+
+cap_mkdb /etc/login.conf
diff --git a/scripts/os/freebsd/20-motd b/scripts/os/freebsd/20-motd
new file mode 100644
index 0000000..9b1eadb
--- /dev/null
+++ b/scripts/os/freebsd/20-motd
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Disable motd.
+sysrc -v update_motd=NO
+rm -f /var/run/motd
diff --git a/scripts/os/freebsd/20-ntp b/scripts/os/freebsd/20-ntp
new file mode 100644
index 0000000..888bab4
--- /dev/null
+++ b/scripts/os/freebsd/20-ntp
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Jails don't need NTP.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" = jail ]; then
+ return
+fi
+
+install_template -m 0644 /etc/ntp.conf
+
+sysrc -v \
+ ntpd_enable=YES \
+ ntpd_sync_on_start=YES
+
+service ntpd restart
diff --git a/scripts/os/freebsd/20-root-ca b/scripts/os/freebsd/20-root-ca
new file mode 100644
index 0000000..1f88c69
--- /dev/null
+++ b/scripts/os/freebsd/20-root-ca
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Create local CA certificates directory.
+install_directory -m 0755 \
+ /usr/local/etc \
+ /usr/local/etc/ssl \
+ /usr/local/etc/ssl/certs
+
+# Install our root CA.
+install_ca_certificate "$site_cacert_path"
+
+certctl rehash
diff --git a/scripts/os/freebsd/20-timezone b/scripts/os/freebsd/20-timezone
new file mode 100644
index 0000000..22a3729
--- /dev/null
+++ b/scripts/os/freebsd/20-timezone
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Set the system timezone.
+cp -v "/usr/share/zoneinfo/${timezone}" /etc/localtime
diff --git a/scripts/os/freebsd/20-zfs b/scripts/os/freebsd/20-zfs
new file mode 100644
index 0000000..aa37c0a
--- /dev/null
+++ b/scripts/os/freebsd/20-zfs
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Every host should have a "state" dataset, which is a ZFS dataset which
+# persists across OS rebuilds.
+[ -n "${state_dataset:-}" ] || die 'state_dataset not defined!'
+create_dataset "$state_dataset"
+
+# If this is baremetal host or a VM, trim the zpools periodically.
+if [ "$BOXCONF_VIRTUALIZATION_TYPE" != jail ]; then
+ install_file -m 0644 /etc/cron.d/zfs-trim
+fi
diff --git a/scripts/os/freebsd/30-mail b/scripts/os/freebsd/30-mail
new file mode 100644
index 0000000..511ce69
--- /dev/null
+++ b/scripts/os/freebsd/30-mail
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ "$BOXCONF_HOSTCLASS" = smtp_server ]; then
+ return
+fi
+
+# Configure local mail agent.
+install_template -m 0644 /etc/dma/dma.conf
+install_template -m 0644 /etc/aliases
diff --git a/scripts/os/freebsd/30-ssh b/scripts/os/freebsd/30-ssh
new file mode 100644
index 0000000..91b1991
--- /dev/null
+++ b/scripts/os/freebsd/30-ssh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Create state dataset to persist SSH host keys across OS rebuilds.
+create_dataset -o "mountpoint=${ssh_host_key_dir}" "${state_dataset}/ssh"
+
+# If the state dataset contains existing host keys, symlink them into
+# /etc/ssh.
+#
+# If not, this is the first time we are building this box, so copy the
+# autogenerated host keys to the state partition.
+for key in \
+ ssh_host_ecdsa_key \
+ ssh_host_ed25519_key \
+ ssh_host_rsa_key
+do
+ [ -f "${ssh_host_key_dir}/${key}" ] || \
+ mv -v "/etc/ssh/${key}" "/etc/ssh/${key}.pub" "$ssh_host_key_dir"
+
+ ln -snvf "${ssh_host_key_dir}/${key}" "/etc/ssh/${key}"
+ ln -snvf "${ssh_host_key_dir}/${key}.pub" "/etc/ssh/${key}.pub"
+done
+
+# Copy SSH configs.
+install_directory -m 0755 /etc/ssh/sshd_config.d
+
+install_template -m 0644 \
+ /etc/ssh/sshd_config \
+ /etc/ssh/ssh_config
+
+# Restart sshd.
+service sshd restart
diff --git a/scripts/os/freebsd/30-syslog b/scripts/os/freebsd/30-syslog
new file mode 100644
index 0000000..6f3dc8c
--- /dev/null
+++ b/scripts/os/freebsd/30-syslog
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Copy syslog configuration.
+install_file -m 0644 /etc/syslog.conf
+
+# Restart syslogd.
+service syslogd restart
diff --git a/scripts/os/freebsd/40-pkg b/scripts/os/freebsd/40-pkg
new file mode 100644
index 0000000..7c1c828
--- /dev/null
+++ b/scripts/os/freebsd/40-pkg
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+case $BOXCONF_HOSTCLASS in
+ pkg_repository)
+ return # Do nothing.
+ ;;
+ freebsd_hypervisor)
+ ;; # Keep default FreeBSD pkg repository.
+ *)
+ # Configure on-prem pkg repository.
+ install_directory -m 0755 \
+ /usr/local/etc/pkg \
+ /usr/local/etc/pkg/repos
+
+ install_file -m 0644 \
+ /usr/local/etc/ssl/repo.crt \
+ /usr/local/etc/pkg/repos/FreeBSD.conf
+
+ install_template -m 0644 /usr/local/etc/pkg/repos/onprem.conf
+ ;;
+esac
+
+# Update packages.
+pkg update -f
+
+# Install default packages.
+if [ -n "${install_packages:-}" ]; then
+ pkg install -y $install_packages
+fi
diff --git a/scripts/os/freebsd/70-pf b/scripts/os/freebsd/70-pf
new file mode 100644
index 0000000..9ec9961
--- /dev/null
+++ b/scripts/os/freebsd/70-pf
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+if [ "$enable_pf" != true ]; then
+ return
+fi
+
+# Enable pf.
+sysrc -v pf_enable=YES
+
+# Copy pf configuration.
+install_template -m 0600 /etc/pf.conf
+
+# Start (or reload) pf.
+service pf status > /dev/null || service pf start
+service pf reload
diff --git a/vars/common b/vars/common
new file mode 100644
index 0000000..bb7c4db
--- /dev/null
+++ b/vars/common
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+domain=idm.example.com
+email_domain=example.com
+locale=en_US.UTF-8
+ntp_pools='pool.ntp.org'
+root_password=changeme
+root_authorized_keys='ssh-ed25519 changeme
+ssh-ed25519 changeme'
+root_mail_alias="you@${email_domain}"
+smtp_host_ip=1.2.3.4
+timezone=America/New_York
+
+
+allowed_tcp_ports=ssh
+bootstrap_resolvers='8.8.8.8 8.8.4.4'
+smtp_host="smtp.${domain}"
+tcp_buffer_size=2097152 # suitable for 1 GigE
diff --git a/vars/hostclass/freebsd_hypervisor b/vars/hostclass/freebsd_hypervisor
new file mode 100644
index 0000000..c38452f
--- /dev/null
+++ b/vars/hostclass/freebsd_hypervisor
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+enable_pf=false
+smtp_host=${smtp_host_ip}
+resolvers=$bootstrap_resolvers
diff --git a/vars/os/freebsd b/vars/os/freebsd
new file mode 100644
index 0000000..0d4a6fb
--- /dev/null
+++ b/vars/os/freebsd
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+allow_wx=1
+allow_proc_debug=0
+cx_lowest=Cmax
+enable_pf=true
+install_packages='sudo tmux vim'
+intel_epp=50
+
+export ASSUME_ALWAYS_YES=yes
+acme_standalone_port=9080
+acme_uid=169
+nfscbd_port=7745
+site_cacert_path=/usr/local/etc/ssl/certs/ca.crt
+ssh_host_key_dir=/var/db/ssh
+
+# For 10 Gbit ethernet, bump up the TCP buffers.
+if ifconfig | grep -q '10Gbase-T'; then
+ tcp_buffer_size=16777216
+fi
+
+case $BOXCONF_VIRTUALIZATION_TYPE in
+ jail)
+ # For jails, the state dataset is delegated to the jail and named "data".
+ state_dataset=$(zfs list -Ho jailed,name | awk '$1 == "on" && $2 ~ /\/data$/ {print $2;exit}')
+ ;;
+ *)
+ # Otherwise, assume the state dataset is named "data" in the root zpool.
+ root_zpool=$(zfs list -Ho name,mountpoint | awk '$2 == "/" {print $1;exit}' | cut -d/ -f1)
+ if [ -n "$root_zpool" ]; then
+ state_dataset="${root_zpool}/data"
+ fi
+ ;;
+esac
--
cgit v1.2.3