--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/content/posts/nas.md	Mon May 12 20:35:51 2025 +0200
@@ -0,0 +1,337 @@
++++
+title = "Media Server with Ubuntu on ZFS"
+description = "Setting up ZFS on Ubuntu"
+date = "2025-05-12"
+updated = "2025-05-12"
+draft = true
+
+[taxonomies]
+categories = ["homelab"]
+tags = ["zfs", "ubuntu", "nas"]
+
+[extra]
+lang = "en"
+toc = true
+comment = false
+copy = true
++++
+
+
+sources:
+* <https://medo64.com/categories/zfs>
+* <https://openzfs.github.io/openzfs-docs/Getting%20Started/Ubuntu/Ubuntu%2022.04%20Root%20on%20ZFS.html>
+
+
+## Partition SSDs
+
+``` bash
+DISK1=/dev/disk/by-id/virtio-ssd0 
+DISK2=/dev/disk/by-id/virtio-ssd1
+```
+
+Remove all data
+``` bash
+blkdiscard -f $DISK1
+blkdiscard -f $DISK2
+sgdisk --zap-all $DISK1
+sgdisk --zap-all $DISK2
+```
+
+Create partitions on each drive:
+
+1. Bootloader:
+``` bash
+sgdisk -n1:1M:+512M -t1:EF00 $DISK1
+sgdisk -n1:1M:+512M -t1:EF00 $DISK2
+```
+
+2. Boot pool partition (2G to allow snapshots and updates)
+``` bash
+sgdisk -n2:0:+2G -t2:BE00 $DISK1
+sgdisk -n2:0:+2G -t2:BE00 $DISK2
+```
+
+3. Special device [^special-device] partition for metadata
+0.3% of total size: 36GB, extra margin to store small files: 128GB
+``` bash
+sgdisk -n3:0:+128G -t3:BF00 $DISK1
+sgdisk -n3:0:+128G -t3:BF00 $DISK2
+```
+
+4. Root pool partition
+``` bash
+sgdisk -n4:0:0 -t4:BF00 $DISK1
+sgdisk -n4:0:0 -t4:BF00 $DISK2
+```
+
+5. Check layout:
+``` bash
+sgdisk --print $DISK1
+```
+
+
+Create boot pool
+``` bash
+zpool create \
+  -o ashift=12 \
+  -o autotrim=on \
+  -o cachefile=/etc/zfs/zpool.cache \
+  -o compatibility=grub2 \
+  -o feature@livelist=enabled \
+  -o feature@zpool_checkpoint=enabled \
+  -O devices=off \
+  -O acltype=posixacl -O xattr=sa \
+  -O compression=lz4 \
+  -O relatime=on \
+  -O canmount=off -O mountpoint=/boot -R /mnt\
+  bpool mirror \
+  $DISK1-part2 $DISK2-part2
+
+zpool status bpool
+```
+
+Create root pool
+``` bash
+zpool create \
+  -o ashift=12 \
+  -o autotrim=on \
+  -O acltype=posixacl -O xattr=sa -O dnodesize=auto \
+  -O compression=lz4 \
+  -O normalization=formD \
+  -O relatime=on \
+  -O canmount=off -O mountpoint=/ -R /mnt \
+  rpool mirror \
+  $DISK1-part4 $DISK2-part4
+
+zpool status rpool
+```
+
+Create filesystems:
+``` bash
+zfs create -o canmount=off -o mountpoint=none rpool/ROOT
+zfs create -o canmount=off -o mountpoint=none bpool/BOOT
+
+UUID=$(dd if=/dev/urandom bs=1 count=100 2>/dev/null |
+    tr -dc 'a-z0-9' | cut -c-6)
+
+zfs create -o mountpoint=/ rpool/ROOT/ubuntu_$UUID
+
+zfs create -o mountpoint=/boot bpool/BOOT/ubuntu_$UUID
+```
+
+``` bash
+zfs create -o canmount=off \
+    rpool/ROOT/ubuntu_$UUID/usr
+zfs create -o  canmount=off \
+    rpool/ROOT/ubuntu_$UUID/var
+zfs create rpool/ROOT/ubuntu_$UUID/var/lib
+zfs create rpool/ROOT/ubuntu_$UUID/var/log
+zfs create rpool/ROOT/ubuntu_$UUID/var/spool
+
+zfs create -o canmount=off -o mountpoint=/ \
+    rpool/USERDATA
+zfs create -o canmount=on -o mountpoint=/root \
+    rpool/USERDATA/root_$UUID
+chmod 700 /mnt/root
+
+zfs create rpool/ROOT/ubuntu_$UUID/var/lib/apt
+zfs create rpool/ROOT/ubuntu_$UUID/var/lib/dpkg
+zfs create rpool/ROOT/ubuntu_$UUID/usr/local
+zfs create rpool/ROOT/ubuntu_$UUID/var/lib/docker
+```
+
+GRUB
+``` bash
+zfs create bpool/grub
+```
+
+Bootstrap
+``` bash
+mkdir /mnt/run
+mount -t tmpfs tmpfs /mnt/run
+mkdir /mnt/run/lock
+```
+
+``` bash
+apt install --yes debootstrap
+debootstrap noble /mnt
+```
+
+``` bash
+mkdir /mnt/etc/zfs
+cp /etc/zfs/zpool.cache /mnt/etc/zfs/
+```
+
+
+## System setup
+``` bash
+hostname Donnager
+hostname > /mnt/etc/hostname
+sed "s/ubuntu/$HOST/" /etc/hosts > /mnt/etc/hosts
+rm /mnt/etc/apt/sources.list
+cp /etc/apt/sources.list.d/ubuntu.sources /mnt/etc/apt/sources.list.d/ubuntu.sources
+mkdir -p /mnt/etc/netplan
+```
+Configure network in `/etc/netplan/00-network.yaml`:
+``` yaml
+network:
+    version: 2
+    renderer: networkd
+    ethernets:
+        en01:
+            dhcp4: true
+            dhcp6: true
+    
+```
+
+Bind mount and chroot:
+``` bash
+mount --rbind /dev /mnt/dev
+mount --rbind /proc /mnt/proc
+mount --rbind /sys /mnt/sys
+
+chroot /mnt \
+  /usr/bin/env DISK1=$DISK1 DISK2=$DISK2 USER=$USER UUID=$UUID \
+  bash --login
+```
+
+Configure locale and timezone
+``` bash
+dpkg-reconfigure locales tzdata
+
+apt install --yes dosfstools
+mkdosfs -F 32 -s 1 -n EFI ${DISK1}-part1
+mkdosfs -F 32 -s 1 -n EFI ${DISK2}-part1
+mkdir /boot/efi
+echo /dev/disk/by-uuid/$(blkid -s UUID -o value ${DISK1}-part1) \
+    /boot/efi vfat defaults 0 0 >> /etc/fstab
+mount /boot/efi
+```
+
+``` bash
+apt install --yes \
+    grub-efi-amd64 grub-efi-amd64-signed linux-image-generic \
+    shim-signed zfs-initramfs
+```
+
+``` bash
+apt purge --yes os-prober
+```
+
+Setup root password
+``` bash
+passwd
+```
+
+Setup tmpfs
+``` bash
+cp /usr/share/systemd/tmp.mount /etc/systemd/system/
+systemctl enable tmp.mount
+```
+
+SSH config
+``` bash
+apt install --yes openssh-server
+
+vi /etc/ssh/sshd_config
+# Set: PermitRootLogin yes
+```
+
+``` bash
+update-initramfs -c -k all
+```
+``` bash
+vi /etc/default/grub
+# Add init_on_alloc=0 to: GRUB_CMDLINE_LINUX_DEFAULT
+
+update-grub
+```
+
+Install boot loader
+``` bash
+grub-install --target=x86_64-efi --efi-directory=/boot/efi \
+    --bootloader-id=ubuntu --recheck --no-floppy
+```
+
+Disable grub-initrd-fallback
+``` bash
+systemctl mask grub-initrd-fallback.service
+```
+
+``` bash
+mkdir /etc/zfs/zfs-list.cache
+touch /etc/zfs/zfs-list.cache/bpool
+touch /etc/zfs/zfs-list.cache/rpool
+zed -F &
+```
+
+Install basics
+``` bash
+apt install --yes ubuntu-server-minimal
+```
+
+Leave chroot `exit`
+
+Unmount all filesystems:
+``` bash
+mount | grep -v zfs | tac | awk '/\/mnt/ {print $3}' | \
+    xargs -i{} umount -lf {}
+zpool export -a
+```
+Force release the mount if needed, search for `zed` processes and kill them
+``` bash
+zpool export -a
+```
+Reboot
+
+## Configuration
+
+``` bash
+dpkg-reconfigure grub-efi-amd64
+```
+
+Create useraccount:
+``` bash
+username=YOUR_USERNAME
+
+UUID=$(dd if=/dev/urandom bs=1 count=100 2>/dev/null |
+    tr -dc 'a-z0-9' | cut -c-6)
+ROOT_DS=$(zfs list -o name | awk '/ROOT\/ubuntu_/{print $1;exit}')
+zfs create -o com.ubuntu.zsys:bootfs-datasets=$ROOT_DS \
+    -o canmount=on -o mountpoint=/home/$username \
+    rpool/USERDATA/${username}_$UUID
+adduser $username
+
+cp -a /etc/skel/. /home/$username
+chown -R $username:$username /home/$username
+usermod -a -G adm,cdrom,dip,lpadmin,lxd,plugdev,sambashare,sudo $username
+```
+
+Disable log compression:
+``` bash
+for file in /etc/logrotate.d/* ; do
+    if grep -Eq "(^|[^#y])compress" "$file" ; then
+        sed -i -r "s/(^|[^#y])(compress)/\1#\2/" "$file"
+    fi
+done
+```
+
+Remove root password:
+``` bash
+sudo usermod -p '*' root
+```
+
+Remove root login over ssh:
+``` bash
+sudo vi /etc/ssh/sshd_config
+# Remove: PermitRootLogin yes
+```
+
+Create a snapshot of the root and boot partitions:
+
+``` bash
+zfs snapshot rpool/ROOT/ubuntu_<UUID>@baseline
+zfs snapshot bpool/BOOT/ubuntu_<UUID>@baseline
+```
+
+[^special-device]: see <https://forum.level1techs.com/t/zfs-metadata-special-device-z/159954>