Monday, April 10, 2023

Yocto - building a dmverity/squashfs ro-rootfs

Configure the kernel with dm-verity support

$ bitbake linux-yocto -c kernel_configme -f
$ bitbake linux-yocto -c menuconfig

Using menuconfig select -

Device Drivers / Multiple device drivers support / Device Mapper Support / DM "dm-mod.create=" parameter support = [*]
Verity target support = [*]
Verity data device root hash signature verification support = [*]
Cryptographic API / Certificates for signature checking / Additional X.509 keys for default system keyring = ("${HOME}/keys/verity_cert.pem")

Generate a config fragment and check the content

$ bitbake linux-yocto -c diffconfig

$ cat tmp/work/qemux86_64-poky-linux/linux-yocto/*/fragment.cfg
CONFIG_DM_INIT=y
CONFIG_DM_VERITY=y
CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y
# CONFIG_DM_VERITY_FEC is not set
CONFIG_SYSTEM_TRUSTED_KEYS="${HOME}/keys/verity_cert.pem"

$ mv 
tmp/work/qemux86_64-poky-linux/linux-yocto/*/fragment.cfg ../meta-mylayer/recipes-kernel/linux/linux-yocto/dmverity.cfg

Add a reference to the fragment in your kernel recipe - meta-mylayer/recipes-kernel/linux/linux-yocto_%.bbappend

SRC_URI += "file://dmverity.cfg"

Create the certificate

Parameter explanation
-nodes
No DES (i.e. don't encrypt the private key with Data Encryption Standard) 
-days
Specifies the number of days to make a certificate valid for. 
-set_serial
Specifies the serial number to use. 
In a certificate, the serial number is chosen by the CA which issued the certificate. It is just written in the certificate. The CA can choose the serial number in any way as it sees fit, not necessarily randomly (and it has to fit in 20 bytes). A CA is supposed to choose unique serial numbers, that is, unique for the CA.  
-subject
https://en.wikipedia.org/wiki/X.509#Sample_X.509_certificates

Command -
$ openssl req -x509 -newkey rsa:1024 -keyout verity_key.pem \
    -out verity_cert.pem -nodes -days 7300 -set_serial 01 -subject /CN=bugwhine.blogspot.com

Move them somewhere safe
$ mv verity_cert.pem ${HOME}/keys
$ mv verity_key.pem ${HOME}/keys

Build the kernel
$ bitbake linux-yocto -c clean
$ bitbake linux-yocto 

Confirm that it was configured correctly -
$ grep -i verity tmp/work/qemux86_64-poky-linux/linux-yocto/*/*/.config
CONFIG_DM_VERITY=y
CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y
# CONFIG_DM_VERITY_FEC is not set
# CONFIG_FS_VERITY is not set
CONFIG_SYSTEM_TRUSTED_KEYS="${HOME}/keys/verity_cert.pem"

Setup yocto to generate verity metadata for the rootfs image

$ git clone git://git.yoctoproject.org/meta-security
$ cd meta-security
$ git checkout --track remotes/origin/dunfell
$ cd ..
$ git clone https://github.com/openembedded/meta-openembedded.git
$ git checkout --track remotes/origin/dunfell

Add these layers to conf/bblayers.conf -

  meta-security \
  meta-openembedded/meta-oe \
  meta-openembedded/meta-python \
  meta-openembedded/meta-networking \
  meta-openembedded/meta-perl \

Create a wic kickstart file (recipes-core/images/my-image-verity-wic.wks) containing -

part / --source rootfs --ondisk sda --fstype=squashfs
part /media/rfs/rw --ondisk sda --fstype=ext4 --label rwoverlay --size 100M
part / --source rawcopy --ondisk sda --sourceparams="file=${IMGDEPLOYDIR}/${DM_VERITY_IMAGE}-${MACHINE}.${DM_VERITY_IMAGE_TYPE}.verity"

Create a new image recipe (recipes-core/images/my-image-verity-wic.bb) containing -

DESCRIPTION = "A console-only image with more full-featured Linux system \
functionality installed."

SRC_URI = "file://${FILE_DIRNAME}/${BPN}.wks"

IMAGE_FEATURES += "splash ssh-server-openssh"

IMAGE_INSTALL = "\
    packagegroup-core-boot \
    packagegroup-core-full-cmdline \
    ${CORE_IMAGE_EXTRA_INSTALL} \
    initscripts-readonly-rootfs-overlay \
    cryptsetup \
    "

DM_VERITY_IMAGE = "my-image-verity-wic"
DM_VERITY_IMAGE_TYPE = "squashfs"
IMAGE_CLASSES += "dm-verity-img"
IMAGE_FSTYPES = "squashfs squashfs.verity wic"

WICVARS_append = " DM_VERITY_IMAGE DM_VERITY_IMAGE_TYPE"

inherit core-image

WKS_FILE = "my-image.wks"

WKS_FILE_DEPENDS = "dosfstools-native mtools-native gptfdisk-native squashfs-tools-native"
WKS_FILE_DEPENDS_append_x86 = " syslinux-native syslinux"
WKS_FILE_DEPENDS_append_x86-64 = " syslinux-native syslinux"
WKS_FILE_DEPENDS_append_x86-x32 = " syslinux-native syslinux"

QB_KERNEL_CMDLINE_APPEND += "root=/dev/vda1 rootrw=/dev/vda2 rootrwoptions=rw,noatime init=/init"
QB_DEFAULT_FSTYPE = "wic"
QB_FSINFO = "wic:no-kernel-in-fs"
#QB_KERNEL_ROOT = "/dev/vda1"

Patch meta-security/classes/dm-verity-img.bbclass adding squashfs support -

VERITY_TYPES = "ext2.verity ext3.verity ext4.verity btrfs.verity squashfs.verity"

Build and note the verity parameters needed to map it

$ bitbake my-image-verity-wic

$ cat tmp/work-shared/qemux86-64/dm-verity/my-image-verity-wic.squashfs.verity.env
UUID=abcab4c0-3187-4b91-bcb5-d804229456e2
HASH_TYPE=1
DATA_BLOCKS=26760
DATA_BLOCK_SIZE=1024
HASH_BLOCK_SIZE=4096
HASH_ALGORITHM=sha256
SALT=5ac714319ab03dfad686c5563dc78b3be03c802c0f62477a423a56148a32c417
ROOT_HASH=6b4478922104f14b6ca6f463dab26b9a4c653e6b2507e493cf2c95f687b19fc8
DATA_SIZE=27402240

Boot in qemu

At this stage we have both the original squashfs image, and squashfs image with verity metadata appended in partitions 1 and 3 of the wic image.

We boot the squashfs image first -

$ runqemu my-image-verity-wic nographic kvm

Poky (Yocto Project Reference Distro) 3.1.24 qemux86-64 ttyS0

qemux86-64 login: root
root@qemux86-64:~#
root@qemux86-64:~# lsblk
NAME   MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
hdc     22:0    1     4G  0 disk
vda    253:0    0 153.1M  0 disk
|-vda1 253:1    0  26.1M  0 part /media/rfs/ro
|-vda2 253:2    0   100M  0 part /media/rfs/rw
`-vda3 253:3    0    27M  0 part

Then map and mount it -

root@qemux86-64:~# veritysetup --data-block-size=1024 --hash-offset=27402240 open /dev/vda3 root /dev/vda3 \
>  6b4478922104f14b6ca6f463dab26b9a4c653e6b2507e493cf2c95f687b19fc8
root@qemux86-64:~# ls /dev/mapper/root
/dev/mapper/root
root@qemux86-64:~# mkdir /tmp/dmroot
root@qemux86-64:~# mount /dev/mapper/root /tmp/dmroot/
mount: /var/volatile/tmp/dmroot: WARNING: device write-protected, mounted read-only.
root@qemux86-64:~#

and confirm that it is readable

root@qemux86-64:~# file /tmp/dmroot/usr/sbin/veritysetup
/tmp/dmroot/usr/sbin/veritysetup: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-x86-64.so.2, BuildID[sha1]=8035280d420b22091cf8319d4ceda8e6d

Now we'll tamper with the image, reboot -

root@qemux86-64:~# dd if=/dev/zero of=/dev/vda3 bs=1024 skip=10 count=1
1+0 records in
1+0 records out
1024 bytes (1.0 kB, 1.0 KiB) copied, 0.00516249 s, 198 kB/s
root@qemux86-64:~# reboot

And check if we can remap/mount it -

root@qemux86-64:~#
root@qemux86-64:~# veritysetup --data-block-size=1024 --hash-offset=27402240 open /dev/vda3 root /dev/vda3 \
>  6b4478922104f14b6ca6f463dab26b9a4c653e6b2507e493cf2c95f687b19fc8
Verity device detected corruption after activation.
root@qemux86-64:~#
root@qemux86-64:~# dmesg |tail
[   46.457801] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.459131] device-mapper: verity: 253:3: data block 0 is corrupted
[   46.459468] device-mapper: verity: 253:3: data block 0 is corrupted
[   46.459475] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.461389] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.462957] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.465372] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.466989] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.468663] Buffer I/O error on dev dm-0, logical block 0, async page read
[   46.544350] device-mapper: verity: 253:3: reached maximum errors
root@qemux86-64:~#
root@qemux86-64:~# mkdir /tmp/dmroot
root@qemux86-64:~# mount /dev/mapper/root /tmp/dmroot/
mount: /var/volatile/tmp/dmroot: can't read superblock on /dev/mapper/root.
root@qemux86-64:~#

Note it complains about logical block 0 as I used 'skip=10' rather than 'seek=10' in my dd command.

As previously, the files are available at -


Next steps will be 
- signing and authenticating the root hash
- boot with authentication

Sunday, April 9, 2023

Yocto - read-only rootfs (squashfs) + ext4 overlay (qemu)

Experimenting with a qemu yocto build that models an Embedded system with a read-only rootfs, and writeable overlay.

Basic yocto setup

Clone git://git.yoctoproject.org/poky

Checkout dunfell branch

Clone https://github.com/marcusfolkesson/meta-readonly-rootfs-overlay.git

Update the compatibility in meta-readonly-rootfs-overlay/conf/layer.conf to match your poky branch

LAYERSERIES_COMPAT_readonly-rootfs-overlay = "honister kirkstone dunfell"

source oe-init-build-env

Add meta-readonly-rootfs-overlay to build/conf/bblayers.conf

Create own layer meta-mylayer (git@github.com:bugwhine/meta-mylayer.git)

and add to build/conf/bblayers.conf

Setup meta-mylayer/conf/layer.conf containing the boilerplate -

# We have a conf and classes directory, add to BBPATH
BBPATH .= ":${LAYERDIR}"

# We have recipes-* directories, add to BBFILES
BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \
            ${LAYERDIR}/recipes-*/*/*.bbappend"

BBFILE_COLLECTIONS += "mylayer"
BBFILE_PATTERN_mylayer = "^${LAYERDIR}/"
BBFILE_PRIORITY_mylayer = "5"
LAYERVERSION_mylayer = "4"
LAYERSERIES_COMPAT_mylayer = "dunfell"

Add squashfs support to the yocto kernel

$ bitbake linux-yocto -c kernel_configme -f
$ bitbake linux-yocto -c menuconfig

Use menuconfig to add squashfs support, and customize options as desired

$ bitbake linux-yocto -c diffconfig
$ mkdir -p meta-mylayer/recipes-kernel/linux
$ mv fragment.cfg ../meta-mylayer/recipes-kernel/linux/linux-yocto/squashfs.cfg

Create a bbappend file meta-mylayer/recipes-kernel/linux/linux-yocto_%.bbappend referencing the fragment

FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"
SRC_URI += "file://squashfs.cfg"

Build the kernel 

$ bitbake linux-yocto

And confirm that the configuration is present

$ grep SQUASHFS `find tmp/work/qemux86_64-poky-linux/linux-yocto/ -name '.config'`
CONFIG_SQUASHFS=y
...
CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE=3

Setup an image (wic)

Create an image recipe meta-mylayer/recipes-core/images/my-image-wic.bb with content (based on core-image-full-cmdline)

DESCRIPTION = "A console-only image with more full-featured Linux system \
functionality installed."

SRC_URI = "file://${FILE_DIRNAME}/${BPN}.wks"

IMAGE_FEATURES += "splash ssh-server-openssh"

IMAGE_INSTALL = "\
    packagegroup-core-boot \
    packagegroup-core-full-cmdline \
    ${CORE_IMAGE_EXTRA_INSTALL} \
    initscripts-readonly-rootfs-overlay \
    "

IMAGE_FSTYPES = "wic"

inherit core-image

WKS_FILE = "my-image-wic.wks"

WKS_FILE_DEPENDS = "dosfstools-native mtools-native gptfdisk-native squashfs-tools-native"
WKS_FILE_DEPENDS_append_x86 = " syslinux-native syslinux"
WKS_FILE_DEPENDS_append_x86-64 = " syslinux-native syslinux"
WKS_FILE_DEPENDS_append_x86-x32 = " syslinux-native syslinux"

QB_KERNEL_CMDLINE_APPEND += "root=/dev/vda1 rootrw=/dev/vda2 rootrwoptions=rw,noatime init=/init"
QB_DEFAULT_FSTYPE = "wic"
QB_FSINFO = "wic:no-kernel-in-fs"
#QB_KERNEL_ROOT = "/dev/vda1"

Create the wks (wick kickstart script) meta-mylayer/recipes-core/images/my-image-wic.wks with content - 

part / --source rootfs --ondisk sda --fstype=squashfs --align 1024
part /media/rfs/rw --ondisk sda --fstype=ext4 --label rwoverlay --size 100M

Build and run

$ bitbake my-image-wic
...
$ runqemu my-image-wic nographic kvm
...
Booting from ROM...
[    0.000000] Linux version 5.4.230-yocto-standard (oe-user@oe-host) (gcc version 9.5.0 (GCC)) #1 SMP PREEMPT Wed Feb 1 15:40:31 UTC 2023
[    0.000000] Command line: root=/dev/vda rw  console=ttyS0 mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 root=/dev/vda1 rootrw=/dev/vda2 rootrwoptions=rw,noatime init=/ini
...
Poky (Yocto Project Reference Distro) 3.1.24 qemux86-64 ttyS0

qemux86-64 login: root

Check the rootfs mounts  

root@qemux86-64:~# lsblk
NAME   MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
hdc     22:0    1     4G  0 disk
vda    253:0    0 126.3M  0 disk
|-vda1 253:1    0  25.3M  0 part /media/rfs/ro
`-vda2 253:2    0   100M  0 part /media/rfs/rw

root@qemux86-64:~# cat /proc/mounts
...
/dev/root /media/rfs/ro squashfs ro,relatime 0 0
/dev/vda2 /media/rfs/rw ext4 rw,noatime 0 0
overlay / overlay rw,relatime,lowerdir=/media/rfs/ro,upperdir=/media/rfs/rw/upperdir,workdir=/media/rfs/rw/work 0 0
...

Confirm that the overlay is working

root@qemux86-64:~# touch /etc/aaa
root@qemux86-64:~# ls /media/rfs/rw/upperdir/etc/aaa
/media/rfs/rw/upperdir/etc/aaa
root@qemux86-64:~# ls /media/rfs/ro/etc/aaa
ls: cannot access '/media/rfs/ro/etc/aaa': No such file or directory