We attempted to enable QEMU's ISA-DEBUG to collect firmware and BIOS/UEFI debug logs by injecting the libvirt XML via sidecar approach, but the VM fails to start.

CNV-v4.20.0.rhel9-242

100%

kubectl create configmap cm-isa-debug --from-file=script=./onDefineDomain.py

metadata:
  annotations:
      hooks.kubevirt.io/hookSidecars: >
        [
            {
                "args": ["--version", "v1alpha3"],
                "configMap": {
                  "name": "cm-isa-debug",
                  "key": "script",
                  "hookPath": "/usr/bin/onDefineDomain"
                }
            }
        ] 

VM CrashLoopBackOff, virt-launcher error

$ oc get pod -w
virt-launcher-rhel9-isa-debug2-tfzc7   0/3     Init:0/1   0          12s
virt-launcher-rhel9-isa-debug2-tfzc7   0/3     Init:0/1   0          13s
virt-launcher-rhel9-isa-debug2-tfzc7   1/3     PodInitializing   0          14s
virt-launcher-rhel9-isa-debug2-tfzc7   3/3     Running           0          15s
virt-launcher-rhel9-isa-debug2-tfzc7   3/3     Running           0          15s
virt-launcher-rhel9-isa-debug2-tfzc7   2/3     Error             0          22s

VM running successful

{"component":"virt-launcher","level":"info","msg":"Re-registered domain memory device size change callback","pos":"libvirt.go:544","timestamp":"2025-10-29T12:49:18.022050Z"}
{"component":"virt-launcher","level":"error","msg":"Re-registered domain and agent callbacks for new connection","pos":"libvirt.go:548","timestamp":"2025-10-29T12:49:18.022137Z"}
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x199473b]
{"component":"virt-launcher-monitor","level":"info","msg":"Reaped Launcher main pid","pos":"virt-launcher-monitor.go:128","timestamp":"2025-10-29T12:49:18.028855Z"}
{"component":"virt-launcher-monitor","level":"info","msg":"Reaped pid 8 with status 512","pos":"virt-launcher-monitor.go:131","timestamp":"2025-10-29T12:49:18.028910Z"}
{"component":"virt-launcher-monitor","level":"error","msg":"dirty virt-launcher shutdown: exit-code 2","pos":"virt-launcher-monitor.go:145","timestamp":"2025-10-29T12:49:18.028924Z"}
{"component":"virt-launcher-monitor","level":"error","msg":"failed to read qemu log directory","pos":"virt-launcher-monitor.go:151","reason":"open /run/kubevirt-private/libvirt/qemu/log: no such file or directory","timestamp":"2025-10-29T12:49:18.028972Z"}
{"component":"virt-launcher-monitor","level":"info","msg":"virt-launcher-monitor: Exiting...","pos":"virt-launcher-monitor.go:91","timestamp":"2025-10-29T12:49:20.032283Z"}

$ echo 0x199473b | go tool addr2line ./virt-launcher
kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap.(*LibvirtDomainManager).buildDevicesMetadata
/remote-source/app/pkg/virt-launcher/virtwrap/manager.go:2285

#!/usr/bin/python3

import xml.etree.ElementTree as ET
import sys

def add_isa_debug(domainxml):
  root = ET.fromstring(domainxml)
  devices = root.find("devices")

  # Bail if already added
  next_port=0
  for serial in devices.findall("serial"):
    t = serial.get('type')
    if t is not None and t == 'null':
      sys.stderr.write("Already inserted.")
      sys.stderr.write(ET.tostring(serial).decode("utf-8"))
      ET.dump(root)
      return

    target = serial.find("target")
    if target is None:
      continue

    port = target.get('port')
    if port is None or int(port) < next_port:
      continue

    next_port = int(port) + 1

  file_serial = ET.fromstring(
    f"""
  <serial type='file'>
    <log file='/tmp/fw.log' append='off'/>
    <target type='isa-debug' port='{next_port}'>
      <model name='isa-debugcon'/>
    </target>
    <address type='isa' iobase='0x402'/>
    <source path='/tmp/DOMAIN-ovmf.log'/>
  </serial>""")

  pty_serial = ET.fromstring(
    f"""
  <serial type='pty'>
    <log file='/tmp/serial.log' append='off'/>
    <target type='isa-serial' port='{next_port + 1}'>
      <model name='isa-serial'/>
    </target>
  </serial>""")

  devices.insert(len(devices), file_serial)
  devices.insert(len(devices), pty_serial)
  ET.dump(root)

if __name__ == "__main__":
  add_isa_debug(sys.argv[4])

apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  annotations:
    kubemacpool.io/transaction-timestamp: '2025-10-29T07:01:49.457096008Z'
    vm.kubevirt.io/validations: |
      [
        {
          "name": "minimal-required-memory",
          "path": "jsonpath::.spec.domain.memory.guest",
          "rule": "integer",
          "message": "This VM requires more memory.",
          "min": 1610612736
        }
      ]
  creationTimestamp: '2025-10-29T07:01:49Z'
  generation: 1
  labels:
    app: rhel9-isa-debug
    kubevirt.io/dynamic-credentials-support: 'true'
    vm.kubevirt.io/template: rhel9-server-small
    vm.kubevirt.io/template.namespace: openshift
    vm.kubevirt.io/template.revision: '1'
    vm.kubevirt.io/template.version: v0.34.1
  managedFields:
    - apiVersion: kubevirt.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        'f:metadata':
          'f:annotations':
            .: {}
            'f:kubemacpool.io/transaction-timestamp': {}
            'f:vm.kubevirt.io/validations': {}
          'f:labels':
            .: {}
            'f:app': {}
            'f:kubevirt.io/dynamic-credentials-support': {}
            'f:vm.kubevirt.io/template': {}
            'f:vm.kubevirt.io/template.namespace': {}
            'f:vm.kubevirt.io/template.revision': {}
            'f:vm.kubevirt.io/template.version': {}
        'f:spec':
          .: {}
          'f:dataVolumeTemplates': {}
          'f:runStrategy': {}
          'f:template':
            .: {}
            'f:metadata':
              .: {}
              'f:annotations':
                .: {}
                'f:vm.kubevirt.io/flavor': {}
                'f:vm.kubevirt.io/os': {}
                'f:vm.kubevirt.io/workload': {}
              'f:creationTimestamp': {}
              'f:labels':
                .: {}
                'f:kubevirt.io/domain': {}
                'f:kubevirt.io/size': {}
            'f:spec':
              .: {}
              'f:architecture': {}
              'f:domain':
                .: {}
                'f:cpu':
                  .: {}
                  'f:cores': {}
                  'f:sockets': {}
                  'f:threads': {}
                'f:devices':
                  .: {}
                  'f:disks': {}
                  'f:interfaces': {}
                  'f:rng': {}
                'f:features':
                  .: {}
                  'f:acpi': {}
                  'f:smm':
                    .: {}
                    'f:enabled': {}
                'f:firmware':
                  .: {}
                  'f:bootloader':
                    .: {}
                    'f:efi': {}
                  'f:serial': {}
                  'f:uuid': {}
                'f:machine':
                  .: {}
                  'f:type': {}
                'f:memory':
                  .: {}
                  'f:guest': {}
                'f:resources': {}
              'f:networks': {}
              'f:terminationGracePeriodSeconds': {}
              'f:volumes': {}
      manager: kubectl-create
      operation: Update
      time: '2025-10-29T07:01:49Z'
  name: rhel9-isa-debug
  namespace: default
  uid: add588c4-28d9-4383-9447-433d874fba5e
spec:
  dataVolumeTemplates:
    - apiVersion: cdi.kubevirt.io/v1beta1
      kind: DataVolume
      metadata:
        creationTimestamp: null
        name: rhel9-isa-debug
      spec:
        sourceRef:
          kind: DataSource
          name: rhel9
          namespace: openshift-virtualization-os-images
        storage:
          accessModes:
            - ReadWriteMany
          resources:
            requests:
              storage: 30Gi
          storageClassName: ocs-storagecluster-ceph-rbd-virtualization
          volumeMode: Block
  runStrategy: RerunOnFailure
  template:
    metadata:
      annotations:
        hooks.kubevirt.io/hookSidecars: >
          [
            {
                "args": ["--version", "v1alpha3"],
                "configMap": {
                  "name": "cm-isa-debug",
                  "key": "script",
                  "hookPath": "/usr/bin/onDefineDomain"
                }
            }
          ]
        vm.kubevirt.io/flavor: small
        vm.kubevirt.io/os: rhel9
        vm.kubevirt.io/workload: server
      creationTimestamp: null
      labels:
        kubevirt.io/domain: rhel9-isa-debug
        kubevirt.io/size: small
    spec:
      architecture: amd64
      domain:
        cpu:
          cores: 1
          sockets: 1
          threads: 1
        devices:
          disks:
            - disk:
                bus: virtio
              name: rootdisk
            - disk:
                bus: virtio
              name: cloudinitdisk
          interfaces:
            - macAddress: '02:66:ed:dc:2b:fc'
              masquerade: {}
              model: virtio
              name: default
          rng: {}
        features:
          acpi: {}
          smm:
            enabled: true
        firmware:
          bootloader:
            efi: {}
          serial: 61b08855-6870-472a-8174-e7be0603befe
          uuid: 728c0e46-8bad-434a-adf2-01e286f97b61
        machine:
          type: pc-q35-rhel9.6.0
        memory:
          guest: 2Gi
        resources: {}
      networks:
        - name: default
          pod: {}
      terminationGracePeriodSeconds: 180
      volumes:
        - dataVolume:
            name: rhel9-isa-debug
          name: rootdisk
        - cloudInitNoCloud:
            userData: |-
              #cloud-config
              user: cloud-user
              password: 8e1n-gqcv-iw2e
              chpasswd: { expire: False }
          name: cloudinitdisk