Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-39462

[RHEL-10]The system cannot restart after configure tuned-adm profile cpu-partitioning

    • Icon: Bug Bug
    • Resolution: Done
    • Icon: Blocker Blocker
    • None
    • rhel-10.0
    • tuned
    • None
    • None
    • rhel-sst-cs-net-perf-services
    • ssg_core_services
    • None
    • False
    • Hide

      None

      Show
      None
    • None
    • None
    • None
    • None
    • None

      What were you trying to do that didn't work?

      [RHEL-10]The system cannot restart after configure tuned-adm profile cpu-partitioning

      Please provide the package NVR for which bug is seen:

      kernel-6.9.0-7.el10

      How reproducible:

      Steps to reproduce

      1. Use "setup_bootopts.sh" configure hugepage and use tuned file cpu-partitioning
      2. ./setup_bootopts.sh --hugepagesz=1G --hugepages=24 --tuned-profiles=cpu-partitioning --isolated_cores=2,30,4,32,6,34,8,36,10,38,12,40,14,42,16,44,18,46,20,48,22,50,24,52,26,54 --extra="intel_iommu=on iommu=pt intel_idle.max_cstate=0 processor.max_cstate=0 intel_pstate=disable pci=realloc"
      3. cat setup_bootopts.sh
      4. dbg_flag=${dbg_flag:-"set +x"}
        $dbg_flag
        hugepagesz=1G
        hugepages=0
        extra=""
        isolated_cores=""
        tuned_profiles=""
        
        usage()
        {
        	cat <<-EOF
        	$0 [options] ...
        	options:
        	-h|--help              show this message
        	-s|--hugepagesz        hugepagesz=1G in default
        	-n|--hugepages         hugepages
        	-e|--extra             extra cmdline options
        	-c|--isolated_cores    isolated_cores for tuned
        	-t|--tuned-profiles    tuned-profiles, cpu-partitioning or realtime
        	EOF
        }
        
        options=$(getopt -o hs:n:e:c:t: --long help,hugepagesz:,hugepages:,extra:,isolated_cores:,tuned-profiles: -- "$@")
        getopt_ret=$?
        [[ ${getopt_ret} -eq 0 ]] || {
        	echo "Incorrect options provided"
        	exit 1
        }
        unset getopt_ret
        
        eval set -- "$options"
        while true
        do
        	case "$1" in
        	-h | --help)
        		usage
        		exit 0
        		;;
        	-s | --hugepagesz)
        		shift
        		hugepagesz="$1"
        		;;
        	-n | --hugepages)
        		shift
        		hugepages="$1"
        		;;
        	-e | --extra)
        		shift
        		extra="$1"
        		;;
        	-c | --isolated_cores)
        		shift
        		isolated_cores="$1"
        		;;
        	-t | --tuned-profiles)
        		shift
        		tuned_profiles="$1"
        		;;
        	--)
        		shift
        		break
        		;;
        	esac
        	shift
        done
        
        echo "hugepagesz=$hugepagesz"
        echo "hugepages=$hugepages"
        echo "tuned-profiles=$tuned_profiles"
        echo "isolated_cores=$isolated_cores"
        echo "extra=$extra"
        
        
        
        update_bootopts=0
        #use for debug
        echo "The /proc/cmdline output"
        cat /proc/cmdline
        #add arm system hugepage setting
        arch=$(lscpu|grep Architecture|awk -F " " '{printf $2}')
        if [ "$arch" == "aarch64" ];then
        	echo "${hugepages}" > /proc/sys/vm/nr_hugepages
        fi
        
        if (( hugepages )) && (\
        	! grep "default_hugepagesz=${hugepagesz}" /proc/cmdline &>/dev/null || \
        	! grep "hugepagesz=${hugepagesz}" /proc/cmdline &>/dev/null || \
        	! grep "hugepages=${hugepages}" /proc/cmdline &>/dev/null)
        then
        	((update_bootopts +=1))
        fi
        
        if [[ -n "$tuned_profiles" ]] && [[ "$tuned_profiles" =~ "cpu-partitioning" ]]; then
        	rpm -q "tuned-profiles-${tuned_profiles}" &>/dev/null || \
        	yum -y install "tuned-profiles-${tuned_profiles}"
        
        	if [[ -n "$isolated_cores" ]] && (\
        		! grep "nohz_full=${isolated_cores}" /proc/cmdline &>/dev/null || \
        		! grep "rcu_nocbs=${isolated_cores}" /proc/cmdline &>/dev/null || \
        		! grep -E "isolcpus=[[:graph:]]*${isolated_cores}" /proc/cmdline &>/dev/null)
        	then
        		sed -i '/^isolated_cores=.*/d' "/etc/tuned/${tuned_profiles}-variables.conf"
        		echo "isolated_cores=$isolated_cores" >> "/etc/tuned/${tuned_profiles}-variables.conf"
        		cat "/etc/tuned/${tuned_profiles}-variables.conf"
        		((update_bootopts +=1))
        	fi
        elif [[ -n "$tuned_profiles" ]] && [[ "$tuned_profiles" =~ "realtime" ]]; then
        	rpm -q tuned-profiles-nfv &>/dev/null || yum -y install tuned-profiles-nfv
        	if [[ -n "$isolated_cores" ]] && (\
        		! grep "nohz_full=${isolated_cores}" /proc/cmdline &>/dev/null || \
        		! grep "rcu_nocbs=${isolated_cores}" /proc/cmdline &>/dev/null || \
        		! grep -E "isolcpus=[[:graph:]]*${isolated_cores}" /proc/cmdline &>/dev/null)
        		then
        			# workaround https://issues.redhat.com/browse/RHEL-17895
        			if [[ $(nproc) -ge 56 ]]; then
        				cat > "/etc/tuned/${tuned_profiles}-variables.conf" <<-EOF
        					isolated_cores=$isolated_cores
        					isolate_managed_irq=Y
        					netdev_queue_count=4
        				EOF
        			else
        				cat > "/etc/tuned/${tuned_profiles}-variables.conf" <<-EOF
        					isolated_cores=$isolated_cores
        					isolate_managed_irq=Y
        				EOF
        			fi
        		cat "/etc/tuned/${tuned_profiles}-variables.conf"
        		((update_bootopts +=1))
        	fi
        fi
        
        if ! tuned-adm active | grep "${tuned_profiles}" &>/dev/null; then
        	tuned-adm profile "${tuned_profiles}"
        	((update_bootopts +=1))
        fi
        
        if [[ -n "$extra" ]] && ! grep "${extra}" /proc/cmdline &>/dev/null
        then
        	((update_bootopts +=1))
        fi
        set -x
        if (( update_bootopts ))
        then
        	#if grub2-editenv list | grep kernelopts &>/dev/null
        	#then
        	#	kernelopts=$(grub2-editenv list | grep kernelopts | sed -n '/^kernelopts=/ s/kernelopts=//p')
        	#	sed -i "s|^options.*|options $kernelopts|" /boot/loader/entries/$(ls /boot/loader/entries/ | grep -e ".*$(uname -r).conf")
        	#fi
        	#[[ -n "$tuned_profiles" ]] && { tuned-adm profile ${tuned_profiles}; }
        	bootopts=""
        	(( hugepages )) && bootopts="default_hugepagesz=${hugepagesz} hugepagesz=${hugepagesz} hugepages=${hugepages}"
        	if [[ -n "$tuned_profiles" ]] && [[ "$tuned_profiles" =~ "cpu-partitioning" ]]; then
        		[[ -n "$isolated_cores" ]] && bootopts="$bootopts isolcpus=${isolated_cores}"
        	elif [[ -n "$tuned_profiles" ]] && [[ "$tuned_profiles" =~ "realtime" ]]; then
        		if ! tuned-adm profile "${tuned_profiles}"; then
        			[[ -n "$isolated_cores" ]] && bootopts="$bootopts isolcpus=${isolated_cores}"
        		fi
        	fi
        	[[ -n "$extra" ]] && bootopts="$bootopts $extra"
        	# workaround a know issue https://access.redhat.com/solutions/5622731
        	rpm -q rpmdevtools &>/dev/null || yum install -y rpmdevtools
        	. /etc/os-release
        	echo "VERSION_ID=${VERSION_ID}"
        	rpmdev-vercmp "${VERSION_ID}" 8.2
        	rhel_82_ret=$?
        	if [[ ${rhel_82_ret} -eq 12 ]] && rpm -q kernel-rt &>/dev/null
        	then
        		# if use below command will occur another issue: https://access.redhat.com/solutions/49144
        		# kernelopts=$(grub2-editenv - list | grep kernelopts | sed -e 's/kernelopts=//g')
        		# grub2-editenv - set kernelopts="$bootopts $kernelopts"
        		# grub2-mkconfig -o /boot/grub2/grub.cfg
        		[[ -n "$tuned_profiles" ]] && { grubby --args="$bootopts" --update-kernel=$(grubby --default-kernel) ; ls -l /boot/grub2; tuned-adm profile "${tuned_profiles}"; ls -l /boot/grub2; }
        	fi
        	[[ -n "${bootopts}" ]] && grubby --args="${bootopts}" --update-kernel=$(grubby --default-kernel)
        	[[ -n "${tuned_profiles}" ]] && { tuned-adm profile "${tuned_profiles}";dracut -f; }
        
        fi
        set +x
        
        systemctl disable irqbalance
        systemctl enable tuned
        
        if ((update_bootopts))
        then
        	efibootmgr -v &>/dev/null && efibootmgr -n $(efibootmgr -v | grep BootCurrent | sed 's/.*: //')
        	sleep 10
        	rhts-reboot
        	exit 0
        fi
        
        systemctl stop irqbalance
        systemctl start tuned
        tuned-adm active
        cat /proc/cmdline
        echo
        cat /proc/meminfo
        
        

      Expected results

      The system can start up successfully after run above script.

      Actual results

      Run following command, and then reboot system, and the system cannot start up after run "tuned-adm profile cpu-partitioning"
      grubby '--args=default_hugepagesz=1G hugepagesz=1G hugepages=24 isolcpus=2,30,4,32,6,34,8,36,10,38,12,40,14,42,16,44,18,46,20,48,22,50,24,52,26,54 intel_iommu=on iommu=pt intel_idle.max_cstate=0 processor.max_cstate=0 intel_pstate=disable pci=realloc' --update-kernel=/boot/vmlinuz-6.9.0-7.el10.x86_64
      tuned-adm profile cpu-partitioning
      dracut -f

      If run "tuned-adm profile realtime-virtual-host" to use realtime-virtual-host, it has no issue. It only has issue when use cpu-partitioning.

      beaker job:
      https://beaker.engineering.redhat.com/jobs/9360382

        1. console_log_dell52
          45 kB
        2. console.log
          148 kB
        3. dell740-57-console-pic.png
          dell740-57-console-pic.png
          260 kB
        4. dmesg.log
          182 kB
        5. image-2024-07-11-15-20-20-719.png
          image-2024-07-11-15-20-20-719.png
          152 kB
        6. image-2024-07-13-18-40-46-802.png
          image-2024-07-13-18-40-46-802.png
          255 kB

              jskarvad Jaroslav Skarvada
              tli@redhat.com Ting Li
              Jaroslav Skarvada Jaroslav Skarvada
              Robin Hack Robin Hack
              Votes:
              0 Vote for this issue
              Watchers:
              8 Start watching this issue

                Created:
                Updated:
                Resolved: