Description of problem:

1. lvconvert -m0 vg00/optoraclevol /dev/mapper/3624a93707581c241a80e426903055aed2
   • • buffer overflow detected ***: lvconvert terminated
       ======= Backtrace: =========
       /lib64/libc.so.6(__fortify_fail+0x37)[0x7fe07c2007a7]
       /lib64/libc.so.6(+0x116922)[0x7fe07c1fe922]
       /lib64/libc.so.6(+0x118707)[0x7fe07c200707]
       /lib64/libdevmapper-event.so.1.02(+0x2328)[0x7fe07d1b2328]
       /lib64/libdevmapper-event.so.1.02(daemon_talk+0xb5)[0x7fe07d1b2a95]
       /lib64/libdevmapper-event.so.1.02(+0x335b)[0x7fe07d1b335b]
       /lib64/libdevmapper-event.so.1.02(dm_event_get_registered_device+0x77)[0x7fe07d1b3a87]
       lvconvert(monitor_dev_for_events+0x3cc)[0x55f65409cb7c]
       lvconvert(lv_suspend_if_active+0x6cc)[0x55f65409e70c]
       lvconvert(+0x144aed)[0x55f65415eaed]
       lvconvert(+0xba1ab)[0x55f6540d41ab]
       lvconvert(lock_vol+0x140)[0x55f6540d4b80]
       lvconvert(+0xc4b79)[0x55f6540deb79]
       lvconvert(+0xee553)[0x55f654108553]
       lvconvert(remove_mirror_images+0x94)[0x55f654108c24]
       lvconvert(+0x45ab7)[0x55f65405fab7]
       lvconvert(+0x46202)[0x55f654060202]
       lvconvert(+0x469fe)[0x55f6540609fe]
       lvconvert(+0x46c8a)[0x55f654060c8a]
       lvconvert(process_each_lv_in_vg+0x12ad)[0x55f654086d7d]
       lvconvert(process_each_lv+0xe7f)[0x55f6540882bf]
       lvconvert(lvconvert_raid_types_cmd+0x128)[0x55f654062fb8]
       lvconvert(lvm_run_command+0x1fe8)[0x55f65406f118]
       lvconvert(lvm2_main+0xa6e)[0x55f65407042e]
       /lib64/libc.so.6(__libc_start_main+0xf5)[0x7fe07c10a555]
       lvconvert(+0x329ae)[0x55f65404c9ae]
       ======= Memory map: ========
       55f65401a000-55f6541fd000 r-xp 00000000 fd:577 17116 /usr/sbin/lvm
       55f6543fc000-55f654406000 r--p 001e2000 fd:577 17116 /usr/sbin/lvm
       55f654406000-55f65443b000 rw-p 001ec000 fd:577 17116 /usr/sbin/lvm
       55f65443b000-55f654673000 rw-p 00000000 00:00 0
       55f655497000-55f656ec0000 rw-p 00000000 00:00 0 [heap]
       7fe073854000-7fe079d97000 r--p 00000000 fd:577 4306 /usr/lib/locale/locale-archive
       7fe079d97000-7fe079df7000 r-xp 00000000 fd:577 4293 /usr/lib64/libpcre.so.1.2.0
       7fe079df7000-7fe079ff7000 ---p 00060000 fd:577 4293 /usr/lib64/libpcre.so.1.2.0
       7fe079ff7000-7fe079ff8000 r--p 00060000 fd:577 4293 /usr/lib64/libpcre.so.1.2.0
       7fe079ff8000-7fe079ff9000 rw-p 00061000 fd:577 4293 /usr/lib64/libpcre.so.1.2.0
       7fe079ff9000-7fe07a008000 r-xp 00000000 fd:577 4583 /usr/lib64/libbz2.so.1.0.6
       7fe07a008000-7fe07a207000 ---p 0000f000 fd:577 4583 /usr/lib64/libbz2.so.1.0.6
       7fe07a207000-7fe07a208000 r--p 0000e000 fd:577 4583 /usr/lib64/libbz2.so.1.0.6
       7fe07a208000-7fe07a209000 rw-p 0000f000 fd:577 4583 /usr/lib64/libbz2.so.1.0.6
       7fe07a209000-7fe07a22e000 r-xp 00000000 fd:577 4359 /usr/lib64/liblzma.so.5.2.2
       7fe07a22e000-7fe07a42d000 ---p 00025000 fd:577 4359 /usr/lib64/liblzma.so.5.2.2
       7fe07a42d000-7fe07a42e000 r--p 00024000 fd:577 4359 /usr/lib64/liblzma.so.5.2.2
       7fe07a42e000-7fe07a42f000 rw-p 00025000 fd:577 4359 /usr/lib64/liblzma.so.5.2.2
       7fe07a42f000-7fe07a444000 r-xp 00000000 fd:577 4344 /usr/lib64/libz.so.1.2.7
       7fe07a444000-7fe07a643000 ---p 00015000 fd:577 4344 /usr/lib64/libz.so.1.2.7
       7fe07a643000-7fe07a644000 r--p 00014000 fd:577 4344 /usr/lib64/libz.so.1.2.7
       7fe07a644000-7fe07a645000 rw-p 00015000 fd:577 4344 /usr/lib64/libz.so.1.2.7
       7fe07a645000-7fe07a65c000 r-xp 00000000 fd:577 4458 /usr/lib64/libelf-0.176.so
       7fe07a65c000-7fe07a85b000 ---p 00017000 fd:577 4458 /usr/lib64/libelf-0.176.so
       7fe07a85b000-7fe07a85c000 r--p 00016000 fd:577 4458 /usr/lib64/libelf-0.176.so
       7fe07a85c000-7fe07a85d000 rw-p 00017000 fd:577 4458 /usr/lib64/libelf-0.176.so
       7fe07a85d000-7fe07a861000 r-xp 00000000 fd:577 4600 /usr/lib64/libattr.so.1.1.0
       7fe07a861000-7fe07aa60000 ---p 00004000 fd:577 4600 /usr/lib64/libattr.so.1.1.0
       7fe07aa60000-7fe07aa61000 r--p 00003000 fd:577 4600 /usr/lib64/libattr.so.1.1.0
       7fe07aa61000-7fe07aa62000 rw-p 00004000 fd:577 4600 /usr/lib64/libattr.so.1.1.0
       7fe07aa62000-7fe07aa87000 r-xp 00000000 fd:577 4291 /usr/lib64/libtinfo.so.5.9
       7fe07aa87000-7fe07ac87000 ---p 00025000 fd:577 4291 /usr/lib64/libtinfo.so.5.9
       7fe07ac87000-7fe07ac8b000 r--p 00025000 fd:577 4291 /usr/lib64/libtinfo.so.5.9
       7fe07ac8b000-7fe07ac8c000 rw-p 00029000 fd:577 4291 /usr/lib64/libtinfo.so.5.9
       7fe07ac8c000-7fe07ad28000 r-xp 00000000 fd:577 25386 /usr/lib64/libsepol.so.1
       7fe07ad28000-7fe07af27000 ---p 0009c000 fd:577 25386 /usr/lib64/libsepol.so.1
       7fe07af27000-7fe07af28000 r--p 0009b000 fd:577 25386 /usr/lib64/libsepol.so.1
       7fe07af28000-7fe07af29000 rw-p 0009c000 fd:577 25386 /usr/lib64/libsepol.so.1
       7fe07af29000-7fe07af2a000 rw-p 00000000 00:00 0
       7fe07af2a000-7fe07af4e000 r-xp 00000000 fd:577 47 /usr/lib64/libselinux.so.1
       7fe07af4e000-7fe07b14d000 ---p 00024000 fd:577 47 /usr/lib64/libselinux.so.1
       7fe07b14d000-7fe07b14e000 r--p 00023000 fd:577 47 /usr/lib64/libselinux.so.1
       7fe07b14e000-7fe07b14f000 rw-p 00024000 fd:577 47 /usr/lib64/libselinux.so.1
       7fe07b14f000-7fe07b151000 rw-p 00000000 00:00 0
       7fe07b151000-7fe07b155000 r-xp 00000000 fd:577 4519 /usr/lib64/libuuid.so.1.3.0
       7fe07b155000-7fe07b354000 ---p 00004000 fd:577 4519 /usr/lib64/libuuid.so.1.3.0
       7fe07b354000-7fe07b355000 r--p 00003000 fd:577 4519 /usr/lib64/libuuid.so.1.3.0
       7fe07b355000-7fe07b356000 rw-p 00004000 fd:577 4519 /usr/lib64/libuuid.so.1.3.0
       7fe07b356000-7fe07b36b000 r-xp 00000000 fd:577 29429 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
       7fe07b36b000-7fe07b56a000 ---p 00015000 fd:577 29429 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
       7fe07b56a000-7fe07b56b000 r--p 00014000 fd:577 29429 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
       7fe07b56b000-7fe07b56c000 rw-p 00015000 fd:577 29429 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
       7fe07b56c000-7fe07b5ba000 r-xp 00000000 fd:577 29026 /usr/lib64/libdw-0.176.so
       7fe07b5ba000-7fe07b7ba000 ---p 0004e000 fd:577 29026 /usr/lib64/libdw-0.176.so
       7fe07b7ba000-7fe07b7bc000 r--p 0004e000 fd:577 29026 /usr/lib64/libdw-0.176.soAborted (core dumped)

Version-Release number of selected component (if applicable):

lvm2-2.02.187-6.el7_9.5.x86_64
device-mapper-1.02.170-6.el7_9.5.x86_64
device-mapper-multipath-0.4.9-135.el7_9.x86_64
device-mapper-persistent-data-0.8.5-3.el7_9.2.x86_64
kernel-3.10.0-1160.76.1.el7.x86_64

Additional info:

• Customer able to work around the issue by removing all but the local luns. From the customer:

~~~~
Hi all, i have just performed a test: on this server there are plenty LUNs ( more than 1000 ) each of which has at least 4 paths x LUN and we launch the command "lvconvert -m0 vg00/optoraclevol /dev/mapper/3624a93707581c241a80e426903055aed2" with strace , we watch which has to do the discovery of all the LUNS and cannot complete (I don't think it is open file, though). I changed the open file with ulimit -n to 4096 but I obtain the same error. So ( with the San Team ) we had disabled the data hba , leaving only the boot hba with only 2 disks, the command lvconvert works!!!! Could you recommend some parameters to modify?
~~~

• There are over 11,000 scsi devices and over 1500 dm devs.

vg00-varmqmvol: 0 4194304 linear 253:20 184877056
vg00-icingavol: 0 2097152 linear 253:20 98574336
vg00-agentoemvol: 0 106496 linear 253:20 100777984
vg00-optoraclevol: 0 383778816 mirror core 1 4096 2 253:1531 0 253:1532 0 1 handle_errors
vg00-perfvol: 0 6291456 linear 253:20 189071360
vg00-oradatavol: 0 106496 linear 253:20 100671488
vg00-optoraclevol_mimage_1: 0 383778816 linear 253:0 8192
vg00-varvol: 0 8388608 linear 253:20 88088576
vg00-varvol: 8388608 12582912 linear 253:20 223674368
vg00-varvol: 20971520 8380416 linear 253:20 598016000
vg00-swapvol: 0 67108864 linear 253:20 8192
vg00-rootvol: 0 8388608 linear 253:20 79699968
vg00-optoraclevol_mimage_0: 0 83886080 linear 253:20 100884480
vg00-optoraclevol_mimage_0: 83886080 230686720 linear 253:20 244645888
vg00-optoraclevol_mimage_0: 314572800 69206016 linear 253:20 485818368
vg00-optvol: 0 8388608 linear 253:20 67117056
vg00-optvol: 8388608 4194304 linear 253:20 198508544
vg00-optvol: 12582912 8388608 linear 253:20 236257280
vg00-usropenvvol: 0 10485760 linear 253:20 202702848
vg00-usropenvvol: 10485760 10485760 linear 253:20 475332608
vg00-vartwsvol: 0 106496 linear 253:20 184770560
vg00-var_vrtsvcsvol: 0 10485760 linear 253:20 213188608
vg00-tmpvol: 0 2097152 linear 253:20 96477184
vg00-tmpvol: 2097152 3145728 linear 253:20 195362816
vg00-tmpvol: 5242880 7340032 linear 253:20 590675968
vg00-homevol: 0 4194304 linear 253:20 75505664

1. grep -c Lun proc/scsi/scsi
   11004

1. cat sos_commands/devicemapper/dmsetup_info_-c | wc -l
   1540

• Below is the backtrace for the app core:

[john@host sosreport-sdlmop115-2023-03-13-dhcynhq]$ cat ../ccpp-2023-03-13-14\:43\:43-159739/core_backtrace
{ "signal": 6
, "executable": "/usr/sbin/lvm"
, "stacktrace":
[ { "crash_thread": true
, "frames":
[

{ "address": 140602130948999 , "build_id": "fc4fa58e47a5acc137eadb7689bce4357c557a96" , "build_id_offset": 222087 , "function_name": "raise" , "file_name": "/lib64/libc.so.6" }

,

{ "address": 140602130954872 , "build_id": "fc4fa58e47a5acc137eadb7689bce4357c557a96" , "build_id_offset": 227960 , "function_name": "abort" , "file_name": "/lib64/libc.so.6" }

,

{ "address": 140602131222375 , "build_id": "fc4fa58e47a5acc137eadb7689bce4357c557a96" , "build_id_offset": 495463 , "function_name": "__libc_message" , "file_name": "/lib64/libc.so.6" }

,

{ "address": 140602131875751 , "build_id": "fc4fa58e47a5acc137eadb7689bce4357c557a96" , "build_id_offset": 1148839 , "function_name": "__fortify_fail" , "file_name": "/lib64/libc.so.6" }

,

{ "address": 140602131867938 , "build_id": "fc4fa58e47a5acc137eadb7689bce4357c557a96" , "build_id_offset": 1141026 , "file_name": "/lib64/libc.so.6" }

,

{ "address": 140602131875591 , "build_id": "fc4fa58e47a5acc137eadb7689bce4357c557a96" , "build_id_offset": 1148679 , "file_name": "/lib64/libc.so.6" }

,

{ "address": 140602148332328 , "build_id": "864c7a4e3e8e47b826dc57ac667c63c4bdac906f" , "build_id_offset": 9000 , "function_name": "_daemon_write.isra.0" , "file_name": "/lib64/libdevmapper-event.so.1.02" }

,

{ "address": 140602148334229 , "build_id": "864c7a4e3e8e47b826dc57ac667c63c4bdac906f" , "build_id_offset": 10901 , "function_name": "daemon_talk" , "file_name": "/lib64/libdevmapper-event.so.1.02" }

,

{ "address": 140602148336475 , "build_id": "864c7a4e3e8e47b826dc57ac667c63c4bdac906f" , "build_id_offset": 13147 , "function_name": "_do_event" , "file_name": "/lib64/libdevmapper-event.so.1.02" }

,

{ "address": 140602148338311 , "build_id": "864c7a4e3e8e47b826dc57ac667c63c4bdac906f" , "build_id_offset": 14983 , "function_name": "dm_event_get_registered_device" , "file_name": "/lib64/libdevmapper-event.so.1.02" }

,

{ "address": 94516460243836 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 535420 , "function_name": "monitor_dev_for_events" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460250892 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 542476 , "function_name": "lv_suspend_if_active" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516461038317 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 1329901 , "function_name": "_file_lock_resource" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460470699 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 762283 , "function_name": "_lock_vol" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460473216 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 764800 , "function_name": "lock_vol" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460514169 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 805753 , "function_name": "_lv_update_and_reload" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460684627 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 976211 , "function_name": "_remove_mirror_images" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460686372 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 977956 , "function_name": "remove_mirror_images" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516459993783 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 285367 , "function_name": "_lvconvert_mirrors_aux.constprop.34" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516459995650 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 287234 , "function_name": "_lvconvert_mirrors" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516459997694 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 289278 , "function_name": "_lvconvert_raid_types" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516459998346 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 289930 , "function_name": "_lvconvert_raid_types_single" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460154237 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 445821 , "function_name": "process_each_lv_in_vg" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460159679 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 451263 , "function_name": "process_each_lv" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460007352 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 298936 , "function_name": "lvconvert_raid_types_cmd" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460056856 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 348440 , "function_name": "lvm_run_command" , "file_name": "/usr/sbin/lvm" }

,

{ "address": 94516460061742 , "build_id": "5be612fea7ecd81eb37f74fc13f9723b0bb9d5cd" , "build_id_offset": 353326 , "function_name": "lvm2_main" , "file_name": "/usr/sbin/lvm" }

]
} ]
}