[sgao@fedora Downloads]$ oc debug node/sgao-vsan-hbwfc-master-0 Temporary namespace openshift-debug-2b62q is created for debugging node... Starting pod/sgao-vsan-hbwfc-master-0-debug ... To use host binaries, run `chroot /host` Pod IP: 172.16.1.34 If you don't see a command prompt, try pressing enter. sh-4.4# chroot /host sh-5.1# podman run --volume /var/lib/etcd:/var/lib/etcd:Z quay.io/openshift-scale/etcd-perf Trying to pull quay.io/openshift-scale/etcd-perf:latest... Getting image source signatures Copying blob c6334716fbaa done Copying blob 4ac1d6819b77 done Copying blob 92456dffb857 done Copying config 14eae71083 done Writing manifest to image destination Storing signatures ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1689240698, "timestamp_ms" : 1689240698943, "time" : "Thu Jul 13 09:31:38 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 48708, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 389640000, "io_kbytes" : 380507, "bw_bytes" : 6493891, "bw" : 6341, "iops" : 811.736471, "runtime" : 60001, "total_ios" : 48705, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 4368, "max" : 7837624, "mean" : 148243.947213, "stddev" : 242632.298771, "N" : 48705, "percentile" : { "1.000000" : 7456, "5.000000" : 8640, "10.000000" : 9408, "20.000000" : 10816, "30.000000" : 39168, "40.000000" : 44800, "50.000000" : 47872, "60.000000" : 51456, "70.000000" : 59648, "80.000000" : 419840, "90.000000" : 440320, "95.000000" : 460800, "99.000000" : 692224, "99.500000" : 1253376, "99.900000" : 2736128, "99.950000" : 3063808, "99.990000" : 4489216 } }, "lat_ns" : { "min" : 4488, "max" : 7837854, "mean" : 148423.856709, "stddev" : 242638.269250, "N" : 48705 }, "bw_min" : 2468, "bw_max" : 9609, "bw_agg" : 99.689508, "bw_mean" : 6322.453782, "bw_dev" : 2044.415725, "bw_samples" : 119, "iops_min" : 316, "iops_max" : 1230, "iops_mean" : 809.327731, "iops_stddev" : 261.685188, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1352, "max" : 31425298, "mean" : 1080972.206763, "stddev" : 520176.435221, "N" : 48708, "percentile" : { "1.000000" : 610304, "5.000000" : 700416, "10.000000" : 749568, "20.000000" : 774144, "30.000000" : 782336, "40.000000" : 790528, "50.000000" : 806912, "60.000000" : 839680, "70.000000" : 1073152, "80.000000" : 1630208, "90.000000" : 1712128, "95.000000" : 1810432, "99.000000" : 2932736, "99.500000" : 3325952, "99.900000" : 4227072, "99.950000" : 5079040, "99.990000" : 9240576 } } }, "job_runtime" : 60000, "usr_cpu" : 0.341667, "sys_cpu" : 1.853333, "ctx" : 153562, "majf" : 0, "minf" : 18, "iodepth_level" : { "1" : 200.006160, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 14.039626, "20" : 13.440099, "50" : 28.670568, "100" : 18.550457, "250" : 0.281285, "500" : 22.979160, "750" : 1.116928, "1000" : 0.252541 }, "latency_ms" : { "2" : 0.379838, "4" : 0.277179, "10" : 0.012319, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 35176, "write_ios" : 76722, "read_merges" : 0, "write_merges" : 3759, "read_ticks" : 46975, "write_ticks" : 64008, "in_queue" : 110983, "util" : 99.673252 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 2932736 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd