[sgao@fedora Downloads]$ oc debug node/sgao-o-z67ws-master-0 Temporary namespace openshift-debug-6ssvr is created for debugging node... Starting pod/sgao-o-z67ws-master-0-debug ... To use host binaries, run `chroot /host` Pod IP: 172.17.1.133 If you don't see a command prompt, try pressing enter. sh-4.4# chroot /host sh-5.1# podman run --volume /var/lib/etcd:/var/lib/etcd:Z quay.io/openshift-scale/etcd-perf Trying to pull quay.io/openshift-scale/etcd-perf:latest... Getting image source signatures Copying blob c6334716fbaa done Copying blob 92456dffb857 done Copying blob 4ac1d6819b77 done Copying config 14eae71083 done Writing manifest to image destination Storing signatures ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1689241097, "timestamp_ms" : 1689241097127, "time" : "Thu Jul 13 09:38:17 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 62088, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 496680000, "io_kbytes" : 485039, "bw_bytes" : 8277862, "bw" : 8083, "iops" : 1034.732754, "runtime" : 60001, "total_ios" : 62085, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 2724, "max" : 24802602, "mean" : 311191.152597, "stddev" : 364308.746095, "N" : 62085, "percentile" : { "1.000000" : 4576, "5.000000" : 5472, "10.000000" : 6496, "20.000000" : 12352, "30.000000" : 296960, "40.000000" : 317440, "50.000000" : 329728, "60.000000" : 346112, "70.000000" : 370688, "80.000000" : 403456, "90.000000" : 452608, "95.000000" : 518144, "99.000000" : 1105920, "99.500000" : 1531904, "99.900000" : 4620288, "99.950000" : 6782976, "99.990000" : 12779520 } }, "lat_ns" : { "min" : 2784, "max" : 24802873, "mean" : 311316.328405, "stddev" : 364306.001777, "N" : 62085 }, "bw_min" : 2468, "bw_max" : 9718, "bw_agg" : 99.865787, "bw_mean" : 8073.722689, "bw_dev" : 1426.389765, "bw_samples" : 119, "iops_min" : 316, "iops_max" : 1244, "iops_mean" : 1033.495798, "iops_stddev" : 182.578559, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1668, "max" : 204911998, "mean" : 653205.238661, "stddev" : 1021750.891291, "N" : 62088, "percentile" : { "1.000000" : 321536, "5.000000" : 395264, "10.000000" : 407552, "20.000000" : 423936, "30.000000" : 436224, "40.000000" : 452608, "50.000000" : 477184, "60.000000" : 509952, "70.000000" : 569344, "80.000000" : 905216, "90.000000" : 1089536, "95.000000" : 1253376, "99.000000" : 2834432, "99.500000" : 3424256, "99.900000" : 7962624, "99.950000" : 10027008, "99.990000" : 20316160 } } }, "job_runtime" : 60000, "usr_cpu" : 0.295000, "sys_cpu" : 1.768333, "ctx" : 200851, "majf" : 0, "minf" : 16, "iodepth_level" : { "1" : 200.004832, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.378513, "10" : 17.352017, "20" : 4.276395, "50" : 0.317307, "100" : 0.014496, "250" : 0.144963, "500" : 71.593783, "750" : 3.984859, "1000" : 0.739309 }, "latency_ms" : { "2" : 0.905211, "4" : 0.172344, "10" : 0.099863, "20" : 0.017718, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 48197, "write_ios" : 91994, "read_merges" : 0, "write_merges" : 8824, "read_ticks" : 19002, "write_ticks" : 79551, "in_queue" : 98554, "util" : 99.353118 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 2834432 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd