master-01.priv.ocp.oraclevcn.com == START_1 [Tue May 23 18:21:57 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866177, "timestamp_ms" : 1684866177479, "time" : "Tue May 23 18:22:57 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60879, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 487000000, "io_kbytes" : 475585, "bw_bytes" : 8116531, "bw" : 7926, "iops" : 1014.566424, "runtime" : 60001, "total_ios" : 60875, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6973, "max" : 29151336, "mean" : 435665.377429, "stddev" : 420855.260798, "N" : 60875, "percentile" : { "1.000000" : 11968, "5.000000" : 313344, "10.000000" : 325632, "20.000000" : 346112, "30.000000" : 362496, "40.000000" : 378880, "50.000000" : 391168, "60.000000" : 411648, "70.000000" : 432128, "80.000000" : 460800, "90.000000" : 514048, "95.000000" : 593920, "99.000000" : 1253376, "99.500000" : 2179072, "99.900000" : 6651904, "99.950000" : 9371648, "99.990000" : 12386304 } }, "lat_ns" : { "min" : 7133, "max" : 29151497, "mean" : 435916.398094, "stddev" : 420864.436257, "N" : 60875 }, "bw_min" : 7062, "bw_max" : 8453, "bw_agg" : 99.996218, "bw_mean" : 7926.336134, "bw_dev" : 306.747930, "bw_samples" : 119, "iops_min" : 904, "iops_max" : 1082, "iops_mean" : 1014.621849, "iops_stddev" : 39.265932, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1493, "max" : 24072658, "mean" : 546289.891506, "stddev" : 295654.705508, "N" : 60879, "percentile" : { "1.000000" : 358400, "5.000000" : 428032, "10.000000" : 456704, "20.000000" : 477184, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 518144, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 618496, "95.000000" : 684032, "99.000000" : 1056768, "99.500000" : 1417216, "99.900000" : 4046848, "99.950000" : 7438336, "99.990000" : 11862016 } } }, "job_runtime" : 60000, "usr_cpu" : 0.720000, "sys_cpu" : 3.476667, "ctx" : 178956, "majf" : 0, "minf" : 17, "iodepth_level" : { "1" : 200.006571, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.364682, "20" : 1.121971, "50" : 0.067351, "100" : 0.010000, "250" : 0.000000, "500" : 86.897741, "750" : 8.891992, "1000" : 1.194251 }, "latency_ms" : { "2" : 0.895277, "4" : 0.382752, "10" : 0.141273, "20" : 0.034497, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59833, "write_ios" : 63352, "read_merges" : 0, "write_merges" : 66, "read_ticks" : 25052, "write_ticks" : 33973, "in_queue" : 59024, "util" : 99.486487 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1056768 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:22:57 UTC 2023 == START_2 [Tue May 23 18:23:57 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866298, "timestamp_ms" : 1684866298107, "time" : "Tue May 23 18:24:58 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60974, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 487768000, "io_kbytes" : 476335, "bw_bytes" : 8129331, "bw" : 7938, "iops" : 1016.166397, "runtime" : 60001, "total_ios" : 60971, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6693, "max" : 13178949, "mean" : 436272.442784, "stddev" : 373479.452762, "N" : 60971, "percentile" : { "1.000000" : 11712, "5.000000" : 313344, "10.000000" : 325632, "20.000000" : 346112, "30.000000" : 362496, "40.000000" : 374784, "50.000000" : 391168, "60.000000" : 407552, "70.000000" : 428032, "80.000000" : 456704, "90.000000" : 509952, "95.000000" : 602112, "99.000000" : 1449984, "99.500000" : 2244608, "99.900000" : 5996544, "99.950000" : 9109504, "99.990000" : 12255232 } }, "lat_ns" : { "min" : 6924, "max" : 13179160, "mean" : 436531.031769, "stddev" : 373486.506258, "N" : 60971 }, "bw_min" : 6687, "bw_max" : 8625, "bw_agg" : 100.000000, "bw_mean" : 7943.630252, "bw_dev" : 318.626205, "bw_samples" : 119, "iops_min" : 856, "iops_max" : 1104, "iops_mean" : 1016.840336, "iops_stddev" : 40.782595, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1222, "max" : 19709848, "mean" : 544010.910109, "stddev" : 306303.677229, "N" : 60974, "percentile" : { "1.000000" : 358400, "5.000000" : 428032, "10.000000" : 456704, "20.000000" : 477184, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 518144, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 618496, "95.000000" : 675840, "99.000000" : 970752, "99.500000" : 1269760, "99.900000" : 3784704, "99.950000" : 7372800, "99.990000" : 15794176 } } }, "job_runtime" : 60000, "usr_cpu" : 0.670000, "sys_cpu" : 3.558333, "ctx" : 179736, "majf" : 0, "minf" : 14, "iodepth_level" : { "1" : 200.004920, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.346066, "20" : 1.133326, "50" : 0.075446, "100" : 0.010000, "250" : 0.000000, "500" : 87.328402, "750" : 7.993964, "1000" : 1.243214 }, "latency_ms" : { "2" : 1.302258, "4" : 0.403471, "10" : 0.136130, "20" : 0.036083, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59935, "write_ios" : 63653, "read_merges" : 0, "write_merges" : 79, "read_ticks" : 25180, "write_ticks" : 34281, "in_queue" : 59461, "util" : 99.574844 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 970752 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:24:58 UTC 2023 == START_3 [Tue May 23 18:25:58 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866418, "timestamp_ms" : 1684866418740, "time" : "Tue May 23 18:26:58 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60545, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 484336000, "io_kbytes" : 472984, "bw_bytes" : 8072132, "bw" : 7882, "iops" : 1009.016516, "runtime" : 60001, "total_ios" : 60542, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6983, "max" : 32098340, "mean" : 435809.675102, "stddev" : 403669.464058, "N" : 60542, "percentile" : { "1.000000" : 11840, "5.000000" : 313344, "10.000000" : 329728, "20.000000" : 350208, "30.000000" : 362496, "40.000000" : 378880, "50.000000" : 395264, "60.000000" : 411648, "70.000000" : 432128, "80.000000" : 460800, "90.000000" : 509952, "95.000000" : 602112, "99.000000" : 1236992, "99.500000" : 2113536, "99.900000" : 6127616, "99.950000" : 9502720, "99.990000" : 14221312 } }, "lat_ns" : { "min" : 7113, "max" : 32098690, "mean" : 436053.697070, "stddev" : 403676.244980, "N" : 60542 }, "bw_min" : 6562, "bw_max" : 8484, "bw_agg" : 100.000000, "bw_mean" : 7886.546218, "bw_dev" : 331.789977, "bw_samples" : 119, "iops_min" : 840, "iops_max" : 1086, "iops_mean" : 1009.529412, "iops_stddev" : 42.470892, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1454, "max" : 41997187, "mean" : 551351.625353, "stddev" : 397593.671687, "N" : 60545, "percentile" : { "1.000000" : 358400, "5.000000" : 432128, "10.000000" : 456704, "20.000000" : 481280, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 626688, "95.000000" : 684032, "99.000000" : 1028096, "99.500000" : 1335296, "99.900000" : 5406720, "99.950000" : 8847360, "99.990000" : 14745600 } } }, "job_runtime" : 60000, "usr_cpu" : 0.776667, "sys_cpu" : 3.396667, "ctx" : 178335, "majf" : 0, "minf" : 17, "iodepth_level" : { "1" : 200.004955, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.391464, "20" : 1.076938, "50" : 0.082587, "100" : 0.010000, "250" : 0.010000, "500" : 87.023884, "750" : 8.712960, "1000" : 1.280103 }, "latency_ms" : { "2" : 0.895246, "4" : 0.355125, "10" : 0.138747, "20" : 0.034687, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59513, "write_ios" : 63149, "read_merges" : 0, "write_merges" : 72, "read_ticks" : 24947, "write_ticks" : 34205, "in_queue" : 59152, "util" : 99.451511 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1028096 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:26:58 UTC 2023 == START_4 [Tue May 23 18:27:58 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866539, "timestamp_ms" : 1684866539348, "time" : "Tue May 23 18:28:59 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 61045, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 488336000, "io_kbytes" : 476890, "bw_bytes" : 8138797, "bw" : 7948, "iops" : 1017.349711, "runtime" : 60001, "total_ios" : 61042, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 7034, "max" : 23781056, "mean" : 433537.761459, "stddev" : 379472.735924, "N" : 61042, "percentile" : { "1.000000" : 11712, "5.000000" : 309248, "10.000000" : 325632, "20.000000" : 346112, "30.000000" : 358400, "40.000000" : 374784, "50.000000" : 387072, "60.000000" : 407552, "70.000000" : 428032, "80.000000" : 456704, "90.000000" : 509952, "95.000000" : 593920, "99.000000" : 1449984, "99.500000" : 2179072, "99.900000" : 6324224, "99.950000" : 8290304, "99.990000" : 12386304 } }, "lat_ns" : { "min" : 7214, "max" : 23781316, "mean" : 433795.267603, "stddev" : 379475.497695, "N" : 61042 }, "bw_min" : 6390, "bw_max" : 8562, "bw_agg" : 100.000000, "bw_mean" : 7950.722689, "bw_dev" : 386.556701, "bw_samples" : 119, "iops_min" : 818, "iops_max" : 1096, "iops_mean" : 1017.747899, "iops_stddev" : 49.481073, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1333, "max" : 31105934, "mean" : 545662.265656, "stddev" : 349904.360950, "N" : 61045, "percentile" : { "1.000000" : 354304, "5.000000" : 428032, "10.000000" : 456704, "20.000000" : 477184, "30.000000" : 489472, "40.000000" : 501760, "50.000000" : 514048, "60.000000" : 528384, "70.000000" : 544768, "80.000000" : 569344, "90.000000" : 618496, "95.000000" : 684032, "99.000000" : 1122304, "99.500000" : 1515520, "99.900000" : 4079616, "99.950000" : 7241728, "99.990000" : 13565952 } } }, "job_runtime" : 60000, "usr_cpu" : 0.686667, "sys_cpu" : 3.496667, "ctx" : 179919, "majf" : 0, "minf" : 18, "iodepth_level" : { "1" : 200.004915, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.399725, "20" : 1.082861, "50" : 0.070443, "100" : 0.010000, "250" : 0.010000, "500" : 87.408669, "750" : 8.169785, "1000" : 1.115625 }, "latency_ms" : { "2" : 1.172963, "4" : 0.411192, "10" : 0.135972, "20" : 0.027850, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59991, "write_ios" : 63498, "read_merges" : 0, "write_merges" : 78, "read_ticks" : 25021, "write_ticks" : 34008, "in_queue" : 59029, "util" : 99.448122 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1122304 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:28:59 UTC 2023 == START_5 [Tue May 23 18:29:59 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866659, "timestamp_ms" : 1684866659965, "time" : "Tue May 23 18:30:59 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60391, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 483096000, "io_kbytes" : 471773, "bw_bytes" : 8051063, "bw" : 7862, "iops" : 1006.382908, "runtime" : 60004, "total_ios" : 60387, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6993, "max" : 17071668, "mean" : 438455.157136, "stddev" : 400332.486343, "N" : 60387, "percentile" : { "1.000000" : 12096, "5.000000" : 313344, "10.000000" : 329728, "20.000000" : 346112, "30.000000" : 362496, "40.000000" : 378880, "50.000000" : 391168, "60.000000" : 411648, "70.000000" : 432128, "80.000000" : 460800, "90.000000" : 514048, "95.000000" : 593920, "99.000000" : 1433600, "99.500000" : 2605056, "99.900000" : 6389760, "99.950000" : 9240576, "99.990000" : 13434880 } }, "lat_ns" : { "min" : 7154, "max" : 17071989, "mean" : 438723.055492, "stddev" : 400340.333070, "N" : 60387 }, "bw_min" : 6609, "bw_max" : 8593, "bw_agg" : 100.000000, "bw_mean" : 7867.991597, "bw_dev" : 385.884994, "bw_samples" : 119, "iops_min" : 846, "iops_max" : 1100, "iops_mean" : 1007.159664, "iops_stddev" : 49.395324, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1853, "max" : 23138424, "mean" : 551380.560845, "stddev" : 347084.228570, "N" : 60391, "percentile" : { "1.000000" : 358400, "5.000000" : 432128, "10.000000" : 456704, "20.000000" : 477184, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 626688, "95.000000" : 692224, "99.000000" : 1019904, "99.500000" : 1335296, "99.900000" : 5013504, "99.950000" : 7700480, "99.990000" : 14614528 } } }, "job_runtime" : 60003, "usr_cpu" : 0.778294, "sys_cpu" : 3.406496, "ctx" : 176762, "majf" : 0, "minf" : 15, "iodepth_level" : { "1" : 200.006624, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.356037, "20" : 1.092950, "50" : 0.096047, "100" : 0.010000, "250" : 0.010000, "500" : 86.846507, "750" : 8.854555, "1000" : 1.131038 }, "latency_ms" : { "2" : 0.912448, "4" : 0.501764, "10" : 0.162287, "20" : 0.034776, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59354, "write_ios" : 63409, "read_merges" : 0, "write_merges" : 70, "read_ticks" : 25056, "write_ticks" : 34399, "in_queue" : 59455, "util" : 99.474947 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1019904 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:31:00 UTC 2023 == START_6 [Tue May 23 18:32:00 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866780, "timestamp_ms" : 1684866780594, "time" : "Tue May 23 18:33:00 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60479, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 483800000, "io_kbytes" : 472460, "bw_bytes" : 8063198, "bw" : 7874, "iops" : 1007.899868, "runtime" : 60001, "total_ios" : 60475, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6832, "max" : 28239807, "mean" : 439714.489905, "stddev" : 417627.464140, "N" : 60475, "percentile" : { "1.000000" : 11840, "5.000000" : 313344, "10.000000" : 329728, "20.000000" : 350208, "30.000000" : 366592, "40.000000" : 378880, "50.000000" : 395264, "60.000000" : 411648, "70.000000" : 432128, "80.000000" : 460800, "90.000000" : 514048, "95.000000" : 602112, "99.000000" : 1368064, "99.500000" : 2408448, "99.900000" : 6324224, "99.950000" : 8847360, "99.990000" : 15269888 } }, "lat_ns" : { "min" : 6983, "max" : 28239968, "mean" : 439964.509996, "stddev" : 417630.752449, "N" : 60475 }, "bw_min" : 6687, "bw_max" : 8546, "bw_agg" : 100.000000, "bw_mean" : 7877.159664, "bw_dev" : 349.756331, "bw_samples" : 119, "iops_min" : 856, "iops_max" : 1094, "iops_mean" : 1008.336134, "iops_stddev" : 44.767058, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1733, "max" : 25027594, "mean" : 548642.745697, "stddev" : 304513.342297, "N" : 60479, "percentile" : { "1.000000" : 358400, "5.000000" : 432128, "10.000000" : 456704, "20.000000" : 477184, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 626688, "95.000000" : 692224, "99.000000" : 1011712, "99.500000" : 1335296, "99.900000" : 4358144, "99.950000" : 7372800, "99.990000" : 12124160 } } }, "job_runtime" : 60000, "usr_cpu" : 0.768333, "sys_cpu" : 3.380000, "ctx" : 178454, "majf" : 0, "minf" : 14, "iodepth_level" : { "1" : 200.006614, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.355519, "20" : 1.117817, "50" : 0.079372, "100" : 0.010000, "250" : 0.010000, "500" : 86.667218, "750" : 8.921042, "1000" : 1.231914 }, "latency_ms" : { "2" : 1.010335, "4" : 0.448119, "10" : 0.130632, "20" : 0.029764, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59442, "write_ios" : 63030, "read_merges" : 0, "write_merges" : 75, "read_ticks" : 25164, "write_ticks" : 33874, "in_queue" : 59038, "util" : 99.481554 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1011712 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:33:00 UTC 2023 == START_7 [Tue May 23 18:34:00 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684866901, "timestamp_ms" : 1684866901212, "time" : "Tue May 23 18:35:01 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60776, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 486176000, "io_kbytes" : 474781, "bw_bytes" : 8102798, "bw" : 7912, "iops" : 1012.849786, "runtime" : 60001, "total_ios" : 60772, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 7034, "max" : 19482436, "mean" : 435120.563779, "stddev" : 356986.386993, "N" : 60772, "percentile" : { "1.000000" : 11712, "5.000000" : 317440, "10.000000" : 329728, "20.000000" : 350208, "30.000000" : 366592, "40.000000" : 378880, "50.000000" : 395264, "60.000000" : 411648, "70.000000" : 432128, "80.000000" : 460800, "90.000000" : 514048, "95.000000" : 602112, "99.000000" : 1220608, "99.500000" : 2211840, "99.900000" : 5210112, "99.950000" : 7503872, "99.990000" : 12648448 } }, "lat_ns" : { "min" : 7165, "max" : 19482666, "mean" : 435372.755200, "stddev" : 356995.154329, "N" : 60772 }, "bw_min" : 7015, "bw_max" : 8609, "bw_agg" : 100.000000, "bw_mean" : 7919.193277, "bw_dev" : 349.157953, "bw_samples" : 119, "iops_min" : 898, "iops_max" : 1102, "iops_mean" : 1013.714286, "iops_stddev" : 44.692005, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1493, "max" : 22811482, "mean" : 548439.110965, "stddev" : 338003.675235, "N" : 60776, "percentile" : { "1.000000" : 358400, "5.000000" : 432128, "10.000000" : 456704, "20.000000" : 481280, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 626688, "95.000000" : 684032, "99.000000" : 987136, "99.500000" : 1286144, "99.900000" : 4145152, "99.950000" : 7700480, "99.990000" : 13434880 } } }, "job_runtime" : 60000, "usr_cpu" : 0.703333, "sys_cpu" : 3.536667, "ctx" : 179082, "majf" : 0, "minf" : 17, "iodepth_level" : { "1" : 200.006582, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.386691, "20" : 1.076153, "50" : 0.087211, "100" : 0.010000, "250" : 0.000000, "500" : 86.623774, "750" : 9.178569, "1000" : 1.169947 }, "latency_ms" : { "2" : 0.923122, "4" : 0.404792, "10" : 0.115185, "20" : 0.027973, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59720, "write_ios" : 63559, "read_merges" : 0, "write_merges" : 70, "read_ticks" : 25021, "write_ticks" : 34530, "in_queue" : 59552, "util" : 99.554815 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 987136 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:35:01 UTC 2023 == START_8 [Tue May 23 18:36:01 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684867021, "timestamp_ms" : 1684867021823, "time" : "Tue May 23 18:37:01 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 60435, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 483448000, "io_kbytes" : 472117, "bw_bytes" : 8057332, "bw" : 7868, "iops" : 1007.166547, "runtime" : 60001, "total_ios" : 60431, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6682, "max" : 202924345, "mean" : 440188.598964, "stddev" : 912933.320051, "N" : 60431, "percentile" : { "1.000000" : 11712, "5.000000" : 313344, "10.000000" : 329728, "20.000000" : 350208, "30.000000" : 366592, "40.000000" : 378880, "50.000000" : 395264, "60.000000" : 411648, "70.000000" : 432128, "80.000000" : 460800, "90.000000" : 509952, "95.000000" : 585728, "99.000000" : 1236992, "99.500000" : 2244608, "99.900000" : 6651904, "99.950000" : 9240576, "99.990000" : 12779520 } }, "lat_ns" : { "min" : 6793, "max" : 202924545, "mean" : 440449.188777, "stddev" : 912939.382305, "N" : 60431 }, "bw_min" : 5234, "bw_max" : 8531, "bw_agg" : 99.993794, "bw_mean" : 7868.420168, "bw_dev" : 410.356387, "bw_samples" : 119, "iops_min" : 670, "iops_max" : 1092, "iops_mean" : 1007.201681, "iops_stddev" : 52.519339, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1953, "max" : 30245778, "mean" : 548976.532738, "stddev" : 336811.111375, "N" : 60435, "percentile" : { "1.000000" : 362496, "5.000000" : 428032, "10.000000" : 460800, "20.000000" : 481280, "30.000000" : 493568, "40.000000" : 509952, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 626688, "95.000000" : 684032, "99.000000" : 995328, "99.500000" : 1269760, "99.900000" : 3948544, "99.950000" : 7503872, "99.990000" : 13172736 } } }, "job_runtime" : 60000, "usr_cpu" : 0.703333, "sys_cpu" : 3.475000, "ctx" : 177578, "majf" : 0, "minf" : 15, "iodepth_level" : { "1" : 200.006619, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.388873, "20" : 1.088845, "50" : 0.072810, "100" : 0.010000, "250" : 0.010000, "500" : 86.832917, "750" : 9.043372, "1000" : 1.102083 }, "latency_ms" : { "2" : 0.903510, "4" : 0.382254, "10" : 0.143966, "20" : 0.029786, "50" : 0.010000, "100" : 0.000000, "250" : 0.010000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 59417, "write_ios" : 63094, "read_merges" : 0, "write_merges" : 70, "read_ticks" : 25145, "write_ticks" : 34225, "in_queue" : 59371, "util" : 99.276715 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 995328 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:37:02 UTC 2023 == START_9 [Tue May 23 18:38:02 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684867142, "timestamp_ms" : 1684867142437, "time" : "Tue May 23 18:39:02 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 57750, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 461976000, "io_kbytes" : 451148, "bw_bytes" : 7699471, "bw" : 7519, "iops" : 962.433959, "runtime" : 60001, "total_ios" : 57747, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6854, "max" : 22887558, "mean" : 483106.235094, "stddev" : 635528.319567, "N" : 57747, "percentile" : { "1.000000" : 11840, "5.000000" : 313344, "10.000000" : 329728, "20.000000" : 350208, "30.000000" : 366592, "40.000000" : 378880, "50.000000" : 395264, "60.000000" : 415744, "70.000000" : 436224, "80.000000" : 468992, "90.000000" : 528384, "95.000000" : 684032, "99.000000" : 2670592, "99.500000" : 4685824, "99.900000" : 9895936, "99.950000" : 11075584, "99.990000" : 14614528 } }, "lat_ns" : { "min" : 6963, "max" : 22887749, "mean" : 483336.457357, "stddev" : 635534.151056, "N" : 57747 }, "bw_min" : 5328, "bw_max" : 8671, "bw_agg" : 100.000000, "bw_mean" : 7528.697479, "bw_dev" : 725.879648, "bw_samples" : 119, "iops_min" : 682, "iops_max" : 1110, "iops_mean" : 963.731092, "iops_stddev" : 92.912178, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1423, "max" : 27303608, "mean" : 552165.251100, "stddev" : 377909.930589, "N" : 57750, "percentile" : { "1.000000" : 358400, "5.000000" : 432128, "10.000000" : 456704, "20.000000" : 477184, "30.000000" : 493568, "40.000000" : 505856, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 552960, "80.000000" : 577536, "90.000000" : 626688, "95.000000" : 692224, "99.000000" : 1073152, "99.500000" : 1433600, "99.900000" : 4882432, "99.950000" : 9240576, "99.990000" : 17694720 } } }, "job_runtime" : 60000, "usr_cpu" : 0.681667, "sys_cpu" : 3.346667, "ctx" : 170522, "majf" : 0, "minf" : 15, "iodepth_level" : { "1" : 200.005195, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.387899, "20" : 1.085771, "50" : 0.076194, "100" : 0.010000, "250" : 0.010000, "500" : 84.899648, "750" : 9.254160, "1000" : 1.414792 }, "latency_ms" : { "2" : 1.466743, "4" : 0.812163, "10" : 0.502191, "20" : 0.091780, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 56766, "write_ios" : 60343, "read_merges" : 0, "write_merges" : 67, "read_ticks" : 26519, "write_ticks" : 32761, "in_queue" : 59281, "util" : 99.333067 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1073152 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:39:02 UTC 2023 == START_10 [Tue May 23 18:40:02 UTC 2023] == ---------------------------------------------------------------- Running fio --------------------------------------------------------------------------- { "fio version" : "fio-3.27", "timestamp" : 1684867263, "timestamp_ms" : 1684867263062, "time" : "Tue May 23 18:41:03 2023", "global options" : { "rw" : "write", "ioengine" : "sync", "fdatasync" : "1", "directory" : "/var/lib/etcd", "size" : "100m", "bs" : "8000" }, "jobs" : [ { "jobname" : "etcd_perf", "groupid" : 0, "error" : 0, "eta" : 0, "elapsed" : 61, "job options" : { "name" : "etcd_perf", "runtime" : "60", "time_based" : "1" }, "read" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 59350, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "write" : { "io_bytes" : 474776000, "io_kbytes" : 463648, "bw_bytes" : 7912801, "bw" : 7727, "iops" : 989.100182, "runtime" : 60001, "total_ios" : 59347, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 6773, "max" : 32196735, "mean" : 451663.865503, "stddev" : 476031.918127, "N" : 59347, "percentile" : { "1.000000" : 11968, "5.000000" : 313344, "10.000000" : 329728, "20.000000" : 350208, "30.000000" : 366592, "40.000000" : 382976, "50.000000" : 399360, "60.000000" : 415744, "70.000000" : 436224, "80.000000" : 468992, "90.000000" : 528384, "95.000000" : 643072, "99.000000" : 1581056, "99.500000" : 2605056, "99.900000" : 6979584, "99.950000" : 10289152, "99.990000" : 16318464 } }, "lat_ns" : { "min" : 6903, "max" : 32197396, "mean" : 451910.291927, "stddev" : 476041.168926, "N" : 59347 }, "bw_min" : 6580, "bw_max" : 8531, "bw_agg" : 100.000000, "bw_mean" : 7728.008403, "bw_dev" : 356.698812, "bw_samples" : 119, "iops_min" : 842, "iops_max" : 1092, "iops_mean" : 989.243697, "iops_stddev" : 45.661980, "iops_samples" : 119 }, "trim" : { "io_bytes" : 0, "io_kbytes" : 0, "bw_bytes" : 0, "bw" : 0, "iops" : 0.000000, "runtime" : 0, "total_ios" : 0, "short_ios" : 0, "drop_ios" : 0, "slat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "clat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "lat_ns" : { "min" : 0, "max" : 0, "mean" : 0.000000, "stddev" : 0.000000, "N" : 0 }, "bw_min" : 0, "bw_max" : 0, "bw_agg" : 0.000000, "bw_mean" : 0.000000, "bw_dev" : 0.000000, "bw_samples" : 0, "iops_min" : 0, "iops_max" : 0, "iops_mean" : 0.000000, "iops_stddev" : 0.000000, "iops_samples" : 0 }, "sync" : { "total_ios" : 0, "lat_ns" : { "min" : 1624, "max" : 26298889, "mean" : 555742.289082, "stddev" : 361697.868476, "N" : 59350, "percentile" : { "1.000000" : 358400, "5.000000" : 432128, "10.000000" : 460800, "20.000000" : 481280, "30.000000" : 497664, "40.000000" : 509952, "50.000000" : 522240, "60.000000" : 536576, "70.000000" : 561152, "80.000000" : 585728, "90.000000" : 634880, "95.000000" : 708608, "99.000000" : 1056768, "99.500000" : 1449984, "99.900000" : 4685824, "99.950000" : 8978432, "99.990000" : 13828096 } } }, "job_runtime" : 60000, "usr_cpu" : 0.696667, "sys_cpu" : 3.381667, "ctx" : 174134, "majf" : 0, "minf" : 15, "iodepth_level" : { "1" : 200.005055, "2" : 0.000000, "4" : 0.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, ">=64" : 0.000000 }, "iodepth_submit" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "iodepth_complete" : { "0" : 0.000000, "4" : 100.000000, "8" : 0.000000, "16" : 0.000000, "32" : 0.000000, "64" : 0.000000, ">=64" : 0.000000 }, "latency_ns" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.000000, "20" : 0.000000, "50" : 0.000000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000 }, "latency_us" : { "2" : 0.000000, "4" : 0.000000, "10" : 0.325206, "20" : 1.150859, "50" : 0.075825, "100" : 0.010000, "250" : 0.000000, "500" : 85.060744, "750" : 9.813470, "1000" : 1.570425 }, "latency_ms" : { "2" : 1.292399, "4" : 0.473486, "10" : 0.180296, "20" : 0.047180, "50" : 0.010000, "100" : 0.000000, "250" : 0.000000, "500" : 0.000000, "750" : 0.000000, "1000" : 0.000000, "2000" : 0.000000, ">=2000" : 0.000000 }, "latency_depth" : 1, "latency_target" : 0, "latency_percentile" : 100.000000, "latency_window" : 0 } ], "disk_util" : [ { "name" : "sda", "read_ios" : 58333, "write_ios" : 61909, "read_merges" : 0, "write_merges" : 70, "read_ticks" : 25387, "write_ticks" : 33793, "in_queue" : 59179, "util" : 99.348225 } ] } -------------------------------------------------------------------------------------------------------------------------------------------------------- INFO: 99th percentile of fsync is 1056768 ns INFO: 99th percentile of the fsync is within the recommended threshold: - 20 ms, the disk can be used to host etcd Tue May 23 18:41:03 UTC 2023