randread_4K: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randread_8K: (g=1): rw=randread, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randread_32K: (g=2): rw=randread, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randread_64K: (g=3): rw=randread, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randread_128K: (g=4): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randwrite_4K: (g=5): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randwrite_8K: (g=6): rw=randwrite, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randwrite_32K: (g=7): rw=randwrite, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randwrite_64K: (g=8): rw=randwrite, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randwrite_128K: (g=9): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_4K: (g=10): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqread_8K: (g=11): rw=read, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqread_32K: (g=12): rw=read, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqread_64K: (g=13): rw=read, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqread_128K: (g=14): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_512K: (g=15): rw=read, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqread_1M: (g=16): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... seqwrite_4K: (g=17): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqwrite_8K: (g=18): rw=write, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqwrite_32K: (g=19): rw=write, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_64K: (g=20): rw=write, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_128K: (g=21): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqwrite_512K: (g=22): rw=write, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqwrite_1M: (g=23): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... randrw_4K: (g=24): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_8K: (g=25): rw=randrw, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randrw_32K: (g=26): rw=randrw, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randrw_64K: (g=27): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randrw_128K: (g=28): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randread_4K_latency: (g=29): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_4K_latency: (g=30): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randwrite_4K_latency: (g=31): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_70R30W_4K: (g=32): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_70R30W_64K: (g=33): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... fio-3.25 Starting 127 processes randread_4K: (groupid=0, jobs=4): err= 0: pid=2403: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=209, BW=836KiB/s (856kB/s)(147MiB/180370msec) slat (usec): min=2, max=324186, avg=18715.95, stdev=50023.70 clat (msec): min=10, max=1748, avg=592.14, stdev=192.52 lat (msec): min=10, max=1748, avg=610.86, stdev=200.59 clat percentiles (msec): | 1.00th=[ 169], 5.00th=[ 305], 10.00th=[ 418], 20.00th=[ 447], | 30.00th=[ 464], 40.00th=[ 498], 50.00th=[ 592], 60.00th=[ 617], | 70.00th=[ 651], 80.00th=[ 751], 90.00th=[ 852], 95.00th=[ 936], | 99.00th=[ 1133], 99.50th=[ 1234], 99.90th=[ 1401], 99.95th=[ 1452], | 99.99th=[ 1569] bw ( KiB/s): min= 368, max= 1297, per=100.00%, avg=837.23, stdev=46.55, samples=1436 iops : min= 92, max= 324, avg=209.30, stdev=11.63, samples=1436 lat (msec) : 20=0.01%, 50=0.06%, 100=0.28%, 250=1.58%, 500=38.66% lat (msec) : 750=39.03%, 1000=17.13%, 2000=3.25% cpu : usr=0.08%, sys=0.07%, ctx=6317, majf=0, minf=180 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=37708,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_8K: (groupid=1, jobs=4): err= 0: pid=2420: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=208, BW=1671KiB/s (1711kB/s)(294MiB/180392msec) slat (usec): min=2, max=212317, avg=19106.83, stdev=50821.20 clat (msec): min=122, max=1492, avg=592.90, stdev=169.92 lat (msec): min=122, max=1542, avg=612.00, stdev=177.23 clat percentiles (msec): | 1.00th=[ 288], 5.00th=[ 334], 10.00th=[ 430], 20.00th=[ 451], | 30.00th=[ 468], 40.00th=[ 498], 50.00th=[ 592], 60.00th=[ 617], | 70.00th=[ 642], 80.00th=[ 743], 90.00th=[ 802], 95.00th=[ 919], | 99.00th=[ 1083], 99.50th=[ 1150], 99.90th=[ 1318], 99.95th=[ 1368], | 99.99th=[ 1418] bw ( KiB/s): min= 948, max= 2052, per=99.93%, avg=1670.02, stdev=57.97, samples=1439 iops : min= 118, max= 256, avg=208.72, stdev= 7.24, samples=1439 lat (msec) : 250=0.05%, 500=40.48%, 750=40.33%, 1000=17.13%, 2000=2.02% cpu : usr=0.05%, sys=0.06%, ctx=4766, majf=0, minf=311 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=37684,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_32K: (groupid=2, jobs=4): err= 0: pid=2428: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=202, BW=6474KiB/s (6629kB/s)(1140MiB/180346msec) slat (usec): min=5, max=218370, avg=19736.75, stdev=52515.65 clat (msec): min=185, max=1574, avg=612.51, stdev=174.81 lat (msec): min=221, max=1574, avg=632.24, stdev=182.58 clat percentiles (msec): | 1.00th=[ 296], 5.00th=[ 342], 10.00th=[ 443], 20.00th=[ 468], | 30.00th=[ 485], 40.00th=[ 514], 50.00th=[ 609], 60.00th=[ 634], | 70.00th=[ 659], 80.00th=[ 768], 90.00th=[ 827], 95.00th=[ 944], | 99.00th=[ 1116], 99.50th=[ 1183], 99.90th=[ 1334], 99.95th=[ 1368], | 99.99th=[ 1452] bw ( KiB/s): min= 2862, max= 8208, per=99.85%, avg=6464.03, stdev=210.16, samples=1440 iops : min= 88, max= 256, avg=201.91, stdev= 6.57, samples=1440 lat (msec) : 250=0.01%, 500=36.48%, 750=40.94%, 1000=20.19%, 2000=2.38% cpu : usr=0.03%, sys=0.07%, ctx=4628, majf=0, minf=1075 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=36484,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_64K: (groupid=3, jobs=4): err= 0: pid=2432: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=195, BW=12.2MiB/s (12.8MB/s)(2207MiB/180442msec) slat (usec): min=3, max=310892, avg=20392.49, stdev=54282.32 clat (msec): min=170, max=1633, avg=632.96, stdev=181.77 lat (msec): min=170, max=1680, avg=653.35, stdev=189.70 clat percentiles (msec): | 1.00th=[ 305], 5.00th=[ 355], 10.00th=[ 460], 20.00th=[ 481], | 30.00th=[ 502], 40.00th=[ 531], 50.00th=[ 634], 60.00th=[ 659], | 70.00th=[ 684], 80.00th=[ 793], 90.00th=[ 860], 95.00th=[ 978], | 99.00th=[ 1150], 99.50th=[ 1217], 99.90th=[ 1351], 99.95th=[ 1418], | 99.99th=[ 1586] bw ( KiB/s): min= 5784, max=16416, per=99.86%, avg=12510.17, stdev=396.34, samples=1440 iops : min= 90, max= 256, avg=195.39, stdev= 6.20, samples=1440 lat (msec) : 250=0.02%, 500=30.29%, 750=45.30%, 1000=20.73%, 2000=3.66% cpu : usr=0.03%, sys=0.06%, ctx=4486, majf=0, minf=2102 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=35319,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_128K: (groupid=4, jobs=4): err= 0: pid=2442: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=184, BW=23.0MiB/s (24.2MB/s)(4158MiB/180415msec) slat (usec): min=4, max=237057, avg=21656.93, stdev=57599.93 clat (msec): min=208, max=1692, avg=672.08, stdev=190.98 lat (msec): min=223, max=1722, avg=693.74, stdev=199.32 clat percentiles (msec): | 1.00th=[ 326], 5.00th=[ 376], 10.00th=[ 489], 20.00th=[ 514], | 30.00th=[ 531], 40.00th=[ 567], 50.00th=[ 676], 60.00th=[ 701], | 70.00th=[ 726], 80.00th=[ 852], 90.00th=[ 911], 95.00th=[ 1036], | 99.00th=[ 1217], 99.50th=[ 1267], 99.90th=[ 1418], 99.95th=[ 1502], | 99.99th=[ 1603] bw ( KiB/s): min=11458, max=32816, per=99.84%, avg=23559.51, stdev=763.69, samples=1440 iops : min= 88, max= 256, avg=183.98, stdev= 5.97, samples=1440 lat (msec) : 250=0.02%, 500=13.68%, 750=60.51%, 1000=18.69%, 2000=7.11% cpu : usr=0.03%, sys=0.07%, ctx=4222, majf=0, minf=2104 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=33260,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_4K: (groupid=5, jobs=4): err= 0: pid=2450: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=550, BW=2201KiB/s (2254kB/s)(387MiB/180108msec); 0 zone resets slat (usec): min=2, max=276261, avg=7257.05, stdev=21946.00 clat (msec): min=4, max=1002, avg=225.20, stdev=114.78 lat (msec): min=4, max=1060, avg=232.46, stdev=119.44 clat percentiles (msec): | 1.00th=[ 16], 5.00th=[ 22], 10.00th=[ 27], 20.00th=[ 180], | 30.00th=[ 199], 40.00th=[ 232], 50.00th=[ 251], 60.00th=[ 259], | 70.00th=[ 271], 80.00th=[ 284], 90.00th=[ 334], 95.00th=[ 401], | 99.00th=[ 535], 99.50th=[ 584], 99.90th=[ 709], 99.95th=[ 768], | 99.99th=[ 877] bw ( KiB/s): min= 704, max=20970, per=99.76%, avg=2196.57, stdev=620.84, samples=1440 iops : min= 176, max= 5242, avg=549.03, stdev=155.20, samples=1440 lat (msec) : 10=0.01%, 20=2.78%, 50=14.68%, 100=0.09%, 250=31.94% lat (msec) : 500=48.72%, 750=1.71%, 1000=0.06%, 2000=0.01% cpu : usr=0.13%, sys=0.16%, ctx=12502, majf=0, minf=60 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,99116,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_8K: (groupid=6, jobs=4): err= 0: pid=2527: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=383, BW=3067KiB/s (3140kB/s)(539MiB/180129msec); 0 zone resets slat (usec): min=2, max=390605, avg=10421.82, stdev=29545.66 clat (msec): min=4, max=1420, avg=323.34, stdev=111.54 lat (msec): min=4, max=1420, avg=333.77, stdev=116.93 clat percentiles (msec): | 1.00th=[ 190], 5.00th=[ 203], 10.00th=[ 213], 20.00th=[ 247], | 30.00th=[ 271], 40.00th=[ 284], 50.00th=[ 292], 60.00th=[ 305], | 70.00th=[ 338], 80.00th=[ 388], 90.00th=[ 477], 95.00th=[ 542], | 99.00th=[ 718], 99.50th=[ 802], 99.90th=[ 969], 99.95th=[ 1062], | 99.99th=[ 1267] bw ( KiB/s): min= 672, max= 4104, per=99.88%, avg=3063.56, stdev=188.28, samples=1440 iops : min= 84, max= 512, avg=382.83, stdev=23.53, samples=1440 lat (msec) : 10=0.02%, 20=0.01%, 50=0.06%, 100=0.02%, 250=20.59% lat (msec) : 500=71.20%, 750=7.39%, 1000=0.64%, 2000=0.08% cpu : usr=0.07%, sys=0.10%, ctx=8702, majf=0, minf=52 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,69052,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_32K: (groupid=7, jobs=4): err= 0: pid=2532: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=314, BW=9.81MiB/s (10.3MB/s)(1769MiB/180250msec); 0 zone resets slat (usec): min=4, max=588627, avg=12718.24, stdev=37229.12 clat (msec): min=25, max=2141, avg=394.73, stdev=167.29 lat (msec): min=25, max=2418, avg=407.45, stdev=174.58 clat percentiles (msec): | 1.00th=[ 226], 5.00th=[ 241], 10.00th=[ 255], 20.00th=[ 296], | 30.00th=[ 317], 40.00th=[ 334], 50.00th=[ 351], 60.00th=[ 368], | 70.00th=[ 397], 80.00th=[ 443], 90.00th=[ 592], 95.00th=[ 735], | 99.00th=[ 1099], 99.50th=[ 1217], 99.90th=[ 1435], 99.95th=[ 1603], | 99.99th=[ 1821] bw ( KiB/s): min= 1686, max=14364, per=100.00%, avg=10050.12, stdev=740.87, samples=1439 iops : min= 52, max= 448, avg=313.93, stdev=23.15, samples=1439 lat (msec) : 50=0.01%, 100=0.01%, 250=8.50%, 500=77.20%, 750=9.67% lat (msec) : 1000=3.11%, 2000=1.50%, >=2000=0.01% cpu : usr=0.06%, sys=0.09%, ctx=7134, majf=0, minf=55 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,56604,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_64K: (groupid=8, jobs=4): err= 0: pid=2540: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=288, BW=18.0MiB/s (18.9MB/s)(3248MiB/180170msec); 0 zone resets slat (usec): min=6, max=422155, avg=13854.28, stdev=38832.37 clat (msec): min=21, max=1496, avg=429.88, stdev=131.68 lat (msec): min=21, max=1496, avg=443.74, stdev=138.43 clat percentiles (msec): | 1.00th=[ 257], 5.00th=[ 275], 10.00th=[ 292], 20.00th=[ 338], | 30.00th=[ 363], 40.00th=[ 380], 50.00th=[ 401], 60.00th=[ 422], | 70.00th=[ 451], 80.00th=[ 498], 90.00th=[ 600], 95.00th=[ 709], | 99.00th=[ 894], 99.50th=[ 944], 99.90th=[ 1116], 99.95th=[ 1183], | 99.99th=[ 1284] bw ( KiB/s): min= 5120, max=24624, per=99.87%, avg=18435.83, stdev=1081.18, samples=1440 iops : min= 80, max= 384, avg=287.93, stdev=16.89, samples=1440 lat (msec) : 50=0.02%, 100=0.02%, 250=0.30%, 500=80.04%, 750=15.91% lat (msec) : 1000=3.44%, 2000=0.27% cpu : usr=0.06%, sys=0.09%, ctx=6557, majf=0, minf=55 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,51964,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_128K: (groupid=9, jobs=4): err= 0: pid=2545: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=235, BW=29.5MiB/s (30.9MB/s)(5309MiB/180217msec); 0 zone resets slat (usec): min=6, max=480055, avg=16954.86, stdev=47183.72 clat (msec): min=25, max=1633, avg=526.14, stdev=150.46 lat (msec): min=25, max=1633, avg=543.09, stdev=158.69 clat percentiles (msec): | 1.00th=[ 313], 5.00th=[ 338], 10.00th=[ 355], 20.00th=[ 418], | 30.00th=[ 447], 40.00th=[ 468], 50.00th=[ 489], 60.00th=[ 523], | 70.00th=[ 567], 80.00th=[ 634], 90.00th=[ 743], 95.00th=[ 827], | 99.00th=[ 995], 99.50th=[ 1083], 99.90th=[ 1284], 99.95th=[ 1351], | 99.99th=[ 1502] bw ( KiB/s): min= 8196, max=41040, per=99.84%, avg=30115.64, stdev=1816.82, samples=1440 iops : min= 64, max= 320, avg=235.18, stdev=14.19, samples=1440 lat (msec) : 50=0.01%, 250=0.08%, 500=53.11%, 750=37.24%, 1000=8.64% lat (msec) : 2000=0.91% cpu : usr=0.07%, sys=0.07%, ctx=5370, majf=0, minf=59 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,42468,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_4K: (groupid=10, jobs=4): err= 0: pid=2562: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=67.3k, BW=263MiB/s (276MB/s)(46.2GiB/180037msec) slat (nsec): min=1859, max=1640.4k, avg=4514.79, stdev=3243.62 clat (usec): min=54, max=2606.6k, avg=1896.17, stdev=59200.12 lat (usec): min=59, max=2606.6k, avg=1900.89, stdev=59200.14 clat percentiles (usec): | 1.00th=[ 231], 5.00th=[ 318], 10.00th=[ 375], | 20.00th=[ 404], 30.00th=[ 416], 40.00th=[ 424], | 50.00th=[ 437], 60.00th=[ 441], 70.00th=[ 449], | 80.00th=[ 457], 90.00th=[ 490], 95.00th=[ 840], | 99.00th=[ 979], 99.50th=[ 1012], 99.90th=[ 2999], | 99.95th=[2499806], 99.99th=[2533360] bw ( KiB/s): min= 48, max=1155489, per=100.00%, avg=614144.81, stdev=117752.75, samples=627 iops : min= 12, max=288872, avg=153535.91, stdev=29438.21, samples=627 lat (usec) : 100=0.01%, 250=1.75%, 500=89.97%, 750=3.07%, 1000=4.58% lat (msec) : 2=0.50%, 4=0.05%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 750=0.01%, 1000=0.01%, 2000=0.01% lat (msec) : >=2000=0.05% cpu : usr=4.25%, sys=8.45%, ctx=2415580, majf=0, minf=193 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=12114918,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_8K: (groupid=11, jobs=4): err= 0: pid=2567: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=34.4k, BW=269MiB/s (282MB/s)(47.3GiB/180037msec) slat (nsec): min=1818, max=217286, avg=5045.63, stdev=2503.12 clat (usec): min=97, max=2587.3k, avg=3709.84, stdev=83740.81 lat (usec): min=118, max=2587.3k, avg=3715.11, stdev=83740.86 clat percentiles (usec): | 1.00th=[ 693], 5.00th=[ 750], 10.00th=[ 791], | 20.00th=[ 816], 30.00th=[ 832], 40.00th=[ 840], | 50.00th=[ 857], 60.00th=[ 873], 70.00th=[ 889], | 80.00th=[ 906], 90.00th=[ 1270], 95.00th=[ 1385], | 99.00th=[ 1434], 99.50th=[ 1450], 99.90th=[2499806], | 99.95th=[2499806], 99.99th=[2499806] bw ( KiB/s): min= 320, max=1155537, per=100.00%, avg=657837.78, stdev=133124.49, samples=602 iops : min= 40, max=144442, avg=82229.42, stdev=16640.56, samples=602 lat (usec) : 100=0.01%, 250=0.01%, 500=0.04%, 750=5.02%, 1000=84.23% lat (msec) : 2=10.52%, 4=0.03%, 10=0.02%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, >=2000=0.11% cpu : usr=2.88%, sys=5.12%, ctx=1485993, majf=0, minf=325 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=6200256,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_32K: (groupid=12, jobs=4): err= 0: pid=2578: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=8627, BW=270MiB/s (283MB/s)(47.4GiB/180027msec) slat (usec): min=4, max=1653, avg= 7.96, stdev= 3.26 clat (usec): min=553, max=2531.0k, avg=14824.98, stdev=164870.19 lat (usec): min=590, max=2531.0k, avg=14833.24, stdev=164870.22 clat percentiles (msec): | 1.00th=[ 3], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 4], | 30.00th=[ 4], 40.00th=[ 4], 50.00th=[ 4], 60.00th=[ 4], | 70.00th=[ 4], 80.00th=[ 5], 90.00th=[ 5], 95.00th=[ 5], | 99.00th=[ 5], 99.50th=[ 14], 99.90th=[ 2500], 99.95th=[ 2500], | 99.99th=[ 2534] bw ( KiB/s): min= 2048, max=1156096, per=100.00%, avg=646321.04, stdev=109159.25, samples=614 iops : min= 64, max=36128, avg=20197.08, stdev=3411.22, samples=614 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.03%, 4=77.90%, 10=21.54%, 20=0.03%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 750=0.01%, 1000=0.02%, 2000=0.03% lat (msec) : >=2000=0.42% cpu : usr=1.18%, sys=2.40%, ctx=403597, majf=0, minf=1086 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=1553174,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_64K: (groupid=13, jobs=4): err= 0: pid=2582: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=4296, BW=269MiB/s (282MB/s)(47.2GiB/180042msec) slat (usec): min=2, max=147, avg= 9.85, stdev= 5.56 clat (usec): min=569, max=2726.7k, avg=29775.98, stdev=234424.12 lat (usec): min=594, max=2726.7k, avg=29786.15, stdev=234424.17 clat percentiles (msec): | 1.00th=[ 5], 5.00th=[ 7], 10.00th=[ 7], 20.00th=[ 8], | 30.00th=[ 8], 40.00th=[ 8], 50.00th=[ 8], 60.00th=[ 8], | 70.00th=[ 8], 80.00th=[ 8], 90.00th=[ 8], 95.00th=[ 8], | 99.00th=[ 42], 99.50th=[ 2500], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2567] bw ( KiB/s): min= 6531, max=1158211, per=100.00%, avg=675314.12, stdev=110946.27, samples=584 iops : min= 102, max=18097, avg=10551.46, stdev=1733.54, samples=584 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.04%, 10=98.81%, 20=0.10%, 50=0.05% lat (msec) : 100=0.04%, 250=0.03%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.02%, >=2000=0.88% cpu : usr=0.61%, sys=1.57%, ctx=200566, majf=0, minf=2114 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=773562,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_128K: (groupid=14, jobs=4): err= 0: pid=2590: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=2148, BW=269MiB/s (282MB/s)(47.2GiB/180071msec) slat (usec): min=4, max=853, avg=12.46, stdev= 7.54 clat (usec): min=873, max=2532.0k, avg=59543.09, stdev=329091.06 lat (usec): min=896, max=2532.1k, avg=59555.91, stdev=329091.08 clat percentiles (msec): | 1.00th=[ 9], 5.00th=[ 14], 10.00th=[ 14], 20.00th=[ 14], | 30.00th=[ 15], 40.00th=[ 15], 50.00th=[ 15], 60.00th=[ 15], | 70.00th=[ 16], 80.00th=[ 16], 90.00th=[ 16], 95.00th=[ 16], | 99.00th=[ 2534], 99.50th=[ 2534], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2534] bw ( KiB/s): min= 1792, max=1159936, per=100.00%, avg=668061.63, stdev=103591.81, samples=591 iops : min= 14, max= 9062, avg=5218.84, stdev=809.32, samples=591 lat (usec) : 1000=0.01% lat (msec) : 2=0.01%, 4=0.01%, 10=1.43%, 20=96.51%, 50=0.10% lat (msec) : 100=0.06%, 250=0.02%, 500=0.02%, 750=0.02%, 1000=0.02% lat (msec) : 2000=0.05%, >=2000=1.76% cpu : usr=0.36%, sys=1.00%, ctx=99359, majf=0, minf=2114 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=386931,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_512K: (groupid=15, jobs=4): err= 0: pid=2609: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=694, BW=347MiB/s (364MB/s)(61.1GiB/180226msec) slat (usec): min=7, max=82241, avg=53.81, stdev=1044.01 clat (msec): min=11, max=3059, avg=184.18, stdev=366.69 lat (msec): min=11, max=3059, avg=184.24, stdev=366.70 clat percentiles (msec): | 1.00th=[ 12], 5.00th=[ 16], 10.00th=[ 23], 20.00th=[ 42], | 30.00th=[ 57], 40.00th=[ 61], 50.00th=[ 61], 60.00th=[ 62], | 70.00th=[ 65], 80.00th=[ 131], 90.00th=[ 567], 95.00th=[ 1028], | 99.00th=[ 2005], 99.50th=[ 2123], 99.90th=[ 2635], 99.95th=[ 3004], | 99.99th=[ 3037] bw ( KiB/s): min= 7168, max=1035261, per=100.00%, avg=396054.78, stdev=77841.13, samples=1244 iops : min= 14, max= 2021, avg=773.23, stdev=151.98, samples=1244 lat (msec) : 20=8.86%, 50=14.54%, 100=54.91%, 250=6.23%, 500=3.27% lat (msec) : 750=6.13%, 1000=0.86%, 2000=4.18%, >=2000=1.02% cpu : usr=0.18%, sys=0.68%, ctx=62376, majf=0, minf=2137 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=125150,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_1M: (groupid=16, jobs=4): err= 0: pid=2617: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=252, BW=253MiB/s (265MB/s)(44.5GiB/180311msec) slat (usec): min=15, max=240938, avg=15109.49, stdev=40473.45 clat (msec): min=9, max=2689, avg=490.63, stdev=150.61 lat (msec): min=9, max=2748, avg=505.74, stdev=152.90 clat percentiles (msec): | 1.00th=[ 144], 5.00th=[ 241], 10.00th=[ 334], 20.00th=[ 401], | 30.00th=[ 456], 40.00th=[ 481], 50.00th=[ 498], 60.00th=[ 514], | 70.00th=[ 531], 80.00th=[ 558], 90.00th=[ 617], 95.00th=[ 676], | 99.00th=[ 978], 99.50th=[ 1150], 99.90th=[ 1737], 99.95th=[ 2265], | 99.99th=[ 2635] bw ( KiB/s): min=106553, max=614978, per=99.80%, avg=258474.82, stdev=13948.67, samples=1439 iops : min= 104, max= 600, avg=252.30, stdev=13.61, samples=1439 lat (msec) : 10=0.01%, 20=0.13%, 50=0.28%, 100=0.17%, 250=4.99% lat (msec) : 500=45.51%, 750=45.82%, 1000=2.21%, 2000=0.82%, >=2000=0.07% cpu : usr=0.03%, sys=0.22%, ctx=8511, majf=0, minf=2170 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=45604,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_4K: (groupid=17, jobs=4): err= 0: pid=2625: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=18.3k, BW=71.6MiB/s (75.1MB/s)(12.6GiB/180003msec); 0 zone resets slat (nsec): min=1745, max=137699, avg=7591.91, stdev=4858.55 clat (usec): min=1432, max=67489, avg=6972.04, stdev=1314.87 lat (usec): min=1468, max=67490, avg=6979.94, stdev=1315.03 clat percentiles (usec): | 1.00th=[ 5276], 5.00th=[ 5604], 10.00th=[ 5800], 20.00th=[ 6063], | 30.00th=[ 6325], 40.00th=[ 6521], 50.00th=[ 6718], 60.00th=[ 6915], | 70.00th=[ 7308], 80.00th=[ 7832], 90.00th=[ 8455], 95.00th=[ 8979], | 99.00th=[10290], 99.50th=[11076], 99.90th=[19006], 99.95th=[26608], | 99.99th=[39060] bw ( KiB/s): min=53922, max=84096, per=100.00%, avg=73344.13, stdev=2161.21, samples=1436 iops : min=13480, max=21024, avg=18335.82, stdev=540.33, samples=1436 lat (msec) : 2=0.01%, 4=0.01%, 10=98.58%, 20=1.33%, 50=0.09% lat (msec) : 100=0.01% cpu : usr=2.50%, sys=5.70%, ctx=881545, majf=0, minf=59 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,3299862,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_8K: (groupid=18, jobs=4): err= 0: pid=2632: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=13.3k, BW=104MiB/s (109MB/s)(18.3GiB/180004msec); 0 zone resets slat (nsec): min=2370, max=82260, avg=9052.87, stdev=5972.49 clat (usec): min=1632, max=41461, avg=9610.14, stdev=1207.94 lat (usec): min=1674, max=41479, avg=9619.54, stdev=1207.91 clat percentiles (usec): | 1.00th=[ 7111], 5.00th=[ 7832], 10.00th=[ 8225], 20.00th=[ 8717], | 30.00th=[ 8979], 40.00th=[ 9241], 50.00th=[ 9503], 60.00th=[ 9765], | 70.00th=[10159], 80.00th=[10421], 90.00th=[11076], 95.00th=[11600], | 99.00th=[12911], 99.50th=[13435], 99.90th=[16057], 99.95th=[18482], | 99.99th=[25822] bw ( KiB/s): min=92784, max=137383, per=100.00%, avg=106431.19, stdev=1495.98, samples=1436 iops : min=11598, max=17172, avg=13303.57, stdev=186.98, samples=1436 lat (msec) : 2=0.01%, 4=0.01%, 10=66.86%, 20=33.10%, 50=0.04% cpu : usr=2.35%, sys=4.88%, ctx=624633, majf=0, minf=53 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,2394515,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_32K: (groupid=19, jobs=4): err= 0: pid=2646: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=8528, BW=267MiB/s (279MB/s)(46.9GiB/180084msec); 0 zone resets slat (usec): min=3, max=134, avg= 9.06, stdev= 2.80 clat (usec): min=633, max=2567.6k, avg=14991.92, stdev=162978.21 lat (usec): min=655, max=2567.6k, avg=15001.28, stdev=162978.24 clat percentiles (msec): | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 4], | 30.00th=[ 4], 40.00th=[ 4], 50.00th=[ 4], 60.00th=[ 4], | 70.00th=[ 4], 80.00th=[ 5], 90.00th=[ 5], 95.00th=[ 5], | 99.00th=[ 27], 99.50th=[ 47], 99.90th=[ 2500], 99.95th=[ 2500], | 99.99th=[ 2500] bw ( KiB/s): min= 768, max=1156163, per=100.00%, avg=652766.52, stdev=109990.55, samples=600 iops : min= 24, max=36130, avg=20398.58, stdev=3437.17, samples=600 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.04%, 4=74.40%, 10=22.11%, 20=2.06%, 50=0.90% lat (msec) : 100=0.02%, 250=0.02%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.02%, >=2000=0.42% cpu : usr=1.50%, sys=2.24%, ctx=394764, majf=0, minf=67 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,1535898,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_64K: (groupid=20, jobs=4): err= 0: pid=2654: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=4279, BW=267MiB/s (280MB/s)(47.0GiB/180045msec); 0 zone resets slat (nsec): min=5620, max=77177, avg=11714.16, stdev=5141.66 clat (usec): min=543, max=2626.3k, avg=29894.58, stdev=233161.35 lat (usec): min=569, max=2626.3k, avg=29906.61, stdev=233161.37 clat percentiles (usec): | 1.00th=[ 1532], 5.00th=[ 6652], 10.00th=[ 6718], | 20.00th=[ 7177], 30.00th=[ 7177], 40.00th=[ 7177], | 50.00th=[ 7242], 60.00th=[ 7635], 70.00th=[ 7635], | 80.00th=[ 7635], 90.00th=[ 7635], 95.00th=[ 7635], | 99.00th=[ 145753], 99.50th=[2499806], 99.90th=[2533360], | 99.95th=[2533360], 99.99th=[2566915] bw ( KiB/s): min= 1512, max=1158467, per=100.00%, avg=623659.58, stdev=111223.98, samples=632 iops : min= 23, max=18101, avg=9744.39, stdev=1737.87, samples=632 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=1.38%, 4=0.28%, 10=96.91%, 20=0.27%, 50=0.07% lat (msec) : 100=0.06%, 250=0.02%, 500=0.02%, 750=0.02%, 1000=0.03% lat (msec) : 2000=0.06%, >=2000=0.85% cpu : usr=0.93%, sys=1.48%, ctx=200618, majf=0, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,770447,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_128K: (groupid=21, jobs=4): err= 0: pid=2658: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=2593, BW=324MiB/s (340MB/s)(57.0GiB/180082msec); 0 zone resets slat (usec): min=5, max=118, avg=16.38, stdev= 7.62 clat (usec): min=756, max=2578.6k, avg=49332.18, stdev=207904.37 lat (usec): min=787, max=2578.6k, avg=49348.92, stdev=207904.38 clat percentiles (msec): | 1.00th=[ 14], 5.00th=[ 14], 10.00th=[ 14], 20.00th=[ 15], | 30.00th=[ 15], 40.00th=[ 37], 50.00th=[ 40], 60.00th=[ 41], | 70.00th=[ 41], 80.00th=[ 41], 90.00th=[ 44], 95.00th=[ 58], | 99.00th=[ 94], 99.50th=[ 2500], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2567] bw ( KiB/s): min= 9746, max=1163077, per=100.00%, avg=462975.85, stdev=69477.08, samples=1030 iops : min= 76, max= 9086, avg=3616.49, stdev=542.79, samples=1030 lat (usec) : 1000=0.01% lat (msec) : 2=0.01%, 4=0.01%, 10=0.13%, 20=39.04%, 50=54.13% lat (msec) : 100=5.81%, 250=0.13%, 500=0.05%, 750=0.01%, >=2000=0.70% cpu : usr=0.80%, sys=1.17%, ctx=120762, majf=0, minf=64 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,466986,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_512K: (groupid=22, jobs=4): err= 0: pid=2666: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=800, BW=400MiB/s (420MB/s)(70.4GiB/180147msec); 0 zone resets slat (usec): min=13, max=128894, avg=1280.71, stdev=4799.55 clat (msec): min=36, max=683, avg=158.56, stdev=25.82 lat (msec): min=36, max=683, avg=159.84, stdev=25.70 clat percentiles (msec): | 1.00th=[ 93], 5.00th=[ 134], 10.00th=[ 142], 20.00th=[ 148], | 30.00th=[ 150], 40.00th=[ 153], 50.00th=[ 153], 60.00th=[ 155], | 70.00th=[ 161], 80.00th=[ 169], 90.00th=[ 190], 95.00th=[ 199], | 99.00th=[ 245], 99.50th=[ 275], 99.90th=[ 342], 99.95th=[ 498], | 99.99th=[ 642] bw ( KiB/s): min=300179, max=455909, per=100.00%, avg=410203.27, stdev=6959.90, samples=1436 iops : min= 586, max= 890, avg=800.75, stdev=13.58, samples=1436 lat (msec) : 50=0.01%, 100=1.63%, 250=97.48%, 500=0.84%, 750=0.05% cpu : usr=0.75%, sys=0.62%, ctx=85066, majf=0, minf=61 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,144246,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_1M: (groupid=23, jobs=4): err= 0: pid=2682: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=244, BW=245MiB/s (257MB/s)(43.1GiB/180249msec); 0 zone resets slat (usec): min=29, max=704032, avg=15821.81, stdev=48172.80 clat (msec): min=41, max=2105, avg=506.73, stdev=152.18 lat (msec): min=41, max=2105, avg=522.55, stdev=155.48 clat percentiles (msec): | 1.00th=[ 213], 5.00th=[ 292], 10.00th=[ 368], 20.00th=[ 443], | 30.00th=[ 456], 40.00th=[ 464], 50.00th=[ 477], 60.00th=[ 485], | 70.00th=[ 502], 80.00th=[ 558], 90.00th=[ 718], 95.00th=[ 827], | 99.00th=[ 1003], 99.50th=[ 1083], 99.90th=[ 1401], 99.95th=[ 1720], | 99.99th=[ 2005] bw ( KiB/s): min=36864, max=454656, per=99.98%, avg=250715.25, stdev=16903.67, samples=1438 iops : min= 36, max= 444, avg=244.72, stdev=16.50, samples=1438 lat (msec) : 50=0.01%, 100=0.09%, 250=1.82%, 500=68.39%, 750=21.25% lat (msec) : 1000=7.45%, 2000=0.98%, >=2000=0.01% cpu : usr=0.32%, sys=0.11%, ctx=7404, majf=0, minf=63 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,44141,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_4K: (groupid=24, jobs=4): err= 0: pid=2689: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=165, BW=661KiB/s (677kB/s)(116MiB/180270msec) slat (nsec): min=1970, max=273828k, avg=12090216.39, stdev=35019287.69 clat (msec): min=64, max=1457, avg=423.09, stdev=196.49 lat (msec): min=64, max=1457, avg=435.18, stdev=200.26 clat percentiles (msec): | 1.00th=[ 138], 5.00th=[ 184], 10.00th=[ 218], 20.00th=[ 257], | 30.00th=[ 296], 40.00th=[ 338], 50.00th=[ 376], 60.00th=[ 422], | 70.00th=[ 485], 80.00th=[ 575], 90.00th=[ 718], 95.00th=[ 818], | 99.00th=[ 986], 99.50th=[ 1053], 99.90th=[ 1183], 99.95th=[ 1250], | 99.99th=[ 1351] bw ( KiB/s): min= 177, max= 1192, per=99.87%, avg=660.06, stdev=47.53, samples=1440 iops : min= 44, max= 298, avg=165.01, stdev=11.88, samples=1440 write: IOPS=164, BW=659KiB/s (675kB/s)(116MiB/180270msec); 0 zone resets slat (usec): min=2, max=318636, avg=12099.20, stdev=35009.76 clat (msec): min=3, max=1180, avg=327.65, stdev=142.24 lat (msec): min=3, max=1180, avg=339.75, stdev=147.32 clat percentiles (msec): | 1.00th=[ 113], 5.00th=[ 142], 10.00th=[ 167], 20.00th=[ 211], | 30.00th=[ 243], 40.00th=[ 271], 50.00th=[ 305], 60.00th=[ 338], | 70.00th=[ 376], 80.00th=[ 430], 90.00th=[ 523], 95.00th=[ 609], | 99.00th=[ 768], 99.50th=[ 827], 99.90th=[ 961], 99.95th=[ 995], | 99.99th=[ 1070] bw ( KiB/s): min= 208, max= 1216, per=99.82%, avg=658.58, stdev=49.50, samples=1440 iops : min= 52, max= 304, avg=164.64, stdev=12.37, samples=1440 lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.21% lat (msec) : 250=25.53%, 500=54.26%, 750=15.28%, 1000=4.23%, 2000=0.46% cpu : usr=0.10%, sys=0.10%, ctx=7507, majf=0, minf=65 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=29783,29709,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_8K: (groupid=25, jobs=4): err= 0: pid=2697: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=116, BW=930KiB/s (952kB/s)(164MiB/180326msec) slat (usec): min=2, max=339038, avg=16686.85, stdev=47455.54 clat (msec): min=12, max=1758, avg=593.40, stdev=246.61 lat (msec): min=12, max=1758, avg=610.09, stdev=251.86 clat percentiles (msec): | 1.00th=[ 167], 5.00th=[ 264], 10.00th=[ 317], 20.00th=[ 376], | 30.00th=[ 435], 40.00th=[ 485], 50.00th=[ 550], 60.00th=[ 617], | 70.00th=[ 709], 80.00th=[ 818], 90.00th=[ 944], 95.00th=[ 1045], | 99.00th=[ 1234], 99.50th=[ 1318], 99.90th=[ 1519], 99.95th=[ 1536], | 99.99th=[ 1603] bw ( KiB/s): min= 184, max= 1844, per=99.79%, avg=928.13, stdev=72.71, samples=1440 iops : min= 22, max= 230, avg=116.01, stdev= 9.09, samples=1440 write: IOPS=118, BW=949KiB/s (972kB/s)(167MiB/180326msec); 0 zone resets slat (usec): min=2, max=355518, avg=16775.93, stdev=47804.05 clat (usec): min=778, max=1864.9k, avg=463234.49, stdev=190624.20 lat (usec): min=801, max=1913.6k, avg=480010.83, stdev=197057.20 clat percentiles (msec): | 1.00th=[ 96], 5.00th=[ 220], 10.00th=[ 251], 20.00th=[ 309], | 30.00th=[ 355], 40.00th=[ 397], 50.00th=[ 435], 60.00th=[ 472], | 70.00th=[ 531], 80.00th=[ 600], 90.00th=[ 718], 95.00th=[ 818], | 99.00th=[ 1036], 99.50th=[ 1133], 99.90th=[ 1318], 99.95th=[ 1435], | 99.99th=[ 1552] bw ( KiB/s): min= 200, max= 2048, per=99.71%, avg=946.82, stdev=72.87, samples=1440 iops : min= 25, max= 256, avg=118.35, stdev= 9.11, samples=1440 lat (usec) : 1000=0.01% lat (msec) : 2=0.01%, 4=0.02%, 10=0.05%, 20=0.05%, 50=0.26% lat (msec) : 100=0.34%, 250=6.06%, 500=46.98%, 750=29.35%, 1000=12.66% lat (msec) : 2000=4.21% cpu : usr=0.06%, sys=0.08%, ctx=6544, majf=0, minf=73 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=20962,21386,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_32K: (groupid=26, jobs=4): err= 0: pid=2701: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=108, BW=3482KiB/s (3565kB/s)(613MiB/180275msec) slat (usec): min=4, max=431407, avg=18205.27, stdev=52600.50 clat (msec): min=89, max=2234, avg=633.00, stdev=266.27 lat (msec): min=108, max=2234, avg=651.21, stdev=272.30 clat percentiles (msec): | 1.00th=[ 220], 5.00th=[ 288], 10.00th=[ 338], 20.00th=[ 397], | 30.00th=[ 456], 40.00th=[ 518], 50.00th=[ 575], 60.00th=[ 659], | 70.00th=[ 751], 80.00th=[ 869], 90.00th=[ 1011], 95.00th=[ 1133], | 99.00th=[ 1351], 99.50th=[ 1469], 99.90th=[ 1687], 99.95th=[ 1770], | 99.99th=[ 1838] bw ( KiB/s): min= 768, max= 7372, per=99.87%, avg=3477.07, stdev=278.63, samples=1439 iops : min= 24, max= 230, avg=108.62, stdev= 8.70, samples=1439 write: IOPS=108, BW=3467KiB/s (3551kB/s)(610MiB/180275msec); 0 zone resets slat (usec): min=4, max=437811, avg=18579.18, stdev=52798.15 clat (msec): min=39, max=2131, avg=508.36, stdev=228.39 lat (msec): min=39, max=2131, avg=526.94, stdev=236.29 clat percentiles (msec): | 1.00th=[ 178], 5.00th=[ 241], 10.00th=[ 266], 20.00th=[ 330], | 30.00th=[ 376], 40.00th=[ 418], 50.00th=[ 460], 60.00th=[ 510], | 70.00th=[ 575], 80.00th=[ 659], 90.00th=[ 802], 95.00th=[ 936], | 99.00th=[ 1301], 99.50th=[ 1485], 99.90th=[ 1821], 99.95th=[ 1854], | 99.99th=[ 2039] bw ( KiB/s): min= 768, max= 7046, per=99.87%, avg=3463.80, stdev=281.36, samples=1440 iops : min= 24, max= 220, avg=108.20, stdev= 8.79, samples=1440 lat (msec) : 50=0.01%, 100=0.02%, 250=4.55%, 500=43.21%, 750=30.91% lat (msec) : 1000=14.07%, 2000=7.24%, >=2000=0.01% cpu : usr=0.04%, sys=0.07%, ctx=4977, majf=0, minf=68 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=19614,19534,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_64K: (groupid=27, jobs=4): err= 0: pid=2720: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=106, BW=6840KiB/s (7004kB/s)(1205MiB/180361msec) slat (usec): min=5, max=307801, avg=18016.27, stdev=51019.07 clat (msec): min=5, max=1855, avg=646.83, stdev=270.72 lat (msec): min=5, max=2062, avg=664.85, stdev=277.15 clat percentiles (msec): | 1.00th=[ 97], 5.00th=[ 264], 10.00th=[ 334], 20.00th=[ 414], | 30.00th=[ 477], 40.00th=[ 542], 50.00th=[ 609], 60.00th=[ 693], | 70.00th=[ 785], 80.00th=[ 894], 90.00th=[ 1028], 95.00th=[ 1133], | 99.00th=[ 1318], 99.50th=[ 1385], 99.90th=[ 1536], 99.95th=[ 1586], | 99.99th=[ 1754] bw ( KiB/s): min= 1280, max=15374, per=99.79%, avg=6826.24, stdev=622.77, samples=1440 iops : min= 20, max= 240, avg=106.61, stdev= 9.73, samples=1440 write: IOPS=108, BW=6935KiB/s (7101kB/s)(1221MiB/180361msec); 0 zone resets slat (usec): min=6, max=384246, avg=17898.33, stdev=50687.26 clat (usec): min=874, max=1543.0k, avg=507204.83, stdev=219778.45 lat (usec): min=902, max=1543.0k, avg=525103.60, stdev=227605.60 clat percentiles (msec): | 1.00th=[ 34], 5.00th=[ 207], 10.00th=[ 259], 20.00th=[ 330], | 30.00th=[ 388], 40.00th=[ 435], 50.00th=[ 477], 60.00th=[ 531], | 70.00th=[ 592], 80.00th=[ 684], 90.00th=[ 802], 95.00th=[ 902], | 99.00th=[ 1133], 99.50th=[ 1200], 99.90th=[ 1368], 99.95th=[ 1418], | 99.99th=[ 1519] bw ( KiB/s): min= 1664, max=17291, per=99.83%, avg=6923.79, stdev=614.52, samples=1440 iops : min= 26, max= 270, avg=108.14, stdev= 9.60, samples=1440 lat (usec) : 1000=0.01% lat (msec) : 2=0.05%, 4=0.10%, 10=0.07%, 20=0.14%, 50=0.57% lat (msec) : 100=0.76%, 250=4.61%, 500=37.82%, 750=32.39%, 1000=16.40% lat (msec) : 2000=7.09% cpu : usr=0.05%, sys=0.08%, ctx=7613, majf=0, minf=74 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=19277,19543,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_128K: (groupid=28, jobs=4): err= 0: pid=2728: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=97, BW=12.2MiB/s (12.7MB/s)(2193MiB/180416msec) slat (usec): min=3, max=366609, avg=20616.84, stdev=57249.16 clat (msec): min=101, max=1985, avg=716.69, stdev=267.57 lat (msec): min=101, max=1985, avg=737.31, stdev=273.70 clat percentiles (msec): | 1.00th=[ 266], 5.00th=[ 342], 10.00th=[ 409], 20.00th=[ 477], | 30.00th=[ 531], 40.00th=[ 600], 50.00th=[ 667], 60.00th=[ 751], | 70.00th=[ 860], 80.00th=[ 969], 90.00th=[ 1099], 95.00th=[ 1200], | 99.00th=[ 1368], 99.50th=[ 1435], 99.90th=[ 1586], 99.95th=[ 1653], | 99.99th=[ 1838] bw ( KiB/s): min= 2817, max=26880, per=99.87%, avg=12429.57, stdev=932.27, samples=1440 iops : min= 22, max= 210, avg=97.06, stdev= 7.28, samples=1440 write: IOPS=96, BW=12.0MiB/s (12.6MB/s)(2169MiB/180416msec); 0 zone resets slat (usec): min=4, max=386643, avg=20658.27, stdev=57496.73 clat (msec): min=37, max=1684, avg=564.31, stdev=210.17 lat (msec): min=37, max=1745, avg=584.97, stdev=217.22 clat percentiles (msec): | 1.00th=[ 230], 5.00th=[ 275], 10.00th=[ 313], 20.00th=[ 388], | 30.00th=[ 451], 40.00th=[ 485], 50.00th=[ 523], 60.00th=[ 575], | 70.00th=[ 642], 80.00th=[ 726], 90.00th=[ 860], 95.00th=[ 969], | 99.00th=[ 1183], 99.50th=[ 1250], 99.90th=[ 1435], 99.95th=[ 1485], | 99.99th=[ 1636] bw ( KiB/s): min= 3328, max=26624, per=99.92%, avg=12300.04, stdev=900.22, samples=1440 iops : min= 26, max= 208, avg=96.05, stdev= 7.03, samples=1440 lat (msec) : 50=0.01%, 250=1.25%, 500=33.21%, 750=36.32%, 1000=18.42% lat (msec) : 2000=10.80% cpu : usr=0.04%, sys=0.07%, ctx=4429, majf=0, minf=67 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=17542,17350,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_4K_latency: (groupid=29, jobs=1): err= 0: pid=2732: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=81, BW=326KiB/s (334kB/s)(57.3MiB/180012msec) slat (nsec): min=12360, max=87157, avg=28468.09, stdev=1610.20 clat (usec): min=1526, max=57344, avg=12230.81, stdev=4345.42 lat (usec): min=1555, max=57374, avg=12260.04, stdev=4345.39 clat percentiles (usec): | 1.00th=[ 3982], 5.00th=[ 5997], 10.00th=[ 7111], 20.00th=[ 8848], | 30.00th=[10028], 40.00th=[11076], 50.00th=[12125], 60.00th=[13042], | 70.00th=[14091], 80.00th=[15401], 90.00th=[17171], 95.00th=[18482], | 99.00th=[21103], 99.50th=[22676], 99.90th=[50594], 99.95th=[52167], | 99.99th=[55837] bw ( KiB/s): min= 272, max= 376, per=99.99%, avg=326.04, stdev=19.43, samples=360 iops : min= 68, max= 94, avg=81.51, stdev= 4.86, samples=360 lat (msec) : 2=0.03%, 4=0.98%, 10=28.75%, 20=68.20%, 50=1.93% lat (msec) : 100=0.12% cpu : usr=0.14%, sys=0.38%, ctx=14674, majf=0, minf=14 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=14673,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_4K_latency: (groupid=30, jobs=1): err= 0: pid=2737: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=79, BW=316KiB/s (324kB/s)(55.6MiB/180019msec) slat (usec): min=12, max=133, avg=28.65, stdev= 1.94 clat (usec): min=1786, max=66267, avg=12228.03, stdev=4124.74 lat (usec): min=1815, max=66300, avg=12257.44, stdev=4124.70 clat percentiles (usec): | 1.00th=[ 3982], 5.00th=[ 5997], 10.00th=[ 7242], 20.00th=[ 8848], | 30.00th=[10028], 40.00th=[11076], 50.00th=[12125], 60.00th=[13173], | 70.00th=[14222], 80.00th=[15401], 90.00th=[17171], 95.00th=[18482], | 99.00th=[20841], 99.50th=[21890], 99.90th=[51119], 99.95th=[56886], | 99.99th=[65274] bw ( KiB/s): min= 264, max= 384, per=99.96%, avg=316.12, stdev=18.61, samples=359 iops : min= 66, max= 96, avg=79.03, stdev= 4.65, samples=359 write: IOPS=78, BW=314KiB/s (321kB/s)(55.2MiB/180019msec); 0 zone resets slat (nsec): min=10860, max=88832, avg=29172.89, stdev=1905.65 clat (usec): min=145, max=7077, avg=353.47, stdev=180.07 lat (usec): min=172, max=7107, avg=383.40, stdev=180.10 clat percentiles (usec): | 1.00th=[ 208], 5.00th=[ 231], 10.00th=[ 249], 20.00th=[ 273], | 30.00th=[ 297], 40.00th=[ 322], 50.00th=[ 343], 60.00th=[ 359], | 70.00th=[ 383], 80.00th=[ 412], 90.00th=[ 457], 95.00th=[ 502], | 99.00th=[ 619], 99.50th=[ 676], 99.90th=[ 1450], 99.95th=[ 5145], | 99.99th=[ 6063] bw ( KiB/s): min= 112, max= 568, per=99.74%, avg=313.55, stdev=69.36, samples=359 iops : min= 28, max= 142, avg=78.39, stdev=17.34, samples=359 lat (usec) : 250=5.19%, 500=42.14%, 750=2.36%, 1000=0.06% lat (msec) : 2=0.02%, 4=0.50%, 10=14.21%, 20=34.51%, 50=0.95% lat (msec) : 100=0.05% cpu : usr=0.31%, sys=0.74%, ctx=28358, majf=0, minf=18 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=14227,14123,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randwrite_4K_latency: (groupid=31, jobs=1): err= 0: pid=2748: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] write: IOPS=481, BW=1926KiB/s (1972kB/s)(339MiB/180001msec); 0 zone resets slat (usec): min=9, max=134, avg=28.77, stdev= 3.27 clat (usec): min=105, max=35352, avg=2040.62, stdev=1090.29 lat (usec): min=115, max=35383, avg=2070.18, stdev=1091.28 clat percentiles (usec): | 1.00th=[ 165], 5.00th=[ 1156], 10.00th=[ 1352], 20.00th=[ 1532], | 30.00th=[ 1663], 40.00th=[ 1795], 50.00th=[ 1893], 60.00th=[ 2008], | 70.00th=[ 2147], 80.00th=[ 2343], 90.00th=[ 2704], 95.00th=[ 3195], | 99.00th=[ 7177], 99.50th=[ 8356], 99.90th=[10552], 99.95th=[16712], | 99.99th=[17171] bw ( KiB/s): min= 672, max=13472, per=99.29%, avg=1912.24, stdev=763.86, samples=359 iops : min= 168, max= 3368, avg=477.95, stdev=190.98, samples=359 lat (usec) : 250=3.11%, 500=0.70%, 750=0.13%, 1000=0.17% lat (msec) : 2=54.79%, 4=38.01%, 10=2.98%, 20=0.11%, 50=0.01% cpu : usr=0.83%, sys=2.43%, ctx=86664, majf=0, minf=14 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,86657,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_70R30W_4K: (groupid=32, jobs=4): err= 0: pid=2753: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=169, BW=677KiB/s (694kB/s)(119MiB/180286msec) slat (nsec): min=1970, max=272000k, avg=16347803.68, stdev=45877178.72 clat (msec): min=60, max=2143, avg=549.69, stdev=195.55 lat (msec): min=60, max=2143, avg=566.04, stdev=200.27 clat percentiles (msec): | 1.00th=[ 213], 5.00th=[ 292], 10.00th=[ 330], 20.00th=[ 388], | 30.00th=[ 426], 40.00th=[ 472], 50.00th=[ 518], 60.00th=[ 567], | 70.00th=[ 625], 80.00th=[ 701], 90.00th=[ 827], 95.00th=[ 927], | 99.00th=[ 1099], 99.50th=[ 1167], 99.90th=[ 1368], 99.95th=[ 1418], | 99.99th=[ 1720] bw ( KiB/s): min= 296, max= 1040, per=99.78%, avg=676.60, stdev=33.07, samples=1439 iops : min= 74, max= 260, avg=169.15, stdev= 8.27, samples=1439 write: IOPS=73, BW=292KiB/s (299kB/s)(51.5MiB/180286msec); 0 zone resets slat (usec): min=2, max=255522, avg=16740.47, stdev=46757.40 clat (msec): min=60, max=1340, avg=421.36, stdev=153.02 lat (msec): min=60, max=1340, avg=438.10, stdev=158.73 clat percentiles (msec): | 1.00th=[ 169], 5.00th=[ 205], 10.00th=[ 239], 20.00th=[ 300], | 30.00th=[ 338], 40.00th=[ 376], 50.00th=[ 405], 60.00th=[ 430], | 70.00th=[ 468], 80.00th=[ 527], 90.00th=[ 617], 95.00th=[ 718], | 99.00th=[ 911], 99.50th=[ 969], 99.90th=[ 1116], 99.95th=[ 1183], | 99.99th=[ 1318] bw ( KiB/s): min= 53, max= 592, per=99.91%, avg=292.04, stdev=24.92, samples=1439 iops : min= 13, max= 148, avg=73.00, stdev= 6.23, samples=1439 lat (msec) : 100=0.04%, 250=5.39%, 500=49.49%, 750=33.00%, 1000=10.20% lat (msec) : 2000=1.87%, >=2000=0.01% cpu : usr=0.09%, sys=0.07%, ctx=5550, majf=0, minf=62 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=30535,13173,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_70R30W_64K: (groupid=33, jobs=4): err= 0: pid=2760: Sat Jan 11 06:13:25 2025 Description : ["Storage megatest by Andrey"] read: IOPS=110, BW=7085KiB/s (7255kB/s)(1248MiB/180321msec) slat (usec): min=4, max=863436, avg=24353.05, stdev=76963.72 clat (msec): min=15, max=3303, avg=757.94, stdev=353.27 lat (msec): min=15, max=3872, avg=782.29, stdev=368.76 clat percentiles (msec): | 1.00th=[ 230], 5.00th=[ 347], 10.00th=[ 409], 20.00th=[ 468], | 30.00th=[ 542], 40.00th=[ 600], 50.00th=[ 676], 60.00th=[ 760], | 70.00th=[ 877], 80.00th=[ 1003], 90.00th=[ 1234], 95.00th=[ 1435], | 99.00th=[ 1938], 99.50th=[ 2140], 99.90th=[ 2400], 99.95th=[ 2567], | 99.99th=[ 2903] bw ( KiB/s): min= 640, max=16896, per=100.00%, avg=7174.17, stdev=855.95, samples=1418 iops : min= 10, max= 264, avg=112.10, stdev=13.37, samples=1418 write: IOPS=47, BW=3070KiB/s (3144kB/s)(541MiB/180321msec); 0 zone resets slat (usec): min=5, max=840060, avg=26543.21, stdev=80370.46 clat (usec): min=1055, max=4800.9k, avg=832912.68, stdev=792582.88 lat (usec): min=1081, max=4819.8k, avg=859456.30, stdev=808521.77 clat percentiles (msec): | 1.00th=[ 140], 5.00th=[ 249], 10.00th=[ 300], 20.00th=[ 372], | 30.00th=[ 430], 40.00th=[ 468], 50.00th=[ 514], 60.00th=[ 592], | 70.00th=[ 684], 80.00th=[ 902], 90.00th=[ 2299], 95.00th=[ 2735], | 99.00th=[ 3473], 99.50th=[ 3742], 99.90th=[ 4329], 99.95th=[ 4463], | 99.99th=[ 4799] bw ( KiB/s): min= 512, max= 9733, per=100.00%, avg=3211.86, stdev=441.96, samples=1374 iops : min= 8, max= 152, avg=50.18, stdev= 6.91, samples=1374 lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.02%, 50=0.08% lat (msec) : 100=0.21%, 250=2.24%, 500=28.93%, 750=31.79%, 1000=17.24% lat (msec) : 2000=15.05%, >=2000=4.42% cpu : usr=0.03%, sys=0.06%, ctx=4120, majf=0, minf=68 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=19963,8651,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 Run status group 0 (all jobs): READ: bw=836KiB/s (856kB/s), 836KiB/s-836KiB/s (856kB/s-856kB/s), io=147MiB (154MB), run=180370-180370msec Run status group 1 (all jobs): READ: bw=1671KiB/s (1711kB/s), 1671KiB/s-1671KiB/s (1711kB/s-1711kB/s), io=294MiB (309MB), run=180392-180392msec Run status group 2 (all jobs): READ: bw=6474KiB/s (6629kB/s), 6474KiB/s-6474KiB/s (6629kB/s-6629kB/s), io=1140MiB (1196MB), run=180346-180346msec Run status group 3 (all jobs): READ: bw=12.2MiB/s (12.8MB/s), 12.2MiB/s-12.2MiB/s (12.8MB/s-12.8MB/s), io=2207MiB (2315MB), run=180442-180442msec Run status group 4 (all jobs): READ: bw=23.0MiB/s (24.2MB/s), 23.0MiB/s-23.0MiB/s (24.2MB/s-24.2MB/s), io=4158MiB (4359MB), run=180415-180415msec Run status group 5 (all jobs): WRITE: bw=2201KiB/s (2254kB/s), 2201KiB/s-2201KiB/s (2254kB/s-2254kB/s), io=387MiB (406MB), run=180108-180108msec Run status group 6 (all jobs): WRITE: bw=3067KiB/s (3140kB/s), 3067KiB/s-3067KiB/s (3140kB/s-3140kB/s), io=539MiB (566MB), run=180129-180129msec Run status group 7 (all jobs): WRITE: bw=9.81MiB/s (10.3MB/s), 9.81MiB/s-9.81MiB/s (10.3MB/s-10.3MB/s), io=1769MiB (1855MB), run=180250-180250msec Run status group 8 (all jobs): WRITE: bw=18.0MiB/s (18.9MB/s), 18.0MiB/s-18.0MiB/s (18.9MB/s-18.9MB/s), io=3248MiB (3406MB), run=180170-180170msec Run status group 9 (all jobs): WRITE: bw=29.5MiB/s (30.9MB/s), 29.5MiB/s-29.5MiB/s (30.9MB/s-30.9MB/s), io=5309MiB (5566MB), run=180217-180217msec Run status group 10 (all jobs): READ: bw=263MiB/s (276MB/s), 263MiB/s-263MiB/s (276MB/s-276MB/s), io=46.2GiB (49.6GB), run=180037-180037msec Run status group 11 (all jobs): READ: bw=269MiB/s (282MB/s), 269MiB/s-269MiB/s (282MB/s-282MB/s), io=47.3GiB (50.8GB), run=180037-180037msec Run status group 12 (all jobs): READ: bw=270MiB/s (283MB/s), 270MiB/s-270MiB/s (283MB/s-283MB/s), io=47.4GiB (50.9GB), run=180027-180027msec Run status group 13 (all jobs): READ: bw=269MiB/s (282MB/s), 269MiB/s-269MiB/s (282MB/s-282MB/s), io=47.2GiB (50.7GB), run=180042-180042msec Run status group 14 (all jobs): READ: bw=269MiB/s (282MB/s), 269MiB/s-269MiB/s (282MB/s-282MB/s), io=47.2GiB (50.7GB), run=180071-180071msec Run status group 15 (all jobs): READ: bw=347MiB/s (364MB/s), 347MiB/s-347MiB/s (364MB/s-364MB/s), io=61.1GiB (65.6GB), run=180226-180226msec Run status group 16 (all jobs): READ: bw=253MiB/s (265MB/s), 253MiB/s-253MiB/s (265MB/s-265MB/s), io=44.5GiB (47.8GB), run=180311-180311msec Run status group 17 (all jobs): WRITE: bw=71.6MiB/s (75.1MB/s), 71.6MiB/s-71.6MiB/s (75.1MB/s-75.1MB/s), io=12.6GiB (13.5GB), run=180003-180003msec Run status group 18 (all jobs): WRITE: bw=104MiB/s (109MB/s), 104MiB/s-104MiB/s (109MB/s-109MB/s), io=18.3GiB (19.6GB), run=180004-180004msec Run status group 19 (all jobs): WRITE: bw=267MiB/s (279MB/s), 267MiB/s-267MiB/s (279MB/s-279MB/s), io=46.9GiB (50.3GB), run=180084-180084msec Run status group 20 (all jobs): WRITE: bw=267MiB/s (280MB/s), 267MiB/s-267MiB/s (280MB/s-280MB/s), io=47.0GiB (50.5GB), run=180045-180045msec Run status group 21 (all jobs): WRITE: bw=324MiB/s (340MB/s), 324MiB/s-324MiB/s (340MB/s-340MB/s), io=57.0GiB (61.2GB), run=180082-180082msec Run status group 22 (all jobs): WRITE: bw=400MiB/s (420MB/s), 400MiB/s-400MiB/s (420MB/s-420MB/s), io=70.4GiB (75.6GB), run=180147-180147msec Run status group 23 (all jobs): WRITE: bw=245MiB/s (257MB/s), 245MiB/s-245MiB/s (257MB/s-257MB/s), io=43.1GiB (46.3GB), run=180249-180249msec Run status group 24 (all jobs): READ: bw=661KiB/s (677kB/s), 661KiB/s-661KiB/s (677kB/s-677kB/s), io=116MiB (122MB), run=180270-180270msec WRITE: bw=659KiB/s (675kB/s), 659KiB/s-659KiB/s (675kB/s-675kB/s), io=116MiB (122MB), run=180270-180270msec Run status group 25 (all jobs): READ: bw=930KiB/s (952kB/s), 930KiB/s-930KiB/s (952kB/s-952kB/s), io=164MiB (172MB), run=180326-180326msec WRITE: bw=949KiB/s (972kB/s), 949KiB/s-949KiB/s (972kB/s-972kB/s), io=167MiB (175MB), run=180326-180326msec Run status group 26 (all jobs): READ: bw=3482KiB/s (3565kB/s), 3482KiB/s-3482KiB/s (3565kB/s-3565kB/s), io=613MiB (643MB), run=180275-180275msec WRITE: bw=3467KiB/s (3551kB/s), 3467KiB/s-3467KiB/s (3551kB/s-3551kB/s), io=610MiB (640MB), run=180275-180275msec Run status group 27 (all jobs): READ: bw=6840KiB/s (7004kB/s), 6840KiB/s-6840KiB/s (7004kB/s-7004kB/s), io=1205MiB (1263MB), run=180361-180361msec WRITE: bw=6935KiB/s (7101kB/s), 6935KiB/s-6935KiB/s (7101kB/s-7101kB/s), io=1221MiB (1281MB), run=180361-180361msec Run status group 28 (all jobs): READ: bw=12.2MiB/s (12.7MB/s), 12.2MiB/s-12.2MiB/s (12.7MB/s-12.7MB/s), io=2193MiB (2299MB), run=180416-180416msec WRITE: bw=12.0MiB/s (12.6MB/s), 12.0MiB/s-12.0MiB/s (12.6MB/s-12.6MB/s), io=2169MiB (2274MB), run=180416-180416msec Run status group 29 (all jobs): READ: bw=326KiB/s (334kB/s), 326KiB/s-326KiB/s (334kB/s-334kB/s), io=57.3MiB (60.1MB), run=180012-180012msec Run status group 30 (all jobs): READ: bw=316KiB/s (324kB/s), 316KiB/s-316KiB/s (324kB/s-324kB/s), io=55.6MiB (58.3MB), run=180019-180019msec WRITE: bw=314KiB/s (321kB/s), 314KiB/s-314KiB/s (321kB/s-321kB/s), io=55.2MiB (57.8MB), run=180019-180019msec Run status group 31 (all jobs): WRITE: bw=1926KiB/s (1972kB/s), 1926KiB/s-1926KiB/s (1972kB/s-1972kB/s), io=339MiB (355MB), run=180001-180001msec Run status group 32 (all jobs): READ: bw=677KiB/s (694kB/s), 677KiB/s-677KiB/s (694kB/s-694kB/s), io=119MiB (125MB), run=180286-180286msec WRITE: bw=292KiB/s (299kB/s), 292KiB/s-292KiB/s (299kB/s-299kB/s), io=51.5MiB (53.0MB), run=180286-180286msec Run status group 33 (all jobs): READ: bw=7085KiB/s (7255kB/s), 7085KiB/s-7085KiB/s (7255kB/s-7255kB/s), io=1248MiB (1308MB), run=180321-180321msec WRITE: bw=3070KiB/s (3144kB/s), 3070KiB/s-3070KiB/s (3144kB/s-3144kB/s), io=541MiB (567MB), run=180321-180321msec Disk stats (read/write): sde: ios=6389801/2885278, merge=15177506/6321483, ticks=157653050/132370337, in_queue=290023388, util=99.84%