randread_4K: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randread_8K: (g=1): rw=randread, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randread_32K: (g=2): rw=randread, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randread_64K: (g=3): rw=randread, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randread_128K: (g=4): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randwrite_4K: (g=5): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randwrite_8K: (g=6): rw=randwrite, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randwrite_32K: (g=7): rw=randwrite, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randwrite_64K: (g=8): rw=randwrite, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randwrite_128K: (g=9): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_4K: (g=10): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqread_8K: (g=11): rw=read, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqread_32K: (g=12): rw=read, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqread_64K: (g=13): rw=read, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqread_128K: (g=14): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_512K: (g=15): rw=read, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqread_1M: (g=16): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... seqwrite_4K: (g=17): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqwrite_8K: (g=18): rw=write, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqwrite_32K: (g=19): rw=write, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_64K: (g=20): rw=write, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_128K: (g=21): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqwrite_512K: (g=22): rw=write, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqwrite_1M: (g=23): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... randrw_4K: (g=24): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_8K: (g=25): rw=randrw, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randrw_32K: (g=26): rw=randrw, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randrw_64K: (g=27): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randrw_128K: (g=28): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randread_4K_latency: (g=29): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_4K_latency: (g=30): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randwrite_4K_latency: (g=31): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_70R30W_4K: (g=32): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_70R30W_64K: (g=33): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... fio-3.25 Starting 127 processes randread_4K: (groupid=0, jobs=4): err= 0: pid=2861: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=203, BW=816KiB/s (835kB/s)(144MiB/180350msec) slat (usec): min=2, max=281058, avg=18982.40, stdev=50859.84 clat (msec): min=8, max=1975, avg=607.51, stdev=213.68 lat (msec): min=8, max=1975, avg=626.49, stdev=222.40 clat percentiles (msec): | 1.00th=[ 133], 5.00th=[ 296], 10.00th=[ 409], 20.00th=[ 456], | 30.00th=[ 477], 40.00th=[ 510], 50.00th=[ 600], 60.00th=[ 625], | 70.00th=[ 667], 80.00th=[ 776], 90.00th=[ 902], 95.00th=[ 995], | 99.00th=[ 1217], 99.50th=[ 1318], 99.90th=[ 1519], 99.95th=[ 1586], | 99.99th=[ 1737] bw ( KiB/s): min= 288, max= 1296, per=99.93%, avg=815.51, stdev=50.61, samples=1437 iops : min= 72, max= 324, avg=203.88, stdev=12.65, samples=1437 lat (msec) : 10=0.01%, 20=0.01%, 50=0.12%, 100=0.43%, 250=2.46% lat (msec) : 500=35.40%, 750=38.09%, 1000=18.59%, 2000=4.90% cpu : usr=0.08%, sys=0.08%, ctx=6989, majf=0, minf=178 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=36773,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_8K: (groupid=1, jobs=4): err= 0: pid=2872: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=203, BW=1630KiB/s (1670kB/s)(287MiB/180314msec) slat (usec): min=3, max=248909, avg=19590.65, stdev=52195.08 clat (msec): min=39, max=1561, avg=607.84, stdev=181.88 lat (msec): min=39, max=1589, avg=627.43, stdev=188.92 clat percentiles (msec): | 1.00th=[ 288], 5.00th=[ 334], 10.00th=[ 439], 20.00th=[ 464], | 30.00th=[ 481], 40.00th=[ 506], 50.00th=[ 600], 60.00th=[ 625], | 70.00th=[ 659], 80.00th=[ 768], 90.00th=[ 844], 95.00th=[ 953], | 99.00th=[ 1133], 99.50th=[ 1234], 99.90th=[ 1401], 99.95th=[ 1435], | 99.99th=[ 1552] bw ( KiB/s): min= 697, max= 2052, per=99.79%, avg=1627.20, stdev=55.78, samples=1440 iops : min= 86, max= 256, avg=203.37, stdev= 6.97, samples=1440 lat (msec) : 50=0.01%, 250=0.08%, 500=38.24%, 750=39.87%, 1000=18.70% lat (msec) : 2000=3.11% cpu : usr=0.05%, sys=0.06%, ctx=4653, majf=0, minf=310 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=36748,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_32K: (groupid=2, jobs=4): err= 0: pid=2876: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=195, BW=6254KiB/s (6404kB/s)(1102MiB/180379msec) slat (usec): min=2, max=389206, avg=20426.74, stdev=54508.42 clat (msec): min=167, max=1807, avg=633.85, stdev=189.73 lat (msec): min=167, max=1807, avg=654.28, stdev=197.68 clat percentiles (msec): | 1.00th=[ 300], 5.00th=[ 359], 10.00th=[ 451], 20.00th=[ 477], | 30.00th=[ 502], 40.00th=[ 542], 50.00th=[ 625], 60.00th=[ 651], | 70.00th=[ 684], 80.00th=[ 793], 90.00th=[ 894], 95.00th=[ 995], | 99.00th=[ 1183], 99.50th=[ 1284], 99.90th=[ 1435], 99.95th=[ 1485], | 99.99th=[ 1586] bw ( KiB/s): min= 1170, max= 8396, per=99.88%, avg=6247.35, stdev=225.93, samples=1440 iops : min= 34, max= 262, avg=195.14, stdev= 7.07, samples=1440 lat (msec) : 250=0.01%, 500=30.11%, 750=45.58%, 1000=19.59%, 2000=4.71% cpu : usr=0.03%, sys=0.06%, ctx=4460, majf=0, minf=1073 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=35255,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_64K: (groupid=3, jobs=4): err= 0: pid=2887: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=192, BW=12.0MiB/s (12.6MB/s)(2170MiB/180409msec) slat (usec): min=5, max=235606, avg=20740.74, stdev=55195.48 clat (msec): min=172, max=1781, avg=643.80, stdev=192.51 lat (msec): min=172, max=1781, avg=664.54, stdev=200.08 clat percentiles (msec): | 1.00th=[ 309], 5.00th=[ 355], 10.00th=[ 464], 20.00th=[ 489], | 30.00th=[ 510], 40.00th=[ 531], 50.00th=[ 642], 60.00th=[ 667], | 70.00th=[ 693], 80.00th=[ 810], 90.00th=[ 885], 95.00th=[ 1011], | 99.00th=[ 1200], 99.50th=[ 1318], 99.90th=[ 1469], 99.95th=[ 1519], | 99.99th=[ 1552] bw ( KiB/s): min= 4716, max=16416, per=99.88%, avg=12304.04, stdev=375.73, samples=1440 iops : min= 71, max= 256, avg=192.16, stdev= 5.88, samples=1440 lat (msec) : 250=0.01%, 500=25.52%, 750=50.49%, 1000=18.25%, 2000=5.73% cpu : usr=0.02%, sys=0.07%, ctx=4399, majf=0, minf=2098 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=34724,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_128K: (groupid=4, jobs=4): err= 0: pid=2901: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=180, BW=22.6MiB/s (23.7MB/s)(4076MiB/180413msec) slat (usec): min=4, max=252925, avg=22089.08, stdev=58782.98 clat (msec): min=162, max=1698, avg=685.62, stdev=203.28 lat (msec): min=162, max=1830, avg=707.71, stdev=211.72 clat percentiles (msec): | 1.00th=[ 330], 5.00th=[ 376], 10.00th=[ 498], 20.00th=[ 518], | 30.00th=[ 542], 40.00th=[ 575], 50.00th=[ 684], 60.00th=[ 709], | 70.00th=[ 743], 80.00th=[ 860], 90.00th=[ 944], 95.00th=[ 1070], | 99.00th=[ 1267], 99.50th=[ 1351], 99.90th=[ 1502], 99.95th=[ 1586], | 99.99th=[ 1670] bw ( KiB/s): min=10145, max=32784, per=99.84%, avg=23096.99, stdev=849.83, samples=1440 iops : min= 77, max= 256, avg=180.36, stdev= 6.64, samples=1440 lat (msec) : 250=0.01%, 500=11.40%, 750=60.40%, 1000=19.42%, 2000=8.77% cpu : usr=0.03%, sys=0.07%, ctx=4135, majf=0, minf=2101 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=32604,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_4K: (groupid=5, jobs=4): err= 0: pid=2909: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=309, BW=1238KiB/s (1268kB/s)(218MiB/180235msec); 0 zone resets slat (usec): min=2, max=221518, avg=12898.71, stdev=34618.33 clat (msec): min=5, max=1251, avg=400.36, stdev=90.15 lat (msec): min=5, max=1251, avg=413.26, stdev=96.98 clat percentiles (msec): | 1.00th=[ 255], 5.00th=[ 279], 10.00th=[ 296], 20.00th=[ 330], | 30.00th=[ 363], 40.00th=[ 384], 50.00th=[ 401], 60.00th=[ 418], | 70.00th=[ 435], 80.00th=[ 456], 90.00th=[ 498], 95.00th=[ 542], | 99.00th=[ 693], 99.50th=[ 776], 99.90th=[ 936], 99.95th=[ 978], | 99.99th=[ 1116] bw ( KiB/s): min= 768, max= 2765, per=99.67%, avg=1234.93, stdev=42.50, samples=1436 iops : min= 192, max= 689, avg=308.73, stdev=10.61, samples=1436 lat (msec) : 10=0.06%, 20=0.40%, 50=0.17%, 100=0.02%, 250=0.25% lat (msec) : 500=89.67%, 750=8.82%, 1000=0.58%, 2000=0.03% cpu : usr=0.10%, sys=0.09%, ctx=7048, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,55788,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_8K: (groupid=6, jobs=4): err= 0: pid=2970: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=305, BW=2444KiB/s (2502kB/s)(430MiB/180227msec); 0 zone resets slat (usec): min=2, max=194074, avg=13073.76, stdev=34966.02 clat (msec): min=2, max=1194, avg=405.75, stdev=85.38 lat (msec): min=2, max=1211, avg=418.82, stdev=92.45 clat percentiles (msec): | 1.00th=[ 262], 5.00th=[ 288], 10.00th=[ 300], 20.00th=[ 330], | 30.00th=[ 368], 40.00th=[ 393], 50.00th=[ 409], 60.00th=[ 422], | 70.00th=[ 439], 80.00th=[ 456], 90.00th=[ 502], 95.00th=[ 542], | 99.00th=[ 676], 99.50th=[ 768], 99.90th=[ 944], 99.95th=[ 1011], | 99.99th=[ 1099] bw ( KiB/s): min= 1494, max= 3490, per=99.89%, avg=2441.06, stdev=74.60, samples=1440 iops : min= 186, max= 435, avg=305.02, stdev= 9.32, samples=1440 lat (msec) : 4=0.01%, 10=0.03%, 20=0.04%, 50=0.08%, 100=0.01% lat (msec) : 250=0.27%, 500=89.66%, 750=9.36%, 1000=0.48%, 2000=0.05% cpu : usr=0.07%, sys=0.09%, ctx=6979, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,55052,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_32K: (groupid=7, jobs=4): err= 0: pid=2977: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=291, BW=9334KiB/s (9558kB/s)(1643MiB/180226msec); 0 zone resets slat (usec): min=3, max=169485, avg=13693.49, stdev=36573.95 clat (msec): min=13, max=1304, avg=425.04, stdev=86.75 lat (msec): min=13, max=1304, avg=438.73, stdev=94.14 clat percentiles (msec): | 1.00th=[ 279], 5.00th=[ 300], 10.00th=[ 317], 20.00th=[ 347], | 30.00th=[ 388], 40.00th=[ 414], 50.00th=[ 426], 60.00th=[ 439], | 70.00th=[ 456], 80.00th=[ 477], 90.00th=[ 523], 95.00th=[ 567], | 99.00th=[ 709], 99.50th=[ 793], 99.90th=[ 961], 99.95th=[ 1028], | 99.99th=[ 1133] bw ( KiB/s): min= 6144, max=12300, per=99.82%, avg=9318.99, stdev=293.12, samples=1440 iops : min= 192, max= 384, avg=291.09, stdev= 9.16, samples=1440 lat (msec) : 20=0.01%, 50=0.04%, 100=0.03%, 250=0.13%, 500=86.08% lat (msec) : 750=13.01%, 1000=0.63%, 2000=0.07% cpu : usr=0.06%, sys=0.09%, ctx=6643, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,52572,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_64K: (groupid=8, jobs=4): err= 0: pid=2998: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=279, BW=17.4MiB/s (18.3MB/s)(3144MiB/180209msec); 0 zone resets slat (usec): min=7, max=201325, avg=14313.20, stdev=38207.51 clat (msec): min=20, max=1319, avg=444.20, stdev=90.75 lat (msec): min=20, max=1319, avg=458.52, stdev=98.51 clat percentiles (msec): | 1.00th=[ 288], 5.00th=[ 317], 10.00th=[ 334], 20.00th=[ 363], | 30.00th=[ 401], 40.00th=[ 426], 50.00th=[ 447], 60.00th=[ 460], | 70.00th=[ 477], 80.00th=[ 502], 90.00th=[ 542], 95.00th=[ 592], | 99.00th=[ 735], 99.50th=[ 835], 99.90th=[ 986], 99.95th=[ 1036], | 99.99th=[ 1133] bw ( KiB/s): min=12288, max=24915, per=99.84%, avg=17835.03, stdev=566.84, samples=1440 iops : min= 192, max= 389, avg=278.55, stdev= 8.85, samples=1440 lat (msec) : 50=0.04%, 100=0.03%, 250=0.11%, 500=80.05%, 750=18.83% lat (msec) : 1000=0.84%, 2000=0.09% cpu : usr=0.06%, sys=0.08%, ctx=6346, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,50300,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_128K: (groupid=9, jobs=4): err= 0: pid=3005: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=255, BW=31.9MiB/s (33.5MB/s)(5751MiB/180240msec); 0 zone resets slat (usec): min=6, max=229678, avg=15651.22, stdev=41825.57 clat (msec): min=24, max=1432, avg=485.77, stdev=101.91 lat (msec): min=24, max=1577, avg=501.42, stdev=110.40 clat percentiles (msec): | 1.00th=[ 309], 5.00th=[ 342], 10.00th=[ 363], 20.00th=[ 397], | 30.00th=[ 435], 40.00th=[ 468], 50.00th=[ 489], 60.00th=[ 506], | 70.00th=[ 523], 80.00th=[ 550], 90.00th=[ 609], 95.00th=[ 651], | 99.00th=[ 818], 99.50th=[ 911], 99.90th=[ 1099], 99.95th=[ 1167], | 99.99th=[ 1267] bw ( KiB/s): min=16384, max=41040, per=99.89%, avg=32633.85, stdev=1111.56, samples=1440 iops : min= 128, max= 320, avg=254.84, stdev= 8.68, samples=1440 lat (msec) : 50=0.04%, 100=0.03%, 250=0.09%, 500=57.46%, 750=40.69% lat (msec) : 1000=1.45%, 2000=0.23% cpu : usr=0.07%, sys=0.09%, ctx=5819, majf=0, minf=56 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,46004,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_4K: (groupid=10, jobs=4): err= 0: pid=3016: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=63.9k, BW=250MiB/s (262MB/s)(43.9GiB/180024msec) slat (nsec): min=1885, max=1634.8k, avg=4496.09, stdev=2838.42 clat (usec): min=48, max=2561.7k, avg=1997.90, stdev=61389.07 lat (usec): min=53, max=2561.7k, avg=2002.59, stdev=61389.10 clat percentiles (usec): | 1.00th=[ 129], 5.00th=[ 314], 10.00th=[ 367], | 20.00th=[ 424], 30.00th=[ 433], 40.00th=[ 453], | 50.00th=[ 461], 60.00th=[ 469], 70.00th=[ 474], | 80.00th=[ 490], 90.00th=[ 519], 95.00th=[ 807], | 99.00th=[ 914], 99.50th=[ 2057], 99.90th=[ 3589], | 99.95th=[2499806], 99.99th=[2533360] bw ( KiB/s): min= 96, max=1075865, per=100.00%, avg=602188.16, stdev=113931.11, samples=605 iops : min= 24, max=268966, avg=150546.80, stdev=28482.78, samples=605 lat (usec) : 50=0.01%, 100=0.01%, 250=1.91%, 500=83.82%, 750=8.82% lat (usec) : 1000=4.66% lat (msec) : 2=0.25%, 4=0.45%, 10=0.02%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.01%, >=2000=0.06% cpu : usr=4.05%, sys=8.23%, ctx=2320178, majf=0, minf=191 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=11499412,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_8K: (groupid=11, jobs=4): err= 0: pid=3023: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=32.4k, BW=253MiB/s (266MB/s)(44.5GiB/180029msec) slat (usec): min=2, max=1548, avg= 5.27, stdev= 3.11 clat (usec): min=63, max=2566.2k, avg=3940.08, stdev=85430.76 lat (usec): min=76, max=2566.2k, avg=3945.57, stdev=85430.79 clat percentiles (usec): | 1.00th=[ 676], 5.00th=[ 742], 10.00th=[ 824], | 20.00th=[ 865], 30.00th=[ 881], 40.00th=[ 914], | 50.00th=[ 930], 60.00th=[ 947], 70.00th=[ 955], | 80.00th=[ 988], 90.00th=[ 1237], 95.00th=[ 1336], | 99.00th=[ 2147], 99.50th=[ 2573], 99.90th=[2499806], | 99.95th=[2499806], 99.99th=[2533360] bw ( KiB/s): min= 208, max=1073088, per=100.00%, avg=589207.53, stdev=110361.17, samples=633 iops : min= 26, max=134136, avg=73650.58, stdev=13795.13, samples=633 lat (usec) : 100=0.01%, 250=0.02%, 500=0.07%, 750=5.29%, 1000=78.11% lat (msec) : 2=15.49%, 4=0.85%, 10=0.03%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.01%, >=2000=0.12% cpu : usr=2.51%, sys=5.06%, ctx=1291491, majf=0, minf=322 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=5837890,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_32K: (groupid=12, jobs=4): err= 0: pid=3039: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=8013, BW=250MiB/s (263MB/s)(44.0GiB/180029msec) slat (usec): min=2, max=124, avg= 8.24, stdev= 3.54 clat (usec): min=312, max=2719.9k, avg=15960.22, stdev=171347.63 lat (usec): min=320, max=2719.9k, avg=15968.75, stdev=171347.64 clat percentiles (msec): | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 4], | 30.00th=[ 4], 40.00th=[ 4], 50.00th=[ 4], 60.00th=[ 4], | 70.00th=[ 5], 80.00th=[ 5], 90.00th=[ 5], 95.00th=[ 5], | 99.00th=[ 6], 99.50th=[ 447], 99.90th=[ 2500], 99.95th=[ 2500], | 99.99th=[ 2534] bw ( KiB/s): min= 704, max=1076928, per=100.00%, avg=602529.84, stdev=103885.69, samples=611 iops : min= 22, max=33654, avg=18828.69, stdev=3246.42, samples=611 lat (usec) : 500=0.01%, 750=0.09%, 1000=0.05% lat (msec) : 2=0.12%, 4=64.48%, 10=34.61%, 20=0.07%, 50=0.03% lat (msec) : 100=0.02%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.01%, >=2000=0.47% cpu : usr=1.12%, sys=2.22%, ctx=385714, majf=0, minf=1088 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=1442742,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_64K: (groupid=13, jobs=4): err= 0: pid=3047: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=4021, BW=251MiB/s (264MB/s)(44.2GiB/180050msec) slat (usec): min=5, max=125, avg=11.59, stdev= 6.95 clat (usec): min=318, max=2542.8k, avg=31810.98, stdev=242400.93 lat (usec): min=338, max=2542.8k, avg=31822.92, stdev=242400.93 clat percentiles (msec): | 1.00th=[ 5], 5.00th=[ 7], 10.00th=[ 8], 20.00th=[ 8], | 30.00th=[ 8], 40.00th=[ 8], 50.00th=[ 8], 60.00th=[ 8], | 70.00th=[ 8], 80.00th=[ 9], 90.00th=[ 9], 95.00th=[ 10], | 99.00th=[ 169], 99.50th=[ 2500], 99.90th=[ 2500], 99.95th=[ 2534], | 99.99th=[ 2534] bw ( KiB/s): min= 5507, max=1076120, per=100.00%, avg=620439.10, stdev=106123.96, samples=596 iops : min= 86, max=16814, avg=9694.06, stdev=1658.20, samples=596 lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.18%, 4=0.71%, 10=97.64%, 20=0.28%, 50=0.08% lat (msec) : 100=0.04%, 250=0.03%, 500=0.02%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.03%, >=2000=0.94% cpu : usr=0.62%, sys=1.89%, ctx=184376, majf=0, minf=2118 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=724064,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_128K: (groupid=14, jobs=4): err= 0: pid=3052: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=2009, BW=251MiB/s (263MB/s)(44.2GiB/180070msec) slat (usec): min=3, max=873, avg=12.60, stdev= 7.73 clat (usec): min=1278, max=2555.8k, avg=63683.12, stdev=343276.63 lat (usec): min=1303, max=2555.8k, avg=63696.07, stdev=343276.66 clat percentiles (msec): | 1.00th=[ 7], 5.00th=[ 10], 10.00th=[ 13], 20.00th=[ 15], | 30.00th=[ 15], 40.00th=[ 16], 50.00th=[ 16], 60.00th=[ 16], | 70.00th=[ 17], 80.00th=[ 17], 90.00th=[ 17], 95.00th=[ 18], | 99.00th=[ 2534], 99.50th=[ 2534], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2567] bw ( KiB/s): min= 1537, max=1076508, per=100.00%, avg=602088.96, stdev=110740.44, samples=614 iops : min= 12, max= 8410, avg=4703.57, stdev=865.17, samples=614 lat (msec) : 2=0.01%, 4=0.05%, 10=6.97%, 20=90.24%, 50=0.48% lat (msec) : 100=0.21%, 250=0.08%, 500=0.01%, 1000=0.02%, 2000=0.03% lat (msec) : >=2000=1.92% cpu : usr=0.31%, sys=0.96%, ctx=91567, majf=0, minf=2108 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=361786,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_512K: (groupid=15, jobs=4): err= 0: pid=3062: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=543, BW=272MiB/s (285MB/s)(47.8GiB/180396msec) slat (usec): min=10, max=74407, avg=113.27, stdev=1632.05 clat (msec): min=11, max=3105, avg=235.32, stdev=397.66 lat (msec): min=11, max=3105, avg=235.44, stdev=397.67 clat percentiles (msec): | 1.00th=[ 16], 5.00th=[ 32], 10.00th=[ 53], 20.00th=[ 60], | 30.00th=[ 63], 40.00th=[ 64], 50.00th=[ 65], 60.00th=[ 67], | 70.00th=[ 146], 80.00th=[ 271], 90.00th=[ 642], 95.00th=[ 1083], | 99.00th=[ 2089], 99.50th=[ 2400], 99.90th=[ 2869], 99.95th=[ 3037], | 99.99th=[ 3104] bw ( KiB/s): min= 8192, max=977404, per=100.00%, avg=294899.88, stdev=72214.50, samples=1353 iops : min= 16, max= 1908, avg=575.73, stdev=140.99, samples=1353 lat (msec) : 20=2.14%, 50=7.09%, 100=58.56%, 250=11.34%, 500=4.13% lat (msec) : 750=9.14%, 1000=1.85%, 2000=4.64%, >=2000=1.11% cpu : usr=0.13%, sys=0.54%, ctx=48166, majf=0, minf=2136 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=97985,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_1M: (groupid=16, jobs=4): err= 0: pid=3070: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=489, BW=489MiB/s (513MB/s)(86.1GiB/180203msec) slat (usec): min=15, max=241216, avg=8165.14, stdev=22837.49 clat (msec): min=87, max=3024, avg=253.34, stdev=332.94 lat (msec): min=119, max=3053, avg=261.51, stdev=333.95 clat percentiles (msec): | 1.00th=[ 120], 5.00th=[ 121], 10.00th=[ 180], 20.00th=[ 180], | 30.00th=[ 180], 40.00th=[ 180], 50.00th=[ 180], 60.00th=[ 180], | 70.00th=[ 239], 80.00th=[ 239], 90.00th=[ 292], 95.00th=[ 380], | 99.00th=[ 2769], 99.50th=[ 2836], 99.90th=[ 2937], 99.95th=[ 2937], | 99.99th=[ 3004] bw ( KiB/s): min=212992, max=594809, per=99.98%, avg=500854.61, stdev=18857.18, samples=1440 iops : min= 208, max= 580, avg=488.90, stdev=18.41, samples=1440 lat (msec) : 100=0.01%, 250=86.55%, 500=10.52%, 750=0.47%, 1000=0.25% lat (msec) : 2000=0.69%, >=2000=1.52% cpu : usr=0.05%, sys=0.40%, ctx=11122, majf=0, minf=2161 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=88154,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_4K: (groupid=17, jobs=4): err= 0: pid=3086: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=17.8k, BW=69.7MiB/s (73.1MB/s)(12.3GiB/180029msec); 0 zone resets slat (nsec): min=1969, max=146157, avg=6967.58, stdev=3566.86 clat (usec): min=1150, max=58790, avg=7165.43, stdev=8585.82 lat (usec): min=1188, max=58796, avg=7172.69, stdev=8586.81 clat percentiles (usec): | 1.00th=[ 2868], 5.00th=[ 3097], 10.00th=[ 3294], 20.00th=[ 3523], | 30.00th=[ 3720], 40.00th=[ 3916], 50.00th=[ 4113], 60.00th=[ 4293], | 70.00th=[ 4555], 80.00th=[ 5145], 90.00th=[18482], 95.00th=[30802], | 99.00th=[41681], 99.50th=[43254], 99.90th=[46400], 99.95th=[47973], | 99.99th=[49546] bw ( KiB/s): min=11103, max=135280, per=100.00%, avg=71501.38, stdev=12813.66, samples=1436 iops : min= 2775, max=33820, avg=17875.02, stdev=3203.44, samples=1436 lat (msec) : 2=0.01%, 4=44.85%, 10=41.38%, 20=4.43%, 50=9.34% lat (msec) : 100=0.01% cpu : usr=2.31%, sys=4.79%, ctx=845025, majf=0, minf=58 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,3211693,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_8K: (groupid=18, jobs=4): err= 0: pid=3097: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=27.4k, BW=214MiB/s (225MB/s)(37.7GiB/180003msec); 0 zone resets slat (nsec): min=1641, max=113454, avg=7114.51, stdev=2988.68 clat (usec): min=142, max=2562.1k, avg=4656.35, stdev=14446.69 lat (usec): min=163, max=2562.1k, avg=4663.75, stdev=14446.72 clat percentiles (usec): | 1.00th=[ 594], 5.00th=[ 3523], 10.00th=[ 3654], 20.00th=[ 3785], | 30.00th=[ 3916], 40.00th=[ 4015], 50.00th=[ 4113], 60.00th=[ 4293], | 70.00th=[ 4555], 80.00th=[ 5145], 90.00th=[ 6456], 95.00th=[ 7701], | 99.00th=[10159], 99.50th=[10945], 99.90th=[12780], 99.95th=[14091], | 99.99th=[34341] bw ( KiB/s): min= 7831, max=652633, per=100.00%, avg=221911.87, stdev=16667.91, samples=1420 iops : min= 978, max=81579, avg=27738.59, stdev=2083.50, samples=1420 lat (usec) : 250=0.01%, 500=0.11%, 750=1.53%, 1000=1.21% lat (msec) : 2=0.32%, 4=36.41%, 10=59.35%, 20=1.04%, 50=0.02% lat (msec) : 100=0.01%, 250=0.01%, 1000=0.01%, 2000=0.01%, >=2000=0.01% cpu : usr=3.46%, sys=7.59%, ctx=1368749, majf=0, minf=65 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,4938116,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_32K: (groupid=19, jobs=4): err= 0: pid=3104: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=7761, BW=243MiB/s (254MB/s)(42.6GiB/180054msec); 0 zone resets slat (usec): min=3, max=117, avg= 9.06, stdev= 2.69 clat (usec): min=352, max=3437.2k, avg=16479.04, stdev=158653.69 lat (usec): min=362, max=3437.2k, avg=16488.41, stdev=158653.77 clat percentiles (usec): | 1.00th=[ 1037], 5.00th=[ 1680], 10.00th=[ 3097], | 20.00th=[ 3490], 30.00th=[ 3687], 40.00th=[ 3752], | 50.00th=[ 3818], 60.00th=[ 3916], 70.00th=[ 4047], | 80.00th=[ 4146], 90.00th=[ 4359], 95.00th=[ 5276], | 99.00th=[ 43779], 99.50th=[1166017], 99.90th=[2533360], | 99.95th=[2533360], 99.99th=[2533360] bw ( KiB/s): min= 640, max=1073114, per=100.00%, avg=453561.18, stdev=96935.88, samples=762 iops : min= 20, max=33534, avg=14173.45, stdev=3029.23, samples=762 lat (usec) : 500=0.01%, 750=0.15%, 1000=0.57% lat (msec) : 2=6.22%, 4=59.40%, 10=31.01%, 20=1.29%, 50=0.38% lat (msec) : 100=0.12%, 250=0.11%, 500=0.09%, 750=0.06%, 1000=0.07% lat (msec) : 2000=0.21%, >=2000=0.33% cpu : usr=1.37%, sys=2.03%, ctx=368064, majf=0, minf=63 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,1397452,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_64K: (groupid=20, jobs=4): err= 0: pid=3116: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=3887, BW=243MiB/s (255MB/s)(42.7GiB/180077msec); 0 zone resets slat (usec): min=4, max=1652, avg=12.95, stdev= 6.91 clat (usec): min=559, max=2577.6k, avg=32901.30, stdev=219214.35 lat (usec): min=574, max=2577.6k, avg=32914.60, stdev=219214.43 clat percentiles (usec): | 1.00th=[ 1254], 5.00th=[ 3884], 10.00th=[ 6325], | 20.00th=[ 7242], 30.00th=[ 7439], 40.00th=[ 7701], | 50.00th=[ 7832], 60.00th=[ 7898], 70.00th=[ 8029], | 80.00th=[ 8291], 90.00th=[ 8979], 95.00th=[ 17957], | 99.00th=[1010828], 99.50th=[2332034], 99.90th=[2533360], | 99.95th=[2533360], 99.99th=[2566915] bw ( KiB/s): min= 768, max=1078528, per=100.00%, avg=443393.70, stdev=98760.24, samples=788 iops : min= 12, max=16852, avg=6927.68, stdev=1543.13, samples=788 lat (usec) : 750=0.04%, 1000=0.21% lat (msec) : 2=1.18%, 4=4.83%, 10=88.10%, 20=1.36%, 50=1.73% lat (msec) : 100=0.56%, 250=0.43%, 500=0.31%, 750=0.13%, 1000=0.11% lat (msec) : 2000=0.36%, >=2000=0.65% cpu : usr=0.86%, sys=1.64%, ctx=168468, majf=0, minf=65 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,700123,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_128K: (groupid=21, jobs=4): err= 0: pid=3128: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=1921, BW=240MiB/s (252MB/s)(42.2GiB/180078msec); 0 zone resets slat (nsec): min=4987, max=67589, avg=15687.60, stdev=7277.38 clat (usec): min=673, max=2607.6k, avg=66567.53, stdev=300855.50 lat (usec): min=699, max=2607.6k, avg=66583.56, stdev=300855.59 clat percentiles (msec): | 1.00th=[ 8], 5.00th=[ 11], 10.00th=[ 15], 20.00th=[ 15], | 30.00th=[ 16], 40.00th=[ 16], 50.00th=[ 16], 60.00th=[ 16], | 70.00th=[ 17], 80.00th=[ 17], 90.00th=[ 18], 95.00th=[ 82], | 99.00th=[ 2198], 99.50th=[ 2534], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2567] bw ( KiB/s): min= 2305, max=1072149, per=100.00%, avg=434381.30, stdev=100204.16, samples=806 iops : min= 18, max= 8376, avg=3393.25, stdev=782.83, samples=806 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.08%, 10=4.03%, 20=88.21%, 50=1.66% lat (msec) : 100=1.23%, 250=1.26%, 500=0.87%, 750=0.34%, 1000=0.26% lat (msec) : 2000=0.83%, >=2000=1.21% cpu : usr=0.58%, sys=0.78%, ctx=83653, majf=0, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,346108,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_512K: (groupid=22, jobs=4): err= 0: pid=3146: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=905, BW=453MiB/s (475MB/s)(79.6GiB/180133msec); 0 zone resets slat (usec): min=12, max=159250, avg=675.81, stdev=3244.47 clat (msec): min=10, max=564, avg=140.70, stdev=13.21 lat (msec): min=10, max=564, avg=141.38, stdev=13.25 clat percentiles (msec): | 1.00th=[ 115], 5.00th=[ 128], 10.00th=[ 134], 20.00th=[ 136], | 30.00th=[ 136], 40.00th=[ 136], 50.00th=[ 140], 60.00th=[ 144], | 70.00th=[ 144], 80.00th=[ 146], 90.00th=[ 150], 95.00th=[ 155], | 99.00th=[ 167], 99.50th=[ 199], 99.90th=[ 317], 99.95th=[ 330], | 99.99th=[ 380] bw ( KiB/s): min=295936, max=510714, per=100.00%, avg=463749.05, stdev=5523.40, samples=1436 iops : min= 578, max= 996, avg=905.28, stdev=10.78, samples=1436 lat (msec) : 20=0.01%, 50=0.01%, 100=0.62%, 250=99.10%, 500=0.25% lat (msec) : 750=0.01% cpu : usr=0.84%, sys=0.67%, ctx=90957, majf=0, minf=53 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,163064,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_1M: (groupid=23, jobs=4): err= 0: pid=3234: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=245, BW=246MiB/s (258MB/s)(43.3GiB/180295msec); 0 zone resets slat (usec): min=26, max=302998, avg=16233.37, stdev=44827.17 clat (msec): min=103, max=2454, avg=504.06, stdev=107.27 lat (msec): min=103, max=2454, avg=520.30, stdev=114.67 clat percentiles (msec): | 1.00th=[ 271], 5.00th=[ 330], 10.00th=[ 380], 20.00th=[ 430], | 30.00th=[ 460], 40.00th=[ 485], 50.00th=[ 502], 60.00th=[ 527], | 70.00th=[ 550], 80.00th=[ 575], 90.00th=[ 625], 95.00th=[ 667], | 99.00th=[ 802], 99.50th=[ 869], 99.90th=[ 1099], 99.95th=[ 1250], | 99.99th=[ 1653] bw ( KiB/s): min=129024, max=522502, per=99.80%, avg=251288.08, stdev=12196.54, samples=1439 iops : min= 126, max= 510, avg=245.28, stdev=11.90, samples=1439 lat (msec) : 250=0.82%, 500=47.91%, 750=49.39%, 1000=1.71%, 2000=0.16% lat (msec) : >=2000=0.01% cpu : usr=0.31%, sys=0.11%, ctx=5757, majf=0, minf=60 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,44333,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_4K: (groupid=24, jobs=4): err= 0: pid=3241: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=122, BW=492KiB/s (504kB/s)(86.6MiB/180330msec) slat (usec): min=2, max=207585, avg=16220.91, stdev=43771.48 clat (msec): min=51, max=1410, avg=561.55, stdev=204.42 lat (msec): min=51, max=1557, avg=577.77, stdev=208.25 clat percentiles (msec): | 1.00th=[ 230], 5.00th=[ 271], 10.00th=[ 347], 20.00th=[ 384], | 30.00th=[ 414], 40.00th=[ 481], 50.00th=[ 523], 60.00th=[ 575], | 70.00th=[ 659], 80.00th=[ 760], 90.00th=[ 877], 95.00th=[ 936], | 99.00th=[ 1053], 99.50th=[ 1083], 99.90th=[ 1183], 99.95th=[ 1217], | 99.99th=[ 1351] bw ( KiB/s): min= 132, max= 967, per=99.82%, avg=491.01, stdev=28.73, samples=1440 iops : min= 32, max= 241, avg=122.74, stdev= 7.18, samples=1440 write: IOPS=122, BW=490KiB/s (502kB/s)(86.3MiB/180330msec); 0 zone resets slat (usec): min=2, max=267233, avg=16274.32, stdev=43674.42 clat (usec): min=537, max=1307.2k, avg=447199.14, stdev=156028.33 lat (usec): min=560, max=1453.6k, avg=463473.85, stdev=161446.72 clat percentiles (msec): | 1.00th=[ 205], 5.00th=[ 239], 10.00th=[ 255], 20.00th=[ 321], | 30.00th=[ 372], 40.00th=[ 393], 50.00th=[ 409], 60.00th=[ 451], | 70.00th=[ 514], 80.00th=[ 558], 90.00th=[ 667], 95.00th=[ 743], | 99.00th=[ 885], 99.50th=[ 944], 99.90th=[ 1083], 99.95th=[ 1133], | 99.99th=[ 1167] bw ( KiB/s): min= 192, max= 1073, per=99.75%, avg=489.30, stdev=28.87, samples=1440 iops : min= 48, max= 268, avg=122.32, stdev= 7.22, samples=1440 lat (usec) : 750=0.01% lat (msec) : 2=0.01%, 4=0.03%, 10=0.01%, 20=0.01%, 50=0.03% lat (msec) : 100=0.06%, 250=5.40%, 500=49.69%, 750=31.82%, 1000=11.69% lat (msec) : 2000=1.26% cpu : usr=0.09%, sys=0.07%, ctx=5674, majf=0, minf=64 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=22176,22100,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_8K: (groupid=25, jobs=4): err= 0: pid=3252: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=120, BW=968KiB/s (991kB/s)(170MiB/180321msec) slat (usec): min=2, max=218577, avg=16185.66, stdev=43657.29 clat (msec): min=5, max=1374, avg=567.36, stdev=210.62 lat (msec): min=5, max=1429, avg=583.55, stdev=215.22 clat percentiles (msec): | 1.00th=[ 222], 5.00th=[ 268], 10.00th=[ 342], 20.00th=[ 388], | 30.00th=[ 414], 40.00th=[ 485], 50.00th=[ 527], 60.00th=[ 584], | 70.00th=[ 667], 80.00th=[ 776], 90.00th=[ 894], 95.00th=[ 944], | 99.00th=[ 1062], 99.50th=[ 1099], 99.90th=[ 1217], 99.95th=[ 1267], | 99.99th=[ 1351] bw ( KiB/s): min= 253, max= 1904, per=99.80%, avg=966.76, stdev=59.34, samples=1439 iops : min= 31, max= 238, avg=120.84, stdev= 7.42, samples=1439 write: IOPS=123, BW=985KiB/s (1008kB/s)(173MiB/180321msec); 0 zone resets slat (usec): min=2, max=228415, avg=16345.06, stdev=43907.31 clat (usec): min=608, max=1365.4k, avg=448571.52, stdev=161029.40 lat (usec): min=632, max=1377.9k, avg=464916.97, stdev=166869.05 clat percentiles (msec): | 1.00th=[ 192], 5.00th=[ 236], 10.00th=[ 255], 20.00th=[ 309], | 30.00th=[ 372], 40.00th=[ 393], 50.00th=[ 409], 60.00th=[ 451], | 70.00th=[ 518], 80.00th=[ 567], 90.00th=[ 667], 95.00th=[ 760], | 99.00th=[ 911], 99.50th=[ 953], 99.90th=[ 1099], 99.95th=[ 1183], | 99.99th=[ 1318] bw ( KiB/s): min= 330, max= 1856, per=99.93%, avg=984.14, stdev=61.29, samples=1439 iops : min= 41, max= 232, avg=123.01, stdev= 7.66, samples=1439 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.03%, 10=0.05%, 20=0.04%, 50=0.09% lat (msec) : 100=0.22%, 250=5.33%, 500=48.91%, 750=31.58%, 1000=12.32% lat (msec) : 2000=1.43% cpu : usr=0.06%, sys=0.07%, ctx=6120, majf=0, minf=72 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=21818,22194,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_32K: (groupid=26, jobs=4): err= 0: pid=3274: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=116, BW=3733KiB/s (3823kB/s)(658MiB/180359msec) slat (usec): min=4, max=228482, avg=17109.62, stdev=46113.88 clat (msec): min=112, max=1403, avg=593.57, stdev=214.15 lat (msec): min=112, max=1473, avg=610.68, stdev=218.59 clat percentiles (msec): | 1.00th=[ 243], 5.00th=[ 284], 10.00th=[ 359], 20.00th=[ 405], | 30.00th=[ 435], 40.00th=[ 506], 50.00th=[ 558], 60.00th=[ 617], | 70.00th=[ 693], 80.00th=[ 802], 90.00th=[ 919], 95.00th=[ 978], | 99.00th=[ 1099], 99.50th=[ 1133], 99.90th=[ 1217], 99.95th=[ 1250], | 99.99th=[ 1301] bw ( KiB/s): min= 1344, max= 6272, per=99.78%, avg=3725.19, stdev=222.68, samples=1440 iops : min= 42, max= 196, avg=116.37, stdev= 6.96, samples=1440 write: IOPS=116, BW=3721KiB/s (3810kB/s)(655MiB/180359msec); 0 zone resets slat (usec): min=5, max=234044, avg=17168.47, stdev=46126.27 clat (msec): min=51, max=1309, avg=470.49, stdev=162.91 lat (msec): min=51, max=1309, avg=487.65, stdev=169.54 clat percentiles (msec): | 1.00th=[ 222], 5.00th=[ 251], 10.00th=[ 271], 20.00th=[ 334], | 30.00th=[ 388], 40.00th=[ 409], 50.00th=[ 430], 60.00th=[ 477], | 70.00th=[ 542], 80.00th=[ 592], 90.00th=[ 701], 95.00th=[ 785], | 99.00th=[ 936], 99.50th=[ 978], 99.90th=[ 1116], 99.95th=[ 1133], | 99.99th=[ 1250] bw ( KiB/s): min= 1344, max= 6403, per=99.82%, avg=3714.96, stdev=224.86, samples=1440 iops : min= 42, max= 200, avg=116.05, stdev= 7.02, samples=1440 lat (msec) : 100=0.03%, 250=3.22%, 500=47.63%, 750=33.72%, 1000=13.27% lat (msec) : 2000=2.12% cpu : usr=0.04%, sys=0.08%, ctx=5340, majf=0, minf=58 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=21042,20970,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_64K: (groupid=27, jobs=4): err= 0: pid=3282: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=111, BW=7141KiB/s (7312kB/s)(1258MiB/180348msec) slat (usec): min=5, max=252144, avg=17430.17, stdev=47431.27 clat (msec): min=117, max=1467, avg=617.17, stdev=220.30 lat (msec): min=117, max=1622, avg=634.60, stdev=225.09 clat percentiles (msec): | 1.00th=[ 253], 5.00th=[ 296], 10.00th=[ 380], 20.00th=[ 422], | 30.00th=[ 451], 40.00th=[ 531], 50.00th=[ 575], 60.00th=[ 651], | 70.00th=[ 726], 80.00th=[ 835], 90.00th=[ 953], 95.00th=[ 1011], | 99.00th=[ 1133], 99.50th=[ 1167], 99.90th=[ 1234], 99.95th=[ 1284], | 99.99th=[ 1385] bw ( KiB/s): min= 3072, max=11781, per=99.86%, avg=7131.74, stdev=428.13, samples=1440 iops : min= 48, max= 184, avg=111.38, stdev= 6.69, samples=1440 write: IOPS=112, BW=7220KiB/s (7393kB/s)(1272MiB/180348msec); 0 zone resets slat (usec): min=5, max=309808, avg=18147.31, stdev=48285.12 clat (msec): min=56, max=1254, avg=488.41, stdev=164.74 lat (msec): min=56, max=1309, avg=506.55, stdev=170.53 clat percentiles (msec): | 1.00th=[ 232], 5.00th=[ 264], 10.00th=[ 284], 20.00th=[ 363], | 30.00th=[ 405], 40.00th=[ 426], 50.00th=[ 447], 60.00th=[ 493], | 70.00th=[ 567], 80.00th=[ 609], 90.00th=[ 726], 95.00th=[ 802], | 99.00th=[ 953], 99.50th=[ 1020], 99.90th=[ 1099], 99.95th=[ 1150], | 99.99th=[ 1217] bw ( KiB/s): min= 2432, max=12806, per=99.90%, avg=7213.37, stdev=432.75, samples=1440 iops : min= 38, max= 200, avg=112.66, stdev= 6.76, samples=1440 lat (msec) : 100=0.01%, 250=1.70%, 500=47.21%, 750=34.14%, 1000=13.69% lat (msec) : 2000=3.25% cpu : usr=0.03%, sys=0.09%, ctx=5127, majf=0, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=20122,20346,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_128K: (groupid=28, jobs=4): err= 0: pid=3293: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=104, BW=13.1MiB/s (13.8MB/s)(2367MiB/180363msec) slat (usec): min=5, max=259600, avg=19210.97, stdev=51449.39 clat (msec): min=140, max=1604, avg=659.10, stdev=236.14 lat (msec): min=140, max=1622, avg=678.32, stdev=241.90 clat percentiles (msec): | 1.00th=[ 271], 5.00th=[ 317], 10.00th=[ 405], 20.00th=[ 451], | 30.00th=[ 485], 40.00th=[ 567], 50.00th=[ 617], 60.00th=[ 676], | 70.00th=[ 776], 80.00th=[ 902], 90.00th=[ 1011], 95.00th=[ 1083], | 99.00th=[ 1217], 99.50th=[ 1250], 99.90th=[ 1368], 99.95th=[ 1418], | 99.99th=[ 1569] bw ( KiB/s): min= 4609, max=23574, per=99.78%, avg=13409.93, stdev=852.45, samples=1440 iops : min= 36, max= 184, avg=104.72, stdev= 6.66, samples=1440 write: IOPS=103, BW=12.0MiB/s (13.6MB/s)(2342MiB/180363msec); 0 zone resets slat (usec): min=6, max=274241, avg=19016.86, stdev=51364.68 clat (msec): min=94, max=1346, avg=526.94, stdev=178.42 lat (msec): min=94, max=1346, avg=545.96, stdev=185.52 clat percentiles (msec): | 1.00th=[ 251], 5.00th=[ 284], 10.00th=[ 305], 20.00th=[ 393], | 30.00th=[ 435], 40.00th=[ 460], 50.00th=[ 485], 60.00th=[ 535], | 70.00th=[ 609], 80.00th=[ 659], 90.00th=[ 776], 95.00th=[ 869], | 99.00th=[ 1036], 99.50th=[ 1083], 99.90th=[ 1183], 99.95th=[ 1250], | 99.99th=[ 1318] bw ( KiB/s): min= 4097, max=24587, per=99.91%, avg=13287.29, stdev=836.37, samples=1440 iops : min= 32, max= 192, avg=103.76, stdev= 6.53, samples=1440 lat (msec) : 100=0.01%, 250=0.60%, 500=43.54%, 750=32.35%, 1000=17.41% lat (msec) : 2000=6.09% cpu : usr=0.04%, sys=0.08%, ctx=4783, majf=0, minf=63 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=18937,18739,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_4K_latency: (groupid=29, jobs=1): err= 0: pid=3300: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=81, BW=328KiB/s (335kB/s)(57.6MiB/180017msec) slat (nsec): min=20418, max=58089, avg=29469.50, stdev=1372.87 clat (usec): min=1449, max=60811, avg=12172.74, stdev=3780.52 lat (usec): min=1479, max=60841, avg=12202.97, stdev=3780.54 clat percentiles (usec): | 1.00th=[ 3884], 5.00th=[ 5932], 10.00th=[ 7308], 20.00th=[ 8848], | 30.00th=[10028], 40.00th=[11207], 50.00th=[12125], 60.00th=[13173], | 70.00th=[14222], 80.00th=[15401], 90.00th=[17171], 95.00th=[18482], | 99.00th=[20579], 99.50th=[21365], 99.90th=[22414], 99.95th=[23987], | 99.99th=[38011] bw ( KiB/s): min= 280, max= 368, per=99.83%, avg=327.60, stdev=17.02, samples=359 iops : min= 70, max= 92, avg=81.90, stdev= 4.26, samples=359 lat (msec) : 2=0.02%, 4=1.06%, 10=28.25%, 20=69.07%, 50=1.59% lat (msec) : 100=0.01% cpu : usr=0.18%, sys=0.48%, ctx=14747, majf=0, minf=13 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=14742,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_4K_latency: (groupid=30, jobs=1): err= 0: pid=3317: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=63, BW=253KiB/s (259kB/s)(44.4MiB/180007msec) slat (nsec): min=13116, max=60020, avg=29114.80, stdev=1560.13 clat (usec): min=2006, max=50148, avg=15113.58, stdev=6933.89 lat (usec): min=2035, max=50178, avg=15143.43, stdev=6933.91 clat percentiles (usec): | 1.00th=[ 4293], 5.00th=[ 6325], 10.00th=[ 7701], 20.00th=[ 9634], | 30.00th=[10945], 40.00th=[12387], 50.00th=[13698], 60.00th=[15270], | 70.00th=[16909], 80.00th=[19792], 90.00th=[25297], 95.00th=[29230], | 99.00th=[36963], 99.50th=[38536], 99.90th=[42730], 99.95th=[45351], | 99.99th=[47973] bw ( KiB/s): min= 192, max= 352, per=99.80%, avg=252.39, stdev=20.25, samples=359 iops : min= 48, max= 88, avg=63.10, stdev= 5.06, samples=359 write: IOPS=62, BW=250KiB/s (256kB/s)(43.9MiB/180007msec); 0 zone resets slat (nsec): min=13495, max=60844, avg=29166.31, stdev=1738.88 clat (usec): min=113, max=15833, avg=660.55, stdev=1170.13 lat (usec): min=143, max=15863, avg=690.48, stdev=1170.09 clat percentiles (usec): | 1.00th=[ 184], 5.00th=[ 215], 10.00th=[ 233], 20.00th=[ 258], | 30.00th=[ 273], 40.00th=[ 289], 50.00th=[ 306], 60.00th=[ 330], | 70.00th=[ 347], 80.00th=[ 388], 90.00th=[ 1762], 95.00th=[ 2868], | 99.00th=[ 5145], 99.50th=[ 9634], 99.90th=[11469], 99.95th=[11863], | 99.99th=[12518] bw ( KiB/s): min= 112, max= 384, per=99.77%, avg=249.63, stdev=47.95, samples=359 iops : min= 28, max= 96, avg=62.41, stdev=11.99, samples=359 lat (usec) : 250=8.38%, 500=33.82%, 750=0.74%, 1000=0.09% lat (msec) : 2=2.22%, 4=3.74%, 10=12.03%, 20=29.12%, 50=9.85% lat (msec) : 100=0.01% cpu : usr=0.25%, sys=0.63%, ctx=22595, majf=0, minf=17 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=11363,11231,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randwrite_4K_latency: (groupid=31, jobs=1): err= 0: pid=3324: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=307, BW=1230KiB/s (1260kB/s)(216MiB/180004msec); 0 zone resets slat (nsec): min=9212, max=59563, avg=29460.70, stdev=1174.33 clat (usec): min=118, max=38217, avg=3214.24, stdev=1357.41 lat (usec): min=148, max=38248, avg=3244.50, stdev=1357.42 clat percentiles (usec): | 1.00th=[ 676], 5.00th=[ 1336], 10.00th=[ 1696], 20.00th=[ 2147], | 30.00th=[ 2474], 40.00th=[ 2769], 50.00th=[ 3064], 60.00th=[ 3392], | 70.00th=[ 3752], 80.00th=[ 4228], 90.00th=[ 4883], 95.00th=[ 5538], | 99.00th=[ 6915], 99.50th=[ 7570], 99.90th=[12518], 99.95th=[12911], | 99.99th=[15008] bw ( KiB/s): min= 896, max= 1550, per=99.99%, avg=1230.47, stdev=107.81, samples=359 iops : min= 224, max= 387, avg=307.51, stdev=26.95, samples=359 lat (usec) : 250=0.10%, 500=0.46%, 750=0.65%, 1000=1.13% lat (msec) : 2=14.11%, 4=59.39%, 10=23.94%, 20=0.21%, 50=0.01% cpu : usr=0.60%, sys=1.82%, ctx=55361, majf=0, minf=13 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,55356,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_70R30W_4K: (groupid=32, jobs=4): err= 0: pid=3327: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=157, BW=629KiB/s (644kB/s)(111MiB/180359msec) slat (usec): min=2, max=282205, avg=17663.52, stdev=47435.34 clat (msec): min=18, max=1901, avg=587.79, stdev=202.13 lat (msec): min=18, max=1901, avg=605.45, stdev=207.99 clat percentiles (msec): | 1.00th=[ 245], 5.00th=[ 305], 10.00th=[ 384], 20.00th=[ 418], | 30.00th=[ 447], 40.00th=[ 514], 50.00th=[ 558], 60.00th=[ 592], | 70.00th=[ 676], 80.00th=[ 743], 90.00th=[ 877], 95.00th=[ 978], | 99.00th=[ 1133], 99.50th=[ 1217], 99.90th=[ 1385], 99.95th=[ 1452], | 99.99th=[ 1703] bw ( KiB/s): min= 200, max= 1128, per=99.85%, avg=628.60, stdev=34.45, samples=1438 iops : min= 50, max= 282, avg=157.15, stdev= 8.61, samples=1438 write: IOPS=68, BW=272KiB/s (279kB/s)(47.0MiB/180359msec); 0 zone resets slat (usec): min=2, max=285335, avg=17574.83, stdev=47751.79 clat (msec): min=2, max=1459, avg=461.26, stdev=164.17 lat (msec): min=2, max=1459, avg=478.84, stdev=170.26 clat percentiles (msec): | 1.00th=[ 192], 5.00th=[ 253], 10.00th=[ 271], 20.00th=[ 313], | 30.00th=[ 397], 40.00th=[ 414], 50.00th=[ 430], 60.00th=[ 456], | 70.00th=[ 502], 80.00th=[ 575], 90.00th=[ 684], 95.00th=[ 785], | 99.00th=[ 978], 99.50th=[ 1045], 99.90th=[ 1200], 99.95th=[ 1267], | 99.99th=[ 1401] bw ( KiB/s): min= 56, max= 616, per=100.00%, avg=273.14, stdev=24.10, samples=1436 iops : min= 14, max= 154, avg=68.28, stdev= 6.02, samples=1436 lat (msec) : 4=0.01%, 10=0.02%, 20=0.02%, 50=0.06%, 100=0.16% lat (msec) : 250=1.93%, 500=45.99%, 750=36.58%, 1000=12.23%, 2000=3.00% cpu : usr=0.08%, sys=0.07%, ctx=5549, majf=0, minf=62 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=28358,12281,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_70R30W_64K: (groupid=33, jobs=4): err= 0: pid=3338: Sat Jan 11 08:14:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=144, BW=9274KiB/s (9496kB/s)(1633MiB/180359msec) slat (usec): min=4, max=291504, avg=19298.40, stdev=51831.85 clat (msec): min=113, max=1906, avg=638.97, stdev=213.04 lat (msec): min=113, max=1906, avg=658.26, stdev=219.04 clat percentiles (msec): | 1.00th=[ 279], 5.00th=[ 334], 10.00th=[ 418], 20.00th=[ 456], | 30.00th=[ 485], 40.00th=[ 558], 50.00th=[ 609], 60.00th=[ 651], | 70.00th=[ 735], 80.00th=[ 810], 90.00th=[ 944], 95.00th=[ 1045], | 99.00th=[ 1234], 99.50th=[ 1301], 99.90th=[ 1452], 99.95th=[ 1502], | 99.99th=[ 1586] bw ( KiB/s): min= 4096, max=14976, per=99.83%, avg=9258.31, stdev=474.32, samples=1440 iops : min= 64, max= 234, avg=144.66, stdev= 7.41, samples=1440 write: IOPS=62, BW=3993KiB/s (4089kB/s)(703MiB/180359msec); 0 zone resets slat (usec): min=6, max=296417, avg=19175.37, stdev=51704.68 clat (msec): min=86, max=1595, avg=502.33, stdev=175.17 lat (msec): min=86, max=1595, avg=521.51, stdev=182.26 clat percentiles (msec): | 1.00th=[ 247], 5.00th=[ 275], 10.00th=[ 296], 20.00th=[ 347], | 30.00th=[ 430], 40.00th=[ 451], 50.00th=[ 468], 60.00th=[ 493], | 70.00th=[ 535], 80.00th=[ 625], 90.00th=[ 751], 95.00th=[ 844], | 99.00th=[ 1099], 99.50th=[ 1150], 99.90th=[ 1301], 99.95th=[ 1418], | 99.99th=[ 1485] bw ( KiB/s): min= 640, max= 8192, per=99.91%, avg=3990.40, stdev=348.01, samples=1440 iops : min= 10, max= 128, avg=62.35, stdev= 5.44, samples=1440 lat (msec) : 100=0.01%, 250=0.53%, 500=41.75%, 750=34.93%, 1000=17.74% lat (msec) : 2000=5.04% cpu : usr=0.03%, sys=0.07%, ctx=4742, majf=0, minf=60 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=26134,11254,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 Run status group 0 (all jobs): READ: bw=816KiB/s (835kB/s), 816KiB/s-816KiB/s (835kB/s-835kB/s), io=144MiB (151MB), run=180350-180350msec Run status group 1 (all jobs): READ: bw=1630KiB/s (1670kB/s), 1630KiB/s-1630KiB/s (1670kB/s-1670kB/s), io=287MiB (301MB), run=180314-180314msec Run status group 2 (all jobs): READ: bw=6254KiB/s (6404kB/s), 6254KiB/s-6254KiB/s (6404kB/s-6404kB/s), io=1102MiB (1155MB), run=180379-180379msec Run status group 3 (all jobs): READ: bw=12.0MiB/s (12.6MB/s), 12.0MiB/s-12.0MiB/s (12.6MB/s-12.6MB/s), io=2170MiB (2276MB), run=180409-180409msec Run status group 4 (all jobs): READ: bw=22.6MiB/s (23.7MB/s), 22.6MiB/s-22.6MiB/s (23.7MB/s-23.7MB/s), io=4076MiB (4273MB), run=180413-180413msec Run status group 5 (all jobs): WRITE: bw=1238KiB/s (1268kB/s), 1238KiB/s-1238KiB/s (1268kB/s-1268kB/s), io=218MiB (229MB), run=180235-180235msec Run status group 6 (all jobs): WRITE: bw=2444KiB/s (2502kB/s), 2444KiB/s-2444KiB/s (2502kB/s-2502kB/s), io=430MiB (451MB), run=180227-180227msec Run status group 7 (all jobs): WRITE: bw=9334KiB/s (9558kB/s), 9334KiB/s-9334KiB/s (9558kB/s-9558kB/s), io=1643MiB (1723MB), run=180226-180226msec Run status group 8 (all jobs): WRITE: bw=17.4MiB/s (18.3MB/s), 17.4MiB/s-17.4MiB/s (18.3MB/s-18.3MB/s), io=3144MiB (3296MB), run=180209-180209msec Run status group 9 (all jobs): WRITE: bw=31.9MiB/s (33.5MB/s), 31.9MiB/s-31.9MiB/s (33.5MB/s-33.5MB/s), io=5751MiB (6030MB), run=180240-180240msec Run status group 10 (all jobs): READ: bw=250MiB/s (262MB/s), 250MiB/s-250MiB/s (262MB/s-262MB/s), io=43.9GiB (47.1GB), run=180024-180024msec Run status group 11 (all jobs): READ: bw=253MiB/s (266MB/s), 253MiB/s-253MiB/s (266MB/s-266MB/s), io=44.5GiB (47.8GB), run=180029-180029msec Run status group 12 (all jobs): READ: bw=250MiB/s (263MB/s), 250MiB/s-250MiB/s (263MB/s-263MB/s), io=44.0GiB (47.3GB), run=180029-180029msec Run status group 13 (all jobs): READ: bw=251MiB/s (264MB/s), 251MiB/s-251MiB/s (264MB/s-264MB/s), io=44.2GiB (47.5GB), run=180050-180050msec Run status group 14 (all jobs): READ: bw=251MiB/s (263MB/s), 251MiB/s-251MiB/s (263MB/s-263MB/s), io=44.2GiB (47.4GB), run=180070-180070msec Run status group 15 (all jobs): READ: bw=272MiB/s (285MB/s), 272MiB/s-272MiB/s (285MB/s-285MB/s), io=47.8GiB (51.4GB), run=180396-180396msec Run status group 16 (all jobs): READ: bw=489MiB/s (513MB/s), 489MiB/s-489MiB/s (513MB/s-513MB/s), io=86.1GiB (92.4GB), run=180203-180203msec Run status group 17 (all jobs): WRITE: bw=69.7MiB/s (73.1MB/s), 69.7MiB/s-69.7MiB/s (73.1MB/s-73.1MB/s), io=12.3GiB (13.2GB), run=180029-180029msec Run status group 18 (all jobs): WRITE: bw=214MiB/s (225MB/s), 214MiB/s-214MiB/s (225MB/s-225MB/s), io=37.7GiB (40.5GB), run=180003-180003msec Run status group 19 (all jobs): WRITE: bw=243MiB/s (254MB/s), 243MiB/s-243MiB/s (254MB/s-254MB/s), io=42.6GiB (45.8GB), run=180054-180054msec Run status group 20 (all jobs): WRITE: bw=243MiB/s (255MB/s), 243MiB/s-243MiB/s (255MB/s-255MB/s), io=42.7GiB (45.9GB), run=180077-180077msec Run status group 21 (all jobs): WRITE: bw=240MiB/s (252MB/s), 240MiB/s-240MiB/s (252MB/s-252MB/s), io=42.2GiB (45.4GB), run=180078-180078msec Run status group 22 (all jobs): WRITE: bw=453MiB/s (475MB/s), 453MiB/s-453MiB/s (475MB/s-475MB/s), io=79.6GiB (85.5GB), run=180133-180133msec Run status group 23 (all jobs): WRITE: bw=246MiB/s (258MB/s), 246MiB/s-246MiB/s (258MB/s-258MB/s), io=43.3GiB (46.5GB), run=180295-180295msec Run status group 24 (all jobs): READ: bw=492KiB/s (504kB/s), 492KiB/s-492KiB/s (504kB/s-504kB/s), io=86.6MiB (90.8MB), run=180330-180330msec WRITE: bw=490KiB/s (502kB/s), 490KiB/s-490KiB/s (502kB/s-502kB/s), io=86.3MiB (90.5MB), run=180330-180330msec Run status group 25 (all jobs): READ: bw=968KiB/s (991kB/s), 968KiB/s-968KiB/s (991kB/s-991kB/s), io=170MiB (179MB), run=180321-180321msec WRITE: bw=985KiB/s (1008kB/s), 985KiB/s-985KiB/s (1008kB/s-1008kB/s), io=173MiB (182MB), run=180321-180321msec Run status group 26 (all jobs): READ: bw=3733KiB/s (3823kB/s), 3733KiB/s-3733KiB/s (3823kB/s-3823kB/s), io=658MiB (690MB), run=180359-180359msec WRITE: bw=3721KiB/s (3810kB/s), 3721KiB/s-3721KiB/s (3810kB/s-3810kB/s), io=655MiB (687MB), run=180359-180359msec Run status group 27 (all jobs): READ: bw=7141KiB/s (7312kB/s), 7141KiB/s-7141KiB/s (7312kB/s-7312kB/s), io=1258MiB (1319MB), run=180348-180348msec WRITE: bw=7220KiB/s (7393kB/s), 7220KiB/s-7220KiB/s (7393kB/s-7393kB/s), io=1272MiB (1333MB), run=180348-180348msec Run status group 28 (all jobs): READ: bw=13.1MiB/s (13.8MB/s), 13.1MiB/s-13.1MiB/s (13.8MB/s-13.8MB/s), io=2367MiB (2482MB), run=180363-180363msec WRITE: bw=12.0MiB/s (13.6MB/s), 12.0MiB/s-12.0MiB/s (13.6MB/s-13.6MB/s), io=2342MiB (2456MB), run=180363-180363msec Run status group 29 (all jobs): READ: bw=328KiB/s (335kB/s), 328KiB/s-328KiB/s (335kB/s-335kB/s), io=57.6MiB (60.4MB), run=180017-180017msec Run status group 30 (all jobs): READ: bw=253KiB/s (259kB/s), 253KiB/s-253KiB/s (259kB/s-259kB/s), io=44.4MiB (46.5MB), run=180007-180007msec WRITE: bw=250KiB/s (256kB/s), 250KiB/s-250KiB/s (256kB/s-256kB/s), io=43.9MiB (46.0MB), run=180007-180007msec Run status group 31 (all jobs): WRITE: bw=1230KiB/s (1260kB/s), 1230KiB/s-1230KiB/s (1260kB/s-1260kB/s), io=216MiB (227MB), run=180004-180004msec Run status group 32 (all jobs): READ: bw=629KiB/s (644kB/s), 629KiB/s-629KiB/s (644kB/s-644kB/s), io=111MiB (116MB), run=180359-180359msec WRITE: bw=272KiB/s (279kB/s), 272KiB/s-272KiB/s (279kB/s-279kB/s), io=47.0MiB (50.3MB), run=180359-180359msec Run status group 33 (all jobs): READ: bw=9274KiB/s (9496kB/s), 9274KiB/s-9274KiB/s (9496kB/s-9496kB/s), io=1633MiB (1713MB), run=180359-180359msec WRITE: bw=3993KiB/s (4089kB/s), 3993KiB/s-3993KiB/s (4089kB/s-4089kB/s), io=703MiB (738MB), run=180359-180359msec Disk stats (read/write): sdd: ios=6061231/3420458, merge=14354741/7835565, ticks=158849061/131085208, in_queue=289934270, util=99.89%