randread_4K: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randread_8K: (g=1): rw=randread, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randread_32K: (g=2): rw=randread, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randread_64K: (g=3): rw=randread, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randread_128K: (g=4): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randwrite_4K: (g=5): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randwrite_8K: (g=6): rw=randwrite, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randwrite_32K: (g=7): rw=randwrite, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randwrite_64K: (g=8): rw=randwrite, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randwrite_128K: (g=9): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_4K: (g=10): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqread_8K: (g=11): rw=read, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqread_32K: (g=12): rw=read, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqread_64K: (g=13): rw=read, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqread_128K: (g=14): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_512K: (g=15): rw=read, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqread_1M: (g=16): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... seqwrite_4K: (g=17): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqwrite_8K: (g=18): rw=write, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqwrite_32K: (g=19): rw=write, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_64K: (g=20): rw=write, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_128K: (g=21): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqwrite_512K: (g=22): rw=write, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqwrite_1M: (g=23): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... randrw_4K: (g=24): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_8K: (g=25): rw=randrw, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randrw_32K: (g=26): rw=randrw, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randrw_64K: (g=27): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randrw_128K: (g=28): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randread_4K_latency: (g=29): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_4K_latency: (g=30): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randwrite_4K_latency: (g=31): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_70R30W_4K: (g=32): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_70R30W_64K: (g=33): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... fio-3.25 Starting 127 processes randread_4K: (groupid=0, jobs=4): err= 0: pid=1182: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=205, BW=823KiB/s (843kB/s)(483MiB/600395msec) slat (usec): min=2, max=293943, avg=19422.69, stdev=51734.79 clat (msec): min=15, max=1684, avg=602.35, stdev=180.05 lat (msec): min=15, max=1699, avg=621.77, stdev=187.29 clat percentiles (msec): | 1.00th=[ 288], 5.00th=[ 334], 10.00th=[ 435], 20.00th=[ 456], | 30.00th=[ 477], 40.00th=[ 506], 50.00th=[ 592], 60.00th=[ 625], | 70.00th=[ 651], 80.00th=[ 751], 90.00th=[ 844], 95.00th=[ 944], | 99.00th=[ 1116], 99.50th=[ 1200], 99.90th=[ 1368], 99.95th=[ 1435], | 99.99th=[ 1569] bw ( KiB/s): min= 405, max= 1024, per=99.87%, avg=822.84, stdev=28.92, samples=4799 iops : min= 101, max= 256, avg=205.71, stdev= 7.23, samples=4799 lat (msec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=38.77% lat (msec) : 750=40.61%, 1000=17.56%, 2000=3.04% cpu : usr=0.04%, sys=0.06%, ctx=15578, majf=0, minf=176 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=123540,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_8K: (groupid=1, jobs=4): err= 0: pid=1212: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=205, BW=1643KiB/s (1682kB/s)(963MiB/600355msec) slat (usec): min=2, max=286496, avg=19464.41, stdev=51849.45 clat (msec): min=123, max=1713, avg=603.65, stdev=180.05 lat (msec): min=123, max=1801, avg=623.11, stdev=187.10 clat percentiles (msec): | 1.00th=[ 288], 5.00th=[ 338], 10.00th=[ 435], 20.00th=[ 456], | 30.00th=[ 477], 40.00th=[ 510], 50.00th=[ 600], 60.00th=[ 625], | 70.00th=[ 651], 80.00th=[ 760], 90.00th=[ 835], 95.00th=[ 944], | 99.00th=[ 1133], 99.50th=[ 1217], 99.90th=[ 1368], 99.95th=[ 1418], | 99.99th=[ 1502] bw ( KiB/s): min= 1002, max= 2052, per=99.95%, avg=1642.42, stdev=56.79, samples=4799 iops : min= 125, max= 256, avg=205.28, stdev= 7.09, samples=4799 lat (msec) : 250=0.02%, 500=38.43%, 750=40.71%, 1000=17.83%, 2000=3.02% cpu : usr=0.03%, sys=0.06%, ctx=15544, majf=0, minf=307 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=123284,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_32K: (groupid=2, jobs=4): err= 0: pid=1294: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=199, BW=6380KiB/s (6533kB/s)(3740MiB/600356msec) slat (usec): min=3, max=416388, avg=20049.84, stdev=53433.85 clat (msec): min=53, max=1616, avg=621.83, stdev=185.89 lat (msec): min=53, max=1774, avg=641.88, stdev=193.48 clat percentiles (msec): | 1.00th=[ 296], 5.00th=[ 347], 10.00th=[ 447], 20.00th=[ 472], | 30.00th=[ 489], 40.00th=[ 523], 50.00th=[ 617], 60.00th=[ 642], | 70.00th=[ 676], 80.00th=[ 776], 90.00th=[ 869], 95.00th=[ 978], | 99.00th=[ 1150], 99.50th=[ 1250], 99.90th=[ 1435], 99.95th=[ 1469], | 99.99th=[ 1552] bw ( KiB/s): min= 3446, max= 8208, per=100.00%, avg=6380.22, stdev=211.44, samples=4799 iops : min= 107, max= 256, avg=199.30, stdev= 6.61, samples=4799 lat (msec) : 100=0.01%, 250=0.02%, 500=33.96%, 750=43.01%, 1000=19.20% lat (msec) : 2000=3.81% cpu : usr=0.03%, sys=0.06%, ctx=15124, majf=0, minf=1071 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=119687,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_64K: (groupid=3, jobs=4): err= 0: pid=1323: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=192, BW=12.0MiB/s (12.6MB/s)(7213MiB/600391msec) slat (usec): min=3, max=287418, avg=20795.15, stdev=55398.16 clat (msec): min=159, max=1708, avg=644.98, stdev=192.90 lat (msec): min=159, max=1750, avg=665.78, stdev=200.65 clat percentiles (msec): | 1.00th=[ 309], 5.00th=[ 359], 10.00th=[ 464], 20.00th=[ 489], | 30.00th=[ 506], 40.00th=[ 542], 50.00th=[ 634], 60.00th=[ 667], | 70.00th=[ 701], 80.00th=[ 810], 90.00th=[ 902], 95.00th=[ 1011], | 99.00th=[ 1200], 99.50th=[ 1301], 99.90th=[ 1469], 99.95th=[ 1519], | 99.99th=[ 1603] bw ( KiB/s): min= 4836, max=16416, per=99.99%, avg=12300.86, stdev=400.03, samples=4800 iops : min= 74, max= 256, avg=192.12, stdev= 6.25, samples=4800 lat (msec) : 250=0.01%, 500=26.67%, 750=48.60%, 1000=19.11%, 2000=5.62% cpu : usr=0.02%, sys=0.07%, ctx=14523, majf=0, minf=2102 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=115404,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_128K: (groupid=4, jobs=4): err= 0: pid=1346: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=180, BW=22.6MiB/s (23.7MB/s)(13.3GiB/600412msec) slat (usec): min=4, max=361148, avg=22100.51, stdev=58821.65 clat (msec): min=161, max=1842, avg=685.46, stdev=203.92 lat (msec): min=161, max=1943, avg=707.57, stdev=211.96 clat percentiles (msec): | 1.00th=[ 330], 5.00th=[ 380], 10.00th=[ 498], 20.00th=[ 518], | 30.00th=[ 542], 40.00th=[ 575], 50.00th=[ 684], 60.00th=[ 709], | 70.00th=[ 743], 80.00th=[ 860], 90.00th=[ 953], 95.00th=[ 1070], | 99.00th=[ 1267], 99.50th=[ 1368], 99.90th=[ 1552], 99.95th=[ 1603], | 99.99th=[ 1720] bw ( KiB/s): min= 7729, max=32832, per=99.98%, avg=23147.00, stdev=840.53, samples=4800 iops : min= 59, max= 256, avg=180.76, stdev= 6.57, samples=4800 lat (msec) : 250=0.01%, 500=11.47%, 750=60.84%, 1000=18.81%, 2000=8.87% cpu : usr=0.02%, sys=0.07%, ctx=13679, majf=0, minf=2103 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=108588,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_4K: (groupid=5, jobs=4): err= 0: pid=1369: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=307, BW=1230KiB/s (1259kB/s)(721MiB/600183msec); 0 zone resets slat (usec): min=2, max=302327, avg=13000.52, stdev=34815.14 clat (msec): min=2, max=1292, avg=403.25, stdev=86.47 lat (msec): min=2, max=1339, avg=416.25, stdev=93.44 clat percentiles (msec): | 1.00th=[ 259], 5.00th=[ 284], 10.00th=[ 296], 20.00th=[ 330], | 30.00th=[ 363], 40.00th=[ 384], 50.00th=[ 401], 60.00th=[ 418], | 70.00th=[ 435], 80.00th=[ 456], 90.00th=[ 498], 95.00th=[ 542], | 99.00th=[ 684], 99.50th=[ 768], 99.90th=[ 927], 99.95th=[ 995], | 99.99th=[ 1116] bw ( KiB/s): min= 696, max= 3012, per=99.85%, avg=1228.64, stdev=40.09, samples=4800 iops : min= 174, max= 752, avg=307.16, stdev=10.02, samples=4800 lat (msec) : 4=0.01%, 10=0.02%, 20=0.08%, 50=0.09%, 100=0.01% lat (msec) : 250=0.26%, 500=89.85%, 750=9.12%, 1000=0.52%, 2000=0.04% cpu : usr=0.07%, sys=0.09%, ctx=23231, majf=0, minf=55 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,184527,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_8K: (groupid=6, jobs=4): err= 0: pid=1398: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=304, BW=2434KiB/s (2493kB/s)(1427MiB/600177msec); 0 zone resets slat (usec): min=2, max=302915, avg=13126.47, stdev=35106.85 clat (msec): min=2, max=1436, avg=407.48, stdev=86.31 lat (msec): min=2, max=1436, avg=420.61, stdev=93.32 clat percentiles (msec): | 1.00th=[ 262], 5.00th=[ 288], 10.00th=[ 300], 20.00th=[ 334], | 30.00th=[ 368], 40.00th=[ 388], 50.00th=[ 405], 60.00th=[ 422], | 70.00th=[ 439], 80.00th=[ 460], 90.00th=[ 502], 95.00th=[ 550], | 99.00th=[ 693], 99.50th=[ 776], 99.90th=[ 927], 99.95th=[ 986], | 99.99th=[ 1116] bw ( KiB/s): min= 1216, max= 4443, per=99.99%, avg=2434.14, stdev=77.41, samples=4800 iops : min= 152, max= 554, avg=304.16, stdev= 9.67, samples=4800 lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.02%, 100=0.02% lat (msec) : 250=0.35%, 500=89.24%, 750=9.74%, 1000=0.56%, 2000=0.04% cpu : usr=0.05%, sys=0.09%, ctx=23269, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,182626,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_32K: (groupid=7, jobs=4): err= 0: pid=1425: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=292, BW=9353KiB/s (9577kB/s)(5482MiB/600242msec); 0 zone resets slat (usec): min=3, max=243129, avg=13677.04, stdev=36515.76 clat (msec): min=11, max=1254, avg=424.23, stdev=86.67 lat (msec): min=11, max=1254, avg=437.91, stdev=94.07 clat percentiles (msec): | 1.00th=[ 275], 5.00th=[ 300], 10.00th=[ 317], 20.00th=[ 347], | 30.00th=[ 384], 40.00th=[ 409], 50.00th=[ 426], 60.00th=[ 439], | 70.00th=[ 456], 80.00th=[ 477], 90.00th=[ 523], 95.00th=[ 567], | 99.00th=[ 701], 99.50th=[ 785], 99.90th=[ 961], 99.95th=[ 1020], | 99.99th=[ 1133] bw ( KiB/s): min= 5632, max=12312, per=100.00%, avg=9353.17, stdev=289.77, samples=4800 iops : min= 176, max= 384, avg=292.16, stdev= 9.05, samples=4800 lat (msec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=0.10%, 500=86.40% lat (msec) : 750=12.78%, 1000=0.63%, 2000=0.06% cpu : usr=0.05%, sys=0.09%, ctx=22041, majf=0, minf=53 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,175436,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_64K: (groupid=8, jobs=4): err= 0: pid=1452: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=279, BW=17.4MiB/s (18.3MB/s)(10.2GiB/600237msec); 0 zone resets slat (usec): min=5, max=349510, avg=14285.85, stdev=38166.28 clat (msec): min=18, max=1544, avg=444.20, stdev=94.15 lat (msec): min=18, max=1544, avg=458.49, stdev=101.83 clat percentiles (msec): | 1.00th=[ 284], 5.00th=[ 313], 10.00th=[ 330], 20.00th=[ 359], | 30.00th=[ 401], 40.00th=[ 426], 50.00th=[ 447], 60.00th=[ 460], | 70.00th=[ 477], 80.00th=[ 498], 90.00th=[ 550], 95.00th=[ 600], | 99.00th=[ 751], 99.50th=[ 835], 99.90th=[ 1011], 99.95th=[ 1083], | 99.99th=[ 1183] bw ( KiB/s): min= 8320, max=28708, per=99.97%, avg=17862.87, stdev=629.03, samples=4800 iops : min= 130, max= 448, avg=278.99, stdev= 9.82, samples=4800 lat (msec) : 20=0.01%, 50=0.02%, 100=0.01%, 250=0.38%, 500=80.00% lat (msec) : 750=18.56%, 1000=0.92%, 2000=0.11% cpu : usr=0.06%, sys=0.09%, ctx=21886, majf=0, minf=50 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,167565,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_128K: (groupid=9, jobs=4): err= 0: pid=1485: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=256, BW=32.1MiB/s (33.6MB/s)(18.8GiB/600290msec); 0 zone resets slat (usec): min=6, max=251822, avg=15562.08, stdev=41531.02 clat (msec): min=30, max=1663, avg=483.50, stdev=101.52 lat (msec): min=30, max=1706, avg=499.06, stdev=109.96 clat percentiles (msec): | 1.00th=[ 309], 5.00th=[ 342], 10.00th=[ 363], 20.00th=[ 393], | 30.00th=[ 435], 40.00th=[ 468], 50.00th=[ 485], 60.00th=[ 502], | 70.00th=[ 518], 80.00th=[ 542], 90.00th=[ 600], 95.00th=[ 651], | 99.00th=[ 818], 99.50th=[ 911], 99.90th=[ 1116], 99.95th=[ 1167], | 99.99th=[ 1351] bw ( KiB/s): min=17664, max=53036, per=99.99%, avg=32825.21, stdev=1153.65, samples=4800 iops : min= 138, max= 414, avg=256.34, stdev= 9.01, samples=4800 lat (msec) : 50=0.01%, 100=0.01%, 250=0.18%, 500=58.54%, 750=39.56% lat (msec) : 1000=1.47%, 2000=0.24% cpu : usr=0.07%, sys=0.09%, ctx=19819, majf=0, minf=51 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,153954,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_4K: (groupid=10, jobs=4): err= 0: pid=1512: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=56.0k, BW=219MiB/s (229MB/s)(128GiB/600013msec) slat (nsec): min=1611, max=1651.6k, avg=4885.46, stdev=2765.56 clat (usec): min=38, max=2543.9k, avg=2277.94, stdev=64784.08 lat (usec): min=52, max=2543.9k, avg=2283.04, stdev=64784.10 clat percentiles (usec): | 1.00th=[ 227], 5.00th=[ 461], 10.00th=[ 498], | 20.00th=[ 519], 30.00th=[ 529], 40.00th=[ 545], | 50.00th=[ 553], 60.00th=[ 562], 70.00th=[ 562], | 80.00th=[ 578], 90.00th=[ 586], 95.00th=[ 611], | 99.00th=[ 742], 99.50th=[ 1156], 99.90th=[ 4752], | 99.95th=[2499806], 99.99th=[2499806] bw ( KiB/s): min= 32, max=1016072, per=100.00%, avg=519540.02, stdev=97791.92, samples=2067 iops : min= 8, max=254018, avg=129884.69, stdev=24447.97, samples=2067 lat (usec) : 50=0.01%, 100=0.01%, 250=1.07%, 500=11.14%, 750=86.81% lat (usec) : 1000=0.32% lat (msec) : 2=0.43%, 4=0.11%, 10=0.02%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.01%, >=2000=0.07% cpu : usr=4.22%, sys=7.91%, ctx=7322949, majf=0, minf=191 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=33618801,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_8K: (groupid=11, jobs=4): err= 0: pid=1533: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=30.0k, BW=242MiB/s (254MB/s)(142GiB/600044msec) slat (nsec): min=1685, max=139211, avg=5160.42, stdev=2091.71 clat (usec): min=50, max=2572.1k, avg=4122.87, stdev=87812.00 lat (usec): min=67, max=2572.1k, avg=4128.25, stdev=87812.05 clat percentiles (usec): | 1.00th=[ 363], 5.00th=[ 725], 10.00th=[ 922], | 20.00th=[ 971], 30.00th=[ 988], 40.00th=[ 996], | 50.00th=[ 1004], 60.00th=[ 1012], 70.00th=[ 1020], | 80.00th=[ 1029], 90.00th=[ 1045], 95.00th=[ 1090], | 99.00th=[ 2040], 99.50th=[ 2180], 99.90th=[2499806], | 99.95th=[2499806], 99.99th=[2533360] bw ( KiB/s): min= 240, max=1059328, per=100.00%, avg=593389.41, stdev=108717.68, samples=2002 iops : min= 30, max=132416, avg=74173.31, stdev=13589.71, samples=2002 lat (usec) : 100=0.01%, 250=0.56%, 500=1.76%, 750=2.98%, 1000=43.71% lat (msec) : 2=48.93%, 4=1.87%, 10=0.04%, 20=0.01%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.01%, >=2000=0.12% cpu : usr=2.59%, sys=4.71%, ctx=4729692, majf=0, minf=322 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=18597901,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_32K: (groupid=12, jobs=4): err= 0: pid=1557: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=8220, BW=257MiB/s (269MB/s)(151GiB/600052msec) slat (usec): min=2, max=130, avg= 8.25, stdev= 2.99 clat (usec): min=323, max=2582.3k, avg=15560.23, stdev=170221.06 lat (usec): min=345, max=2582.3k, avg=15568.78, stdev=170221.08 clat percentiles (msec): | 1.00th=[ 3], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 4], | 30.00th=[ 4], 40.00th=[ 4], 50.00th=[ 4], 60.00th=[ 4], | 70.00th=[ 4], 80.00th=[ 4], 90.00th=[ 5], 95.00th=[ 5], | 99.00th=[ 6], 99.50th=[ 37], 99.90th=[ 2500], 99.95th=[ 2500], | 99.99th=[ 2534] bw ( KiB/s): min= 320, max=1089825, per=100.00%, avg=615957.70, stdev=111430.20, samples=2050 iops : min= 10, max=34057, avg=19248.26, stdev=3482.18, samples=2050 lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.76%, 4=88.81%, 10=9.90%, 20=0.02%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01% lat (msec) : >=2000=0.46% cpu : usr=1.14%, sys=2.35%, ctx=1260715, majf=0, minf=1087 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=4932529,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_64K: (groupid=13, jobs=4): err= 0: pid=1582: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=4122, BW=258MiB/s (270MB/s)(151GiB/600044msec) slat (usec): min=2, max=1671, avg=11.78, stdev= 7.06 clat (usec): min=540, max=2538.4k, avg=31030.05, stdev=240151.62 lat (usec): min=562, max=2538.5k, avg=31042.20, stdev=240151.62 clat percentiles (msec): | 1.00th=[ 6], 5.00th=[ 8], 10.00th=[ 8], 20.00th=[ 8], | 30.00th=[ 8], 40.00th=[ 8], 50.00th=[ 8], 60.00th=[ 8], | 70.00th=[ 8], 80.00th=[ 8], 90.00th=[ 9], 95.00th=[ 9], | 99.00th=[ 24], 99.50th=[ 2500], 99.90th=[ 2500], 99.95th=[ 2534], | 99.99th=[ 2534] bw ( KiB/s): min= 512, max=1099008, per=100.00%, avg=655701.25, stdev=110278.84, samples=1931 iops : min= 8, max=17172, avg=10245.02, stdev=1723.10, samples=1931 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.03%, 10=98.70%, 20=0.25%, 50=0.05% lat (msec) : 100=0.01%, 250=0.01%, 500=0.01%, 1000=0.01%, 2000=0.01% lat (msec) : >=2000=0.93% cpu : usr=0.66%, sys=1.96%, ctx=636601, majf=0, minf=2115 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=2473941,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_128K: (groupid=14, jobs=4): err= 0: pid=1596: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=2061, BW=258MiB/s (270MB/s)(151GiB/600074msec) slat (usec): min=2, max=805, avg=11.94, stdev= 6.64 clat (usec): min=1462, max=2762.7k, avg=62064.31, stdev=337736.29 lat (usec): min=1486, max=2762.7k, avg=62076.59, stdev=337736.36 clat percentiles (msec): | 1.00th=[ 10], 5.00th=[ 15], 10.00th=[ 15], 20.00th=[ 15], | 30.00th=[ 15], 40.00th=[ 16], 50.00th=[ 16], 60.00th=[ 16], | 70.00th=[ 16], 80.00th=[ 17], 90.00th=[ 17], 95.00th=[ 17], | 99.00th=[ 2534], 99.50th=[ 2534], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2567] bw ( KiB/s): min= 2305, max=1105705, per=100.00%, avg=654667.45, stdev=110031.05, samples=1934 iops : min= 18, max= 8638, avg=5114.33, stdev=859.62, samples=1934 lat (msec) : 2=0.01%, 4=0.01%, 10=1.41%, 20=96.45%, 50=0.10% lat (msec) : 100=0.05%, 250=0.08%, 500=0.02%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.03%, >=2000=1.84% cpu : usr=0.31%, sys=0.92%, ctx=315115, majf=0, minf=2108 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=1237221,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_512K: (groupid=15, jobs=4): err= 0: pid=1633: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=490, BW=245MiB/s (257MB/s)(144GiB/600301msec) slat (usec): min=8, max=117414, avg=80.12, stdev=1424.77 clat (msec): min=18, max=3141, avg=260.59, stdev=407.03 lat (msec): min=18, max=3141, avg=260.67, stdev=407.03 clat percentiles (msec): | 1.00th=[ 31], 5.00th=[ 56], 10.00th=[ 59], 20.00th=[ 60], | 30.00th=[ 62], 40.00th=[ 63], 50.00th=[ 67], 60.00th=[ 118], | 70.00th=[ 201], 80.00th=[ 376], 90.00th=[ 651], 95.00th=[ 1099], | 99.00th=[ 2165], 99.50th=[ 2500], 99.90th=[ 3004], 99.95th=[ 3037], | 99.99th=[ 3071] bw ( KiB/s): min= 8192, max=1026556, per=100.00%, avg=265853.01, stdev=65782.16, samples=4538 iops : min= 16, max= 2004, avg=519.02, stdev=128.43, samples=4538 lat (msec) : 20=0.01%, 50=3.37%, 100=52.54%, 250=19.21%, 500=6.28% lat (msec) : 750=10.89%, 1000=1.88%, 2000=4.56%, >=2000=1.26% cpu : usr=0.13%, sys=0.49%, ctx=146529, majf=0, minf=2137 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=294717,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_1M: (groupid=16, jobs=4): err= 0: pid=1656: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=234, BW=235MiB/s (246MB/s)(138GiB/600287msec) slat (usec): min=10, max=308901, avg=16375.27, stdev=43867.34 clat (msec): min=14, max=3247, avg=529.03, stdev=215.77 lat (msec): min=14, max=3367, avg=545.40, stdev=216.57 clat percentiles (msec): | 1.00th=[ 86], 5.00th=[ 255], 10.00th=[ 338], 20.00th=[ 405], | 30.00th=[ 447], 40.00th=[ 506], 50.00th=[ 527], 60.00th=[ 542], | 70.00th=[ 567], 80.00th=[ 600], 90.00th=[ 701], 95.00th=[ 818], | 99.00th=[ 1301], 99.50th=[ 1636], 99.90th=[ 2869], 99.95th=[ 2937], | 99.99th=[ 3037] bw ( KiB/s): min=72953, max=571392, per=99.99%, avg=240228.56, stdev=11780.21, samples=4799 iops : min= 71, max= 558, avg=234.50, stdev=11.50, samples=4799 lat (msec) : 20=0.01%, 50=0.44%, 100=0.81%, 250=3.30%, 500=34.41% lat (msec) : 750=53.93%, 1000=4.68%, 2000=2.14%, >=2000=0.29% cpu : usr=0.03%, sys=0.19%, ctx=26911, majf=0, minf=130 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=140836,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_4K: (groupid=17, jobs=4): err= 0: pid=1684: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=22.9k, BW=89.5MiB/s (93.8MB/s)(52.4GiB/600020msec); 0 zone resets slat (nsec): min=1444, max=1642.8k, avg=6048.96, stdev=3898.68 clat (usec): min=37, max=2573.2k, avg=5578.83, stdev=48928.16 lat (usec): min=62, max=2573.2k, avg=5585.15, stdev=48928.34 clat percentiles (usec): | 1.00th=[ 322], 5.00th=[ 515], 10.00th=[ 519], | 20.00th=[ 545], 30.00th=[ 578], 40.00th=[ 611], | 50.00th=[ 734], 60.00th=[ 4228], 70.00th=[ 4555], | 80.00th=[ 4883], 90.00th=[ 6456], 95.00th=[ 25822], | 99.00th=[ 40633], 99.50th=[ 43779], 99.90th=[ 49021], | 99.95th=[ 87557], 99.99th=[2499806] bw ( KiB/s): min= 128, max=916864, per=100.00%, avg=105227.67, stdev=46888.37, samples=4167 iops : min= 32, max=229216, avg=26306.59, stdev=11722.10, samples=4167 lat (usec) : 50=0.01%, 100=0.01%, 250=0.53%, 500=3.28%, 750=47.12% lat (usec) : 1000=1.51% lat (msec) : 2=0.85%, 4=3.29%, 10=34.00%, 20=2.82%, 50=6.52% lat (msec) : 100=0.03%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec) : 2000=0.01%, >=2000=0.04% cpu : usr=2.46%, sys=4.61%, ctx=2890221, majf=0, minf=67 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,13746577,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_8K: (groupid=18, jobs=4): err= 0: pid=1722: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=14.3k, BW=111MiB/s (117MB/s)(65.3GiB/600006msec); 0 zone resets slat (usec): min=2, max=1502, avg=11.24, stdev= 5.47 clat (usec): min=1225, max=62535, avg=8954.89, stdev=8047.02 lat (usec): min=1241, max=62543, avg=8966.44, stdev=8047.66 clat percentiles (usec): | 1.00th=[ 5407], 5.00th=[ 5669], 10.00th=[ 5800], 20.00th=[ 6063], | 30.00th=[ 6259], 40.00th=[ 6456], 50.00th=[ 6652], 60.00th=[ 6849], | 70.00th=[ 7046], 80.00th=[ 7373], 90.00th=[10159], 95.00th=[30016], | 99.00th=[46400], 99.50th=[49021], 99.90th=[51643], 99.95th=[52691], | 99.99th=[54264] bw ( KiB/s): min=19568, max=164738, per=100.00%, avg=114196.25, stdev=13660.79, samples=4796 iops : min= 2446, max=20592, avg=14274.12, stdev=1707.59, samples=4796 lat (msec) : 2=0.01%, 4=0.01%, 10=89.97%, 20=2.52%, 50=7.22% lat (msec) : 100=0.29% cpu : usr=2.04%, sys=5.61%, ctx=2278705, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,8563146,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_32K: (groupid=19, jobs=4): err= 0: pid=1752: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=6582, BW=206MiB/s (216MB/s)(121GiB/600019msec); 0 zone resets slat (usec): min=2, max=1015, avg=10.46, stdev= 6.04 clat (usec): min=1060, max=79909, avg=19430.97, stdev=2173.32 lat (usec): min=1109, max=79919, avg=19441.77, stdev=2173.32 clat percentiles (usec): | 1.00th=[17171], 5.00th=[17171], 10.00th=[17433], 20.00th=[17695], | 30.00th=[17957], 40.00th=[18220], 50.00th=[18744], 60.00th=[19268], | 70.00th=[20055], 80.00th=[21103], 90.00th=[22676], 95.00th=[23987], | 99.00th=[26608], 99.50th=[27132], 99.90th=[28181], 99.95th=[28967], | 99.99th=[31065] bw ( KiB/s): min=194187, max=224448, per=100.00%, avg=210768.06, stdev=1075.82, samples=4796 iops : min= 6068, max= 7013, avg=6586.21, stdev=33.60, samples=4796 lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=70.08%, 50=29.91% lat (msec) : 100=0.01% cpu : usr=1.30%, sys=2.46%, ctx=1018382, majf=0, minf=56 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,3949838,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_64K: (groupid=20, jobs=4): err= 0: pid=1776: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=3624, BW=227MiB/s (238MB/s)(133GiB/600033msec); 0 zone resets slat (usec): min=4, max=885, avg=13.21, stdev= 7.06 clat (usec): min=1271, max=2554.1k, avg=35295.95, stdev=17011.94 lat (usec): min=1323, max=2554.1k, avg=35309.52, stdev=17011.97 clat percentiles (msec): | 1.00th=[ 33], 5.00th=[ 34], 10.00th=[ 34], 20.00th=[ 34], | 30.00th=[ 34], 40.00th=[ 34], 50.00th=[ 35], 60.00th=[ 35], | 70.00th=[ 36], 80.00th=[ 37], 90.00th=[ 39], 95.00th=[ 40], | 99.00th=[ 50], 99.50th=[ 55], 99.90th=[ 60], 99.95th=[ 63], | 99.99th=[ 157] bw ( KiB/s): min=83712, max=443254, per=100.00%, avg=232686.79, stdev=6633.74, samples=4784 iops : min= 1308, max= 6923, avg=3634.87, stdev=103.65, samples=4784 lat (msec) : 2=0.01%, 4=0.01%, 10=0.47%, 20=0.01%, 50=98.60% lat (msec) : 100=0.90%, 250=0.01%, >=2000=0.01% cpu : usr=0.86%, sys=1.55%, ctx=558939, majf=0, minf=63 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,2174992,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_128K: (groupid=21, jobs=4): err= 0: pid=1794: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=1905, BW=238MiB/s (250MB/s)(140GiB/600066msec); 0 zone resets slat (usec): min=5, max=133, avg=16.65, stdev= 7.67 clat (usec): min=1342, max=508790, avg=67140.24, stdev=3423.19 lat (usec): min=1400, max=508823, avg=67157.25, stdev=3423.19 clat percentiles (msec): | 1.00th=[ 65], 5.00th=[ 65], 10.00th=[ 65], 20.00th=[ 65], | 30.00th=[ 66], 40.00th=[ 66], 50.00th=[ 67], 60.00th=[ 67], | 70.00th=[ 68], 80.00th=[ 69], 90.00th=[ 71], 95.00th=[ 73], | 99.00th=[ 77], 99.50th=[ 78], 99.90th=[ 79], 99.95th=[ 80], | 99.99th=[ 114] bw ( KiB/s): min=197325, max=288807, per=100.00%, avg=244058.98, stdev=1538.89, samples=4799 iops : min= 1541, max= 2256, avg=1905.76, stdev=12.04, samples=4799 lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.05% lat (msec) : 100=99.94%, 250=0.01%, 500=0.01%, 750=0.01% cpu : usr=0.62%, sys=0.85%, ctx=295127, majf=0, minf=60 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,1143650,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_512K: (groupid=22, jobs=4): err= 0: pid=1820: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=490, BW=245MiB/s (257MB/s)(144GiB/600255msec); 0 zone resets slat (usec): min=17, max=91738, avg=1083.99, stdev=5650.52 clat (msec): min=18, max=1066, avg=259.81, stdev=25.94 lat (msec): min=18, max=1066, avg=260.89, stdev=25.70 clat percentiles (msec): | 1.00th=[ 188], 5.00th=[ 226], 10.00th=[ 236], 20.00th=[ 255], | 30.00th=[ 257], 40.00th=[ 259], 50.00th=[ 259], 60.00th=[ 262], | 70.00th=[ 264], 80.00th=[ 268], 90.00th=[ 279], 95.00th=[ 288], | 99.00th=[ 305], 99.50th=[ 376], 99.90th=[ 550], 99.95th=[ 634], | 99.99th=[ 818] bw ( KiB/s): min=168271, max=278943, per=100.00%, avg=251231.89, stdev=2010.58, samples=4800 iops : min= 328, max= 544, avg=490.45, stdev= 3.93, samples=4800 lat (msec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=16.98%, 500=82.84% lat (msec) : 750=0.15%, 1000=0.02%, 2000=0.01% cpu : usr=0.44%, sys=0.38%, ctx=157871, majf=0, minf=56 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,294478,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_1M: (groupid=23, jobs=4): err= 0: pid=1845: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=249, BW=249MiB/s (261MB/s)(146GiB/600271msec); 0 zone resets slat (usec): min=24, max=405107, avg=16026.98, stdev=42292.25 clat (msec): min=47, max=1338, avg=497.26, stdev=53.92 lat (msec): min=47, max=1373, avg=513.29, stdev=55.51 clat percentiles (msec): | 1.00th=[ 384], 5.00th=[ 384], 10.00th=[ 384], 20.00th=[ 510], | 30.00th=[ 514], 40.00th=[ 514], 50.00th=[ 514], 60.00th=[ 514], | 70.00th=[ 514], 80.00th=[ 514], 90.00th=[ 514], 95.00th=[ 514], | 99.00th=[ 642], 99.50th=[ 642], 99.90th=[ 751], 99.95th=[ 844], | 99.99th=[ 1133] bw ( KiB/s): min=111580, max=287113, per=100.00%, avg=255305.59, stdev=5162.78, samples=4800 iops : min= 108, max= 280, avg=249.20, stdev= 5.04, samples=4800 lat (msec) : 50=0.01%, 100=0.01%, 250=0.03%, 500=15.11%, 750=84.75% lat (msec) : 1000=0.07%, 2000=0.03% cpu : usr=0.31%, sys=0.10%, ctx=19066, majf=0, minf=60 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,149663,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_4K: (groupid=24, jobs=4): err= 0: pid=1872: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=123, BW=492KiB/s (504kB/s)(289MiB/600283msec) slat (usec): min=2, max=261353, avg=16243.39, stdev=43550.82 clat (msec): min=14, max=1865, avg=561.09, stdev=206.05 lat (msec): min=14, max=1909, avg=577.33, stdev=210.62 clat percentiles (msec): | 1.00th=[ 224], 5.00th=[ 266], 10.00th=[ 342], 20.00th=[ 384], | 30.00th=[ 409], 40.00th=[ 481], 50.00th=[ 523], 60.00th=[ 584], | 70.00th=[ 659], 80.00th=[ 760], 90.00th=[ 877], 95.00th=[ 936], | 99.00th=[ 1045], 99.50th=[ 1083], 99.90th=[ 1183], 99.95th=[ 1250], | 99.99th=[ 1368] bw ( KiB/s): min= 136, max= 1072, per=99.94%, avg=492.22, stdev=29.37, samples=4798 iops : min= 34, max= 268, avg=123.05, stdev= 7.34, samples=4798 write: IOPS=123, BW=493KiB/s (505kB/s)(289MiB/600283msec); 0 zone resets slat (usec): min=2, max=237610, avg=16086.31, stdev=43447.36 clat (usec): min=507, max=1303.3k, avg=445126.56, stdev=155567.06 lat (usec): min=529, max=1303.3k, avg=461213.26, stdev=161126.13 clat percentiles (msec): | 1.00th=[ 203], 5.00th=[ 236], 10.00th=[ 255], 20.00th=[ 317], | 30.00th=[ 368], 40.00th=[ 388], 50.00th=[ 409], 60.00th=[ 451], | 70.00th=[ 514], 80.00th=[ 558], 90.00th=[ 659], 95.00th=[ 743], | 99.00th=[ 885], 99.50th=[ 936], 99.90th=[ 1045], 99.95th=[ 1083], | 99.99th=[ 1183] bw ( KiB/s): min= 168, max= 1184, per=99.91%, avg=493.55, stdev=29.90, samples=4798 iops : min= 42, max= 296, avg=123.39, stdev= 7.47, samples=4798 lat (usec) : 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.03%, 50=0.08% lat (msec) : 100=0.15%, 250=5.66%, 500=49.47%, 750=31.72%, 1000=11.64% lat (msec) : 2000=1.22% cpu : usr=0.05%, sys=0.07%, ctx=19813, majf=0, minf=65 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=73878,74054,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_8K: (groupid=25, jobs=4): err= 0: pid=1911: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=121, BW=975KiB/s (998kB/s)(571MiB/600361msec) slat (usec): min=2, max=264513, avg=16401.49, stdev=43963.96 clat (msec): min=33, max=1395, avg=565.40, stdev=205.54 lat (msec): min=33, max=1505, avg=581.80, stdev=209.99 clat percentiles (msec): | 1.00th=[ 232], 5.00th=[ 271], 10.00th=[ 347], 20.00th=[ 384], | 30.00th=[ 414], 40.00th=[ 485], 50.00th=[ 527], 60.00th=[ 584], | 70.00th=[ 659], 80.00th=[ 768], 90.00th=[ 885], 95.00th=[ 944], | 99.00th=[ 1053], 99.50th=[ 1083], 99.90th=[ 1183], 99.95th=[ 1234], | 99.99th=[ 1284] bw ( KiB/s): min= 288, max= 1760, per=99.94%, avg=974.10, stdev=53.44, samples=4800 iops : min= 36, max= 220, avg=121.76, stdev= 6.68, samples=4800 write: IOPS=122, BW=984KiB/s (1007kB/s)(577MiB/600361msec); 0 zone resets slat (usec): min=2, max=320990, avg=16234.81, stdev=43739.97 clat (msec): min=6, max=1538, avg=447.99, stdev=154.66 lat (msec): min=6, max=1538, avg=464.23, stdev=160.48 clat percentiles (msec): | 1.00th=[ 215], 5.00th=[ 241], 10.00th=[ 257], 20.00th=[ 321], | 30.00th=[ 372], 40.00th=[ 388], 50.00th=[ 409], 60.00th=[ 451], | 70.00th=[ 514], 80.00th=[ 558], 90.00th=[ 667], 95.00th=[ 743], | 99.00th=[ 894], 99.50th=[ 944], 99.90th=[ 1083], 99.95th=[ 1116], | 99.99th=[ 1234] bw ( KiB/s): min= 302, max= 1824, per=99.91%, avg=983.76, stdev=55.84, samples=4800 iops : min= 37, max= 228, avg=122.97, stdev= 6.98, samples=4800 lat (msec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.02%, 250=5.26% lat (msec) : 500=49.55%, 750=31.95%, 1000=11.95%, 2000=1.26% cpu : usr=0.04%, sys=0.07%, ctx=18646, majf=0, minf=70 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=73135,73835,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_32K: (groupid=26, jobs=4): err= 0: pid=1945: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=118, BW=3782KiB/s (3873kB/s)(2217MiB/600285msec) slat (usec): min=2, max=237863, avg=17003.98, stdev=45573.81 clat (msec): min=120, max=1377, avg=585.22, stdev=210.75 lat (msec): min=120, max=1433, avg=602.23, stdev=215.17 clat percentiles (msec): | 1.00th=[ 243], 5.00th=[ 279], 10.00th=[ 363], 20.00th=[ 401], | 30.00th=[ 430], 40.00th=[ 502], 50.00th=[ 550], 60.00th=[ 600], | 70.00th=[ 684], 80.00th=[ 793], 90.00th=[ 911], 95.00th=[ 969], | 99.00th=[ 1083], 99.50th=[ 1116], 99.90th=[ 1217], 99.95th=[ 1250], | 99.99th=[ 1318] bw ( KiB/s): min= 1280, max= 6403, per=99.97%, avg=3781.43, stdev=215.30, samples=4800 iops : min= 40, max= 200, avg=118.13, stdev= 6.73, samples=4800 write: IOPS=117, BW=3763KiB/s (3854kB/s)(2206MiB/600285msec); 0 zone resets slat (usec): min=3, max=310144, avg=16898.67, stdev=45444.26 clat (msec): min=57, max=1386, avg=466.07, stdev=159.71 lat (msec): min=57, max=1408, avg=482.97, stdev=165.85 clat percentiles (msec): | 1.00th=[ 224], 5.00th=[ 251], 10.00th=[ 271], 20.00th=[ 342], | 30.00th=[ 388], 40.00th=[ 405], 50.00th=[ 426], 60.00th=[ 468], | 70.00th=[ 535], 80.00th=[ 584], 90.00th=[ 693], 95.00th=[ 776], | 99.00th=[ 919], 99.50th=[ 978], 99.90th=[ 1099], 99.95th=[ 1150], | 99.99th=[ 1284] bw ( KiB/s): min= 1024, max= 6592, per=99.96%, avg=3762.65, stdev=221.34, samples=4800 iops : min= 32, max= 206, avg=117.54, stdev= 6.91, samples=4800 lat (msec) : 100=0.01%, 250=3.31%, 500=48.16%, 750=33.94%, 1000=12.84% lat (msec) : 2000=1.75% cpu : usr=0.04%, sys=0.07%, ctx=17804, majf=0, minf=59 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=70952,70599,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_64K: (groupid=27, jobs=4): err= 0: pid=1973: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=113, BW=7245KiB/s (7419kB/s)(4248MiB/600321msec) slat (usec): min=5, max=244426, avg=17824.96, stdev=47396.76 clat (msec): min=104, max=1594, avg=607.95, stdev=218.16 lat (msec): min=104, max=1594, avg=625.77, stdev=223.04 clat percentiles (msec): | 1.00th=[ 251], 5.00th=[ 292], 10.00th=[ 376], 20.00th=[ 414], | 30.00th=[ 443], 40.00th=[ 523], 50.00th=[ 567], 60.00th=[ 634], | 70.00th=[ 718], 80.00th=[ 827], 90.00th=[ 944], 95.00th=[ 1003], | 99.00th=[ 1116], 99.50th=[ 1150], 99.90th=[ 1250], 99.95th=[ 1301], | 99.99th=[ 1401] bw ( KiB/s): min= 2304, max=12928, per=99.95%, avg=7242.91, stdev=428.22, samples=4800 iops : min= 36, max= 202, avg=113.13, stdev= 6.69, samples=4800 write: IOPS=114, BW=7319KiB/s (7495kB/s)(4291MiB/600321msec); 0 zone resets slat (usec): min=5, max=239734, avg=17307.73, stdev=46908.18 clat (msec): min=42, max=1406, avg=482.39, stdev=166.40 lat (msec): min=42, max=1451, avg=499.70, stdev=172.63 clat percentiles (msec): | 1.00th=[ 230], 5.00th=[ 257], 10.00th=[ 275], 20.00th=[ 347], | 30.00th=[ 401], 40.00th=[ 422], 50.00th=[ 443], 60.00th=[ 485], | 70.00th=[ 558], 80.00th=[ 600], 90.00th=[ 718], 95.00th=[ 810], | 99.00th=[ 969], 99.50th=[ 1011], 99.90th=[ 1150], 99.95th=[ 1183], | 99.99th=[ 1334] bw ( KiB/s): min= 2304, max=12300, per=99.96%, avg=7316.17, stdev=441.42, samples=4800 iops : min= 36, max= 192, avg=114.27, stdev= 6.89, samples=4800 lat (msec) : 50=0.01%, 100=0.01%, 250=2.19%, 500=47.34%, 750=34.31% lat (msec) : 1000=13.19%, 2000=2.96% cpu : usr=0.04%, sys=0.07%, ctx=17147, majf=0, minf=68 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=67961,68651,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_128K: (groupid=28, jobs=4): err= 0: pid=1997: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=105, BW=13.2MiB/s (13.8MB/s)(7924MiB/600344msec) slat (usec): min=3, max=238974, avg=18907.53, stdev=50852.59 clat (msec): min=123, max=1595, avg=654.46, stdev=232.70 lat (msec): min=123, max=1668, avg=673.37, stdev=237.57 clat percentiles (msec): | 1.00th=[ 271], 5.00th=[ 317], 10.00th=[ 405], 20.00th=[ 447], | 30.00th=[ 481], 40.00th=[ 567], 50.00th=[ 617], 60.00th=[ 676], | 70.00th=[ 768], 80.00th=[ 894], 90.00th=[ 1011], 95.00th=[ 1083], | 99.00th=[ 1200], 99.50th=[ 1234], 99.90th=[ 1351], 99.95th=[ 1385], | 99.99th=[ 1452] bw ( KiB/s): min= 4610, max=24832, per=99.98%, avg=13513.57, stdev=816.81, samples=4800 iops : min= 36, max= 194, avg=105.53, stdev= 6.38, samples=4800 write: IOPS=105, BW=13.2MiB/s (13.8MB/s)(7904MiB/600344msec); 0 zone resets slat (usec): min=5, max=275938, avg=18995.75, stdev=50847.08 clat (msec): min=81, max=1609, avg=521.07, stdev=178.15 lat (msec): min=81, max=1609, avg=540.07, stdev=184.71 clat percentiles (msec): | 1.00th=[ 251], 5.00th=[ 279], 10.00th=[ 300], 20.00th=[ 384], | 30.00th=[ 430], 40.00th=[ 456], 50.00th=[ 477], 60.00th=[ 523], | 70.00th=[ 600], 80.00th=[ 651], 90.00th=[ 768], 95.00th=[ 869], | 99.00th=[ 1036], 99.50th=[ 1099], 99.90th=[ 1234], 99.95th=[ 1267], | 99.99th=[ 1351] bw ( KiB/s): min= 3584, max=25123, per=99.99%, avg=13480.07, stdev=837.65, samples=4800 iops : min= 28, max= 196, avg=105.27, stdev= 6.54, samples=4800 lat (msec) : 100=0.01%, 250=0.64%, 500=44.99%, 750=31.67%, 1000=16.79% lat (msec) : 2000=5.91% cpu : usr=0.04%, sys=0.08%, ctx=15912, majf=0, minf=59 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=63390,63230,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_4K_latency: (groupid=29, jobs=1): err= 0: pid=2049: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=81, BW=327KiB/s (335kB/s)(192MiB/600001msec) slat (nsec): min=12217, max=66282, avg=28986.50, stdev=1328.50 clat (usec): min=203, max=119565, avg=12198.21, stdev=3830.81 lat (usec): min=232, max=119594, avg=12228.02, stdev=3830.81 clat percentiles (usec): | 1.00th=[ 4015], 5.00th=[ 5997], 10.00th=[ 7177], 20.00th=[ 8848], | 30.00th=[10159], 40.00th=[11207], 50.00th=[12125], 60.00th=[13173], | 70.00th=[14222], 80.00th=[15533], 90.00th=[17171], 95.00th=[18482], | 99.00th=[20579], 99.50th=[21365], 99.90th=[22676], 99.95th=[23725], | 99.99th=[35390] bw ( KiB/s): min= 256, max= 384, per=99.72%, avg=326.91, stdev=17.11, samples=1199 iops : min= 64, max= 96, avg=81.73, stdev= 4.28, samples=1199 lat (usec) : 250=0.01% lat (msec) : 2=0.03%, 4=0.96%, 10=27.99%, 20=69.24%, 50=1.77% lat (msec) : 100=0.01%, 250=0.01% cpu : usr=0.16%, sys=0.41%, ctx=49056, majf=0, minf=15 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=49039,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_4K_latency: (groupid=30, jobs=1): err= 0: pid=2060: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=62, BW=251KiB/s (257kB/s)(147MiB/600002msec) slat (nsec): min=12858, max=63385, avg=29068.00, stdev=1502.47 clat (usec): min=175, max=84892, avg=15210.62, stdev=6801.91 lat (usec): min=206, max=84922, avg=15240.48, stdev=6801.91 clat percentiles (usec): | 1.00th=[ 4359], 5.00th=[ 6456], 10.00th=[ 7898], 20.00th=[ 9765], | 30.00th=[11207], 40.00th=[12518], 50.00th=[13829], 60.00th=[15270], | 70.00th=[17171], 80.00th=[20055], 90.00th=[25297], 95.00th=[28967], | 99.00th=[34866], 99.50th=[37487], 99.90th=[43779], 99.95th=[48497], | 99.99th=[61604] bw ( KiB/s): min= 104, max= 328, per=99.78%, avg=250.56, stdev=21.05, samples=1199 iops : min= 26, max= 82, avg=62.64, stdev= 5.26, samples=1199 write: IOPS=62, BW=251KiB/s (257kB/s)(147MiB/600002msec); 0 zone resets slat (nsec): min=13112, max=62560, avg=29172.13, stdev=1625.10 clat (usec): min=132, max=105497, avg=679.12, stdev=1274.34 lat (usec): min=149, max=105522, avg=709.12, stdev=1274.28 clat percentiles (usec): | 1.00th=[ 204], 5.00th=[ 229], 10.00th=[ 249], 20.00th=[ 273], | 30.00th=[ 293], 40.00th=[ 310], 50.00th=[ 326], 60.00th=[ 338], | 70.00th=[ 351], 80.00th=[ 408], 90.00th=[ 1893], 95.00th=[ 2933], | 99.00th=[ 4752], 99.50th=[ 6128], 99.90th=[11338], 99.95th=[11863], | 99.99th=[33424] bw ( KiB/s): min= 96, max= 456, per=99.75%, avg=250.56, stdev=50.58, samples=1199 iops : min= 24, max= 114, avg=62.64, stdev=12.64, samples=1199 lat (usec) : 250=5.32%, 500=36.80%, 750=0.67%, 1000=0.12% lat (msec) : 2=2.39%, 4=4.19%, 10=11.25%, 20=29.20%, 50=10.05% lat (msec) : 100=0.02%, 250=0.01% cpu : usr=0.22%, sys=0.66%, ctx=75196, majf=0, minf=18 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=37584,37594,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randwrite_4K_latency: (groupid=31, jobs=1): err= 0: pid=2095: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] write: IOPS=308, BW=1233KiB/s (1262kB/s)(722MiB/600002msec); 0 zone resets slat (usec): min=9, max=136, avg=29.74, stdev= 1.61 clat (usec): min=130, max=63052, avg=3207.25, stdev=1445.79 lat (usec): min=151, max=63083, avg=3237.82, stdev=1445.78 clat percentiles (usec): | 1.00th=[ 717], 5.00th=[ 1352], 10.00th=[ 1696], 20.00th=[ 2147], | 30.00th=[ 2474], 40.00th=[ 2769], 50.00th=[ 3064], 60.00th=[ 3359], | 70.00th=[ 3720], 80.00th=[ 4146], 90.00th=[ 4817], 95.00th=[ 5473], | 99.00th=[ 6915], 99.50th=[ 7504], 99.90th=[11994], 99.95th=[12780], | 99.99th=[45876] bw ( KiB/s): min= 872, max= 1584, per=99.94%, avg=1232.96, stdev=107.85, samples=1199 iops : min= 218, max= 396, avg=308.14, stdev=26.96, samples=1199 lat (usec) : 250=0.03%, 500=0.41%, 750=0.67%, 1000=1.06% lat (msec) : 2=14.26%, 4=60.18%, 10=23.21%, 20=0.16%, 50=0.02% lat (msec) : 100=0.01% cpu : usr=0.57%, sys=1.74%, ctx=184962, majf=0, minf=13 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,184906,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_70R30W_4K: (groupid=32, jobs=4): err= 0: pid=2116: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=157, BW=631KiB/s (646kB/s)(370MiB/600318msec) slat (usec): min=2, max=283084, avg=17688.21, stdev=47613.48 clat (msec): min=6, max=2934, avg=587.91, stdev=198.39 lat (msec): min=6, max=3055, avg=605.60, stdev=203.72 clat percentiles (msec): | 1.00th=[ 257], 5.00th=[ 309], 10.00th=[ 388], 20.00th=[ 422], | 30.00th=[ 447], 40.00th=[ 514], 50.00th=[ 558], 60.00th=[ 592], | 70.00th=[ 684], 80.00th=[ 743], 90.00th=[ 869], 95.00th=[ 978], | 99.00th=[ 1133], 99.50th=[ 1200], 99.90th=[ 1334], 99.95th=[ 1401], | 99.99th=[ 1502] bw ( KiB/s): min= 159, max= 1048, per=99.86%, avg=630.44, stdev=31.52, samples=4800 iops : min= 39, max= 262, avg=157.61, stdev= 7.88, samples=4800 write: IOPS=67, BW=271KiB/s (277kB/s)(159MiB/600318msec); 0 zone resets slat (usec): min=3, max=268400, avg=17806.59, stdev=47771.65 clat (usec): min=833, max=1446.7k, avg=460941.48, stdev=157144.12 lat (usec): min=859, max=1446.8k, avg=478748.47, stdev=163511.98 clat percentiles (msec): | 1.00th=[ 228], 5.00th=[ 255], 10.00th=[ 275], 20.00th=[ 326], | 30.00th=[ 397], 40.00th=[ 418], 50.00th=[ 430], 60.00th=[ 451], | 70.00th=[ 493], 80.00th=[ 575], 90.00th=[ 684], 95.00th=[ 760], | 99.00th=[ 969], 99.50th=[ 1028], 99.90th=[ 1200], 99.95th=[ 1234], | 99.99th=[ 1334] bw ( KiB/s): min= 32, max= 608, per=99.64%, avg=270.89, stdev=23.29, samples=4800 iops : min= 8, max= 152, avg=67.72, stdev= 5.82, samples=4800 lat (usec) : 1000=0.01% lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec) : 100=0.02%, 250=1.56%, 500=46.77%, 750=36.61%, 1000=12.13% lat (msec) : 2000=2.90%, >=2000=0.01% cpu : usr=0.05%, sys=0.07%, ctx=17095, majf=0, minf=68 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=94683,40668,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_70R30W_64K: (groupid=33, jobs=4): err= 0: pid=2142: Sat Jan 11 02:53:45 2025 Description : ["Storage megatest by Andrey"] read: IOPS=146, BW=9384KiB/s (9609kB/s)(5502MiB/600338msec) slat (usec): min=3, max=304851, avg=19030.43, stdev=51227.46 clat (msec): min=107, max=1711, avg=633.98, stdev=211.62 lat (msec): min=107, max=1711, avg=653.01, stdev=217.47 clat percentiles (msec): | 1.00th=[ 279], 5.00th=[ 334], 10.00th=[ 418], 20.00th=[ 451], | 30.00th=[ 481], 40.00th=[ 558], 50.00th=[ 600], 60.00th=[ 642], | 70.00th=[ 735], 80.00th=[ 802], 90.00th=[ 936], 95.00th=[ 1045], | 99.00th=[ 1217], 99.50th=[ 1267], 99.90th=[ 1418], 99.95th=[ 1485], | 99.99th=[ 1603] bw ( KiB/s): min= 3201, max=15488, per=99.97%, avg=9381.17, stdev=471.63, samples=4800 iops : min= 50, max= 242, avg=146.57, stdev= 7.37, samples=4800 write: IOPS=62, BW=4019KiB/s (4115kB/s)(2356MiB/600338msec); 0 zone resets slat (usec): min=5, max=424902, avg=19218.18, stdev=51498.60 clat (msec): min=99, max=1580, avg=494.05, stdev=166.91 lat (msec): min=99, max=1580, avg=513.27, stdev=173.57 clat percentiles (msec): | 1.00th=[ 249], 5.00th=[ 275], 10.00th=[ 296], 20.00th=[ 347], | 30.00th=[ 426], 40.00th=[ 447], 50.00th=[ 464], 60.00th=[ 485], | 70.00th=[ 523], 80.00th=[ 609], 90.00th=[ 726], 95.00th=[ 818], | 99.00th=[ 1036], 99.50th=[ 1116], 99.90th=[ 1267], 99.95th=[ 1301], | 99.99th=[ 1452] bw ( KiB/s): min= 512, max= 9220, per=99.96%, avg=4017.93, stdev=353.69, samples=4799 iops : min= 8, max= 144, avg=62.77, stdev= 5.53, samples=4799 lat (msec) : 100=0.01%, 250=0.45%, 500=43.41%, 750=34.41%, 1000=16.75% lat (msec) : 2000=4.97% cpu : usr=0.03%, sys=0.07%, ctx=15785, majf=0, minf=63 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=88027,37695,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 Run status group 0 (all jobs): READ: bw=823KiB/s (843kB/s), 823KiB/s-823KiB/s (843kB/s-843kB/s), io=483MiB (506MB), run=600395-600395msec Run status group 1 (all jobs): READ: bw=1643KiB/s (1682kB/s), 1643KiB/s-1643KiB/s (1682kB/s-1682kB/s), io=963MiB (1010MB), run=600355-600355msec Run status group 2 (all jobs): READ: bw=6380KiB/s (6533kB/s), 6380KiB/s-6380KiB/s (6533kB/s-6533kB/s), io=3740MiB (3922MB), run=600356-600356msec Run status group 3 (all jobs): READ: bw=12.0MiB/s (12.6MB/s), 12.0MiB/s-12.0MiB/s (12.6MB/s-12.6MB/s), io=7213MiB (7563MB), run=600391-600391msec Run status group 4 (all jobs): READ: bw=22.6MiB/s (23.7MB/s), 22.6MiB/s-22.6MiB/s (23.7MB/s-23.7MB/s), io=13.3GiB (14.2GB), run=600412-600412msec Run status group 5 (all jobs): WRITE: bw=1230KiB/s (1259kB/s), 1230KiB/s-1230KiB/s (1259kB/s-1259kB/s), io=721MiB (756MB), run=600183-600183msec Run status group 6 (all jobs): WRITE: bw=2434KiB/s (2493kB/s), 2434KiB/s-2434KiB/s (2493kB/s-2493kB/s), io=1427MiB (1496MB), run=600177-600177msec Run status group 7 (all jobs): WRITE: bw=9353KiB/s (9577kB/s), 9353KiB/s-9353KiB/s (9577kB/s-9577kB/s), io=5482MiB (5749MB), run=600242-600242msec Run status group 8 (all jobs): WRITE: bw=17.4MiB/s (18.3MB/s), 17.4MiB/s-17.4MiB/s (18.3MB/s-18.3MB/s), io=10.2GiB (10.0GB), run=600237-600237msec Run status group 9 (all jobs): WRITE: bw=32.1MiB/s (33.6MB/s), 32.1MiB/s-32.1MiB/s (33.6MB/s-33.6MB/s), io=18.8GiB (20.2GB), run=600290-600290msec Run status group 10 (all jobs): READ: bw=219MiB/s (229MB/s), 219MiB/s-219MiB/s (229MB/s-229MB/s), io=128GiB (138GB), run=600013-600013msec Run status group 11 (all jobs): READ: bw=242MiB/s (254MB/s), 242MiB/s-242MiB/s (254MB/s-254MB/s), io=142GiB (152GB), run=600044-600044msec Run status group 12 (all jobs): READ: bw=257MiB/s (269MB/s), 257MiB/s-257MiB/s (269MB/s-269MB/s), io=151GiB (162GB), run=600052-600052msec Run status group 13 (all jobs): READ: bw=258MiB/s (270MB/s), 258MiB/s-258MiB/s (270MB/s-270MB/s), io=151GiB (162GB), run=600044-600044msec Run status group 14 (all jobs): READ: bw=258MiB/s (270MB/s), 258MiB/s-258MiB/s (270MB/s-270MB/s), io=151GiB (162GB), run=600074-600074msec Run status group 15 (all jobs): READ: bw=245MiB/s (257MB/s), 245MiB/s-245MiB/s (257MB/s-257MB/s), io=144GiB (155GB), run=600301-600301msec Run status group 16 (all jobs): READ: bw=235MiB/s (246MB/s), 235MiB/s-235MiB/s (246MB/s-246MB/s), io=138GiB (148GB), run=600287-600287msec Run status group 17 (all jobs): WRITE: bw=89.5MiB/s (93.8MB/s), 89.5MiB/s-89.5MiB/s (93.8MB/s-93.8MB/s), io=52.4GiB (56.3GB), run=600020-600020msec Run status group 18 (all jobs): WRITE: bw=111MiB/s (117MB/s), 111MiB/s-111MiB/s (117MB/s-117MB/s), io=65.3GiB (70.1GB), run=600006-600006msec Run status group 19 (all jobs): WRITE: bw=206MiB/s (216MB/s), 206MiB/s-206MiB/s (216MB/s-216MB/s), io=121GiB (129GB), run=600019-600019msec Run status group 20 (all jobs): WRITE: bw=227MiB/s (238MB/s), 227MiB/s-227MiB/s (238MB/s-238MB/s), io=133GiB (143GB), run=600033-600033msec Run status group 21 (all jobs): WRITE: bw=238MiB/s (250MB/s), 238MiB/s-238MiB/s (250MB/s-250MB/s), io=140GiB (150GB), run=600066-600066msec Run status group 22 (all jobs): WRITE: bw=245MiB/s (257MB/s), 245MiB/s-245MiB/s (257MB/s-257MB/s), io=144GiB (154GB), run=600255-600255msec Run status group 23 (all jobs): WRITE: bw=249MiB/s (261MB/s), 249MiB/s-249MiB/s (261MB/s-261MB/s), io=146GiB (157GB), run=600271-600271msec Run status group 24 (all jobs): READ: bw=492KiB/s (504kB/s), 492KiB/s-492KiB/s (504kB/s-504kB/s), io=289MiB (303MB), run=600283-600283msec WRITE: bw=493KiB/s (505kB/s), 493KiB/s-493KiB/s (505kB/s-505kB/s), io=289MiB (303MB), run=600283-600283msec Run status group 25 (all jobs): READ: bw=975KiB/s (998kB/s), 975KiB/s-975KiB/s (998kB/s-998kB/s), io=571MiB (599MB), run=600361-600361msec WRITE: bw=984KiB/s (1007kB/s), 984KiB/s-984KiB/s (1007kB/s-1007kB/s), io=577MiB (605MB), run=600361-600361msec Run status group 26 (all jobs): READ: bw=3782KiB/s (3873kB/s), 3782KiB/s-3782KiB/s (3873kB/s-3873kB/s), io=2217MiB (2325MB), run=600285-600285msec WRITE: bw=3763KiB/s (3854kB/s), 3763KiB/s-3763KiB/s (3854kB/s-3854kB/s), io=2206MiB (2313MB), run=600285-600285msec Run status group 27 (all jobs): READ: bw=7245KiB/s (7419kB/s), 7245KiB/s-7245KiB/s (7419kB/s-7419kB/s), io=4248MiB (4454MB), run=600321-600321msec WRITE: bw=7319KiB/s (7495kB/s), 7319KiB/s-7319KiB/s (7495kB/s-7495kB/s), io=4291MiB (4499MB), run=600321-600321msec Run status group 28 (all jobs): READ: bw=13.2MiB/s (13.8MB/s), 13.2MiB/s-13.2MiB/s (13.8MB/s-13.8MB/s), io=7924MiB (8309MB), run=600344-600344msec WRITE: bw=13.2MiB/s (13.8MB/s), 13.2MiB/s-13.2MiB/s (13.8MB/s-13.8MB/s), io=7904MiB (8288MB), run=600344-600344msec Run status group 29 (all jobs): READ: bw=327KiB/s (335kB/s), 327KiB/s-327KiB/s (335kB/s-335kB/s), io=192MiB (201MB), run=600001-600001msec Run status group 30 (all jobs): READ: bw=251KiB/s (257kB/s), 251KiB/s-251KiB/s (257kB/s-257kB/s), io=147MiB (154MB), run=600002-600002msec WRITE: bw=251KiB/s (257kB/s), 251KiB/s-251KiB/s (257kB/s-257kB/s), io=147MiB (154MB), run=600002-600002msec Run status group 31 (all jobs): WRITE: bw=1233KiB/s (1262kB/s), 1233KiB/s-1233KiB/s (1262kB/s-1262kB/s), io=722MiB (757MB), run=600002-600002msec Run status group 32 (all jobs): READ: bw=631KiB/s (646kB/s), 631KiB/s-631KiB/s (646kB/s-646kB/s), io=370MiB (388MB), run=600318-600318msec WRITE: bw=271KiB/s (277kB/s), 271KiB/s-271KiB/s (277kB/s-277kB/s), io=159MiB (167MB), run=600318-600318msec Run status group 33 (all jobs): READ: bw=9384KiB/s (9609kB/s), 9384KiB/s-9384KiB/s (9609kB/s-9609kB/s), io=5502MiB (5769MB), run=600338-600338msec WRITE: bw=4019KiB/s (4115kB/s), 4019KiB/s-4019KiB/s (4115kB/s-4115kB/s), io=2356MiB (2470MB), run=600338-600338msec Disk stats (read/write): sdb: ios=17520261/9196404, merge=44986952/22341280, ticks=529458731/436514917, in_queue=965973648, util=100.00%