randread_4K: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randread_8K: (g=1): rw=randread, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randread_32K: (g=2): rw=randread, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randread_64K: (g=3): rw=randread, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randread_128K: (g=4): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randwrite_4K: (g=5): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randwrite_8K: (g=6): rw=randwrite, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randwrite_32K: (g=7): rw=randwrite, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randwrite_64K: (g=8): rw=randwrite, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randwrite_128K: (g=9): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_4K: (g=10): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqread_8K: (g=11): rw=read, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqread_32K: (g=12): rw=read, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqread_64K: (g=13): rw=read, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqread_128K: (g=14): rw=read, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqread_512K: (g=15): rw=read, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqread_1M: (g=16): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... seqwrite_4K: (g=17): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... seqwrite_8K: (g=18): rw=write, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... seqwrite_32K: (g=19): rw=write, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_64K: (g=20): rw=write, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... seqwrite_128K: (g=21): rw=write, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... seqwrite_512K: (g=22): rw=write, bs=(R) 512KiB-512KiB, (W) 512KiB-512KiB, (T) 512KiB-512KiB, ioengine=libaio, iodepth=32 ... seqwrite_1M: (g=23): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=32 ... randrw_4K: (g=24): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_8K: (g=25): rw=randrw, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=32 ... randrw_32K: (g=26): rw=randrw, bs=(R) 32.0KiB-32.0KiB, (W) 32.0KiB-32.0KiB, (T) 32.0KiB-32.0KiB, ioengine=libaio, iodepth=32 ... randrw_64K: (g=27): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... randrw_128K: (g=28): rw=randrw, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=32 ... randread_4K_latency: (g=29): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_4K_latency: (g=30): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randwrite_4K_latency: (g=31): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1 randrw_70R30W_4K: (g=32): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32 ... randrw_70R30W_64K: (g=33): rw=randrw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=32 ... fio-3.25 Starting 127 processes randread_4K: (groupid=0, jobs=4): err= 0: pid=1251: Tue Jan 14 10:23:49 2025 Description : ["Storage megatest by Andrey"] read: IOPS=208, BW=832KiB/s (852kB/s)(16.6MiB/20394msec) slat (usec): min=3, max=189111, avg=15646.68, stdev=42456.14 clat (msec): min=17, max=2232, avg=588.46, stdev=315.09 lat (msec): min=17, max=2343, avg=604.11, stdev=327.56 clat percentiles (msec): | 1.00th=[ 54], 5.00th=[ 125], 10.00th=[ 176], 20.00th=[ 275], | 30.00th=[ 401], 40.00th=[ 481], 50.00th=[ 584], 60.00th=[ 667], | 70.00th=[ 776], 80.00th=[ 869], 90.00th=[ 1011], 95.00th=[ 1116], | 99.00th=[ 1334], 99.50th=[ 1469], 99.90th=[ 1938], 99.95th=[ 1972], | 99.99th=[ 2232] bw ( KiB/s): min= 360, max= 1248, per=99.35%, avg=827.28, stdev=68.31, samples=158 iops : min= 90, max= 312, avg=206.80, stdev=17.08, samples=158 lat (msec) : 20=0.02%, 50=0.80%, 100=2.50%, 250=13.81%, 500=25.61% lat (msec) : 750=24.34%, 1000=22.08%, 2000=10.79%, >=2000=0.05% cpu : usr=0.50%, sys=0.14%, ctx=2065, majf=0, minf=179 IO depths : 1=0.1%, 2=0.2%, 4=0.4%, 8=0.8%, 16=1.5%, 32=97.1%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=4244,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_8K: (groupid=1, jobs=4): err= 0: pid=1257: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=207, BW=1660KiB/s (1700kB/s)(32.0MiB/20338msec) slat (usec): min=2, max=224777, avg=17293.30, stdev=46765.21 clat (msec): min=17, max=1629, avg=589.95, stdev=252.52 lat (msec): min=17, max=1629, avg=607.25, stdev=262.90 clat percentiles (msec): | 1.00th=[ 69], 5.00th=[ 167], 10.00th=[ 271], 20.00th=[ 426], | 30.00th=[ 460], 40.00th=[ 493], 50.00th=[ 584], 60.00th=[ 634], | 70.00th=[ 693], 80.00th=[ 802], 90.00th=[ 927], 95.00th=[ 1028], | 99.00th=[ 1250], 99.50th=[ 1301], 99.90th=[ 1469], 99.95th=[ 1485], | 99.99th=[ 1636] bw ( KiB/s): min= 967, max= 2577, per=100.00%, avg=1660.62, stdev=137.68, samples=157 iops : min= 120, max= 322, avg=207.52, stdev=17.21, samples=157 lat (msec) : 20=0.02%, 50=0.59%, 100=1.49%, 250=6.61%, 500=32.51% lat (msec) : 750=33.60%, 1000=18.91%, 2000=6.26% cpu : usr=0.29%, sys=0.10%, ctx=1344, majf=0, minf=312 IO depths : 1=0.1%, 2=0.2%, 4=0.4%, 8=0.8%, 16=1.5%, 32=97.1%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=4220,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_32K: (groupid=2, jobs=4): err= 0: pid=1262: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=200, BW=6429KiB/s (6583kB/s)(128MiB/20369msec) slat (usec): min=5, max=206845, avg=19611.31, stdev=52471.52 clat (msec): min=199, max=1450, avg=613.18, stdev=184.30 lat (msec): min=231, max=1513, avg=632.80, stdev=190.81 clat percentiles (msec): | 1.00th=[ 296], 5.00th=[ 342], 10.00th=[ 439], 20.00th=[ 464], | 30.00th=[ 485], 40.00th=[ 514], 50.00th=[ 609], 60.00th=[ 634], | 70.00th=[ 667], 80.00th=[ 768], 90.00th=[ 844], 95.00th=[ 969], | 99.00th=[ 1167], 99.50th=[ 1250], 99.90th=[ 1368], 99.95th=[ 1385], | 99.99th=[ 1452] bw ( KiB/s): min= 3072, max= 8204, per=98.30%, avg=6319.35, stdev=250.25, samples=160 iops : min= 95, max= 256, avg=197.37, stdev= 7.83, samples=160 lat (msec) : 250=0.07%, 500=36.44%, 750=41.18%, 1000=18.62%, 2000=3.69% cpu : usr=0.08%, sys=0.08%, ctx=560, majf=0, minf=1078 IO depths : 1=0.1%, 2=0.2%, 4=0.4%, 8=0.8%, 16=1.6%, 32=97.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=4092,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_64K: (groupid=3, jobs=4): err= 0: pid=1266: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=193, BW=12.1MiB/s (12.7MB/s)(247MiB/20402msec) slat (usec): min=6, max=229240, avg=20348.91, stdev=54458.15 clat (msec): min=217, max=1536, avg=636.89, stdev=186.70 lat (msec): min=217, max=1540, avg=657.24, stdev=195.05 clat percentiles (msec): | 1.00th=[ 313], 5.00th=[ 372], 10.00th=[ 464], 20.00th=[ 485], | 30.00th=[ 502], 40.00th=[ 542], 50.00th=[ 634], 60.00th=[ 659], | 70.00th=[ 684], 80.00th=[ 802], 90.00th=[ 885], 95.00th=[ 995], | 99.00th=[ 1167], 99.50th=[ 1250], 99.90th=[ 1401], 99.95th=[ 1502], | 99.99th=[ 1536] bw ( KiB/s): min= 6112, max=16408, per=98.12%, avg=12152.80, stdev=432.23, samples=160 iops : min= 95, max= 256, avg=189.78, stdev= 6.77, samples=160 lat (msec) : 250=0.08%, 500=29.23%, 750=46.99%, 1000=19.00%, 2000=4.71% cpu : usr=0.04%, sys=0.08%, ctx=533, majf=0, minf=2101 IO depths : 1=0.1%, 2=0.2%, 4=0.4%, 8=0.8%, 16=1.6%, 32=96.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=3948,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_128K: (groupid=4, jobs=4): err= 0: pid=1270: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=183, BW=22.9MiB/s (24.0MB/s)(467MiB/20370msec) slat (usec): min=4, max=235198, avg=21488.74, stdev=57547.19 clat (msec): min=182, max=1651, avg=673.36, stdev=206.81 lat (msec): min=182, max=1651, avg=694.85, stdev=214.07 clat percentiles (msec): | 1.00th=[ 321], 5.00th=[ 359], 10.00th=[ 477], 20.00th=[ 510], | 30.00th=[ 531], 40.00th=[ 575], 50.00th=[ 667], 60.00th=[ 701], | 70.00th=[ 735], 80.00th=[ 844], 90.00th=[ 953], 95.00th=[ 1070], | 99.00th=[ 1267], 99.50th=[ 1301], 99.90th=[ 1569], 99.95th=[ 1653], | 99.99th=[ 1653] bw ( KiB/s): min= 8724, max=30748, per=98.37%, avg=23068.80, stdev=991.61, samples=160 iops : min= 68, max= 240, avg=180.15, stdev= 7.75, samples=160 lat (msec) : 250=0.11%, 500=17.07%, 750=56.59%, 1000=17.85%, 2000=8.39% cpu : usr=0.03%, sys=0.09%, ctx=511, majf=0, minf=2105 IO depths : 1=0.1%, 2=0.2%, 4=0.4%, 8=0.9%, 16=1.7%, 32=96.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=3732,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_4K: (groupid=5, jobs=4): err= 0: pid=1274: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=1701, BW=6805KiB/s (6968kB/s)(134MiB/20115msec); 0 zone resets slat (usec): min=2, max=78489, avg=2325.84, stdev=9927.61 clat (msec): min=2, max=540, avg=72.53, stdev=87.01 lat (msec): min=2, max=540, avg=74.86, stdev=90.13 clat percentiles (msec): | 1.00th=[ 13], 5.00th=[ 13], 10.00th=[ 17], 20.00th=[ 17], | 30.00th=[ 18], 40.00th=[ 22], 50.00th=[ 23], 60.00th=[ 26], | 70.00th=[ 46], 80.00th=[ 184], 90.00th=[ 220], 95.00th=[ 236], | 99.00th=[ 288], 99.50th=[ 313], 99.90th=[ 384], 99.95th=[ 414], | 99.99th=[ 493] bw ( KiB/s): min= 2049, max=25784, per=99.08%, avg=6742.49, stdev=2158.86, samples=156 iops : min= 512, max= 6446, avg=1685.41, stdev=539.72, samples=156 lat (msec) : 4=0.02%, 10=0.15%, 20=32.00%, 50=39.23%, 100=2.19% lat (msec) : 250=22.90%, 500=3.50%, 750=0.01% cpu : usr=0.70%, sys=0.49%, ctx=4358, majf=0, minf=58 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,34220,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_8K: (groupid=6, jobs=4): err= 0: pid=1281: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=506, BW=4050KiB/s (4147kB/s)(80.2MiB/20266msec); 0 zone resets slat (usec): min=2, max=164912, avg=7804.06, stdev=22395.74 clat (msec): min=4, max=910, avg=244.17, stdev=105.45 lat (msec): min=4, max=974, avg=251.97, stdev=110.07 clat percentiles (msec): | 1.00th=[ 35], 5.00th=[ 157], 10.00th=[ 161], 20.00th=[ 167], | 30.00th=[ 207], 40.00th=[ 213], 50.00th=[ 218], 60.00th=[ 224], | 70.00th=[ 239], 80.00th=[ 279], 90.00th=[ 380], 95.00th=[ 481], | 99.00th=[ 617], 99.50th=[ 676], 99.90th=[ 802], 99.95th=[ 852], | 99.99th=[ 902] bw ( KiB/s): min= 1440, max= 6855, per=99.48%, avg=4029.58, stdev=300.88, samples=160 iops : min= 180, max= 855, avg=503.47, stdev=37.57, samples=160 lat (msec) : 10=0.14%, 20=0.08%, 50=1.88%, 100=0.63%, 250=70.01% lat (msec) : 500=24.07%, 750=2.94%, 1000=0.24% cpu : usr=0.30%, sys=0.17%, ctx=1337, majf=0, minf=59 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.3%, 16=0.6%, 32=98.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,10260,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_32K: (groupid=7, jobs=4): err= 0: pid=1288: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=61, BW=1967KiB/s (2014kB/s)(39.9MiB/20757msec); 0 zone resets slat (usec): min=4, max=651152, avg=63491.90, stdev=173607.84 clat (msec): min=313, max=4138, avg=1995.71, stdev=461.07 lat (msec): min=542, max=4138, avg=2059.20, stdev=491.33 clat percentiles (msec): | 1.00th=[ 810], 5.00th=[ 1183], 10.00th=[ 1452], 20.00th=[ 1603], | 30.00th=[ 1720], 40.00th=[ 2039], 50.00th=[ 2089], 60.00th=[ 2140], | 70.00th=[ 2232], 80.00th=[ 2299], 90.00th=[ 2366], 95.00th=[ 2769], | 99.00th=[ 3138], 99.50th=[ 3339], 99.90th=[ 3910], 99.95th=[ 4144], | 99.99th=[ 4144] bw ( KiB/s): min= 256, max= 3074, per=100.00%, avg=2007.15, stdev=103.17, samples=147 iops : min= 8, max= 96, avg=62.70, stdev= 3.22, samples=147 lat (msec) : 500=0.24%, 750=0.55%, 1000=2.59%, 2000=34.25%, >=2000=62.38% cpu : usr=0.06%, sys=0.03%, ctx=213, majf=0, minf=56 IO depths : 1=0.3%, 2=0.6%, 4=1.3%, 8=2.5%, 16=5.0%, 32=90.3%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.3%, 64=0.0%, >=64=0.0% issued rwts: total=0,1276,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_64K: (groupid=8, jobs=4): err= 0: pid=1294: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=44, BW=2866KiB/s (2935kB/s)(60.2MiB/21526msec); 0 zone resets slat (usec): min=6, max=1112.6k, avg=84939.96, stdev=235866.20 clat (msec): min=681, max=6956, avg=2724.67, stdev=712.16 lat (msec): min=784, max=6956, avg=2809.61, stdev=748.42 clat percentiles (msec): | 1.00th=[ 1418], 5.00th=[ 1854], 10.00th=[ 1921], 20.00th=[ 2123], | 30.00th=[ 2500], 40.00th=[ 2567], 50.00th=[ 2601], 60.00th=[ 2769], | 70.00th=[ 2869], 80.00th=[ 3171], 90.00th=[ 3540], 95.00th=[ 4111], | 99.00th=[ 5000], 99.50th=[ 5269], 99.90th=[ 6946], 99.95th=[ 6946], | 99.99th=[ 6946] bw ( KiB/s): min= 512, max= 4104, per=100.00%, avg=3912.37, stdev=194.84, samples=110 iops : min= 8, max= 64, avg=61.11, stdev= 3.04, samples=110 lat (msec) : 750=0.10%, 1000=0.31%, 2000=15.66%, >=2000=83.92% cpu : usr=0.04%, sys=0.02%, ctx=169, majf=0, minf=50 IO depths : 1=0.4%, 2=0.8%, 4=1.7%, 8=3.3%, 16=6.6%, 32=87.1%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.5%, 8=0.0%, 16=0.0%, 32=0.5%, 64=0.0%, >=64=0.0% issued rwts: total=0,964,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randwrite_128K: (groupid=9, jobs=4): err= 0: pid=1303: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=80, BW=10.1MiB/s (10.6MB/s)(204MiB/20209msec); 0 zone resets slat (usec): min=8, max=1777.5k, avg=49274.74, stdev=195039.58 clat (msec): min=109, max=9255, avg=1489.02, stdev=1473.17 lat (msec): min=210, max=9255, avg=1538.29, stdev=1520.91 clat percentiles (msec): | 1.00th=[ 222], 5.00th=[ 305], 10.00th=[ 393], 20.00th=[ 468], | 30.00th=[ 642], 40.00th=[ 785], 50.00th=[ 978], 60.00th=[ 1099], | 70.00th=[ 1301], 80.00th=[ 1989], 90.00th=[ 4044], 95.00th=[ 5134], | 99.00th=[ 6074], 99.50th=[ 6409], 99.90th=[ 7684], 99.95th=[ 9194], | 99.99th=[ 9194] bw ( KiB/s): min= 3073, max=38912, per=100.00%, avg=14638.70, stdev=2134.10, samples=100 iops : min= 24, max= 304, avg=114.32, stdev=16.67, samples=100 lat (msec) : 250=1.72%, 500=20.58%, 750=15.85%, 1000=13.21%, 2000=28.75% lat (msec) : >=2000=19.90% cpu : usr=0.06%, sys=0.02%, ctx=252, majf=0, minf=62 IO depths : 1=0.2%, 2=0.5%, 4=1.0%, 8=2.0%, 16=3.9%, 32=92.4%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.3%, 64=0.0%, >=64=0.0% issued rwts: total=0,1628,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_4K: (groupid=10, jobs=4): err= 0: pid=1310: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=63.5k, BW=248MiB/s (260MB/s)(4997MiB/20156msec) slat (usec): min=2, max=1178, avg= 4.37, stdev= 2.84 clat (usec): min=38, max=2545.7k, avg=2003.22, stdev=61747.22 lat (usec): min=45, max=2545.7k, avg=2007.78, stdev=61747.27 clat percentiles (usec): | 1.00th=[ 101], 5.00th=[ 131], 10.00th=[ 151], | 20.00th=[ 388], 30.00th=[ 408], 40.00th=[ 420], | 50.00th=[ 437], 60.00th=[ 445], 70.00th=[ 453], | 80.00th=[ 461], 90.00th=[ 498], 95.00th=[ 807], | 99.00th=[ 2024], 99.50th=[ 3392], 99.90th=[ 9241], | 99.95th=[2466251], 99.99th=[2533360] bw ( KiB/s): min= 400, max=1111280, per=100.00%, avg=545523.11, stdev=117581.75, samples=71 iops : min= 100, max=277820, avg=136380.31, stdev=29395.45, samples=71 lat (usec) : 50=0.01%, 100=0.95%, 250=10.93%, 500=78.64%, 750=3.35% lat (usec) : 1000=4.31% lat (msec) : 2=0.79%, 4=0.68%, 10=0.25%, 20=0.02%, 50=0.01% lat (msec) : 100=0.01%, 250=0.01%, 2000=0.01%, >=2000=0.06% cpu : usr=4.08%, sys=8.04%, ctx=271009, majf=0, minf=197 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=1279251,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_8K: (groupid=11, jobs=4): err= 0: pid=1314: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=35.0k, BW=281MiB/s (295MB/s)(5651MiB/20115msec) slat (usec): min=3, max=108, avg= 5.22, stdev= 2.32 clat (usec): min=145, max=2573.9k, avg=3540.58, stdev=79952.05 lat (usec): min=151, max=2573.0k, avg=3546.03, stdev=79952.11 clat percentiles (usec): | 1.00th=[ 668], 5.00th=[ 717], 10.00th=[ 750], | 20.00th=[ 799], 30.00th=[ 840], 40.00th=[ 865], | 50.00th=[ 889], 60.00th=[ 906], 70.00th=[ 955], | 80.00th=[ 1090], 90.00th=[ 1123], 95.00th=[ 1254], | 99.00th=[ 2343], 99.50th=[ 2540], 99.90th=[2365588], | 99.95th=[2499806], 99.99th=[2533360] bw ( KiB/s): min= 640, max=1117742, per=100.00%, avg=626372.12, stdev=120550.03, samples=72 iops : min= 80, max=139717, avg=78296.32, stdev=15068.75, samples=72 lat (usec) : 250=0.05%, 500=0.07%, 750=10.10%, 1000=61.28% lat (msec) : 2=27.44%, 4=0.77%, 10=0.11%, 20=0.02%, 50=0.04% lat (msec) : 100=0.01%, 250=0.01%, >=2000=0.10% cpu : usr=2.96%, sys=5.81%, ctx=160685, majf=0, minf=325 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=723388,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_32K: (groupid=12, jobs=4): err= 0: pid=1318: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=8289, BW=259MiB/s (272MB/s)(5214MiB/20129msec) slat (nsec): min=4328, max=50922, avg=8176.77, stdev=4314.79 clat (usec): min=534, max=2538.4k, avg=15402.96, stdev=169322.00 lat (usec): min=546, max=2538.4k, avg=15411.45, stdev=169322.09 clat percentiles (msec): | 1.00th=[ 3], 5.00th=[ 3], 10.00th=[ 3], 20.00th=[ 4], | 30.00th=[ 4], 40.00th=[ 4], 50.00th=[ 4], 60.00th=[ 4], | 70.00th=[ 4], 80.00th=[ 4], 90.00th=[ 5], 95.00th=[ 5], | 99.00th=[ 8], 99.50th=[ 194], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2534] bw ( KiB/s): min= 3840, max=1109120, per=100.00%, avg=566329.42, stdev=114952.78, samples=72 iops : min= 120, max=34660, avg=17697.28, stdev=3592.27, samples=72 lat (usec) : 750=0.13%, 1000=0.05% lat (msec) : 2=0.51%, 4=84.31%, 10=14.14%, 20=0.13%, 50=0.09% lat (msec) : 100=0.12%, 250=0.06%, >=2000=0.47% cpu : usr=1.06%, sys=2.40%, ctx=34559, majf=0, minf=1091 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=166857,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_64K: (groupid=13, jobs=4): err= 0: pid=1322: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=4111, BW=257MiB/s (269MB/s)(5151MiB/20045msec) slat (nsec): min=5164, max=79940, avg=11067.63, stdev=6293.82 clat (usec): min=458, max=2552.0k, avg=31086.50, stdev=230792.75 lat (usec): min=478, max=2552.0k, avg=31097.93, stdev=230792.74 clat percentiles (msec): | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 7], | 30.00th=[ 8], 40.00th=[ 8], 50.00th=[ 8], 60.00th=[ 8], | 70.00th=[ 8], 80.00th=[ 8], 90.00th=[ 8], 95.00th=[ 9], | 99.00th=[ 1020], 99.50th=[ 2500], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2534] bw ( KiB/s): min=13568, max=1117618, per=100.00%, avg=537489.40, stdev=105844.81, samples=76 iops : min= 212, max=17462, avg=8397.72, stdev=1653.83, samples=76 lat (usec) : 500=0.01%, 750=0.02%, 1000=0.05% lat (msec) : 2=0.17%, 4=3.11%, 10=94.15%, 20=0.46%, 50=0.46% lat (msec) : 100=0.34%, 250=0.04%, 500=0.05%, 750=0.04%, 1000=0.06% lat (msec) : 2000=0.27%, >=2000=0.77% cpu : usr=0.63%, sys=1.80%, ctx=21305, majf=0, minf=2113 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=82420,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_128K: (groupid=14, jobs=4): err= 0: pid=1326: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=2064, BW=258MiB/s (271MB/s)(5207MiB/20175msec) slat (usec): min=2, max=920, avg=12.55, stdev=11.46 clat (usec): min=1207, max=2593.2k, avg=61799.72, stdev=337083.07 lat (usec): min=1222, max=2593.3k, avg=61812.64, stdev=337083.19 clat percentiles (msec): | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 10], | 30.00th=[ 13], 40.00th=[ 14], 50.00th=[ 15], 60.00th=[ 15], | 70.00th=[ 16], 80.00th=[ 16], 90.00th=[ 16], 95.00th=[ 17], | 99.00th=[ 2500], 99.50th=[ 2534], 99.90th=[ 2567], 99.95th=[ 2567], | 99.99th=[ 2601] bw ( KiB/s): min=11776, max=1108736, per=100.00%, avg=581165.64, stdev=114729.48, samples=70 iops : min= 92, max= 8662, avg=4539.96, stdev=896.35, samples=70 lat (msec) : 2=0.20%, 4=10.43%, 10=9.97%, 20=75.25%, 50=1.28% lat (msec) : 100=0.41%, 250=0.34%, 500=0.01%, 750=0.18%, 2000=0.17% lat (msec) : >=2000=1.77% cpu : usr=0.32%, sys=1.01%, ctx=10565, majf=0, minf=2116 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=41656,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_512K: (groupid=15, jobs=4): err= 0: pid=1333: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=1006, BW=503MiB/s (528MB/s)(9.96GiB/20268msec) slat (usec): min=9, max=72061, avg=444.75, stdev=2835.40 clat (msec): min=31, max=2720, avg=126.41, stdev=303.50 lat (msec): min=31, max=2720, avg=126.86, stdev=303.65 clat percentiles (msec): | 1.00th=[ 45], 5.00th=[ 45], 10.00th=[ 53], 20.00th=[ 56], | 30.00th=[ 60], 40.00th=[ 68], 50.00th=[ 86], 60.00th=[ 97], | 70.00th=[ 120], 80.00th=[ 120], 90.00th=[ 120], 95.00th=[ 133], | 99.00th=[ 2567], 99.50th=[ 2635], 99.90th=[ 2702], 99.95th=[ 2702], | 99.99th=[ 2735] bw ( KiB/s): min=327680, max=556867, per=100.00%, avg=517878.35, stdev=15565.10, samples=160 iops : min= 640, max= 1086, avg=1011.00, stdev=30.35, samples=160 lat (msec) : 50=6.45%, 100=55.19%, 250=35.32%, 500=1.59%, >=2000=1.45% cpu : usr=0.21%, sys=1.13%, ctx=10521, majf=0, minf=2138 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.3%, 32=99.4%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=20400,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqread_1M: (groupid=16, jobs=4): err= 0: pid=1337: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=243, BW=244MiB/s (256MB/s)(4931MiB/20218msec) slat (usec): min=19, max=277733, avg=15575.40, stdev=42878.01 clat (msec): min=17, max=2089, avg=505.07, stdev=167.16 lat (msec): min=36, max=2089, avg=520.65, stdev=168.10 clat percentiles (msec): | 1.00th=[ 65], 5.00th=[ 249], 10.00th=[ 330], 20.00th=[ 397], | 30.00th=[ 435], 40.00th=[ 477], 50.00th=[ 502], 60.00th=[ 531], | 70.00th=[ 558], 80.00th=[ 600], 90.00th=[ 676], 95.00th=[ 768], | 99.00th=[ 1053], 99.50th=[ 1167], 99.90th=[ 1469], 99.95th=[ 1552], | 99.99th=[ 2089] bw ( KiB/s): min=119505, max=397789, per=98.43%, avg=245833.83, stdev=14851.82, samples=160 iops : min= 115, max= 388, avg=239.93, stdev=14.52, samples=160 lat (msec) : 20=0.02%, 50=0.79%, 100=0.69%, 250=3.57%, 500=44.01% lat (msec) : 750=44.98%, 1000=4.50%, 2000=1.42%, >=2000=0.02% cpu : usr=0.03%, sys=0.26%, ctx=1052, majf=0, minf=2165 IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.3%, 32=97.5%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=4931,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_4K: (groupid=17, jobs=4): err= 0: pid=1341: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=10.9k, BW=42.4MiB/s (44.5MB/s)(849MiB/20027msec); 0 zone resets slat (nsec): min=1504, max=60232, avg=8337.17, stdev=5857.96 clat (usec): min=1211, max=78830, avg=11780.29, stdev=7646.72 lat (usec): min=1235, max=78836, avg=11788.98, stdev=7647.24 clat percentiles (usec): | 1.00th=[ 4228], 5.00th=[ 4555], 10.00th=[ 4883], 20.00th=[ 5538], | 30.00th=[ 6128], 40.00th=[ 7177], 50.00th=[ 9241], 60.00th=[11207], | 70.00th=[14746], 80.00th=[17695], 90.00th=[23987], 95.00th=[26346], | 99.00th=[33817], 99.50th=[39060], 99.90th=[60556], 99.95th=[66847], | 99.99th=[78119] bw ( KiB/s): min=23800, max=96032, per=100.00%, avg=43439.60, stdev=4633.53, samples=160 iops : min= 5950, max=24008, avg=10859.60, stdev=1158.34, samples=160 lat (msec) : 2=0.01%, 4=0.04%, 10=54.86%, 20=31.25%, 50=13.60% lat (msec) : 100=0.25% cpu : usr=1.88%, sys=3.62%, ctx=56247, majf=0, minf=62 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,217377,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_8K: (groupid=18, jobs=4): err= 0: pid=1347: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=10.0k, BW=78.2MiB/s (82.0MB/s)(1565MiB/20010msec); 0 zone resets slat (nsec): min=4657, max=59484, avg=11872.08, stdev=6536.46 clat (usec): min=88, max=692976, avg=12770.22, stdev=28579.37 lat (usec): min=101, max=692985, avg=12782.42, stdev=28579.59 clat percentiles (usec): | 1.00th=[ 330], 5.00th=[ 529], 10.00th=[ 644], 20.00th=[ 1811], | 30.00th=[ 6063], 40.00th=[ 6980], 50.00th=[ 8160], 60.00th=[ 10028], | 70.00th=[ 11863], 80.00th=[ 16319], 90.00th=[ 19530], 95.00th=[ 28181], | 99.00th=[135267], 99.50th=[204473], 99.90th=[421528], 99.95th=[505414], | 99.99th=[692061] bw ( KiB/s): min= 3008, max=288752, per=100.00%, avg=80424.15, stdev=12483.92, samples=156 iops : min= 376, max=36094, avg=10052.74, stdev=1560.50, samples=156 lat (usec) : 100=0.01%, 250=0.81%, 500=2.38%, 750=8.46%, 1000=4.40% lat (msec) : 2=5.00%, 4=2.83%, 10=36.16%, 20=30.63%, 50=6.99% lat (msec) : 100=1.12%, 250=0.89%, 500=0.27%, 750=0.06% cpu : usr=1.40%, sys=4.24%, ctx=48873, majf=0, minf=67 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,200327,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_32K: (groupid=19, jobs=4): err= 0: pid=1354: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=7678, BW=240MiB/s (252MB/s)(4809MiB/20040msec); 0 zone resets slat (nsec): min=2134, max=56827, avg=13659.69, stdev=4659.47 clat (usec): min=682, max=2519.8k, avg=16639.65, stdev=125718.25 lat (usec): min=698, max=2519.8k, avg=16653.64, stdev=125718.34 clat percentiles (usec): | 1.00th=[ 1614], 5.00th=[ 3228], 10.00th=[ 3392], | 20.00th=[ 3523], 30.00th=[ 3621], 40.00th=[ 3752], | 50.00th=[ 3916], 60.00th=[ 4080], 70.00th=[ 12125], | 80.00th=[ 13566], 90.00th=[ 19268], 95.00th=[ 25297], | 99.00th=[ 56361], 99.50th=[ 214959], 99.90th=[2499806], | 99.95th=[2533360], 99.99th=[2533360] bw ( KiB/s): min= 4352, max=841347, per=100.00%, avg=326339.91, stdev=70545.10, samples=108 iops : min= 136, max=26291, avg=10197.74, stdev=2204.50, samples=108 lat (usec) : 750=0.01%, 1000=0.25% lat (msec) : 2=2.21%, 4=52.42%, 10=9.79%, 20=25.97%, 50=8.23% lat (msec) : 100=0.48%, 250=0.17%, 500=0.05%, 750=0.06%, 1000=0.04% lat (msec) : 2000=0.13%, >=2000=0.20% cpu : usr=1.44%, sys=3.37%, ctx=41471, majf=0, minf=69 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,153886,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_64K: (groupid=20, jobs=4): err= 0: pid=1362: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=4067, BW=254MiB/s (267MB/s)(5103MiB/20074msec); 0 zone resets slat (nsec): min=4760, max=52703, avg=12022.28, stdev=5942.77 clat (usec): min=181, max=2534.6k, avg=31399.09, stdev=212182.21 lat (usec): min=207, max=2534.6k, avg=31411.45, stdev=212182.31 clat percentiles (usec): | 1.00th=[ 627], 5.00th=[ 701], 10.00th=[ 5407], | 20.00th=[ 6718], 30.00th=[ 7111], 40.00th=[ 7242], | 50.00th=[ 7373], 60.00th=[ 7504], 70.00th=[ 7635], | 80.00th=[ 7767], 90.00th=[ 8979], 95.00th=[ 27919], | 99.00th=[ 834667], 99.50th=[2432697], 99.90th=[2533360], | 99.95th=[2533360], 99.99th=[2533360] bw ( KiB/s): min= 2432, max=1096858, per=100.00%, avg=429205.58, stdev=97044.45, samples=92 iops : min= 38, max=17138, avg=6705.97, stdev=1516.30, samples=92 lat (usec) : 250=0.01%, 500=0.36%, 750=4.82%, 1000=0.04% lat (msec) : 2=0.42%, 4=2.20%, 10=83.08%, 20=1.02%, 50=5.83% lat (msec) : 100=0.41%, 250=0.35%, 500=0.28%, 750=0.18%, 1000=0.16% lat (msec) : 2000=0.19%, >=2000=0.67% cpu : usr=0.92%, sys=1.58%, ctx=20731, majf=0, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,81645,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_128K: (groupid=21, jobs=4): err= 0: pid=1369: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=2109, BW=264MiB/s (277MB/s)(5296MiB/20082msec); 0 zone resets slat (nsec): min=6520, max=71977, avg=15832.91, stdev=7460.22 clat (usec): min=1124, max=2556.9k, avg=60549.97, stdev=289802.59 lat (usec): min=1154, max=2556.9k, avg=60566.17, stdev=289802.74 clat percentiles (msec): | 1.00th=[ 9], 5.00th=[ 13], 10.00th=[ 14], 20.00th=[ 15], | 30.00th=[ 15], 40.00th=[ 15], 50.00th=[ 16], 60.00th=[ 16], | 70.00th=[ 16], 80.00th=[ 17], 90.00th=[ 41], 95.00th=[ 64], | 99.00th=[ 2500], 99.50th=[ 2534], 99.90th=[ 2534], 99.95th=[ 2534], | 99.99th=[ 2567] bw ( KiB/s): min= 5378, max=877359, per=100.00%, avg=411906.32, stdev=76030.62, samples=89 iops : min= 42, max= 6854, avg=3217.51, stdev=593.99, samples=89 lat (msec) : 2=0.04%, 4=0.09%, 10=1.93%, 20=83.79%, 50=7.81% lat (msec) : 100=2.37%, 250=0.89%, 500=1.16%, 750=0.19%, 1000=0.06% lat (msec) : 2000=0.40%, >=2000=1.27% cpu : usr=0.70%, sys=0.85%, ctx=9553, majf=0, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.7%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,42371,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_512K: (groupid=22, jobs=4): err= 0: pid=1376: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=808, BW=404MiB/s (424MB/s)(8152MiB/20162msec); 0 zone resets slat (usec): min=13, max=110417, avg=1274.58, stdev=4759.13 clat (msec): min=22, max=307, avg=156.89, stdev=21.63 lat (msec): min=22, max=307, avg=158.16, stdev=21.49 clat percentiles (msec): | 1.00th=[ 120], 5.00th=[ 138], 10.00th=[ 142], 20.00th=[ 148], | 30.00th=[ 150], 40.00th=[ 153], 50.00th=[ 153], 60.00th=[ 155], | 70.00th=[ 157], 80.00th=[ 163], 90.00th=[ 178], 95.00th=[ 190], | 99.00th=[ 259], 99.50th=[ 271], 99.90th=[ 292], 99.95th=[ 296], | 99.99th=[ 305] bw ( KiB/s): min=325792, max=445884, per=99.59%, avg=412313.72, stdev=7567.95, samples=160 iops : min= 636, max= 870, avg=804.82, stdev=14.79, samples=160 lat (msec) : 50=0.21%, 100=0.41%, 250=97.91%, 500=1.47% cpu : usr=0.83%, sys=0.57%, ctx=9955, majf=0, minf=54 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=99.2%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,16303,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 seqwrite_1M: (groupid=23, jobs=4): err= 0: pid=1386: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=254, BW=255MiB/s (267MB/s)(5157MiB/20233msec); 0 zone resets slat (usec): min=25, max=396569, avg=15509.54, stdev=44612.86 clat (msec): min=43, max=1150, avg=484.70, stdev=142.84 lat (msec): min=107, max=1235, avg=500.21, stdev=149.19 clat percentiles (msec): | 1.00th=[ 192], 5.00th=[ 275], 10.00th=[ 300], 20.00th=[ 372], | 30.00th=[ 414], 40.00th=[ 447], 50.00th=[ 485], 60.00th=[ 518], | 70.00th=[ 550], 80.00th=[ 592], 90.00th=[ 667], 95.00th=[ 735], | 99.00th=[ 869], 99.50th=[ 927], 99.90th=[ 1133], 99.95th=[ 1133], | 99.99th=[ 1150] bw ( KiB/s): min=129024, max=442560, per=98.45%, avg=256951.32, stdev=16489.56, samples=160 iops : min= 126, max= 432, avg=250.70, stdev=16.10, samples=160 lat (msec) : 50=0.02%, 250=4.11%, 500=51.29%, 750=40.55%, 1000=3.88% lat (msec) : 2000=0.16% cpu : usr=0.28%, sys=0.17%, ctx=728, majf=0, minf=59 IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=97.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=0,5157,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_4K: (groupid=24, jobs=4): err= 0: pid=1403: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=186, BW=745KiB/s (763kB/s)(14.7MiB/20239msec) slat (usec): min=2, max=198865, avg=10704.74, stdev=30689.61 clat (msec): min=49, max=1251, avg=378.17, stdev=171.67 lat (msec): min=49, max=1251, avg=388.88, stdev=174.62 clat percentiles (msec): | 1.00th=[ 117], 5.00th=[ 165], 10.00th=[ 194], 20.00th=[ 236], | 30.00th=[ 271], 40.00th=[ 305], 50.00th=[ 347], 60.00th=[ 384], | 70.00th=[ 435], 80.00th=[ 502], 90.00th=[ 617], 95.00th=[ 726], | 99.00th=[ 894], 99.50th=[ 978], 99.90th=[ 1133], 99.95th=[ 1133], | 99.99th=[ 1250] bw ( KiB/s): min= 357, max= 1128, per=99.21%, avg=739.40, stdev=43.71, samples=160 iops : min= 88, max= 282, avg=184.80, stdev=10.95, samples=160 write: IOPS=184, BW=737KiB/s (755kB/s)(14.6MiB/20239msec); 0 zone resets slat (usec): min=2, max=196613, avg=10551.73, stdev=29734.20 clat (msec): min=2, max=830, avg=285.98, stdev=118.14 lat (msec): min=2, max=830, avg=296.53, stdev=122.28 clat percentiles (msec): | 1.00th=[ 80], 5.00th=[ 125], 10.00th=[ 148], 20.00th=[ 182], | 30.00th=[ 215], 40.00th=[ 241], 50.00th=[ 271], 60.00th=[ 300], | 70.00th=[ 342], 80.00th=[ 384], 90.00th=[ 443], 95.00th=[ 506], | 99.00th=[ 617], 99.50th=[ 651], 99.90th=[ 735], 99.95th=[ 793], | 99.99th=[ 835] bw ( KiB/s): min= 336, max= 1136, per=99.54%, avg=734.53, stdev=50.30, samples=160 iops : min= 84, max= 284, avg=183.60, stdev=12.58, samples=160 lat (msec) : 4=0.11%, 10=0.01%, 50=0.13%, 100=0.91%, 250=32.56% lat (msec) : 500=53.56%, 750=10.59%, 1000=1.97%, 2000=0.16% cpu : usr=0.51%, sys=0.11%, ctx=1012, majf=0, minf=66 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.4%, 16=0.9%, 32=98.3%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=3769,3731,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_8K: (groupid=25, jobs=4): err= 0: pid=1410: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=196, BW=1568KiB/s (1606kB/s)(30.0MiB/20214msec) slat (usec): min=2, max=141925, avg=9626.68, stdev=27162.12 clat (msec): min=7, max=954, avg=349.82, stdev=155.05 lat (msec): min=7, max=988, avg=359.45, stdev=157.89 clat percentiles (msec): | 1.00th=[ 96], 5.00th=[ 155], 10.00th=[ 186], 20.00th=[ 228], | 30.00th=[ 251], 40.00th=[ 284], 50.00th=[ 317], 60.00th=[ 355], | 70.00th=[ 405], 80.00th=[ 468], 90.00th=[ 567], 95.00th=[ 676], | 99.00th=[ 802], 99.50th=[ 844], 99.90th=[ 919], 99.95th=[ 944], | 99.99th=[ 953] bw ( KiB/s): min= 688, max= 2593, per=99.85%, avg=1566.14, stdev=90.05, samples=159 iops : min= 85, max= 324, avg=195.73, stdev=11.27, samples=159 write: IOPS=201, BW=1615KiB/s (1654kB/s)(31.9MiB/20214msec); 0 zone resets slat (usec): min=2, max=157376, avg=9975.36, stdev=27291.55 clat (usec): min=644, max=736021, avg=268394.06, stdev=106382.41 lat (usec): min=657, max=793078, avg=278369.85, stdev=109700.75 clat percentiles (msec): | 1.00th=[ 55], 5.00th=[ 125], 10.00th=[ 144], 20.00th=[ 176], | 30.00th=[ 209], 40.00th=[ 234], 50.00th=[ 249], 60.00th=[ 279], | 70.00th=[ 313], 80.00th=[ 355], 90.00th=[ 414], 95.00th=[ 464], | 99.00th=[ 567], 99.50th=[ 592], 99.90th=[ 651], 99.95th=[ 684], | 99.99th=[ 735] bw ( KiB/s): min= 825, max= 2753, per=100.00%, avg=1617.93, stdev=100.37, samples=159 iops : min= 102, max= 344, avg=202.19, stdev=12.55, samples=159 lat (usec) : 750=0.02%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.05%, 10=0.15%, 20=0.06%, 50=0.34% lat (msec) : 100=1.06%, 250=38.46%, 500=50.57%, 750=8.07%, 1000=1.19% cpu : usr=0.30%, sys=0.14%, ctx=1266, majf=0, minf=71 IO depths : 1=0.1%, 2=0.1%, 4=0.2%, 8=0.4%, 16=0.8%, 32=98.5%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=3963,4081,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_32K: (groupid=26, jobs=4): err= 0: pid=1417: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=137, BW=4405KiB/s (4511kB/s)(88.0MiB/20680msec) slat (usec): min=4, max=466725, avg=13485.59, stdev=45137.52 clat (msec): min=10, max=2658, avg=506.05, stdev=375.80 lat (msec): min=10, max=2742, avg=519.53, stdev=385.80 clat percentiles (msec): | 1.00th=[ 58], 5.00th=[ 148], 10.00th=[ 194], 20.00th=[ 251], | 30.00th=[ 296], 40.00th=[ 342], 50.00th=[ 397], 60.00th=[ 460], | 70.00th=[ 535], 80.00th=[ 667], 90.00th=[ 953], 95.00th=[ 1368], | 99.00th=[ 1955], 99.50th=[ 2072], 99.90th=[ 2366], 99.95th=[ 2433], | 99.99th=[ 2668] bw ( KiB/s): min= 704, max= 9155, per=99.63%, avg=4389.02, stdev=587.51, samples=160 iops : min= 22, max= 286, avg=137.07, stdev=18.34, samples=160 write: IOPS=135, BW=4340KiB/s (4445kB/s)(87.7MiB/20680msec); 0 zone resets slat (usec): min=5, max=424047, avg=14207.86, stdev=48115.69 clat (usec): min=592, max=2117.3k, avg=397890.98, stdev=309366.89 lat (usec): min=617, max=2117.3k, avg=412099.29, stdev=320493.74 clat percentiles (msec): | 1.00th=[ 28], 5.00th=[ 120], 10.00th=[ 155], 20.00th=[ 203], | 30.00th=[ 241], 40.00th=[ 271], 50.00th=[ 309], 60.00th=[ 355], | 70.00th=[ 414], 80.00th=[ 498], 90.00th=[ 709], 95.00th=[ 1133], | 99.00th=[ 1620], 99.50th=[ 1703], 99.90th=[ 2056], 99.95th=[ 2123], | 99.99th=[ 2123] bw ( KiB/s): min= 384, max=10035, per=100.00%, avg=4355.94, stdev=594.56, samples=159 iops : min= 12, max= 313, avg=136.02, stdev=18.56, samples=159 lat (usec) : 750=0.02% lat (msec) : 2=0.05%, 4=0.11%, 10=0.19%, 20=0.12%, 50=0.88% lat (msec) : 100=1.63%, 250=23.35%, 500=46.55%, 750=14.23%, 1000=4.72% lat (msec) : 2000=7.59%, >=2000=0.55% cpu : usr=0.09%, sys=0.14%, ctx=1167, majf=0, minf=72 IO depths : 1=0.1%, 2=0.1%, 4=0.3%, 8=0.6%, 16=1.1%, 32=97.8%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=2847,2805,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_64K: (groupid=27, jobs=4): err= 0: pid=1424: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=36, BW=2340KiB/s (2396kB/s)(46.7MiB/20433msec) slat (usec): min=5, max=699574, avg=49073.60, stdev=148972.15 clat (msec): min=292, max=4318, avg=1803.79, stdev=858.79 lat (msec): min=326, max=4318, avg=1852.87, stdev=873.12 clat percentiles (msec): | 1.00th=[ 376], 5.00th=[ 502], 10.00th=[ 651], 20.00th=[ 944], | 30.00th=[ 1284], 40.00th=[ 1536], 50.00th=[ 1804], 60.00th=[ 2005], | 70.00th=[ 2232], 80.00th=[ 2500], 90.00th=[ 3004], 95.00th=[ 3373], | 99.00th=[ 3842], 99.50th=[ 4144], 99.90th=[ 4329], 99.95th=[ 4329], | 99.99th=[ 4329] bw ( KiB/s): min= 640, max= 7040, per=100.00%, avg=2525.36, stdev=356.66, samples=138 iops : min= 10, max= 110, avg=39.45, stdev= 5.57, samples=138 write: IOPS=38, BW=2434KiB/s (2492kB/s)(48.6MiB/20433msec); 0 zone resets slat (usec): min=6, max=710321, avg=56205.54, stdev=159618.30 clat (msec): min=230, max=4392, avg=1478.03, stdev=746.88 lat (msec): min=279, max=4392, avg=1534.23, stdev=780.18 clat percentiles (msec): | 1.00th=[ 309], 5.00th=[ 456], 10.00th=[ 514], 20.00th=[ 785], | 30.00th=[ 1053], 40.00th=[ 1217], 50.00th=[ 1452], 60.00th=[ 1603], | 70.00th=[ 1838], 80.00th=[ 1972], 90.00th=[ 2500], 95.00th=[ 2869], | 99.00th=[ 3775], 99.50th=[ 3977], 99.90th=[ 4396], 99.95th=[ 4396], | 99.99th=[ 4396] bw ( KiB/s): min= 599, max= 7043, per=100.00%, avg=2621.57, stdev=351.00, samples=140 iops : min= 9, max= 110, avg=40.93, stdev= 5.49, samples=140 lat (msec) : 250=0.07%, 500=7.22%, 750=8.40%, 1000=9.38%, 2000=45.14% lat (msec) : >=2000=29.79% cpu : usr=0.04%, sys=0.03%, ctx=244, majf=0, minf=68 IO depths : 1=0.3%, 2=0.5%, 4=1.0%, 8=2.1%, 16=4.2%, 32=91.9%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.3%, 64=0.0%, >=64=0.0% issued rwts: total=747,777,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_128K: (groupid=28, jobs=4): err= 0: pid=1431: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=65, BW=8386KiB/s (8587kB/s)(167MiB/20408msec) slat (usec): min=4, max=462880, avg=29298.87, stdev=84163.56 clat (msec): min=202, max=2766, avg=1051.35, stdev=441.70 lat (msec): min=202, max=2766, avg=1080.65, stdev=453.59 clat percentiles (msec): | 1.00th=[ 309], 5.00th=[ 430], 10.00th=[ 502], 20.00th=[ 651], | 30.00th=[ 768], 40.00th=[ 894], 50.00th=[ 995], 60.00th=[ 1150], | 70.00th=[ 1267], 80.00th=[ 1401], 90.00th=[ 1620], 95.00th=[ 1854], | 99.00th=[ 2232], 99.50th=[ 2366], 99.90th=[ 2500], 99.95th=[ 2769], | 99.99th=[ 2769] bw ( KiB/s): min= 1256, max=15879, per=96.03%, avg=8053.18, stdev=881.17, samples=160 iops : min= 7, max= 124, avg=62.80, stdev= 6.93, samples=160 write: IOPS=66, BW=8448KiB/s (8651kB/s)(168MiB/20408msec); 0 zone resets slat (usec): min=5, max=483317, avg=30460.61, stdev=85375.17 clat (msec): min=148, max=2660, avg=824.92, stdev=369.92 lat (msec): min=148, max=2660, avg=855.38, stdev=384.36 clat percentiles (msec): | 1.00th=[ 230], 5.00th=[ 334], 10.00th=[ 422], 20.00th=[ 510], | 30.00th=[ 592], 40.00th=[ 676], 50.00th=[ 768], 60.00th=[ 860], | 70.00th=[ 953], 80.00th=[ 1133], 90.00th=[ 1318], 95.00th=[ 1519], | 99.00th=[ 1888], 99.50th=[ 1972], 99.90th=[ 2668], 99.95th=[ 2668], | 99.99th=[ 2668] bw ( KiB/s): min= 2048, max=21524, per=98.31%, avg=8306.43, stdev=968.57, samples=160 iops : min= 16, max= 168, avg=64.80, stdev= 7.57, samples=160 lat (msec) : 250=0.89%, 500=13.56%, 750=24.14%, 1000=23.85%, 2000=35.84% lat (msec) : >=2000=1.71% cpu : usr=0.05%, sys=0.04%, ctx=386, majf=0, minf=65 IO depths : 1=0.1%, 2=0.3%, 4=0.6%, 8=1.2%, 16=2.4%, 32=95.4%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.0%, 32=0.2%, 64=0.0%, >=64=0.0% issued rwts: total=1337,1347,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randread_4K_latency: (groupid=29, jobs=1): err= 0: pid=1438: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=81, BW=326KiB/s (334kB/s)(6520KiB/20001msec) slat (nsec): min=13042, max=58451, avg=29168.70, stdev=1593.09 clat (usec): min=2058, max=55906, avg=12220.80, stdev=4420.49 lat (usec): min=2088, max=55936, avg=12250.79, stdev=4420.44 clat percentiles (usec): | 1.00th=[ 3556], 5.00th=[ 5997], 10.00th=[ 7308], 20.00th=[ 8717], | 30.00th=[ 9896], 40.00th=[11076], 50.00th=[11994], 60.00th=[12911], | 70.00th=[14091], 80.00th=[15401], 90.00th=[17171], 95.00th=[18482], | 99.00th=[20841], 99.50th=[21890], 99.90th=[53216], 99.95th=[55837], | 99.99th=[55837] bw ( KiB/s): min= 284, max= 368, per=100.00%, avg=326.26, stdev=18.23, samples=39 iops : min= 71, max= 92, avg=81.56, stdev= 4.56, samples=39 lat (msec) : 4=1.29%, 10=29.51%, 20=67.42%, 50=1.60%, 100=0.18% cpu : usr=0.21%, sys=0.54%, ctx=1630, majf=0, minf=14 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=1630,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_4K_latency: (groupid=30, jobs=1): err= 0: pid=1441: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=77, BW=309KiB/s (317kB/s)(6188KiB/20008msec) slat (nsec): min=11920, max=59272, avg=28984.43, stdev=1986.76 clat (usec): min=2040, max=56257, avg=12536.90, stdev=5286.14 lat (usec): min=2070, max=56286, avg=12566.73, stdev=5286.19 clat percentiles (usec): | 1.00th=[ 4047], 5.00th=[ 5932], 10.00th=[ 6980], 20.00th=[ 8717], | 30.00th=[ 9896], 40.00th=[11076], 50.00th=[12125], 60.00th=[13173], | 70.00th=[14353], 80.00th=[15795], 90.00th=[17695], 95.00th=[19006], | 99.00th=[36963], 99.50th=[45351], 99.90th=[55313], 99.95th=[56361], | 99.99th=[56361] bw ( KiB/s): min= 240, max= 360, per=99.91%, avg=309.21, stdev=23.11, samples=39 iops : min= 60, max= 90, avg=77.28, stdev= 5.79, samples=39 write: IOPS=78, BW=313KiB/s (321kB/s)(6272KiB/20008msec); 0 zone resets slat (usec): min=12, max=133, avg=29.18, stdev= 3.39 clat (usec): min=122, max=4323, avg=303.85, stdev=121.42 lat (usec): min=145, max=4352, avg=333.88, stdev=121.54 clat percentiles (usec): | 1.00th=[ 188], 5.00th=[ 212], 10.00th=[ 227], 20.00th=[ 247], | 30.00th=[ 265], 40.00th=[ 281], 50.00th=[ 302], 60.00th=[ 318], | 70.00th=[ 330], 80.00th=[ 347], 90.00th=[ 375], 95.00th=[ 400], | 99.00th=[ 478], 99.50th=[ 537], 99.90th=[ 1090], 99.95th=[ 4293], | 99.99th=[ 4293] bw ( KiB/s): min= 160, max= 456, per=100.00%, avg=316.10, stdev=70.58, samples=39 iops : min= 40, max= 114, avg=79.03, stdev=17.64, samples=39 lat (usec) : 250=10.91%, 500=39.04%, 750=0.26%, 1000=0.06% lat (msec) : 2=0.03%, 4=0.48%, 10=14.67%, 20=32.84%, 50=1.57% lat (msec) : 100=0.13% cpu : usr=0.46%, sys=0.70%, ctx=3115, majf=0, minf=18 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=1547,1568,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randwrite_4K_latency: (groupid=31, jobs=1): err= 0: pid=1444: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] write: IOPS=1649, BW=6598KiB/s (6756kB/s)(129MiB/20001msec); 0 zone resets slat (nsec): min=9070, max=60314, avg=16703.53, stdev=5322.86 clat (usec): min=95, max=10708, avg=584.52, stdev=653.83 lat (usec): min=125, max=10738, avg=601.69, stdev=656.94 clat percentiles (usec): | 1.00th=[ 125], 5.00th=[ 135], 10.00th=[ 147], 20.00th=[ 186], | 30.00th=[ 206], 40.00th=[ 229], 50.00th=[ 258], 60.00th=[ 355], | 70.00th=[ 906], 80.00th=[ 1090], 90.00th=[ 1385], 95.00th=[ 1680], | 99.00th=[ 2245], 99.50th=[ 5080], 99.90th=[ 5866], 99.95th=[ 6521], | 99.99th=[ 7701] bw ( KiB/s): min= 2288, max=11687, per=100.00%, avg=6616.51, stdev=3099.79, samples=39 iops : min= 572, max= 2921, avg=1654.03, stdev=774.86, samples=39 lat (usec) : 100=0.01%, 250=48.93%, 500=20.05%, 750=0.02%, 1000=5.61% lat (msec) : 2=23.59%, 4=1.20%, 10=0.59%, 20=0.01% cpu : usr=1.49%, sys=4.89%, ctx=32998, majf=0, minf=14 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,32992,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=1 randrw_70R30W_4K: (groupid=32, jobs=4): err= 0: pid=1447: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=179, BW=719KiB/s (736kB/s)(14.3MiB/20424msec) slat (usec): min=2, max=262517, avg=15268.90, stdev=42919.42 clat (msec): min=25, max=1810, avg=506.51, stdev=192.62 lat (msec): min=25, max=1810, avg=521.78, stdev=197.58 clat percentiles (msec): | 1.00th=[ 176], 5.00th=[ 251], 10.00th=[ 305], 20.00th=[ 347], | 30.00th=[ 397], 40.00th=[ 435], 50.00th=[ 472], 60.00th=[ 518], | 70.00th=[ 575], 80.00th=[ 651], 90.00th=[ 768], 95.00th=[ 877], | 99.00th=[ 1083], 99.50th=[ 1167], 99.90th=[ 1485], 99.95th=[ 1519], | 99.99th=[ 1804] bw ( KiB/s): min= 364, max= 1120, per=99.87%, avg=718.26, stdev=40.31, samples=159 iops : min= 91, max= 280, avg=179.54, stdev=10.08, samples=159 write: IOPS=77, BW=311KiB/s (319kB/s)(6356KiB/20424msec); 0 zone resets slat (usec): min=2, max=265035, avg=14921.57, stdev=42825.69 clat (msec): min=5, max=1072, avg=403.50, stdev=163.12 lat (msec): min=5, max=1072, avg=418.42, stdev=166.68 clat percentiles (msec): | 1.00th=[ 82], 5.00th=[ 188], 10.00th=[ 209], 20.00th=[ 266], | 30.00th=[ 313], 40.00th=[ 338], 50.00th=[ 376], 60.00th=[ 426], | 70.00th=[ 464], 80.00th=[ 535], 90.00th=[ 625], 95.00th=[ 709], | 99.00th=[ 869], 99.50th=[ 944], 99.90th=[ 1028], 99.95th=[ 1070], | 99.99th=[ 1070] bw ( KiB/s): min= 128, max= 520, per=99.94%, avg=311.36, stdev=24.33, samples=159 iops : min= 32, max= 130, avg=77.83, stdev= 6.08, samples=159 lat (msec) : 10=0.19%, 20=0.02%, 50=0.10%, 100=0.32%, 250=8.16% lat (msec) : 500=52.68%, 750=29.75%, 1000=7.43%, 2000=1.35% cpu : usr=0.46%, sys=0.10%, ctx=742, majf=0, minf=65 IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=97.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=3671,1589,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 randrw_70R30W_64K: (groupid=33, jobs=4): err= 0: pid=1454: Tue Jan 14 10:23:50 2025 Description : ["Storage megatest by Andrey"] read: IOPS=64, BW=4105KiB/s (4203kB/s)(82.7MiB/20628msec) slat (usec): min=4, max=819891, avg=43656.78, stdev=136503.35 clat (msec): min=209, max=3994, avg=1409.96, stdev=753.28 lat (msec): min=209, max=3994, avg=1453.62, stdev=778.76 clat percentiles (msec): | 1.00th=[ 279], 5.00th=[ 347], 10.00th=[ 426], 20.00th=[ 600], | 30.00th=[ 877], 40.00th=[ 1167], 50.00th=[ 1435], 60.00th=[ 1586], | 70.00th=[ 1838], 80.00th=[ 2056], 90.00th=[ 2467], 95.00th=[ 2668], | 99.00th=[ 3205], 99.50th=[ 3473], 99.90th=[ 3809], 99.95th=[ 4010], | 99.99th=[ 4010] bw ( KiB/s): min= 512, max=13184, per=100.00%, avg=4174.43, stdev=634.59, samples=151 iops : min= 8, max= 206, avg=65.23, stdev= 9.92, samples=151 write: IOPS=27, BW=1765KiB/s (1808kB/s)(35.6MiB/20628msec); 0 zone resets slat (usec): min=7, max=844420, avg=40289.42, stdev=122112.43 clat (msec): min=158, max=3343, avg=1194.33, stdev=683.35 lat (msec): min=158, max=3343, avg=1234.62, stdev=703.15 clat percentiles (msec): | 1.00th=[ 163], 5.00th=[ 262], 10.00th=[ 334], 20.00th=[ 468], | 30.00th=[ 709], 40.00th=[ 1003], 50.00th=[ 1183], 60.00th=[ 1351], | 70.00th=[ 1536], 80.00th=[ 1687], 90.00th=[ 2005], 95.00th=[ 2500], | 99.00th=[ 3071], 99.50th=[ 3272], 99.90th=[ 3339], 99.95th=[ 3339], | 99.99th=[ 3339] bw ( KiB/s): min= 512, max= 7168, per=100.00%, avg=2097.39, stdev=396.63, samples=134 iops : min= 8, max= 112, avg=32.77, stdev= 6.20, samples=134 lat (msec) : 250=1.69%, 500=15.43%, 750=10.89%, 1000=8.19%, 2000=45.03% lat (msec) : >=2000=18.76% cpu : usr=0.06%, sys=0.02%, ctx=286, majf=0, minf=68 IO depths : 1=0.2%, 2=0.4%, 4=0.8%, 8=1.7%, 16=3.4%, 32=93.4%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.0%, 32=0.2%, 64=0.0%, >=64=0.0% issued rwts: total=1323,569,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 Run status group 0 (all jobs): READ: bw=832KiB/s (852kB/s), 832KiB/s-832KiB/s (852kB/s-852kB/s), io=16.6MiB (17.4MB), run=20394-20394msec Run status group 1 (all jobs): READ: bw=1660KiB/s (1700kB/s), 1660KiB/s-1660KiB/s (1700kB/s-1700kB/s), io=32.0MiB (34.6MB), run=20338-20338msec Run status group 2 (all jobs): READ: bw=6429KiB/s (6583kB/s), 6429KiB/s-6429KiB/s (6583kB/s-6583kB/s), io=128MiB (134MB), run=20369-20369msec Run status group 3 (all jobs): READ: bw=12.1MiB/s (12.7MB/s), 12.1MiB/s-12.1MiB/s (12.7MB/s-12.7MB/s), io=247MiB (259MB), run=20402-20402msec Run status group 4 (all jobs): READ: bw=22.9MiB/s (24.0MB/s), 22.9MiB/s-22.9MiB/s (24.0MB/s-24.0MB/s), io=467MiB (489MB), run=20370-20370msec Run status group 5 (all jobs): WRITE: bw=6805KiB/s (6968kB/s), 6805KiB/s-6805KiB/s (6968kB/s-6968kB/s), io=134MiB (140MB), run=20115-20115msec Run status group 6 (all jobs): WRITE: bw=4050KiB/s (4147kB/s), 4050KiB/s-4050KiB/s (4147kB/s-4147kB/s), io=80.2MiB (84.0MB), run=20266-20266msec Run status group 7 (all jobs): WRITE: bw=1967KiB/s (2014kB/s), 1967KiB/s-1967KiB/s (2014kB/s-2014kB/s), io=39.9MiB (41.8MB), run=20757-20757msec Run status group 8 (all jobs): WRITE: bw=2866KiB/s (2935kB/s), 2866KiB/s-2866KiB/s (2935kB/s-2935kB/s), io=60.2MiB (63.2MB), run=21526-21526msec Run status group 9 (all jobs): WRITE: bw=10.1MiB/s (10.6MB/s), 10.1MiB/s-10.1MiB/s (10.6MB/s-10.6MB/s), io=204MiB (213MB), run=20209-20209msec Run status group 10 (all jobs): READ: bw=248MiB/s (260MB/s), 248MiB/s-248MiB/s (260MB/s-260MB/s), io=4997MiB (5240MB), run=20156-20156msec Run status group 11 (all jobs): READ: bw=281MiB/s (295MB/s), 281MiB/s-281MiB/s (295MB/s-295MB/s), io=5651MiB (5926MB), run=20115-20115msec Run status group 12 (all jobs): READ: bw=259MiB/s (272MB/s), 259MiB/s-259MiB/s (272MB/s-272MB/s), io=5214MiB (5468MB), run=20129-20129msec Run status group 13 (all jobs): READ: bw=257MiB/s (269MB/s), 257MiB/s-257MiB/s (269MB/s-269MB/s), io=5151MiB (5401MB), run=20045-20045msec Run status group 14 (all jobs): READ: bw=258MiB/s (271MB/s), 258MiB/s-258MiB/s (271MB/s-271MB/s), io=5207MiB (5460MB), run=20175-20175msec Run status group 15 (all jobs): READ: bw=503MiB/s (528MB/s), 503MiB/s-503MiB/s (528MB/s-528MB/s), io=9.96GiB (10.7GB), run=20268-20268msec Run status group 16 (all jobs): READ: bw=244MiB/s (256MB/s), 244MiB/s-244MiB/s (256MB/s-256MB/s), io=4931MiB (5171MB), run=20218-20218msec Run status group 17 (all jobs): WRITE: bw=42.4MiB/s (44.5MB/s), 42.4MiB/s-42.4MiB/s (44.5MB/s-44.5MB/s), io=849MiB (890MB), run=20027-20027msec Run status group 18 (all jobs): WRITE: bw=78.2MiB/s (82.0MB/s), 78.2MiB/s-78.2MiB/s (82.0MB/s-82.0MB/s), io=1565MiB (1641MB), run=20010-20010msec Run status group 19 (all jobs): WRITE: bw=240MiB/s (252MB/s), 240MiB/s-240MiB/s (252MB/s-252MB/s), io=4809MiB (5043MB), run=20040-20040msec Run status group 20 (all jobs): WRITE: bw=254MiB/s (267MB/s), 254MiB/s-254MiB/s (267MB/s-267MB/s), io=5103MiB (5351MB), run=20074-20074msec Run status group 21 (all jobs): WRITE: bw=264MiB/s (277MB/s), 264MiB/s-264MiB/s (277MB/s-277MB/s), io=5296MiB (5554MB), run=20082-20082msec Run status group 22 (all jobs): WRITE: bw=404MiB/s (424MB/s), 404MiB/s-404MiB/s (424MB/s-424MB/s), io=8152MiB (8547MB), run=20162-20162msec Run status group 23 (all jobs): WRITE: bw=255MiB/s (267MB/s), 255MiB/s-255MiB/s (267MB/s-267MB/s), io=5157MiB (5408MB), run=20233-20233msec Run status group 24 (all jobs): READ: bw=745KiB/s (763kB/s), 745KiB/s-745KiB/s (763kB/s-763kB/s), io=14.7MiB (15.4MB), run=20239-20239msec WRITE: bw=737KiB/s (755kB/s), 737KiB/s-737KiB/s (755kB/s-755kB/s), io=14.6MiB (15.3MB), run=20239-20239msec Run status group 25 (all jobs): READ: bw=1568KiB/s (1606kB/s), 1568KiB/s-1568KiB/s (1606kB/s-1606kB/s), io=30.0MiB (32.5MB), run=20214-20214msec WRITE: bw=1615KiB/s (1654kB/s), 1615KiB/s-1615KiB/s (1654kB/s-1654kB/s), io=31.9MiB (33.4MB), run=20214-20214msec Run status group 26 (all jobs): READ: bw=4405KiB/s (4511kB/s), 4405KiB/s-4405KiB/s (4511kB/s-4511kB/s), io=88.0MiB (93.3MB), run=20680-20680msec WRITE: bw=4340KiB/s (4445kB/s), 4340KiB/s-4340KiB/s (4445kB/s-4445kB/s), io=87.7MiB (91.9MB), run=20680-20680msec Run status group 27 (all jobs): READ: bw=2340KiB/s (2396kB/s), 2340KiB/s-2340KiB/s (2396kB/s-2396kB/s), io=46.7MiB (48.0MB), run=20433-20433msec WRITE: bw=2434KiB/s (2492kB/s), 2434KiB/s-2434KiB/s (2492kB/s-2492kB/s), io=48.6MiB (50.9MB), run=20433-20433msec Run status group 28 (all jobs): READ: bw=8386KiB/s (8587kB/s), 8386KiB/s-8386KiB/s (8587kB/s-8587kB/s), io=167MiB (175MB), run=20408-20408msec WRITE: bw=8448KiB/s (8651kB/s), 8448KiB/s-8448KiB/s (8651kB/s-8651kB/s), io=168MiB (177MB), run=20408-20408msec Run status group 29 (all jobs): READ: bw=326KiB/s (334kB/s), 326KiB/s-326KiB/s (334kB/s-334kB/s), io=6520KiB (6676kB), run=20001-20001msec Run status group 30 (all jobs): READ: bw=309KiB/s (317kB/s), 309KiB/s-309KiB/s (317kB/s-317kB/s), io=6188KiB (6337kB), run=20008-20008msec WRITE: bw=313KiB/s (321kB/s), 313KiB/s-313KiB/s (321kB/s-321kB/s), io=6272KiB (6423kB), run=20008-20008msec Run status group 31 (all jobs): WRITE: bw=6598KiB/s (6756kB/s), 6598KiB/s-6598KiB/s (6756kB/s-6756kB/s), io=129MiB (135MB), run=20001-20001msec Run status group 32 (all jobs): READ: bw=719KiB/s (736kB/s), 719KiB/s-719KiB/s (736kB/s-736kB/s), io=14.3MiB (15.0MB), run=20424-20424msec WRITE: bw=311KiB/s (319kB/s), 311KiB/s-311KiB/s (319kB/s-319kB/s), io=6356KiB (6509kB), run=20424-20424msec Run status group 33 (all jobs): READ: bw=4105KiB/s (4203kB/s), 4105KiB/s-4105KiB/s (4203kB/s-4203kB/s), io=82.7MiB (86.7MB), run=20628-20628msec WRITE: bw=1765KiB/s (1808kB/s), 1765KiB/s-1765KiB/s (1808kB/s-1808kB/s), io=35.6MiB (37.3MB), run=20628-20628msec Disk stats (read/write): sdc: ios=644900/291623, merge=1716931/523354, ticks=17733726/14717844, in_queue=32451571, util=98.62%