root@truenas[~]# fio --filename=/fiotest --rw=write --bs=1024Mi --rwmixread=30 -
-iodepth=10 --numjobs=5 --runtime=90s --name=1024ktest --size=10G
1024ktest: (g=0): rw=write, bs=(R) 1024MiB-1024MiB, (W) 1024MiB-1024MiB, (T) 102
4MiB-1024MiB, ioengine=psync, iodepth=10
...
fio-3.28
Starting 5 processes
1024ktest: Laying out IO file (1 file / 10240MiB)
1024ktest: Laying out IO file (1 file / 10240MiB)1024ktest: Laying out IO file (1 file / 10240MiB)
1024ktest: Laying out IO file (1 file / 10240MiB)
1024ktest: Laying out IO file (1 file / 10240MiB)
Jobs: 2 (f=0): [E(1),f(1),E(1),f(1),E(1)][100.0%][w=3835MiB/s][w=3 IOPS][eta 00m
:00s]
1024ktest: (groupid=0, jobs=1): err= 0: pid=4130: Sun Jun 11 16:18:01 2023
write: IOPS=0, BW=66.8MiB/s (70.0MB/s)(6144MiB/91994msec); 0 zone resets
clat (msec): min=468, max=59598, avg=15292.00, stdev=22232.07
lat (msec): min=501, max=59635, avg=15332.26, stdev=22231.64
clat percentiles (msec):
| 1.00th=[ 468], 5.00th=[ 468], 10.00th=[ 468], 20.00th=[ 743],
| 30.00th=[ 743], 40.00th=[ 9060], 50.00th=[ 9060], 60.00th=[10939], | 70.00th=[10939], 80.00th=[10939], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113], | 99.99th=[17113]
bw ( MiB/s): min= 1984, max= 2043, per=100.00%, avg=2026.40, stdev=26.41, samples=5
iops : min= 1, max= 1, avg= 1.00, stdev= 0.00, samples=5
lat (msec) : 500=16.67%, 750=16.67%, >=2000=66.67%
cpu : usr=0.33%, sys=1.57%, ctx=32763, majf=0, minf=0
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=10
1024ktest: (groupid=0, jobs=1): err= 0: pid=4131: Sun Jun 11 16:18:01 2023
write: IOPS=0, BW=34.1MiB/s (35.7MB/s)(3072MiB/90125msec); 0 zone resets
clat (msec): min=10971, max=58440, avg=29115.17, stdev=25633.91
lat (msec): min=11007, max=58478, avg=29153.00, stdev=25633.93
clat percentiles (msec):
| 1.00th=[10939], 5.00th=[10939], 10.00th=[10939], 20.00th=[10939],
| 30.00th=[10939], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( MiB/s): min= 1984, max= 2043, per=100.00%, avg=2014.20, stdev=42.01, samples=2
iops : min= 1, max= 1, avg= 1.00, stdev= 0.00, samples=2
lat (msec) : >=2000=100.00%
cpu : usr=0.13%, sys=0.79%, ctx=24586, majf=0, minf=0
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,3,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=10
1024ktest: (groupid=0, jobs=1): err= 0: pid=4132: Sun Jun 11 16:18:01 2023
write: IOPS=0, BW=77.9MiB/s (81.7MB/s)(7168MiB/92048msec); 0 zone resets
clat (msec): min=237, max=59643, avg=13113.09, stdev=21011.78
lat (msec): min=265, max=59687, avg=13149.67, stdev=21015.19
clat percentiles (msec):
| 1.00th=[ 239], 5.00th=[ 239], 10.00th=[ 239], 20.00th=[ 768],
| 30.00th=[ 2165], 40.00th=[ 2165], 50.00th=[ 7080], 60.00th=[10939],
| 70.00th=[10939], 80.00th=[10939], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( MiB/s): min= 1984, max= 2048, per=100.00%, avg=2026.61, stdev=23.49, samples=6
iops : min= 1, max= 2, avg= 1.17, stdev= 0.41, samples=6
lat (msec) : 250=14.29%, 1000=14.29%, >=2000=71.43%
cpu : usr=0.32%, sys=1.84%, ctx=34698, majf=0, minf=0
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,7,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=10
1024ktest: (groupid=0, jobs=1): err= 0: pid=4133: Sun Jun 11 16:18:01 2023
write: IOPS=0, BW=44.8MiB/s (46.9MB/s)(4096MiB/91495msec); 0 zone resets
clat (msec): min=9018, max=59690, avg=22656.52, stdev=24706.26
lat (msec): min=9054, max=59738, avg=22699.64, stdev=24709.50
clat percentiles (msec):
| 1.00th=[ 9060], 5.00th=[ 9060], 10.00th=[ 9060], 20.00th=[ 9060],
| 30.00th=[10939], 40.00th=[10939], 50.00th=[10939], 60.00th=[10939],
| 70.00th=[10939], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( MiB/s): min= 1984, max= 2043, per=100.00%, avg=2024.11, stdev=34.30, samples=3
iops : min= 1, max= 1, avg= 1.00, stdev= 0.00, samples=3
lat (msec) : >=2000=100.00%
cpu : usr=0.17%, sys=1.13%, ctx=32751, majf=0, minf=0
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,4,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=10
1024ktest: (groupid=0, jobs=1): err= 0: pid=4134: Sun Jun 11 16:18:01 2023
write: IOPS=0, BW=55.6MiB/s (58.3MB/s)(5120MiB/92049msec); 0 zone resets
clat (msec): min=742, max=59684, avg=18271.55, stdev=23531.94
lat (msec): min=794, max=59729, avg=18317.13, stdev=23531.10
clat percentiles (msec):
| 1.00th=[ 743], 5.00th=[ 743], 10.00th=[ 743], 20.00th=[ 743],
| 30.00th=[ 9060], 40.00th=[ 9060], 50.00th=[10939], 60.00th=[10939],
| 70.00th=[10939], 80.00th=[10939], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( MiB/s): min= 1984, max= 2043, per=100.00%, avg=2018.98, stdev=25.72, samples=4
iops : min= 1, max= 1, avg= 1.00, stdev= 0.00, samples=4
lat (msec) : 750=20.00%, >=2000=80.00%
cpu : usr=0.23%, sys=1.35%, ctx=32747, majf=0, minf=0
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,5,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=10
Run status group 0 (all jobs):
WRITE: bw=278MiB/s (292MB/s), 34.1MiB/s-77.9MiB/s (35.7MB/s-81.7MB/s), io=25.0GiB (26.8GB), run=90125-92049msec
root@truenas[~]#