Inference Performance
Model | Imported from | Batch Size | Latency (msec) | Throughput (samples per sec) | Precision | Model source | |
---|---|---|---|---|---|---|---|
inception V3 | TF | 1 | 0.437 | 3432 | INT8 | Source | |
4 | 1.155 | 4422 | INT8 | ||||
8 | 2.198 | 4447 | INT8 | ||||
10 | 2.674 | 4457 | INT8 | ||||
bninception | ONNX | 1 | CF | CF | INT8 | ||
4 | CF | CF | INT8 | ||||
8 | CF | CF | INT8 | ||||
10 | CF | CF | INT8 | ||||
20 | CF | CF | INT8 | ||||
resnet18 V1 224,224 | ONNX | 1 | 0.152 | 14547 | INT8 | Source | |
4 | 0.272 | 27669 | INT8 | ||||
8 | 0.514 | 30690 | INT8 | ||||
10 | 0.57 | 31571 | INT8 | ||||
resnet18 V2 224,224 | ONNX | 1 | 0.214 | 8701 | INT8 | Source | |
4 | 0.517 | 13547 | INT8 | ||||
8 | 0.823 | 14607 | INT8 | ||||
10 | 0.987 | 14823 | INT8 | ||||
resnet34 V1 224,224 | ONNX | 1 | 285 | INT8 | Source | ||
4 | 0.429 | 15784 | INT8 | ||||
8 | 0.702 | 17400 | INT8 | ||||
10 | 0.823 | 18220 | INT8 | ||||
resnet34 V2 224,224 | ONNX | 1 | 0.316 | 5460 | INT8 | Source | |
4 | 0.656 | 9107 | INT8 | ||||
8 | 1.109 | 9976 | INT8 | ||||
10 | 1.385 | 10206 | INT8 | ||||
resnet50 V1 224,224 | TF | 1 | 0.232 | 7238 | INT8 | Source | |
4 | 0.501 | 12873 | INT8 | ||||
8 | 0.824 | 14027 | INT8 | ||||
10 | 0.951 | 14662 | INT8 | ||||
ONNX | 1 | 0.223 | 7505 | INT8 | |||
4 | 0.481 | 13571 | INT8 | ||||
8 | 0.793 | 14768 | INT8 | ||||
10 | 0.905 | 15491 | INT8 | ||||
resnet50 V1 160x160 | ONNX | 1 | 0.201 | N/A | INT8 | Source | |
4 | 0.322 | N/A | INT8 | ||||
8 | 0.525 | N/A | INT8 | ||||
10 | 0.58 | N/A | INT8 | ||||
resnet50 V1 slim 224,224 | TF | 1 | 0.223 | 7439 | INT8 | Source | |
4 | 0.468 | 14477 | INT8 | ||||
8 | 0.759 | 15972 | INT8 | ||||
10 | 0.871 | 16799 | INT8 | ||||
resnet50 V2 224,224 | TF | 1 | 0.362 | 4694 | INT8 | Source | |
4 | 0.727 | 7472 | INT8 | ||||
8 | 1.335 | 8155 | INT8 | ||||
ONNX | 1 | 0.366 | 4468 | INT8 | Source | ||
4 | 0.81 | 6830 | INT8 | ||||
8 | 1.488 | 7363 | INT8 | ||||
resnet101 V1 224,224 | ONNX | 1 | 0.342 | 4582 | INT8 | Source | |
4 | 0.687 | 8214 | INT8 | ||||
8 | 1.244 | 9171 | INT8 | ||||
10 | 1.403 | 9802 | INT8 | ||||
resnet101 V2 224,224 | ONNX | 1 | 0.541 | 2764 | INT8 | Source | |
4 | 1.294 | 4164 | INT8 | ||||
8 | 2.145 | 4511 | INT8 | ||||
10 | 2.605 | 4594 | INT8 | ||||
resnet152 v1 224,224 | TF | 1 | 0.495 | 1566 | INT8 | open source model | |
4 | 1.001 | 3379 | INT8 | ||||
8 | 1.763 | 3527 | INT8 | ||||
10 | 1.946 | 3565 | INT8 | ||||
ONNX | 1 | 0.7 | 2792 | INT8 | Source | ||
4 | 1.641 | 5256 | INT8 | ||||
8 | 2.926 | 5823 | INT8 | ||||
10 | 3.548 | 6176 | INT8 | ||||
resnet152 V1 slim 224,224 | TF | 1 | 0.753 | 1724 | INT8 | Source | |
4 | 0.999 | 4931 | INT8 | ||||
8 | 1.55 | 6612 | INT8 | ||||
10 | 1.773 | 7099 | INT8 | ||||
resnet152 V2 224,224 | ONNX | 1 | 0.7 | 1929 | INT8 | Source | |
4 | 1.641 | 2944 | INT8 | ||||
8 | 2.926 | 3218 | INT8 | ||||
10 | 3.548 | 3267 | INT8 | ||||
ResNext50-32_4d 224,224 | ONNX | 1 | 0.403 | 3825 | INT8 | open source model | |
4 | 0.875 | 6004 | INT8 | ||||
8 | 1.56 | 6522 | INT8 | ||||
10 | 1.889 | 6657 | INT8 | ||||
resnext101_32_4d 224,224 | ONNX | 1 | 0.496 | 3064 | INT8 | open source model | |
4 | 1.134 | 4616 | INT8 | ||||
8 | 1.936 | 5094 | INT8 | ||||
10 | 2.187 | 5433 | INT8 | ||||
tiny yolo v2 | 10881920 | Pytorch | 1 | 2.107 | N/A | INT8 | Source |
1 | 0.199 | N/A | INT8 | Source | |||
320320 | |||||||
1 | 0.26 | N/A | INT8 | Source | |||
416416 | |||||||
1 | 0.499 | N/A | INT8 | Source | |||
608608 | |||||||
960960 | 1 | 0.933 | N/A | INT8 | Source | ||
Yolo V2 | 1088_1920 | Pytorch | 1 | 6.618 | N/A | INT8 | Source |
320320 | 1 | 6.593 | N/A | INT8 | Source | ||
416416 | 1 | 0.578 | N/A | INT8 | Source | ||
544736 | 1 | 0.791 | N/A | INT8 | Source | ||
608608 | 1 | 1.514 | N/A | INT8 | Source | ||
960960 | 1 | 1.454 | N/A | INT8 | Source | ||
yolo v3 | 416416 | Pytorch | 1 | 1.179 | N/A | INT8 | Source |
960960 | Pytorch | 4 | N/A | INT8 | Source | ||
bert squad | MX | 1 | 13.524 | N/A | INT8 | Source | |
LARGE | |||||||
max sequnce length = ??? Check with Dror | 24 | 35.705 | N/A | INT8 | |||
bert mrpc | MX | 1 | 2.841 | N/A | INT8 | ||
BASE | 4 | 2.911 | N/A | INT8 | |||
max sequnce length = 128 | 8 | 2.952 | N/A | INT8 | |||
10 | 3.468 | N/A | INT8 | ||||
10 | 5.934 | N/A | INT16 | ||||
12 | 7.003 | N/A | INT16 | ||||
bert squad | MX | 1 | 2.872 | N/A | INT8 | ||
BASE | 4 | 2.897 | N/A | INT8 | |||
max sequnce length = ??? Dror? | 8 | 2.936 | N/A | INT8 | |||
10 | 5.953 | N/A | INT8 | ||||
10 | 3.488 | N/A | Int16 | ||||
12 | N/A | INT8 | |||||
24 | N/A | INT8 | |||||
bvlc_googlenet 224,224 | ONNX | 1 | 0.296 | NC | INT8 | Source | |
googlenet_bn_no_lrn 224,224 | ONNX | 1 | 0.159 | 12809 | INT8 | Developed inhouse based on Googlenet with batch norm and w/o LRN | |
4 | 0.423 | 17542 | INT8 | ||||
8 | 0.701 | 18431 | INT8 | ||||
10 | 0.823 | 18531 | INT8 | ||||
squeezenet1.1 224,224 | ONNX | 1 | 887 | INT8 | Source | ||
4 | 0.237 | 36420 | INT8 | ||||
8 | 0.426 | 37840 | INT8 | ||||
10 | 0.547 | 37927 | INT8 | ||||
ssd-vgg16 300,300 | MX | 1 | 0.9 | N/A | INT8 | Source |