Inference Performance

Inference Performance

Model		Imported from	Batch Size	Latency (msec)	Throughput (samples per sec)	Precision	Model source
inception V3		TF	1	0.437	3432	INT8	Source
			4	1.155	4422	INT8
			8	2.198	4447	INT8
			10	2.674	4457	INT8
bninception		ONNX	1	CF	CF	INT8
			4	CF	CF	INT8
			8	CF	CF	INT8
			10	CF	CF	INT8
			20	CF	CF	INT8
resnet18 V1 224,224		ONNX	1	0.152	14547	INT8	Source
			4	0.272	27669	INT8
			8	0.514	30690	INT8
			10	0.57	31571	INT8
resnet18 V2 224,224		ONNX	1	0.214	8701	INT8	Source
			4	0.517	13547	INT8
			8	0.823	14607	INT8
			10	0.987	14823	INT8
resnet34 V1 224,224		ONNX	1		285	INT8	Source
			4	0.429	15784	INT8
			8	0.702	17400	INT8
			10	0.823	18220	INT8
resnet34 V2 224,224		ONNX	1	0.316	5460	INT8	Source
			4	0.656	9107	INT8
			8	1.109	9976	INT8
			10	1.385	10206	INT8
resnet50 V1 224,224		TF	1	0.232	7238	INT8	Source
			4	0.501	12873	INT8
			8	0.824	14027	INT8
			10	0.951	14662	INT8
		ONNX	1	0.223	7505	INT8
			4	0.481	13571	INT8
			8	0.793	14768	INT8
			10	0.905	15491	INT8
resnet50 V1 160x160		ONNX	1	0.201	N/A	INT8	Source
			4	0.322	N/A	INT8
			8	0.525	N/A	INT8
			10	0.58	N/A	INT8
resnet50 V1 slim 224,224		TF	1	0.223	7439	INT8	Source
			4	0.468	14477	INT8
			8	0.759	15972	INT8
			10	0.871	16799	INT8
resnet50 V2 224,224		TF	1	0.362	4694	INT8	Source
			4	0.727	7472	INT8
			8	1.335	8155	INT8
		ONNX	1	0.366	4468	INT8	Source
			4	0.81	6830	INT8
			8	1.488	7363	INT8
resnet101 V1 224,224		ONNX	1	0.342	4582	INT8	Source
			4	0.687	8214	INT8
			8	1.244	9171	INT8
			10	1.403	9802	INT8
resnet101 V2 224,224		ONNX	1	0.541	2764	INT8	Source
			4	1.294	4164	INT8
			8	2.145	4511	INT8
			10	2.605	4594	INT8
resnet152 v1 224,224		TF	1	0.495	1566	INT8	open source model
			4	1.001	3379	INT8
			8	1.763	3527	INT8
			10	1.946	3565	INT8
		ONNX	1	0.7	2792	INT8	Source
			4	1.641	5256	INT8
			8	2.926	5823	INT8
			10	3.548	6176	INT8
resnet152 V1 slim 224,224		TF	1	0.753	1724	INT8	Source
			4	0.999	4931	INT8
			8	1.55	6612	INT8
			10	1.773	7099	INT8
resnet152 V2 224,224		ONNX	1	0.7	1929	INT8	Source
			4	1.641	2944	INT8
			8	2.926	3218	INT8
			10	3.548	3267	INT8
ResNext50-32_4d 224,224		ONNX	1	0.403	3825	INT8	open source model
			4	0.875	6004	INT8
			8	1.56	6522	INT8
			10	1.889	6657	INT8
resnext101_32_4d 224,224		ONNX	1	0.496	3064	INT8	open source model
			4	1.134	4616	INT8
			8	1.936	5094	INT8
			10	2.187	5433	INT8
tiny yolo v2	10881920	Pytorch	1	2.107	N/A	INT8	Source
			1	0.199	N/A	INT8	Source
	320320
			1	0.26	N/A	INT8	Source
	416416
			1	0.499	N/A	INT8	Source
	608608
	960960		1	0.933	N/A	INT8	Source
Yolo V2	1088_1920	Pytorch	1	6.618	N/A	INT8	Source
	320320		1	6.593	N/A	INT8	Source
	416416		1	0.578	N/A	INT8	Source
	544736		1	0.791	N/A	INT8	Source
	608608		1	1.514	N/A	INT8	Source
	960960		1	1.454	N/A	INT8	Source
yolo v3	416416	Pytorch	1	1.179	N/A	INT8	Source
	960960	Pytorch	4		N/A	INT8	Source
bert squad		MX	1	13.524	N/A	INT8	Source
LARGE
max sequnce length = ??? Check with Dror			24	35.705	N/A	INT8
bert mrpc		MX	1	2.841	N/A	INT8
BASE			4	2.911	N/A	INT8
max sequnce length = 128			8	2.952	N/A	INT8
			10	3.468	N/A	INT8
			10	5.934	N/A	INT16
			12	7.003	N/A	INT16
bert squad		MX	1	2.872	N/A	INT8
BASE			4	2.897	N/A	INT8
max sequnce length = ??? Dror?			8	2.936	N/A	INT8
			10	5.953	N/A	INT8
			10	3.488	N/A	Int16
			12		N/A	INT8
			24		N/A	INT8
bvlc_googlenet 224,224		ONNX	1	0.296	NC	INT8	Source

googlenet_bn_no_lrn 224,224		ONNX	1	0.159	12809	INT8	Developed inhouse based on Googlenet with batch norm and w/o LRN
			4	0.423	17542	INT8
			8	0.701	18431	INT8
			10	0.823	18531	INT8
squeezenet1.1 224,224		ONNX	1		887	INT8	Source
			4	0.237	36420	INT8
			8	0.426	37840	INT8
			10	0.547	37927	INT8
ssd-vgg16 300,300		MX	1	0.9	N/A	INT8	Source