From 8aa9a5a9054d85f3c10f73f7916125e62bb39b92 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 15 Feb 2021 16:30:33 +0000 Subject: [PATCH] Disable onnx tests and pipeline until we forward-port onnx work. --- ci/docker/runtime_functions.sh | 10 +- ci/jenkins/Jenkinsfile_unix_cpu | 2 + tests/python/unittest/onnx/README.md | 50 ----- tests/python/unittest/onnx/backend.py | 130 ----------- tests/python/unittest/onnx/backend_rep.py | 137 ------------ tests/python/unittest/onnx/backend_test.py | 93 -------- .../python/unittest/onnx/mxnet_export_test.py | 138 ------------ tests/python/unittest/onnx/test_cases.py | 130 ----------- tests/python/unittest/onnx/test_node.py | 210 ------------------ .../python/unittest/onnx/test_onnxruntime.py | 165 -------------- 10 files changed, 7 insertions(+), 1058 deletions(-) delete mode 100644 tests/python/unittest/onnx/README.md delete mode 100644 tests/python/unittest/onnx/backend.py delete mode 100644 tests/python/unittest/onnx/backend_rep.py delete mode 100644 tests/python/unittest/onnx/backend_test.py delete mode 100644 tests/python/unittest/onnx/mxnet_export_test.py delete mode 100644 tests/python/unittest/onnx/test_cases.py delete mode 100644 tests/python/unittest/onnx/test_node.py delete mode 100644 tests/python/unittest/onnx/test_onnxruntime.py diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index cfff03140bb6..34245bbd7d6c 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -881,12 +881,12 @@ integrationtest_ubuntu_cpu_onnx() { set -ex export PYTHONPATH=./python/ export MXNET_SUBGRAPH_VERBOSE=0 - export DMLC_LOG_STACK_TRACE_DEPTH=100 + export DMLC_LOG_STACK_TRACE_DEPTH=100 python3 tests/python/unittest/onnx/backend_test.py - OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py - OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py - OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py - OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_onnxruntime.py + #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py + #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py + #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py + #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_onnxruntime.py } integrationtest_ubuntu_cpu_dist_kvstore() { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 67d5afc4cc76..2b2f5bc0271a 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -48,7 +48,9 @@ core_logic: { custom_steps.test_unix_python3_mkl_cpu('cpu_mkl'), custom_steps.test_unix_python3_mkldnn_cpu('mkldnn_cpu'), custom_steps.test_unix_python3_mkldnn_mkl_cpu('mkldnn_mkl_cpu'), + /* disable onnx tests for now, until onnx work is forwarded-ported to master custom_steps.test_unix_onnx_cpu('cpu'), + */ /* Disabled due to master build failure: * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ * https://github.com/apache/incubator-mxnet/issues/11801 diff --git a/tests/python/unittest/onnx/README.md b/tests/python/unittest/onnx/README.md deleted file mode 100644 index b9b497ba05da..000000000000 --- a/tests/python/unittest/onnx/README.md +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - - - - - - - - -# ONNX tests - -## Directory structure: - -```bash -. -├── README.md -├── backend.py -├── backend_rep.py -├── backend_test.py -├── gluon_backend_test.py -├── mxnet_backend_test.py -├── mxnet_export_test.py -├── test_cases.py -├── test_models.py -└── test_node.py -``` - -* `backend.py` - MXNetBackend. This file contains prepare(). \ -This class can be used for both, MXNet and Gluon backend. -* `backend_rep.py` - MXNetBackendRep and GluonBackendRep for running inference -* `backend_test.py` - prepare tests by including tests from `test_cases.py` -* `gluon_backend_test.py` - Set backend as gluon and execute ONNX tests for ONNX->Gluon import. -* `mxnet_backend_test.py` - Set backend as gluon and add tests for ONNX->MXNet import/export. -Since MXNetBackend for export, tests both import and export, the test list in this file is -a union of tests that execute for import and export, export alone, and import alone. -* `mxnet_export_test.py` - Execute unit tests for testing MXNet export code - this is not specific to -any operator. -* `test_cases.py` - list of test cases for operators/models that are supported -for "both", import and export, "import" alone, or "export" alone. -* `test_models.py` - custom tests for models -* `test_node.py` - custom tests for operators. These tests are written independent of ONNX tests, in case -ONNX doesn't have tests yet or for MXNet specific operators. diff --git a/tests/python/unittest/onnx/backend.py b/tests/python/unittest/onnx/backend.py deleted file mode 100644 index eb803f790332..000000000000 --- a/tests/python/unittest/onnx/backend.py +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -"""MXNet/Gluon backend wrapper for onnx test infrastructure""" - -from mxnet.contrib.onnx.onnx2mx.import_onnx import GraphProto -from mxnet.contrib.onnx.mx2onnx.export_onnx import MXNetGraph -import mxnet as mx -import numpy as np - -try: - from onnx import helper, TensorProto, mapping - from onnx.backend.base import Backend - from onnx.defs import onnx_opset_version -except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") -from backend_rep import MXNetBackendRep, GluonBackendRep - - -# MXNetBackend class will take an ONNX model with inputs, perform a computation, -# and then return the output. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md - -class MXNetBackend(Backend): - """MXNet/Gluon backend for ONNX""" - - backend = 'mxnet' - operation = 'import' - - @classmethod - def set_params(cls, backend, operation): - cls.backend = backend - cls.operation = operation - - @staticmethod - def perform_import_export(sym, arg_params, aux_params, input_shape): - """ Import ONNX model to mxnet model and then export to ONNX model - and then import it back to mxnet for verifying the result""" - graph = GraphProto() - - params = {} - params.update(arg_params) - params.update(aux_params) - # use the latest opset version supported by the onnx library - opset_version = onnx_opset_version() - # exporting to onnx graph proto format - converter = MXNetGraph() - graph_proto = converter.create_onnx_graph_proto(sym, params, in_shape=input_shape, - in_type=mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype('float32')], - opset_version=opset_version) - - # importing back to MXNET for verifying result. - sym, arg_params, aux_params = graph.from_onnx(graph_proto, opset_version) - - return sym, arg_params, aux_params - - @classmethod - def prepare(cls, model, device='CPU', **kwargs): - """For running end to end model(used for onnx test backend) - - Parameters - ---------- - model : onnx ModelProto object - loaded onnx graph - device : 'CPU' - specifying device to run test on - kwargs : - other arguments - - Returns - ------- - MXNetBackendRep : object - Returns object of MXNetBackendRep class which will be in turn - used to run inference on the input model and return the result for comparison. - """ - backend = kwargs.get('backend', cls.backend) - operation = kwargs.get('operation', cls.operation) - - graph = GraphProto() - if device == 'CPU': - ctx = mx.cpu() - else: - raise NotImplementedError("ONNX tests are run only for CPU context.") - - # determine opset version model uses - model_opset_version = max([x.version for x in model.opset_import]) - - if backend == 'mxnet': - sym, arg_params, aux_params = graph.from_onnx(model.graph, model_opset_version) - if operation == 'export': - metadata = graph.get_graph_metadata(model.graph) - input_data = metadata['input_tensor_data'] - input_shape = [data[1] for data in input_data] - sym, arg_params, aux_params = MXNetBackend.perform_import_export(sym, arg_params, aux_params, - input_shape) - - return MXNetBackendRep(sym, arg_params, aux_params, device) - elif backend == 'gluon': - if operation == 'import': - net = graph.graph_to_gluon(model.graph, ctx, model_opset_version) - return GluonBackendRep(net, device) - elif operation == 'export': - raise NotImplementedError("Gluon->ONNX export not implemented.") - - @classmethod - def supports_device(cls, device): - """Supports only CPU for testing""" - return device == 'CPU' - - -prepare = MXNetBackend.prepare - -supports_device = MXNetBackend.supports_device diff --git a/tests/python/unittest/onnx/backend_rep.py b/tests/python/unittest/onnx/backend_rep.py deleted file mode 100644 index 597b1c701877..000000000000 --- a/tests/python/unittest/onnx/backend_rep.py +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# coding: utf-8 -"""MXNet backend rep for onnx test infrastructure""" -try: - from onnx.backend.base import BackendRep -except ImportError: - raise ImportError("Onnx and protobuf need to be installed. Instructions to" - + " install - https://github.com/onnx/onnx#installation") -import numpy as np -import mxnet as mx -from mxnet import nd - -# Using these functions for onnx test infrastructure. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md -# MXNetBackendRep object will be returned by MXNetBackend's prepare method which is used to -# execute a model repeatedly. -# Inputs will be passed to the run method of MXNetBackendRep class, it will perform computation and -# retrieve the corresponding results for comparison to the onnx backend. -# https://github.com/onnx/onnx/blob/master/onnx/backend/test/runner/__init__.py. - -class MXNetBackendRep(BackendRep): - """Running model inference on mxnet engine and return the result - to onnx test infrastructure for comparison.""" - def __init__(self, symbol, arg_params, aux_params, device): - self.symbol = symbol - self.arg_params = arg_params - self.aux_params = aux_params - self.device = device - - def run(self, inputs, **kwargs): - """Run model inference and return the result - - Parameters - ---------- - inputs : numpy array - input to run a layer on - - Returns - ------- - params : numpy array - result obtained after running the inference on mxnet - """ - # create module, passing cpu context - if self.device == 'CPU': - ctx = mx.cpu() - else: - raise NotImplementedError("ONNX tests are run only for CPU context.") - - # To fetch the data names of the input to the model we list the inputs of the symbol graph - # and exclude the argument and auxiliary parameters from the list - data_names = [graph_input for graph_input in self.symbol.list_inputs() - if graph_input not in self.arg_params and graph_input not in self.aux_params] - - data_forward = [] - for idx, input_name in enumerate(data_names): - val = inputs[idx] - data_forward.append(mx.nd.array(val)) - - if self.arg_params: - for idx, input_name in enumerate(self.arg_params): - val = self.arg_params[input_name] - data_names.append(input_name) - data_forward.append(mx.nd.array(val)) - - args = dict(zip(data_names, data_forward)) - exe = self.symbol._bind(ctx, args=args, aux_states=self.aux_params) - exe.forward(is_train=False) - result = [] - for output in exe.outputs: - result.append(output.asnumpy()) - return result - - -# GluonBackendRep object will be returned by GluonBackend's prepare method which is used to -# execute a model repeatedly. -# Inputs will be passed to the run method of MXNetBackendRep class, it will perform computation and -# retrieve the corresponding results for comparison to the onnx backend. -# https://github.com/onnx/onnx/blob/master/onnx/backend/test/runner/__init__.py. -# Implemented by following onnx docs guide: -# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md - -class GluonBackendRep(BackendRep): - """Running model inference on gluon backend and return the result - to onnx test infrastructure for comparison.""" - def __init__(self, net, device): - self.net = net - self.device = device - - def run(self, inputs, **kwargs): - """Run model inference and return the result - - Parameters - ---------- - inputs : numpy array - input to run a layer on - - Returns - ------- - params : numpy array - result obtained after running the inference on mxnet - """ - # create module, passing cpu context - if self.device == 'CPU': - ctx = mx.cpu() - else: - raise NotImplementedError("ONNX tests are run only for CPU context.") - - # run inference - net_inputs = [nd.array(input_data, ctx=ctx) for input_data in inputs] - net_outputs = self.net(*net_inputs) - results = [] - if isinstance(net_outputs, list): - for output in net_outputs: - results.append(output.asnumpy()) - result = results - else: - results.extend([o for o in net_outputs.asnumpy()]) - result = [np.array(results)] - - return result diff --git a/tests/python/unittest/onnx/backend_test.py b/tests/python/unittest/onnx/backend_test.py deleted file mode 100644 index cf3cdc1bfb54..000000000000 --- a/tests/python/unittest/onnx/backend_test.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""ONNX test backend wrapper""" - -def build_test_suite(backend_tests): # type: () -> unittest.TestSuite - ''' - TestSuite that can be run by TestRunner - This has been borrowed from onnx/onnx/backend/test/runner/__init__.py, - since Python3 cannot sort objects of type 'Type' as Runner.test_suite() - expects. - ''' - suite = unittest.TestSuite() - for case in backend_tests.test_cases.values(): - suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(case)) - return suite - - -def prepare_tests(backend, oper): - """ - Prepare the test list - :param backend: mxnet/gluon backend - :param oper: str. export or import - :return: backend test list - """ - BACKEND_TESTS = onnx.backend.test.BackendTest(backend, __name__) - implemented_ops = test_cases.IMPLEMENTED_OPERATORS_TEST.get('both', []) + \ - test_cases.IMPLEMENTED_OPERATORS_TEST.get(oper, []) - - for op_test in implemented_ops: - BACKEND_TESTS.include(op_test) - - basic_models = test_cases.BASIC_MODEL_TESTS.get('both', []) + \ - test_cases.BASIC_MODEL_TESTS.get(oper, []) - - for basic_model_test in basic_models: - BACKEND_TESTS.include(basic_model_test) - - std_models = test_cases.STANDARD_MODEL.get('both', []) + \ - test_cases.STANDARD_MODEL.get(oper, []) - - for std_model_test in std_models: - BACKEND_TESTS.include(std_model_test) - - # Tests for scalar ops are in test_node.py - BACKEND_TESTS.exclude('.*scalar.*') - - return BACKEND_TESTS - - -if __name__ == '__main__': - try: - import onnx.backend.test - except ImportError: - raise ImportError("Onnx and protobuf need to be installed") - - import test_cases - import unittest - import backend as mxnet_backend - import logging - - operations = ['import', 'export'] - backends = ['mxnet', 'gluon'] - # This is a pytest magic variable to load extra plugins - pytest_plugins = "onnx.backend.test.report", - - for bkend in backends: - for operation in operations: - log = logging.getLogger(bkend + operation) - if bkend == 'gluon' and operation == 'export': - log.warning('Gluon->ONNX export not implemented. Skipping tests...') - continue - log.info('Executing tests for ' + bkend + ' backend: ' + operation) - mxnet_backend.MXNetBackend.set_params(bkend, operation) - BACKEND_TESTS = prepare_tests(mxnet_backend, operation) - unittest.TextTestRunner().run(build_test_suite(BACKEND_TESTS.enable_report())) diff --git a/tests/python/unittest/onnx/mxnet_export_test.py b/tests/python/unittest/onnx/mxnet_export_test.py deleted file mode 100644 index baa74768e612..000000000000 --- a/tests/python/unittest/onnx/mxnet_export_test.py +++ /dev/null @@ -1,138 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# pylint: disable=too-many-locals,wrong-import-position,import-error -import os, sys -import unittest -import logging -import tempfile -curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(curr_path, '..')) -from mxnet import nd, sym -from mxnet.test_utils import set_default_context -from mxnet.gluon import nn -from mxnet.gluon import HybridBlock -import mxnet as mx - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -def _assert_sym_equal(lhs, rhs): - assert lhs.list_inputs() == rhs.list_inputs() # input names must be identical - assert len(lhs.list_outputs()) == len(rhs.list_outputs()) # number of outputs must be identical - - -def _force_list(output): - if isinstance(output, nd.NDArray): - return [output] - return list(output) - - -def _optional_group(symbols, group=False): - if group: - return sym.Group(symbols) - else: - return symbols - - -def _check_onnx_export(net, group_outputs=False, shape_type=tuple, extra_params={}): - net.initialize() - data = nd.random.uniform(0, 1, (1, 1024)) - output = _force_list(net(data)) # initialize weights - net_sym = _optional_group(net(sym.Variable('data')), group_outputs) - net_params = {param.var().name: param._reduce() for param in net.collect_params().values()} - net_params.update(extra_params) - with tempfile.TemporaryDirectory() as tmpdirname: - onnx_file_path = os.path.join(tmpdirname, 'net.onnx') - export_path = mx.contrib.onnx.export_model( - sym=net_sym, - params=net_params, - input_shape=[shape_type(data.shape)], - onnx_file_path=onnx_file_path) - assert export_path == onnx_file_path - # Try importing the model to symbol - _assert_sym_equal(net_sym, mx.contrib.onnx.import_model(export_path)[0]) - - # Try importing the model to gluon - imported_net = mx.contrib.onnx.import_to_gluon(export_path, ctx=None) - _assert_sym_equal(net_sym, _optional_group(imported_net(sym.Variable('data')), group_outputs)) - - # Confirm network outputs are the same - imported_net_output = _force_list(imported_net(data)) - for out, imp_out in zip(output, imported_net_output): - mx.test_utils.assert_almost_equal(out, imp_out, atol=1e-5, rtol=1e-5) - - -class SplitConcatBlock(HybridBlock): - """Block which creates two splits and later concatenates them""" - def __init__(self): - super(SplitConcatBlock, self).__init__() - - def hybrid_forward(self, F, x): - splits = F.split(x, axis=1, num_outputs=2) - return F.concat(*splits) - - -class TestExport(unittest.TestCase): - """ Tests ONNX export. - """ - - def setUp(self): - set_default_context(mx.cpu(0)) - - def test_onnx_export_single_output(self): - net = nn.HybridSequential() - net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) - _check_onnx_export(net) - - def test_onnx_export_multi_output(self): - class MultiOutputBlock(nn.HybridBlock): - def __init__(self): - super(MultiOutputBlock, self).__init__() - self.net = nn.HybridSequential() - for i in range(10): - self.net.add(nn.Dense(100 + i * 10, activation='relu')) - - def hybrid_forward(self, F, x): - out = tuple(block()(x) for block in self.net._children.values()) - return out - - net = MultiOutputBlock() - assert len(sym.Group(net(sym.Variable('data'))).list_outputs()) == 10 - _check_onnx_export(net, group_outputs=True) - - def test_onnx_export_list_shape(self): - net = nn.HybridSequential() - net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) - _check_onnx_export(net, shape_type=list) - - def test_onnx_export_extra_params(self): - net = nn.HybridSequential() - net.add(nn.Dense(100, activation='relu'), nn.Dense(10)) - _check_onnx_export(net, extra_params={'extra_param': nd.array([1, 2])}) - - def test_onnx_export_slice(self): - net = nn.HybridSequential() - net.add(nn.Dense(100, activation='relu'), SplitConcatBlock(), nn.Dense(10)) - _check_onnx_export(net) - - def test_onnx_export_slice_changing_shape(self): - net = nn.HybridSequential() - net.add(nn.Dense(100, activation='relu'), SplitConcatBlock(), - nn.Dense(50, activation='relu'), SplitConcatBlock(), nn.Dense(10)) - _check_onnx_export(net) diff --git a/tests/python/unittest/onnx/test_cases.py b/tests/python/unittest/onnx/test_cases.py deleted file mode 100644 index 932310281bd6..000000000000 --- a/tests/python/unittest/onnx/test_cases.py +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -IMPLEMENTED_OPERATORS_TEST = { - 'both': ['test_add', - 'test_sub', - 'test_mul', - 'test_div', - 'test_neg', - 'test_abs', - 'test_sum', - 'test_tanh', - 'test_ceil', - 'test_floor', - 'test_concat', - 'test_identity', - 'test_sigmoid', - 'test_relu', - 'test_constant_pad', - 'test_edge_pad', - 'test_reflect_pad', - 'test_softmax_example', - 'test_softmax_large_number', - 'test_softmax_axis_2', - 'test_transpose', - 'test_globalmaxpool', - 'test_globalaveragepool', - 'test_reciprocal', - 'test_sqrt', - 'test_pow', - 'test_exp_', - 'test_argmax', - 'test_argmin', - 'test_min', - # pytorch operator tests - 'test_exp_', - 'test_operator_maxpool', - 'test_operator_params', - 'test_operator_permute2', - 'test_cos[^h]', - 'test_sin[^h]', - 'test_tan', - 'test_acos[^h]', - 'test_asin[^h]', - 'test_atan[^h]', - 'test_squeeze', - 'test_matmul_', - 'test_depthtospace', - 'test_hardsigmoid', - 'test_instancenorm', - 'test_shape', - 'test_cast((?!STRING).)*$', - 'test_clip', - 'test_size', - 'test_dropout', - 'test_unsqueeze', - 'test_log_', - 'test_flatten_default_axis', - 'test_leakyrelu', - 'test_selu_default', - 'test_elu', - 'test_max_', - 'test_softplus', - 'test_reduce_', - 'test_split_equal', - 'test_gather' - ], - 'import': ['test_softsign', - 'test_mean', - 'test_averagepool_1d', - 'test_averagepool_2d_pads_count_include_pad', - 'test_averagepool_2d_precomputed_pads_count_include_pad', - 'test_averagepool_2d_precomputed_strides', - 'test_averagepool_2d_strides', - 'test_averagepool_3d', - 'test_hardmax' - ], - 'export': ['test_random_uniform', - 'test_random_normal', - 'test_reduce_min', - 'test_reduce_max', - 'test_reduce_mean', - 'test_reduce_prod', - 'test_reduce_sum_d', - 'test_reduce_sum_keepdims_random', - 'test_lrn' - ] -} - -BASIC_MODEL_TESTS = { - 'both': ['test_AvgPool2D', - 'test_BatchNorm', - 'test_ConstantPad2d' - 'test_Conv2d', - 'test_MaxPool', - 'test_PReLU', - 'test_Softmax', - 'test_softmax_functional', - 'test_softmax_lastdim', - ], - 'export': [] -} - -STANDARD_MODEL = { - 'both': ['test_bvlc_alexnet', - 'test_densenet121', - # 'test_inception_v1', - # 'test_inception_v2', - 'test_resnet50', - # 'test_shufflenet', - 'test_squeezenet', - 'test_vgg19' - ], - 'import': ['test_zfnet512'], - 'export': ['test_vgg16'] -} diff --git a/tests/python/unittest/onnx/test_node.py b/tests/python/unittest/onnx/test_node.py deleted file mode 100644 index fabba1a9e555..000000000000 --- a/tests/python/unittest/onnx/test_node.py +++ /dev/null @@ -1,210 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Tests for individual operators -This module contains operator tests which currently do not exist on -ONNX backend test framework. Once we have PRs on the ONNX repo and get -those PRs merged, this file will get EOL'ed. -""" -# pylint: disable=too-many-locals,wrong-import-position,import-error -import sys -import os -import unittest -import logging -import tarfile -from collections import namedtuple -import numpy as np -import numpy.testing as npt -from onnx import checker, numpy_helper, helper, load_model -from onnx import TensorProto -from mxnet.test_utils import download -import mxnet as mx -import backend - -CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(CURR_PATH, '../../python/unittest')) - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -def get_rnd(shape, low=-1.0, high=1.0, dtype=np.float32): - if dtype == np.float32: - return (np.random.uniform(low, high, - np.prod(shape)).reshape(shape).astype(np.float32)) - elif dtype == np.int32: - return (np.random.randint(low, high, - np.prod(shape)).reshape(shape).astype(np.float32)) - elif dtype == np.bool_: - return np.random.choice(a=[False, True], size=shape).astype(np.float32) - - -def _fix_attributes(attrs, attribute_mapping): - new_attrs = attrs - attr_modify = attribute_mapping.get('modify', {}) - for k, v in attr_modify.items(): - new_attrs[v] = new_attrs.pop(k, None) - - attr_add = attribute_mapping.get('add', {}) - for k, v in attr_add.items(): - new_attrs[k] = v - - attr_remove = attribute_mapping.get('remove', []) - for k in attr_remove: - if k in new_attrs: - del new_attrs[k] - - return new_attrs - - -def get_input_tensors(input_data): - input_tensor = [] - input_names = [] - input_sym = [] - for idx, ip in enumerate(input_data): - name = "input" + str(idx + 1) - input_sym.append(mx.sym.Variable(name)) - input_names.append(name) - input_tensor.append(helper.make_tensor_value_info(name, - TensorProto.FLOAT, shape=np.shape(ip))) - return input_names, input_tensor, input_sym - - -def get_onnx_graph(testname, input_names, inputs, output_name, output_shape, attr): - outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=output_shape)] - - nodes = [helper.make_node(output_name, input_names, ["output"], **attr)] - - graph = helper.make_graph(nodes, testname, inputs, outputs) - - model = helper.make_model(graph) - return model - -class TestNode(unittest.TestCase): - """ Tests for models. - Tests are dynamically added. - Therefore edit test_models to add more tests. - """ - def test_imports(self): - for bk in ['mxnet', 'gluon']: - for test in import_test_cases: - test_name, onnx_name, inputs, np_op, attrs = test - with self.subTest(test_name): - names, input_tensors, inputsym = get_input_tensors(inputs) - np_out = [np_op(*inputs, **attrs)] - output_shape = np.shape(np_out) - onnx_model = get_onnx_graph(test_name, names, input_tensors, onnx_name, output_shape, attrs) - bkd_rep = backend.prepare(onnx_model, operation='import', backend=bk) - mxnet_out = bkd_rep.run(inputs) - npt.assert_almost_equal(np_out, mxnet_out, decimal=4) - - def test_exports(self): - for test in export_test_cases: - test_name, onnx_name, mx_op, input_shape, attrs = test - input_sym = mx.sym.var('data') - if isinstance(mx_op, type) and issubclass(mx_op, (mx.gluon.HybridBlock, mx.gluon.SymbolBlock)): - mx_op = mx_op(**attrs) - mx_op.initialize() - mx_op(mx.nd.zeros(input_shape)) - params = {p.var().name: p.data() for p in mx_op.collect_params().values()} - outsym = mx_op(input_sym) - else: - params = {} - outsym = mx_op(input_sym, **attrs) - converted_model = mx.contrib.onnx.export_model(outsym, params, [input_shape], np.float32, - onnx_file_path=outsym.name + ".onnx") - model = load_model(converted_model) - checker.check_model(model) - - -# test_case = ("test_case_name", mxnet op, "ONNX_op_name", [input_list], attribute map, MXNet_specific=True/False, -# fix_attributes = {'modify': {mxnet_attr_name: onnx_attr_name}, -# 'remove': [attr_name], -# 'add': {attr_name: value}, -# check_value=True/False, check_shape=True/False) -test_cases = [ - ("test_equal", mx.sym.broadcast_equal, "Equal", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False, {}, True, - False), - ("test_greater", mx.sym.broadcast_greater, "Greater", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False, {}, True, - False), - ("test_less", mx.sym.broadcast_lesser, "Less", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False, {}, True, - False), - ("test_and", mx.sym.broadcast_logical_and, "And", - [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False), - ("test_xor", mx.sym.broadcast_logical_xor, "Xor", - [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False), - ("test_or", mx.sym.broadcast_logical_or, "Or", - [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False), - ("test_not", mx.sym.logical_not, "Not", [get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False), - ("test_square", mx.sym.square, "Pow", [get_rnd((2, 3), dtype=np.int32)], {}, True, {}, True, False), - ("test_spacetodepth", mx.sym.space_to_depth, "SpaceToDepth", [get_rnd((1, 1, 4, 6))], - {'block_size': 2}, False, {}, True, False), - ("test_fullyconnected", mx.sym.FullyConnected, "Gemm", [get_rnd((4, 3)), get_rnd((4, 3)), get_rnd(4)], - {'num_hidden': 4, 'name': 'FC'}, True, {}, True, False), - ("test_lppool1", mx.sym.Pooling, "LpPool", [get_rnd((2, 3, 20, 20))], - {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 1, 'pool_type': 'lp'}, False, - {'modify': {'kernel': 'kernel_shape', 'pad': 'pads', 'stride': 'strides', 'p_value': 'p'}, - 'remove': ['pool_type']}, True, False), - ("test_lppool2", mx.sym.Pooling, "LpPool", [get_rnd((2, 3, 20, 20))], - {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 2, 'pool_type': 'lp'}, False, - {'modify': {'kernel': 'kernel_shape', 'pad': 'pads', 'stride': 'strides', 'p_value': 'p'}, - 'remove': ['pool_type']}, True, False), - ("test_globallppool1", mx.sym.Pooling, "GlobalLpPool", [get_rnd((2, 3, 20, 20))], - {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 1, 'pool_type': 'lp', 'global_pool': True}, False, - {'modify': {'p_value': 'p'}, - 'remove': ['pool_type', 'kernel', 'pad', 'stride', 'global_pool']}, True, False), - ("test_globallppool2", mx.sym.Pooling, "GlobalLpPool", [get_rnd((2, 3, 20, 20))], - {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 2, 'pool_type': 'lp', 'global_pool': True}, False, - {'modify': {'p_value': 'p'}, - 'remove': ['pool_type', 'kernel', 'pad', 'stride', 'global_pool']}, True, False), - ("test_roipool", mx.sym.ROIPooling, "MaxRoiPool", - [[[get_rnd(shape=(8, 6), low=1, high=100, dtype=np.int32)]], [[0, 0, 0, 4, 4]]], - {'pooled_size': (2, 2), 'spatial_scale': 0.7}, False, - {'modify': {'pooled_size': 'pooled_shape'}}, True, False), - - # since results would be random, checking for shape alone - ("test_multinomial", mx.sym.sample_multinomial, "Multinomial", - [np.array([0, 0.1, 0.2, 0.3, 0.4]).astype("float32")], - {'shape': (10,)}, False, {'modify': {'shape': 'sample_size'}}, False, True), - ("test_random_normal", mx.sym.random_normal, "RandomNormal", [], - {'shape': (2, 2), 'loc': 0, 'scale': 1}, False, {'modify': {'loc': 'mean'}}, False, True), - ("test_random_uniform", mx.sym.random_uniform, "RandomUniform", [], - {'shape': (2, 2), 'low': 0.5, 'high': 1.0}, False, {}, False, True) -] - -test_scalar_ops = ['Add', 'Sub', 'rSub' 'Mul', 'Div', 'Pow'] - -# test_case = ("test_case_name", "ONNX_op_name", [input_list], np_op, attribute map) -import_test_cases = [ - ("test_lpnormalization_default", "LpNormalization", [get_rnd([5, 3, 3, 2])], np.linalg.norm, {'ord':2, 'axis':-1}), - ("test_lpnormalization_ord1", "LpNormalization", [get_rnd([5, 3, 3, 2])], np.linalg.norm, {'ord':1, 'axis':-1}), - ("test_lpnormalization_ord2", "LpNormalization", [get_rnd([5, 3, 3, 2])], np.linalg.norm, {'ord':2, 'axis':1}) -] - -# test_case = ("test_case_name", "ONNX_op_name", mxnet_op, input_shape, attribute map) -export_test_cases = [ - ("test_expand", "Expand", mx.sym.broadcast_to, (2,1,3,1), {'shape': (2,1,3,1)}), - ("test_tile", "Tile", mx.sym.tile, (2,1,3,1), {'reps': (2,3)}), - ("test_topk", "TopK", mx.sym.topk, (2, 10, 2), {'k': 3, 'axis': 1, 'ret_typ': 'both', 'dtype': np.int64}), - ("test_slice_axis", "Slice", mx.sym.slice_axis, (2, 10, 2), {'begin': 3, 'end': 7, 'axis': 1}), - ("test_LSTM", "LSTM", mx.gluon.rnn.LSTM, (3,1,2), {'hidden_size': 3}), - ("test_BiLSTM", "LSTM", mx.gluon.rnn.LSTM, (3,1,2), {'hidden_size': 3, 'bidirectional': True}), -] - -if __name__ == '__main__': - unittest.main() diff --git a/tests/python/unittest/onnx/test_onnxruntime.py b/tests/python/unittest/onnx/test_onnxruntime.py deleted file mode 100644 index 3737b7ad0a79..000000000000 --- a/tests/python/unittest/onnx/test_onnxruntime.py +++ /dev/null @@ -1,165 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import mxnet as mx -import numpy as np -import onnxruntime - -import json -import os -import shutil - -import pytest - - -def run_cv_model_test(model, tmpdir): - def get_gluon_cv_model(model_name, tmp): - tmpfile = os.path.join(tmp, model_name) - ctx = mx.cpu(0) - model = mx.gluon.model_zoo.vision.get_model(model_name, pretrained=True, ctx=ctx, root=tmp) - model.hybridize() - data = mx.nd.zeros((2,3,224,224), dtype='float32', ctx=ctx) - model(data) - model.export(tmpfile, epoch=0) - sym_file = tmpfile + '-symbol.json' - params_file = tmpfile + '-0000.params' - return sym_file, params_file - - def export_model_to_onnx(sym_file, params_file): - input_shape = (1,3,224,224) - onnx_file = os.path.join(os.path.dirname(sym_file), "model.onnx") - converted_model_path = mx.contrib.onnx.export_model(sym_file, params_file, [input_shape], - np.float32, onnx_file) - return onnx_file - - def normalize_image(imgfile): - image = mx.image.imread(imgfile).asnumpy() - image_data = np.array(image).transpose(2, 0, 1) - img_data = image_data.astype('float32') - mean_vec = np.array([0.485, 0.456, 0.406]) - stddev_vec = np.array([0.229, 0.224, 0.225]) - norm_img_data = np.zeros(img_data.shape).astype('float32') - for i in range(img_data.shape[0]): - norm_img_data[i,:,:] = (img_data[i,:,:]/255 - mean_vec[i]) / stddev_vec[i] - return norm_img_data.reshape(1, 3, 224, 224).astype('float32') - - def get_prediction(model, image): - pass - - def softmax(x): - x = x.reshape(-1) - e_x = np.exp(x - np.max(x)) - return e_x / e_x.sum(axis=0) - - def load_imgnet_labels(tmpdir): - tmpfile = os.path.join(tmpdir, 'image_net_labels.json') - mx.test_utils.download('https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/image_net_labels.json', - fname=tmpfile) - return np.array(json.load(open(tmpfile, 'r'))) - - def download_test_images(tmpdir): - test_images = [ - ['dog.jpg',['boxer']], - ['apron.jpg', ['apron', 'maillot']], - ['dolphin.jpg', ['great white shark','grey whale']], - ['hammerheadshark.jpg', ['tiger shark']], - ['lotus.jpg', ['pinwheel','pot']] - ] - for f,_ in test_images: - mx.test_utils.download('https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/onnx/images/'+f+'?raw=true', - fname=os.path.join(tmpdir, f)) - return test_images - - labels = load_imgnet_labels(tmpdir) - test_images = download_test_images(tmpdir) - sym_file, params_file = get_gluon_cv_model(model, tmpdir) - onnx_file = export_model_to_onnx(sym_file, params_file) - #print("exported onnx file: ",onnx_file) - - # create onnxruntime session using the generated onnx file - ses_opt = onnxruntime.SessionOptions() - ses_opt.log_severity_level = 3 - session = onnxruntime.InferenceSession(onnx_file, ses_opt) - input_name = session.get_inputs()[0].name - - for img,classes in test_images: - img_data = normalize_image(os.path.join(tmpdir, img)) - raw_result = session.run([], {input_name: img_data}) - res = softmax(np.array(raw_result)).tolist() - class_idx = np.argmax(res) - #print("Image top classification:",labels[class_idx]) - sort_idx = np.flip(np.squeeze(np.argsort(res))) - #print("\tTop labels: " + ",".join(labels[sort_idx[:5]])) - correct_classification = False - for label in labels[sort_idx[:5]]: - for c in classes: - if c in label: - correct_classification = True - assert correct_classification == True - - # cleanup - shutil.rmtree(tmpdir) - -@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580") -def test_cv_model_inference_onnxruntime_mobilenet0_5(tmp_path): - run_cv_model_test('mobilenet0.5', tmp_path) - -def test_cv_model_inference_onnxruntime_mobilenetv2_1_0(tmp_path): - run_cv_model_test('mobilenetv2_1.0', tmp_path) - -def test_cv_model_inference_onnxruntime_resnet18_v1(tmp_path): - run_cv_model_test('resnet18_v1', tmp_path) - -def test_cv_model_inference_onnxruntime_resnet18_v2(tmp_path): - run_cv_model_test('resnet18_v2', tmp_path) - -def test_cv_model_inference_onnxruntime_resnet101_v1(tmp_path): - run_cv_model_test('resnet101_v1', tmp_path) - -def test_cv_model_inference_onnxruntime_resnet101_v2(tmp_path): - run_cv_model_test('resnet101_v2', tmp_path) - -def test_cv_model_inference_onnxruntime_resnet152_v1(tmp_path): - run_cv_model_test('resnet152_v1', tmp_path) - -def test_cv_model_inference_onnxruntime_resnet152_v2(tmp_path): - run_cv_model_test('resnet152_v2', tmp_path) - -@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580") -def test_cv_model_inference_onnxruntime_squeezenet1_0(tmp_path): - run_cv_model_test('squeezenet1.0', tmp_path) - -@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580") -def test_cv_model_inference_onnxruntime_squeezenet1_1(tmp_path): - run_cv_model_test('squeezenet1.1', tmp_path) - -@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580") -def test_cv_model_inference_onnxruntime_vgg11(tmp_path): - run_cv_model_test('vgg11', tmp_path) - -@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580") -def test_cv_model_inference_onnxruntime_vgg11_bn(tmp_path): - run_cv_model_test('vgg11_bn', tmp_path) - -def test_cv_model_inference_onnxruntime_vgg19(tmp_path): - run_cv_model_test('vgg19', tmp_path) - -def test_cv_model_inference_onnxruntime_vgg19_bn(tmp_path): - run_cv_model_test('vgg19_bn', tmp_path) - - -