Merge pull request #14 from kumardeepakamd/main

support onnx operator tests

e2eshark/README.md

@@ -43,7 +43,10 @@
             export HF_HOME="your path"/HF_HOME
 
  - tools/onnxutil.py : Allows examining an ONNX protobuf file
- - tools/stubrunmodel.py : This is concatenated to 'model.py' in test directory to form a runmodel.py runnable model
+ - tools/stubs/onnxmodel.py : This is concatenated to 'model.py' in test directory to form a 
+                              runmodel.py for onnx input model
+ - tools/stubs/pytorchmodel.py : This is concatenated to 'model.py' in test directory to form a 
+                                 runmodel.py for pytorch input model
 
  The logs are created as .log files in the test run sub directory. Examine the logs to find and fix 
  cause of any failure.
@@ -66,6 +69,11 @@ total count of operator instances
 python ./tools/onnxutil.py onnx/models/resnet50_vaiq_int8/model.onnx -f
 ```
 
+Run given test onnx/combinations/constant_constantofshape upto inference on target backend
+```
+python ./run.py -c ../../torch-mlir/build --frameworks onnx --upto inference -i ../../iree-build/ --tests onnx/combinations/constant_constantofshape
+```
+
 ### Adding new tests
 
 Let us say you wanted to add a new test to the framework "pytorch" to test maxpool  i.e. start with
@@ -77,21 +85,24 @@ https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html provides descr
 Now take following steps:
 1. Use an appropriate suffix to describe what you are intending to test say maxpool_2d_large
 2. mkdir -p pytorch/operators/maxpool_2d_large
-3. cp -pr pytorch/test_template/operators.template model.py
-4. modify model.py (check other existing tests for clue)
-5. once your model.py is ready, you can go to root of the e2eshark test directory and run test as below 
+3. cd pytorch/operators/maxpool_2d_large
+4. cp -pr pytorch/operators/conv2d/model.py .
+5. modify model.py
+6. You can run 'python ./model.py' to see input and output values printed to test 
+7. 
+8. once your model.py is ready, you can go to root of the e2eshark test directory and run test as below 
    ```
    python ./run.py --upto torch-mlir -c "your torch mlir build dir" --tests pytorch/operators/maxpool_2d_large --mode direct
    ```
-   Rerun above with --mode onnx if you want ONNX to ge generated from pytorch and tested in addition.
+   Rerun above with --mode onnx if you want ONNX to get generated from pytorch and tested in addition.
 
    If you want to test up to inference, then provide your iree build in addition as -i option and run as
 
    ```
    python ./run.py --upto inference -c "your torch mlir build dir" -i "your iree build dir" --tests pytorch/operators/maxpool_2d_large --mode direct
    ```
 
-   Rerun above with --mode onnx if you want ONNX to ge generated from pytorch and tested in addition.
+   Rerun above with --mode onnx if you want ONNX to get generated from pytorch and tested in addition.
 
    You will see test-run/pytorch/operators/maxpool_2d_large and logs created. Examine that to see errors. 
    Iterate over fixing your model.py and examing logs till you get it right. Given state of tools, tests may fail.

e2eshark/onnx/models/resnet50_vaiq_int8/model.py

@@ -1,36 +1,27 @@
-import numpy as np
+import numpy
 import onnxruntime
 
 
-# the generated or checked in onnx file must always be called
-# model.onnx
-# This file forms middle of the runmodel.py for this test
-# which is generated by run.py script in root of e2eshark
-# test dir
-# <e2eshark>/tools/stubs/onnxstartmodel.py
-# this model.py
-# <e2eshark>/tools/stubs/onnxendmodel.py
-# are concatenated in that order to form
-# <test-run dir>/onnx/<test category>/<test name>/runmodel.py
-# which is run to run the model to generate output
-# Leave the above comment in the file
+# The generated or checked in onnx file must always be called model.onnx
+# the tools/stubs/onnxmodel.py is appended to model.py
+# to form runmodel.py in the rundirectory which is then taken
+# through flow
+
 
 # insert here any onnx API call to generate onnx file if
 # not using a checked in onnx model
 
-
-# to locally test, can uncomment the line below
-# comment it back for launching from run.py as this will be set
-# with full path to onnx to allow running in a separate run dir
-# session = onnxruntime.InferenceSession("model.onnx", None)
-
+# start an onnxrt session
+session = onnxruntime.InferenceSession("model.onnx", None)
 
 # fill the lines that set test_input and onnx_output
 # these two are special names and should not be changed
-test_input = np.random.rand(1, 3, 224, 224).astype(np.float32)
+test_input = numpy.random.rand(1, 3, 224, 224).astype(numpy.float32)
+print("Input:", test_input)
 
 # Get the name of the input of the model
 input_name = session.get_inputs()[0].name
 
 # call inference session
-onnx_output = [session.run([], {input_name: test_input})[0]]
+test_output = [session.run([], {input_name: test_input})[0]]
+print("Onput:", test_output)

e2eshark/pytorch/combinations/mlp/model.py

@@ -23,3 +23,6 @@ def name(self):
 
 model = mlp()
 test_input = torch.randn(8, 3)
+test_output = model(test_input)
+print("Input:", test_input)
+print("Onput:", test_output)

e2eshark/pytorch/models/llama2-7b/model.py

@@ -25,3 +25,6 @@ def name(self):
 model = model_llama2_7b_hf()
 tokenizer = LlamaTokenizer.from_pretrained(test_model_name)
 test_input = tokenizer.encode("The llama goes to graze grass", return_tensors="pt")
+test_output = model(test_input)
+print("Input:", test_input)
+print("Onput:", test_output)

e2eshark/pytorch/models/opt-125M/model.py

@@ -29,3 +29,6 @@ def name(self):
 model = model_opt_125M()
 tokenizer = AutoTokenizer.from_pretrained(test_model_name)
 test_input = torch.tensor([tokenizer.encode("The test prommpt")])
+test_output = model(test_input)
+print("Input:", test_input)
+print("Onput:", test_output)

e2eshark/pytorch/operators/conv2d/model.py

@@ -20,3 +20,6 @@ def name(self):
 
 model = op_conv2d()
 test_input = torch.randn(2, 8, 12, 16)
+test_output = model(test_input)
+print("Input:", test_input)
+print("Onput:", test_output)

e2eshark/pytorch/operators/linear/model.py

@@ -18,3 +18,4 @@ def name(self):
 
 model = op_linear()
 test_input = torch.randn(8, 3)
+test_output = model(test_input)