download mini data in OSS CI, and enable more tests

Summary: Pull Request resolved: facebookresearch#3689

Differential Revision: D32410043

Pulled By: ppwwyyxx

fbshipit-source-id: 6e75bddfb6899a9a294a3f2d927cdf806db54120

.circleci/config.yml

@@ -45,6 +45,8 @@ version_parameters: &version_parameters
     TORCHVISION_VERSION: << parameters.torchvision_version >>
     PYTORCH_INDEX: << parameters.pytorch_index >>
     PYTHON_VERSION: << parameters.python_version>>
+    # point datasets to ~/.torch so it's cached in CI
+    DETECTRON2_DATASETS: ~/.torch/datasets
 
 # -------------------------------------------------------------------------------------
 # Re-usable commands
@@ -105,10 +107,13 @@ install_linux_dep: &install_linux_dep
   - run:
       name: Install Dependencies
       command: |
+        # disable crash coredump, so unittests fail fast
+        sudo systemctl stop apport.service
         # install from github to get latest; install iopath first since fvcore depends on it
         pip install --progress-bar off -U 'git+https://github.com/facebookresearch/iopath'
         pip install --progress-bar off -U 'git+https://github.com/facebookresearch/fvcore'
-        pip install --progress-bar off ninja opencv-python-headless pytest-xdist tensorboard pycocotools
+        # Don't use pytest-xdist: cuda tests are unstable under multi-process workers.
+        pip install --progress-bar off ninja opencv-python-headless pytest tensorboard pycocotools
         pip install --progress-bar off torch==$PYTORCH_VERSION -f $PYTORCH_INDEX
         if [[ "$TORCHVISION_VERSION" == "master" ]]; then
           pip install git+https://github.com/pytorch/vision.git
@@ -123,16 +128,17 @@ install_detectron2: &install_detectron2
   - run:
       name: Install Detectron2
       command: |
-        # Remove if it's in cache
+        # Remove first, in case it's in the CI cache
         pip uninstall -y detectron2
         pip install --progress-bar off -e .[all]
         python -m detectron2.utils.collect_env
+        ./datasets/prepare_for_tests.sh
 
 run_unittests: &run_unittests
   - run:
       name: Run Unit Tests
       command: |
-        pytest -n 1 -v --durations=15 tests  # parallel causes some random failures
+        pytest -v --durations=15 tests  # parallel causes some random failures
 
 # -------------------------------------------------------------------------------------
 # Jobs to run

.github/workflows/workflow.yml

@@ -45,6 +45,9 @@ jobs:
             torchvision: "0.10"
           - torch: "1.10"
             torchvision: "0.11.1"
+    env:
+      # point datasets to ~/.torch so it's cached by CI
+      DETECTRON2_DATASETS: ~/.torch/datasets
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -73,5 +76,6 @@ jobs:
         run: |
           CC=clang CXX=clang++ python -m pip install -e .[all]
           python -m detectron2.utils.collect_env
+          ./datasets/prepare_for_tests.sh
       - name: Run unittests
-        run: python -m pytest -n 4 -v tests/
+        run: python -m pytest -n 4 --durations=15 -v tests/

datasets/prepare_for_tests.sh

@@ -1,22 +1,31 @@
 #!/bin/bash -e
 # Copyright (c) Facebook, Inc. and its affiliates.
 
-# Download some files needed for running tests.
+# Download the mini dataset (coco val2017_100, with only 100 images)
+# to be used in unittests & integration tests.
 
 cd "${0%/*}"
 
 BASE=https://dl.fbaipublicfiles.com/detectron2
-mkdir -p coco/annotations
+ROOT=${DETECTRON2_DATASETS:-./}
+ROOT=${ROOT/#\~/$HOME}   # expand ~ to HOME
+mkdir -p $ROOT/coco/annotations
 
 for anno in instances_val2017_100 \
-  person_keypoints_val2017_100 \
-  instances_minival2014_100 \
-  person_keypoints_minival2014_100; do
+  person_keypoints_val2017_100 ; do
 
-  dest=coco/annotations/$anno.json
+  dest=$ROOT/coco/annotations/$anno.json
   [[ -s $dest ]] && {
     echo "$dest exists. Skipping ..."
   } || {
     wget $BASE/annotations/coco/$anno.json -O $dest
   }
 done
+
+dest=$ROOT/coco/val2017_100.tgz
+[[ -d $ROOT/coco/val2017 ]] && {
+  echo "$ROOT/coco/val2017 exists. Skipping ..."
+} || {
+  wget $BASE/annotations/coco/val2017_100.tgz -O $dest
+  tar xzf $dest -C $ROOT/coco/ && rm -f $dest
+}

detectron2/data/datasets/builtin.py

@@ -36,7 +36,6 @@
     "coco_2014_train": ("coco/train2014", "coco/annotations/instances_train2014.json"),
     "coco_2014_val": ("coco/val2014", "coco/annotations/instances_val2014.json"),
     "coco_2014_minival": ("coco/val2014", "coco/annotations/instances_minival2014.json"),
-    "coco_2014_minival_100": ("coco/val2014", "coco/annotations/instances_minival2014_100.json"),
     "coco_2014_valminusminival": (
         "coco/val2014",
         "coco/annotations/instances_valminusminival2014.json",
@@ -62,10 +61,6 @@
         "coco/val2014",
         "coco/annotations/person_keypoints_valminusminival2014.json",
     ),
-    "keypoints_coco_2014_minival_100": (
-        "coco/val2014",
-        "coco/annotations/person_keypoints_minival2014_100.json",
-    ),
     "keypoints_coco_2017_train": (
         "coco/train2017",
         "coco/annotations/person_keypoints_train2017.json",
@@ -255,7 +250,7 @@ def register_all_ade20k(root):
 # Internally at fb, we register them elsewhere
 if __name__.endswith(".builtin"):
     # Assume pre-defined datasets live in `./datasets`.
-    _root = os.getenv("DETECTRON2_DATASETS", "datasets")
+    _root = os.path.expanduser(os.getenv("DETECTRON2_DATASETS", "datasets"))
     register_all_coco(_root)
     register_all_lvis(_root)
     register_all_cityscapes(_root)

detectron2/utils/testing.py

@@ -55,7 +55,9 @@ def get_sample_coco_image(tensor=True):
             raise FileNotFoundError()
     except IOError:
         # for public CI to run
-        file_name = "http://images.cocodataset.org/train2017/000000000009.jpg"
+        file_name = PathManager.get_local_path(
+            "http://images.cocodataset.org/train2017/000000000009.jpg"
+        )
     ret = read_image(file_name, format="BGR")
     if tensor:
         ret = torch.from_numpy(np.ascontiguousarray(ret.transpose(2, 0, 1)))

projects/DensePose/setup.py

@@ -4,7 +4,7 @@
 
 try:
     import torch  # noqa: F401
-except ImportError:
+except ImportError as e:
     raise Exception(
         """
 You must install PyTorch prior to installing DensePose:
@@ -13,7 +13,7 @@
 For more information:
     https://pytorch.org/get-started/locally/
     """
-    )
+    ) from e
 
 
 def get_detectron2_current_version():

tests/data/test_coco_evaluation.py

@@ -123,7 +123,6 @@ def test_fast_eval(self):
                         msg = "%s: comparing COCO APIs, %s differs by %f" % (name, k, abs_diff)
                         self.assertTrue(abs_diff < 1e-4, msg=msg)
 
-    @unittest.skipIf(os.environ.get("CI"), "Require COCO data.")
     def test_unknown_category(self):
         dataset = "coco_2017_val_100"
         evaluator = COCOEvaluator(dataset)

tests/data/test_dataset.py

@@ -73,7 +73,6 @@ def test_pickleability(self):
         self.assertEqual(ds[0], 2)
 
 
-@unittest.skipIf(os.environ.get("CI"), "Skipped OSS testing due to COCO data requirement.")
 class TestDataLoader(unittest.TestCase):
     def _get_kwargs(self):
         # get kwargs of build_detection_train_loader

tests/test_engine.py

@@ -84,7 +84,6 @@ def test_writer_hooks(self):
 
             self.assertIn("eta: 0:00:00", logs.output[-1], "Last ETA must be 0!")
 
-    @unittest.skipIf(os.environ.get("CI"), "Require COCO data.")
     def test_default_trainer(self):
         # TODO: this test requires manifold access, so changed device to CPU. see: T88318502
         cfg = get_cfg()

tests/test_export_caffe2.py

@@ -8,19 +8,19 @@
 import torch
 
 from detectron2 import model_zoo
+from detectron2.export import Caffe2Model, Caffe2Tracer, add_export_config
 from detectron2.utils.logger import setup_logger
 from detectron2.utils.testing import get_sample_coco_image
 
 
-@unittest.skipIf(os.environ.get("CI"), "Require COCO data and model zoo.")
+# TODO: this test requires manifold access, see: T88318502
+# Running it on CircleCI causes crash, not sure why.
+@unittest.skipIf(os.environ.get("CIRCLECI"), "Caffe2 tests crash on CircleCI.")
 class TestCaffe2Export(unittest.TestCase):
     def setUp(self):
         setup_logger()
 
     def _test_model(self, config_path, device="cpu"):
-        # requires extra dependencies
-        from detectron2.export import Caffe2Model, add_export_config, Caffe2Tracer
-
         cfg = model_zoo.get_config(config_path)
         add_export_config(cfg)
         cfg.MODEL.DEVICE = device
@@ -29,27 +29,25 @@ def _test_model(self, config_path, device="cpu"):
         inputs = [{"image": get_sample_coco_image()}]
         tracer = Caffe2Tracer(cfg, model, copy.deepcopy(inputs))
 
-        c2_model = tracer.export_caffe2()
-
         with tempfile.TemporaryDirectory(prefix="detectron2_unittest") as d:
-            c2_model.save_protobuf(d)
-            c2_model.save_graph(os.path.join(d, "test.svg"), inputs=copy.deepcopy(inputs))
+            if not os.environ.get("CI"):
+                # This requires onnx, which is not yet available on public CI
+                c2_model = tracer.export_caffe2()
+                c2_model.save_protobuf(d)
+                c2_model.save_graph(os.path.join(d, "test.svg"), inputs=copy.deepcopy(inputs))
 
-            c2_model = Caffe2Model.load_protobuf(d)
-            c2_model(inputs)[0]["instances"]
+                c2_model = Caffe2Model.load_protobuf(d)
+                c2_model(inputs)[0]["instances"]
 
             ts_model = tracer.export_torchscript()
             ts_model.save(os.path.join(d, "model.ts"))
 
     def testMaskRCNN(self):
-        # TODO: this test requires manifold access, see: T88318502
         self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
 
     @unittest.skipIf(not torch.cuda.is_available(), "CUDA not available")
     def testMaskRCNNGPU(self):
-        # TODO: this test requires manifold access, see: T88318502
         self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", device="cuda")
 
     def testRetinaNet(self):
-        # TODO: this test requires manifold access, see: T88318502
         self._test_model("COCO-Detection/retinanet_R_50_FPN_3x.yaml")

tests/test_export_torchscript.py

@@ -18,24 +18,30 @@
 from detectron2.modeling.postprocessing import detector_postprocess
 from detectron2.modeling.roi_heads import KRCNNConvDeconvUpsampleHead
 from detectron2.structures import Boxes, Instances
+from detectron2.utils.env import TORCH_VERSION
 from detectron2.utils.testing import (
     assert_instances_allclose,
     convert_scripted_instances,
     get_sample_coco_image,
     random_boxes,
 )
 
-
 """
 https://detectron2.readthedocs.io/tutorials/deployment.html
 contains some explanations of this file.
 """
 
+SLOW_PUBLIC_CPU_TEST = unittest.skipIf(
+    os.environ.get("CI") and not torch.cuda.is_available(),
+    "The test is too slow on CPUs and will be executed on CircleCI's GPU jobs.",
+)
+
 
 class TestScripting(unittest.TestCase):
     def testMaskRCNNFPN(self):
         self._test_rcnn_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
 
+    @SLOW_PUBLIC_CPU_TEST
     def testMaskRCNNC4(self):
         self._test_rcnn_model("COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml")
 
@@ -100,13 +106,16 @@ def inference_func(model, image):
 
         self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", inference_func)
 
+    @SLOW_PUBLIC_CPU_TEST
     def testMaskRCNNC4(self):
         def inference_func(model, image):
             inputs = [{"image": image}]
             return model.inference(inputs, do_postprocess=False)[0]
 
         self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml", inference_func)
 
+    # bug fixed by https://github.com/pytorch/pytorch/pull/67734
+    @unittest.skipIf(TORCH_VERSION == (1, 10) and os.environ.get("CI"), "1.10 has bugs.")
     def testRetinaNet(self):
         def inference_func(model, image):
             return model.forward([{"image": image}])[0]["instances"]