|
39 | 39 | # "bert_base_uncased": {"model": cm.BertModule(), "path": "trace"}, |
40 | 40 | } |
41 | 41 |
|
| 42 | +# torchvision checkpoints that more than one test file in the same suite asks for |
| 43 | +# with pretrained=True: resnet18 in tests/py/ts/{api,models,integrations} and |
| 44 | +# mobilenet_v2 in tests/py/ts/api. Checkpoints that only one file uses cannot be |
| 45 | +# downloaded twice at once, so they are left to that file. IMAGENET1K_V1 is the |
| 46 | +# set of weights pretrained=True selects. |
| 47 | +PRETRAINED_WEIGHTS = ( |
| 48 | + "MobileNet_V2_Weights", |
| 49 | + "ResNet18_Weights", |
| 50 | +) |
| 51 | + |
| 52 | + |
| 53 | +def download_pretrained_weights(): |
| 54 | + """Fetch the shared torchvision checkpoints before pytest forks its workers. |
| 55 | +
|
| 56 | + pytest runs one test file per xdist worker (--dist=loadfile), and several |
| 57 | + files in the same suite ask torchvision for the same checkpoint, so two |
| 58 | + workers can download one URL at the same time. torch.hub finishes a download |
| 59 | + with shutil.move, and on Windows that is a plain copy whenever the |
| 60 | + destination already exists, so the second worker rewrites the file while the |
| 61 | + first one is reading it and torch.load fails on truncated pickle data. |
| 62 | + Downloading here, in one process before the workers start, leaves them with |
| 63 | + nothing to do but read. |
| 64 | + """ |
| 65 | + if importlib.util.find_spec("torchvision") is None: |
| 66 | + print("torchvision is not installed, skipping pretrained weight download") |
| 67 | + return |
| 68 | + |
| 69 | + import torchvision.models as tv_models |
| 70 | + |
| 71 | + for name in PRETRAINED_WEIGHTS: |
| 72 | + weights = getattr(tv_models, name).IMAGENET1K_V1 |
| 73 | + print("Downloading {}".format(weights)) |
| 74 | + try: |
| 75 | + # check_hash is what torchvision itself passes, and it is the only |
| 76 | + # thing that stops a bad download from being cached and reused. |
| 77 | + weights.get_state_dict(progress=False, check_hash=True) |
| 78 | + except Exception as e: |
| 79 | + # Warming the cache is an optimization, so a failure here must not |
| 80 | + # take the whole suite down. The tests still fetch what they need. |
| 81 | + print("Could not pre-download {}: {}".format(name, e)) |
| 82 | + |
42 | 83 |
|
43 | 84 | def get(n, m, manifest): |
44 | 85 | print("Downloading {}".format(n)) |
@@ -93,6 +134,8 @@ def download_models(version_matches, manifest): |
93 | 134 |
|
94 | 135 | def main(): |
95 | 136 |
|
| 137 | + download_pretrained_weights() |
| 138 | + |
96 | 139 | manifest = None |
97 | 140 | version_matches = False |
98 | 141 | manifest_exists = False |
|
0 commit comments