|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# -*- coding: utf-8 -*- |
| 3 | +# Standard Library |
| 4 | +import glob |
| 5 | +import logging |
| 6 | +import os |
| 7 | +import platform |
| 8 | +import shutil |
| 9 | +import sys |
| 10 | +import tempfile |
| 11 | +import time |
| 12 | +import urllib |
| 13 | +import urllib.request |
| 14 | +from subprocess import check_call |
| 15 | +from zipfile import ZipFile |
| 16 | + |
| 17 | + |
| 18 | +def script_name() -> str: |
| 19 | + """:returns: script name with leading paths removed""" |
| 20 | + return os.path.split(sys.argv[0])[1] |
| 21 | + |
| 22 | + |
| 23 | +def configure_logging(): |
| 24 | + logging.getLogger().setLevel(logging.INFO) |
| 25 | + logging.basicConfig(format="{}: %(asctime)sZ %(levelname)s %(message)s".format(script_name())) |
| 26 | + logging.Formatter.converter = time.gmtime |
| 27 | + |
| 28 | + |
| 29 | +def _get_system_details(): |
| 30 | + return platform.system() |
| 31 | + |
| 32 | + |
| 33 | +def _get_machine_details(): |
| 34 | + return platform.machine() |
| 35 | + |
| 36 | + |
| 37 | +def get_protoc_download_url(): |
| 38 | + """ |
| 39 | + Returns an archive with the binary protoc distro for the platform |
| 40 | + """ |
| 41 | + |
| 42 | + system = _get_system_details() |
| 43 | + machine = _get_machine_details() |
| 44 | + if system == "Darwin": |
| 45 | + archive_url = "https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-osx-x86_64.zip" |
| 46 | + logging.info("Downloading protoc for Darwin: %s", archive_url) |
| 47 | + elif system == "Linux": |
| 48 | + archive_url = f"https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-{machine}.zip" |
| 49 | + logging.info("Downloading protoc for Linux: %s", archive_url) |
| 50 | + else: |
| 51 | + system = platform.system() |
| 52 | + raise RuntimeError( |
| 53 | + f"Could not find protoc for System: {system} Machine: {machine}.\ |
| 54 | + Please install it manually by running sh protoc_downloader.sh" |
| 55 | + ) |
| 56 | + return archive_url |
| 57 | + |
| 58 | + |
| 59 | +def get_protoc(): |
| 60 | + """make sure protoc is available, otherwise download it and return a tuple with the protoc |
| 61 | + binary and a temporary dir if it needed to be downloaded""" |
| 62 | + if shutil.which("protoc"): |
| 63 | + return shutil.which("protoc"), None |
| 64 | + |
| 65 | + archive_url = get_protoc_download_url() |
| 66 | + (fname, headers) = urllib.request.urlretrieve(archive_url) |
| 67 | + tmpdir = tempfile.mkdtemp(prefix="protoc_smdebug") |
| 68 | + with ZipFile(fname, "r") as zipf: |
| 69 | + zipf.extractall(tmpdir) |
| 70 | + protoc_bin = os.path.join(tmpdir, "bin", "protoc") |
| 71 | + |
| 72 | + # Make the binary executable |
| 73 | + os.chmod(protoc_bin, 0o755) |
| 74 | + return protoc_bin, tmpdir |
| 75 | + |
| 76 | + |
| 77 | +def compile_protobuf(): |
| 78 | + """ |
| 79 | + Compile protobuf files for smdebug |
| 80 | + """ |
| 81 | + logging.info("Compile protobuf") |
| 82 | + logging.info("================") |
| 83 | + (protoc_bin, tmpdir) = get_protoc() |
| 84 | + cmd = [protoc_bin] |
| 85 | + proto_files = glob.glob("smdebug/core/tfevent/proto/*.proto") |
| 86 | + cmd.extend(proto_files) |
| 87 | + cmd.append("--python_out=.") |
| 88 | + logging.info("Call to protoc: %s", " ".join(cmd)) |
| 89 | + check_call(cmd) |
| 90 | + if tmpdir: |
| 91 | + shutil.rmtree(tmpdir, ignore_errors=True) |
0 commit comments