feature: Add release to PyPI. Change package name to sagemaker-tensorflow-training. (aws#277)

nadiaya · web-flow · commit ff6a810fc154 · 2020-02-11T18:02:43.000-08:00
* feature: Add release to PyPI. Change package name to sagemaker-tensroflow-training.

* Update latest dockerfiles to use correct binary.

* Fix typo.
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ dist
 **/*.egg-info
 .DS_Store
 .idea/
+.cache/
 *.iml
 **/.ipynb_checkpoints
 **/.python-version
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,10 @@
+recursive-include src/sagemaker_tensorflow_container *
+
 include VERSION
 include LICENSE
-include README.md
+include README.rst
+
+prune test
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
diff --git a/buildspec-deploy.yml b/buildspec-deploy.yml
@@ -0,0 +1,9 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      - PACKAGE_FILE="$CODEBUILD_SRC_DIR_ARTIFACT_1/sagemaker_tensorflow_training-*.tar.gz"
+
+      # publish to pypi
+      - publish-pypi-package $PACKAGE_FILE
diff --git a/buildspec-release.yml b/buildspec-release.yml
@@ -1,92 +1,34 @@
 version: 0.2
 
-env:
-  variables:
-    FRAMEWORK_VERSION: '1.15.0'
-    GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
-    SETUP_FILE: 'setup_cmds.sh'
-    SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]'
-
 phases:
-  pre_build:
-    commands:
-      - start-dockerd
-      - ACCOUNT=$(aws --region $AWS_DEFAULT_REGION sts --endpoint-url https://sts.$AWS_DEFAULT_REGION.amazonaws.com get-caller-identity --query 'Account' --output text)
-
   build:
     commands:
       # prepare the release (update versions, changelog etc.)
-      - git-release --min-version 2.0.0 --prepare
+      - git-release --prepare
 
-      # run linter
-      - tox -e flake8
+      # run linters
+      - tox -e flake8,twine
 
       # run unit tests
-      - tox -e py36,py27 test/unit
-
-      # Create pip archive
-      - build_dir="docker/$FRAMEWORK_VERSION"
-      - python3 setup.py sdist
-      - tar_name=$(ls dist)
-      - cp dist/$tar_name $build_dir
-
-      # Build all images
-      - python3 scripts/build_all.py --account $ACCOUNT --region $AWS_DEFAULT_REGION
+      - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
+        AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
+        tox -e py27,py36 -- test/unit
 
-      # Publish all images
-      - python3 scripts/publish_all.py --account $ACCOUNT --region $AWS_DEFAULT_REGION
+      # run local integ tests
+      #- $(aws ecr get-login --no-include-email --region us-west-2)
+      #- IGNORE_COVERAGE=- tox -e py27,py36 -- test/integ/local
 
-      - base_name="$ACCOUNT.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/sagemaker-tensorflow-scriptmode"
-      # run local cpu integ tests
-      - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
-      - IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --docker-base-name $base_name --framework-version $FRAMEWORK_VERSION --processor cpu
-      - IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --docker-base-name $base_name --py-version 2 --framework-version $FRAMEWORK_VERSION --processor cpu
+      # run sagemaker integ tests
+      #- IGNORE_COVERAGE=- tox -e py27,py36 -- test/integ/sagemaker
 
-      # launch remote gpu instance
-      - prefix='ml.'
-      - instance_type=${GPU_INSTANCE_TYPE#"$prefix"}
-      - create-key-pair
-      - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
-      - printf "$SETUP_CMDS" > $SETUP_FILE
-
-      - py2_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --docker-base-name $base_name --py-version 2 --framework-version $FRAMEWORK_VERSION --processor gpu"
-      - remote-test --github-repo sagemaker-tensorflow-container --setup-file $SETUP_FILE --branch script-mode --test-cmd "$py2_cmd"
-      - py3_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --docker-base-name $base_name --framework-version $FRAMEWORK_VERSION --processor gpu"
-      - remote-test --github-repo sagemaker-tensorflow-container --branch script-mode --test-cmd "$py3_cmd" --skip-setup
-
-      - IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n 24 --region $AWS_DEFAULT_REGION --account-id 142577830533 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version $FRAMEWORK_VERSION --processor cpu,gpu --py-version 2,3
-
-      - |
-        echo '[{
-          "repository": "sagemaker-tensorflow-scriptmode",
-          "tags": [{
-            "source": "1.15.0-cpu-py2",
-            "dest": ["1.15.0-cpu-py2", "1.15-cpu-py2", "1.15.0-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
-          },{
-            "source": "1.15.0-cpu-py3",
-            "dest": ["1.15.0-cpu-py3", "1.15-cpu-py3", "1.15.0-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
-          },{
-            "source": "1.15.0-gpu-py2",
-            "dest": ["1.15.0-gpu-py2", "1.15-gpu-py2", "1.15.0-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
-          },{
-            "source": "1.15.0-gpu-py3",
-            "dest": ["1.15.0-gpu-py3", "1.15-gpu-py3", "1.15.0-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
-          }],
-          "test": [
-            "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor cpu --py-version 2,3",
-            "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor gpu --py-version 2,3"
-          ]
-        }]' > deployments.json
+      # generate the distribution package
+      - python3 setup.py sdist
 
       # publish the release to github
       - git-release --publish
 
-    finally:
-      # shut down remote gpu instance
-      - cleanup-gpu-instances
-      - cleanup-key-pairs
-
 artifacts:
   files:
-    - deployments.json
+    - dist/sagemaker_tensorflow_training-*.tar.gz
   name: ARTIFACT_1
+  discard-paths: yes
diff --git a/docker/1.15.0/py2/Dockerfile.cpu b/docker/1.15.0/py2/Dockerfile.cpu
@@ -22,7 +22,7 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_container*.tar.gz
+ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15/AmazonLinux/cpu/final/tensorflow-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl
 
 RUN apt-get update \
diff --git a/docker/1.15.0/py2/Dockerfile.gpu b/docker/1.15.0/py2/Dockerfile.gpu
@@ -19,7 +19,7 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_container*.tar.gz
+ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15/AmazonLinux/gpu/final/tensorflow_gpu-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl
 
 RUN apt-get update \
diff --git a/docker/1.15.0/py3/Dockerfile.cpu b/docker/1.15.0/py3/Dockerfile.cpu
@@ -22,7 +22,7 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_container*.tar.gz
+ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15/AmazonLinux/cpu/final/tensorflow-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl
 
 RUN apt-get update \
diff --git a/docker/1.15.0/py3/Dockerfile.gpu b/docker/1.15.0/py3/Dockerfile.gpu
@@ -19,7 +19,7 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_container*.tar.gz
+ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15/AmazonLinux/gpu/final/tensorflow_gpu-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl
 
 RUN apt-get update \
diff --git a/setup.py b/setup.py
@@ -29,7 +29,7 @@ def read_version():
 
 
 setup(
-    name='sagemaker_tensorflow_container',
+    name='sagemaker_tensorflow_training',
     version=read_version(),
     description='Open source library for creating '
                 'TensorFlow containers to run on Amazon SageMaker.',
diff --git a/tox.ini b/tox.ini
@@ -71,3 +71,14 @@ deps =
     flake8-future-import
     flake8-import-order
 commands = flake8
+
+
+[testenv:twine]
+basepython = python3
+# https://github.com/pypa/twine/blob/master/docs/changelog.rst
+deps =
+    twine>=1.12.0
+# https://packaging.python.org/guides/making-a-pypi-friendly-readme/#validating-restructuredtext-markup
+commands =
+    python setup.py sdist
+    twine check dist/*.tar.gz
