Remove upload to GCS (#387)

* Remove upload to GCS

Signed-off-by: Abolfazl Shahbazi <[email protected]>

* remove gcs option from the shell script

Signed-off-by: Abolfazl Shahbazi <[email protected]>

Author: ashahba

datasets/imagenet/README.md

@@ -30,8 +30,7 @@ logging in.
    source tf_env/bin/activate
    pip install --upgrade pip==19.3.1
    pip install intel-tensorflow
-   pip install google-cloud-storage
-   pip uninstall urllib3 && pip install urllib3
+   pip install -I urllib3
    pip install wget
    ```
 

datasets/imagenet/imagenet_to_gcs.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 #
 
-
 # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,13 +29,9 @@
 # limitations under the License.
 # ==============================================================================
 
-r"""Script to process the Imagenet dataset and upload to gcs.
+r"""Script to process the Imagenet dataset
 
 To run the script setup a virtualenv with the following libraries installed.
-- `gcloud`: Follow the instructions on
-  [cloud SDK docs](https://cloud.google.com/sdk/downloads) followed by
-  installing the python api using `pip install gcloud`.
-- `google-cloud-storage`: Install with `pip install google-cloud-storage`
 - `tensorflow`: Install with `pip install tensorflow`
 
 Once you have all the above libraries setup, you should register on the
@@ -46,16 +41,6 @@
 - Validation Images: validation/ILSVRC2012_val_00000001.JPEG
 - Validation Labels: synset_labels.txt
 
-To run the script to preprocess the raw dataset as TFRecords and upload to gcs,
-run the following command:
-
-```
-python imagenet_to_gcs.py \
-  --project="TEST_PROJECT" \
-  --gcs_output_path="gs://TEST_BUCKET/IMAGENET_DIR" \
-  --raw_data_dir="path/to/imagenet"
-```
-
 """
 
 import math
@@ -72,19 +57,11 @@
 tf.disable_eager_execution()
 
 
-from google.cloud import storage
-
-flags.DEFINE_string(
-    'project', None, 'Google cloud project id for uploading the dataset.')
-flags.DEFINE_string(
-    'gcs_output_path', None, 'GCS path for uploading the dataset.')
 flags.DEFINE_string(
     'local_scratch_dir', None, 'Scratch directory path for temporary files.')
 flags.DEFINE_string(
     'raw_data_dir', None, 'Directory path for raw Imagenet dataset. '
     'Should have train and validation subdirectories inside it.')
-flags.DEFINE_boolean(
-    'gcs_upload', True, 'Set to false to not upload to gcs.')
 
 FLAGS = flags.FLAGS
 
@@ -425,50 +402,9 @@ def make_shuffle_idx(n):
   return training_records, validation_records
 
 
-def upload_to_gcs(training_records, validation_records):
-  """Upload TF-Record files to GCS, at provided path."""
-
-  # Find the GCS bucket_name and key_prefix for dataset files
-  path_parts = FLAGS.gcs_output_path[5:].split('/', 1)
-  bucket_name = path_parts[0]
-  if len(path_parts) == 1:
-    key_prefix = ''
-  elif path_parts[1].endswith('/'):
-    key_prefix = path_parts[1]
-  else:
-    key_prefix = path_parts[1] + '/'
-
-  client = storage.Client(project=FLAGS.project)
-  bucket = client.get_bucket(bucket_name)
-
-  def _upload_files(filenames):
-    """Upload a list of files into a specifc subdirectory."""
-    for i, filename in enumerate(sorted(filenames)):
-      blob = bucket.blob(key_prefix + os.path.basename(filename))
-      blob.upload_from_filename(filename)
-      if not i % 20:
-        logging.info('Finished uploading file: %s', filename)
-
-  # Upload training dataset
-  logging.info('Uploading the training data.')
-  _upload_files(training_records)
-
-  # Upload validation dataset
-  logging.info('Uploading the validation data.')
-  _upload_files(validation_records)
-
-
 def main(argv):  # pylint: disable=unused-argument
   logging.set_verbosity(logging.INFO)
 
-  if FLAGS.gcs_upload and FLAGS.project is None:
-    raise ValueError('GCS Project must be provided.')
-
-  if FLAGS.gcs_upload and FLAGS.gcs_output_path is None:
-    raise ValueError('GCS output path must be provided.')
-  elif FLAGS.gcs_upload and not FLAGS.gcs_output_path.startswith('gs://'):
-    raise ValueError('GCS output path must start with gs://')
-
   if FLAGS.local_scratch_dir is None:
     raise ValueError('Scratch directory path must be provided.')
 
@@ -482,10 +418,6 @@ def main(argv):  # pylint: disable=unused-argument
   # Convert the raw data into tf-records
   training_records, validation_records = convert_to_tf_records(raw_data_dir)
 
-  # Upload to GCS
-  if FLAGS.gcs_upload:
-    upload_to_gcs(training_records, validation_records)
-
 
 if __name__ == '__main__':
   app.run(main)

datasets/imagenet/imagenet_to_tfrecords.sh

@@ -71,8 +71,7 @@ if [[ ! -f "${WORKDIR}/imagenet_to_gcs.py" ]]; then
 fi
 python3 ${WORKDIR}/imagenet_to_gcs.py \
   --raw_data_dir=$IMAGENET_HOME \
-  --local_scratch_dir=$IMAGENET_HOME/tf_records \
-  --nogcs_upload
+  --local_scratch_dir=$IMAGENET_HOME/tf_records
 
 # Clean up the user work directory, remove the previously downloaded script `imagenet_to_gcs.py`
 rm ${WORKDIR}/imagenet_to_gcs.py