07_Datasets_S3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "def test_s3_read_singlemachine(worker_index, \n",
    "                               num_workers, \n",
    "                               data_dir, \n",
    "                               num_readers,\n",
    "                               shuffle_buffer_size, \n",
    "                               batch_size, \n",
    "                               num_prefetch_batches, \n",
    "                               num_steps):\n",
    "\n",
    "    os.environ[\"S3_REGION\"] = 'us-east-1' # region of the dataset\n",
    "    filename_pattern = os.path.join(s3_data_bucket, 'train-*')\n",
    "    print(\"Reading training data from files matching glob pattern %s\" % filename_pattern)\n",
    "    d = tf.data.Dataset.list_files(filename_pattern)\n",
    "    d = d.shard(num_workers, worker_index)\n",
    "    d = d.repeat()\n",
    "    d = d.shuffle(shuffle_buffer_size)\n",
    "    d = d.repeat()\n",
    "    d = d.interleave(tf.data.TextLineDataset,\n",
    "                     cycle_length=num_readers, block_length=1)\n",
    "    d = d.batch(batch_size)\n",
    "    d = d.prefetch(num_prefetch_batches)\n",
    "    iterator = d.make_initializable_iterator()\n",
    "    it_op = iterator.get_next()\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(iterator.initializer)\n",
    "        for i in range(num_steps):\n",
    "              res = sess.run(it_op)\n",
    "              if i % 30 == 0:\n",
    "                    print(\"Finished step %s, time: %s, result length: %s\" % (i, datetime.now().strftime(\"%H:%M:%S %D UTC\"), len(res)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "s3_data_bucket = \"s3://datapalooza/mnist\"\n",
    "\n",
    "test_s3_read_singlemachine(worker_index=0, \n",
    "                           num_workers=20, \n",
    "                           data_dir=s3_data_bucket, \n",
    "                           num_readers=10, \n",
    "                           shuffle_buffer_size=32, \n",
    "                           batch_size=32, \n",
    "                           num_prefetch_batches=1, \n",
    "                           num_steps=int(100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}