Using-Cactus/Checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<h1>Checkpointing and Restarting</h1>\n",
    "There are a number of reasons why you might not be able to run a simulation all at once.\n",
    "Your simulation may take a week to run, but the maximum time in the queue might be only a few hours. Alternatively,\n",
    "your job might crash after running several hours but only be half finished. Was the preceeding time simply lost? Not\n",
    "if you make use of the checkpoint restart feature of Cactus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd ~/CactusFW2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we're going to start by creating a file named wave3.par. It will differ only slightly from wave.par used in the\n",
    "compiling and running notebook. You'll find the new bits that relate to checkpointing at the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile wave3.par\n",
    "\n",
    "#Reorder the parameters for easy comparison to the input.txt in example 3\n",
    "ActiveThorns = \"\n",
    "  CoordBase FunWave FunwaveCoord CartGrid3D Carpet CarpetIOASCII\n",
    "  CartGrid3D IOUtil CarpetIOBasic CarpetSlab Boundary SymBase MoL\n",
    "  CarpetReduce LocalReduce InitBase CarpetLib LoopControl Tridiagonal\n",
    "  CarpetIOScalar \"\n",
    "\n",
    "#----------------------------------------------------\n",
    "# Flesh and CCTK parameters\n",
    "#----------------------------------------------------\n",
    "\n",
    "# flesh\n",
    "Cactus::cctk_run_title = \"Test Run\"\n",
    "Cactus::cctk_show_schedule = \"yes\"\n",
    "Cactus::cctk_itlast = 300\n",
    "Cactus::allow_mixeddim_gfs = \"yes\"\n",
    "\n",
    "# CartGrid3D\n",
    "CartGrid3D::type = \"coordbase\"\n",
    "CartGrid3D::avoid_origin = \"no\"\n",
    "CoordBase::domainsize = \"minmax\"\n",
    "CoordBase::spacing    = \"gridspacing\"\n",
    "CoordBase::xmin =  0\n",
    "CoordBase::xmax =  30\n",
    "CoordBase::ymin =  0\n",
    "CoordBase::ymax =  30\n",
    "CoordBase::zmin =  0.0\n",
    "CoordBase::zmax =  0.0\n",
    "CoordBase::dx   =  0.25\n",
    "CoordBase::dy   =  0.25\n",
    "\n",
    "CoordBase::boundary_size_x_lower     = 3\n",
    "CoordBase::boundary_size_x_upper     = 3\n",
    "CoordBase::boundary_size_y_lower     = 3\n",
    "CoordBase::boundary_size_y_upper     = 3\n",
    "CoordBase::boundary_size_z_lower     = 0\n",
    "CoordBase::boundary_size_z_upper     = 0\n",
    "CoordBase::boundary_shiftout_x_lower = 1\n",
    "CoordBase::boundary_shiftout_x_upper = 1\n",
    "CoordBase::boundary_shiftout_y_lower = 1\n",
    "CoordBase::boundary_shiftout_y_upper = 1\n",
    "CoordBase::boundary_shiftout_z_lower = 1\n",
    "CoordBase::boundary_shiftout_z_upper = 1\n",
    "\n",
    "# Carpet\n",
    "Carpet::domain_from_coordbase = \"yes\"\n",
    "Carpet::ghost_size_x = 3\n",
    "Carpet::ghost_size_y = 3\n",
    "Carpet::ghost_size_z = 1\n",
    "carpet::adaptive_stepsize = yes\n",
    "\n",
    "# MoL\n",
    "MoL::ODE_Method = \"RK3\"\n",
    "MoL::disable_prolongation        = \"yes\"\n",
    "\n",
    "# the output dir will be named after the parameter file name\n",
    "IO::out_dir = $parfile\n",
    "IO::out_fileinfo=\"none\"\n",
    "IOBasic::outInfo_every = 1\n",
    "IOBasic::outInfo_vars = \"FunWave::eta FunWave::u FunWave::v\"\n",
    "\n",
    "#IOASCII::out1D_every = 1\n",
    "#IOASCII::out1d_vars = \"FunWave::eta Funwave::depth\"\n",
    "CarpetIOASCII::compact_format = false\n",
    "IOASCII::out2D_every = 30\n",
    "IOASCII::out2D_xyplane_z = 0\n",
    "IOASCII::out2D_vars = \"FunWave::eta FunWave::u FunWave::v\"\n",
    "IOASCII::out2D_xz = \"no\"\n",
    "IOASCII::out2D_yz = \"no\"\n",
    "IOASCII::output_ghost_points = \"no\"\n",
    "\n",
    "IOScalar::outScalar_every = 1\n",
    "IOScalar::outScalar_vars = \"FunWave::eta FunWave::u FunWave::v\"\n",
    "\n",
    "#& = \"Funwave::eta\"\n",
    "\n",
    "#----------------------------------------------------\n",
    "# Funwave parameters\n",
    "#----------------------------------------------------\n",
    "\n",
    "# Funwave depth \n",
    "FunWave::depth_file_offset_x = 3\n",
    "FunWave::depth_file_offset_y = 3\n",
    "FunWave::depth_type = \"flat\"\n",
    "FunWave::depth_format = \"ele\"\n",
    "FunWave::depth_file = \"/tmp/__depth__.txt\"\n",
    "FunWave::depth_flat = 0.8\n",
    "#Funwave::test_depth_shore_x = 80\n",
    "#Funwave::test_depth_island_x = 40\n",
    "#Funwave::test_depth_island_y = 40\n",
    "FunWave::depth_xslp = 10.0\n",
    "FunWave::depth_slope = 0.05\n",
    "FunWave::dt_size = 0\n",
    "Funwave::generate_test_depth_data = true\n",
    "Funwave::num_wave_components = 1\n",
    "Funwave::wave_component_file = \"/home/sbrandt/workspace/shi_funwave/example_2/fft/wavemk_per_amp_pha.txt\"\n",
    "Funwave::peak_period = 1\n",
    "\n",
    "# import\n",
    "Funwave::time_ramp = 1.0\n",
    "Funwave::delta_wk = 0.5\n",
    "Funwave::dep_wk = 0.45\n",
    "Funwave::xc_wk = 3.0\n",
    "Funwave::ywidth_wk = 10000.0\n",
    "Funwave::tperiod = 1.0\n",
    "Funwave::amp_wk = 0.0232\n",
    "Funwave::theta_wk = 0.0\n",
    "Funwave::freqpeak = 0.2\n",
    "Funwave::freqmin = 0.1\n",
    "Funwave::freqmax = 0.4\n",
    "Funwave::hmo = 1.0\n",
    "Funwave::gammatma = 5.0\n",
    "Funwave::thetapeak = 10.0\n",
    "Funwave::sigma_theta = 15.0\n",
    "\n",
    "# Funwave wind forcing\n",
    "Funwave::wind_force = false\n",
    "Funwave::use_wind_mask = false\n",
    "Funwave::num_time_wind_data = 2\n",
    "Funwave::timewind[0] = 0\n",
    "Funwave::wu[0] = 25\n",
    "Funwave::wv[0] = 50\n",
    "Funwave::timewind[1] = 1000\n",
    "Funwave::wu[1] = 100\n",
    "Funwave::wv[1] = 100\n",
    "Funwave::boundary = funwave\n",
    "\n",
    "# Funwave wave maker\n",
    "FunWave::wavemaker_type = \"ini_gau\"\n",
    "FunWave::xc = 26.5\n",
    "FunWave::yc = 26.9\n",
    "FunWave::amp =  2.0\n",
    "FunWave::wid =  1\n",
    "Funwave::wdep = 0.78\n",
    "Funwave::xwavemaker = 25.0\n",
    "\n",
    "# Funwave sponge \n",
    "FunWave::sponge_on = false\n",
    "FunWave::sponge_west_width = 2.0\n",
    "FunWave::sponge_east_width = 2.0\n",
    "FunWave::sponge_north_width = 0.0\n",
    "FunWave::sponge_south_width = 0.0\n",
    "FunWave::sponge_decay_rate = 0.9\n",
    "FunWave::sponge_damping_magnitude = 5.0\n",
    "\n",
    "# Funwave dispersion (example 3 enables dispersion)\n",
    "FunWave::dispersion_on = \"true\"\n",
    "FunWave::gamma1 = 1.0\n",
    "FunWave::gamma2 = 1.0\n",
    "FunWave::gamma3 = 1.0\n",
    "FunWave::beta_ref = -0.531\n",
    "FunWave::swe_eta_dep = 0.80\n",
    "FunWave::cd = 0.0\n",
    "\n",
    "# Funwave numerics (MoL parameter controls time integration scheme)\n",
    "FunWave::reconstruction_scheme = \"fourth\"\n",
    "FunWave::riemann_solver = \"HLLC\"\n",
    "FunWave::dtfac = 0.5\n",
    "FunWave::froudecap = 10.0\n",
    "FunWave::mindepth = 0.001\n",
    "FunWave::mindepthfrc = 0.001\n",
    "FunWave::enable_masks = \"true\"\n",
    "Funwave::estimate_dt_on = \"true\"\n",
    "\n",
    "FunwaveCoord::spherical_coordinates = false\n",
    "\n",
    "ActiveThorns = \"CarpetIOHDF5\"\n",
    "IOHDF5::out2D_xyplane_z = 0 \n",
    "IOHDF5::out2D_every = 10\n",
    "IOHDF5::out2D_vars = \" \n",
    "  FunWave::eta\n",
    "  FunWave::u\n",
    "  FunWave::v\n",
    "  Grid::Coordinates{out_every=1000000000}\n",
    "\"\n",
    "IOHDF5::out2D_xz = no\n",
    "IOHDF5::out2D_yz = no\n",
    "\n",
    "# Turn checkpointing on\n",
    "IOHDF5::checkpoint = yes\n",
    "\n",
    "# If you have a long running simulation,\n",
    "# you might want to set this to checkpoint\n",
    "# frequently enough that you don't either\n",
    "# spend too much time checkpointing, or (in\n",
    "# the event of a crash) spend too much time\n",
    "# recalculating.\n",
    "IO::checkpoint_every_walltime_hours = 0.5\n",
    "        \n",
    "# If you think you might want to continue\n",
    "# your simulation.\n",
    "IO::checkpoint_on_terminate = yes\n",
    "\n",
    "# This setting tells Cactus to resume from\n",
    "# a checkpoint file if one exists\n",
    "IO::recover = autoprobe\n",
    "\n",
    "# This setting tells Cactus where to write\n",
    "# its checkpoint files.\n",
    "IO::checkpoint_dir = \"..\"\n",
    "# This setting tells Cactus where to read\n",
    "# its checkpoint files from.\n",
    "IO::recover_dir = \"..\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we studied compiling and running cactus we used \"run-submit\" to run our job interactively. If you are going to\n",
    "make use of a supercomputer, you cannot work this way, you have to submit to a job queue and wait for your job to finish.\n",
    "Simfactory simplifies this task as well, so that you don't need to know which job scheduler a given machine has or\n",
    "what its quirks are. If you are running on a machine without a queueing system, however, Simfactory simply runs the job in the background. That's what happens here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -fr ~/simulations/wave3\n",
    "!./simfactory/bin/sim create-submit wave3.par --procs=2 --num-threads=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next command tells you the status of a job. You can run it repeatedly...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!./simfactory/bin/sim list-sim wave3 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can create a small python script that checks for you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import time\n",
    "while True:\n",
    "    c = os.popen(\"./simfactory/bin/sim list-sim wave3\").read()\n",
    "    sys.stdout.write(c)\n",
    "    sys.stdout.flush()\n",
    "    time.sleep(3)\n",
    "    if re.search(\"FINISHED\",c):\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All the next command does is change the number of iterations the simulation requires. You don't have to use perl, but\n",
    "I happen to like it. Remember, changing a file in place with perl is as easy as pie (i.e. \"perl -p -i -e\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!perl -p -i -e 's{Cactus::cctk_itlast\\s+=\\s+\\d+}{Cactus::cctk_itlast = 400}' wave3.par"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Unfortunately, for the purpose of restarting the job, Simfactory won't see the change we just made unless we\n",
    "copy it to the appropriate directory. Of course, we could have run the perl command on that file instead."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp wave3.par ~/simulations/wave3/SIMFACTORY/par/wave3.par"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next command restarts from the checkpoint file and continues."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!./simfactory/bin/sim submit wave3.par --procs=2 --num-threads=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the job number is now 0001 instead of 0000. It increments for each restart, as many as you have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import time\n",
    "while True:\n",
    "    c = os.popen(\"./simfactory/bin/sim list-sim wave3\").read()\n",
    "    sys.stdout.write(c)\n",
    "    sys.stdout.flush()\n",
    "    time.sleep(3)\n",
    "    if re.search(\"FINISHED\",c):\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code snippets below to plot the results were modified slightly from the compiling and running notebook. The x and y\n",
    "data files are taken from the 0000 directory, but the data is taken from the 0001 directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell enables inline plotting in the notebook\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "# https://matplotlib.org/examples/color/colormaps_reference.html\n",
    "cmap = cm.gist_rainbow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd ~/simulations/wave3/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f5x = h5py.File(\"output-0000/wave3/x.xy.h5\")\n",
    "f5y = h5py.File(\"output-0000/wave3/y.xy.h5\")\n",
    "x_coords = {}\n",
    "y_coords = {}\n",
    "for nm in f5x:\n",
    "    print(nm)\n",
    "    m = re.search(r'rl=.*c=\\d+',nm)\n",
    "    if m:\n",
    "        k = m.group(0)\n",
    "        x_coords[k]=np.copy(f5x[nm])\n",
    "for nm in f5y:\n",
    "    m = re.search(r'rl=.*c=\\d+',nm)\n",
    "    if m:\n",
    "        k = m.group(0)\n",
    "        y_coords[k]=np.copy(f5y[nm])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f5 = h5py.File(\"output-0001/wave3/eta.xy.h5\")\n",
    "mn,mx = None,None\n",
    "\n",
    "# Compute the min and max\n",
    "for nm in f5:\n",
    "    if not hasattr(f5[nm],\"shape\"):\n",
    "        continue\n",
    "    d5 = np.copy(f5[nm])\n",
    "    tmin = np.min(d5)\n",
    "    tmax = np.max(d5)\n",
    "    if mn == None:\n",
    "        mn,mx = tmin,tmax\n",
    "    else:\n",
    "        if tmin < mn:\n",
    "            mn = tmin\n",
    "        if tmax > mx:\n",
    "            mx = tmax\n",
    "            \n",
    "# Collect all the pieces into the d5_tl dictionary\n",
    "d5_tl = {}            \n",
    "for nm in f5:\n",
    "    if not hasattr(f5[nm],\"shape\"):\n",
    "        continue\n",
    "    # Parse the string nm...\n",
    "    m = re.search(r'it=(\\d+)\\s+tl=\\d+\\s+(rl=(\\d+)\\s+c=(\\d+))',nm)\n",
    "    if not m:\n",
    "        print(\"nm=\",nm)\n",
    "        continue\n",
    "    # group(1) is the iteration number\n",
    "    # group(2) is \"rl={number} c={number}\"\n",
    "    # group(3) is the number in \"rl={number}\"\n",
    "    # group(4) is the number in \"c={number}\"\n",
    "    grid = int(m.group(1))\n",
    "    comp = int(m.group(4))\n",
    "    k = m.group(2)\n",
    "    if grid in d5_tl:\n",
    "        d5_tl[grid][\"x\"] += [x_coords[k]] # append to the x array\n",
    "        d5_tl[grid][\"y\"] += [y_coords[k]] # append to the y array\n",
    "        d5_tl[grid][\"D\"] += [f5[nm]] # append to the data array\n",
    "    else:\n",
    "        d5_tl[grid] = {\n",
    "            \"x\":[x_coords[k]],\n",
    "            \"y\":[y_coords[k]],\n",
    "            \"D\":[f5[nm]]\n",
    "        }\n",
    "\n",
    "# Sort the keys so that we display time levels in order\n",
    "def keysetf(d):\n",
    "    a = [] # create an empty list\n",
    "    for k in d: # for each key in d\n",
    "        a.append(k) # append it to the list\n",
    "    return a\n",
    "kys = keysetf(d5_tl.keys())\n",
    "kys.sort()\n",
    "\n",
    "# Show the figures, combing data from the same time level\n",
    "for index in kys:\n",
    "    data = d5_tl[index]\n",
    "    print(\"iteration=\",index)\n",
    "    plt.figure() # put this before the plots you wish to combine\n",
    "    plt.pcolor(data[\"x\"][0],data[\"y\"][0],data[\"D\"][0],vmin=mn,vmax=mx)\n",
    "    plt.pcolor(data[\"x\"][1],data[\"y\"][1],data[\"D\"][1],vmin=mn,vmax=mx)\n",
    "    plt.show() # show the plot.f5 = h5py.File(\"u.xy.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Exercise:\n",
    "Using the existing checkpoint files, continue the evolution to timestemp 500 and plot the results."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<table><tr><td>This work sponsored by NSF grants <a href=\"https://www.nsf.gov/awardsearch/showAward?AWD_ID=1550551\"> OAC 1550551</a> and <a href=\"https://www.nsf.gov/awardsearch/showAward?AWD_ID=1539567\"> CCF 1539567</a></td><td><img src=\"https://www.nsf.gov/awardsearch/images/common/nsf_logo_bottom.png\"></tr></table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}