FDM Zoom-in simulation

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/Input__DumpTable

@@ -0,0 +1,42 @@
+#Dump ID             Dump Time
+      -6    0.0003124024000000
+      -5    0.0005000000000000
+      -4    0.0010000000000000
+      -3    0.0050000000000000
+      -2    0.0100000000000000
+      -1    0.0500000000000000
+       0    0.0909090920000000
+       1    0.1022642090403268
+       2    0.1142926268126806
+       3    0.1269969647105442
+       4    0.1403806580959482
+       5    0.1544481509848806
+       6    0.1692051241492372
+       7    0.1846587623059424
+       8    0.2008180659019670
+       9    0.2176942139307767
+      10    0.2353009853594673
+      11    0.2536552481440784
+      12    0.2727775264961512
+      13    0.2926926590557780
+      14    0.3134305629162865
+      15    0.3350271209736094
+      16    0.3575252126958853
+      17    0.3809759108571344
+      18    0.4054398686204628
+      19    0.4309889220031104
+      20    0.4577079316383287
+      21    0.4856968849564935
+      22    0.5150732779850334
+      23    0.5459748043551692
+      24    0.5785624241065474
+      25    0.6130240298474579
+      26    0.6495793081783160
+      27    0.6884872598419996
+      28    0.7300595373040358
+      29    0.7746853173110612
+      30    0.8228743661061253
+      31    0.8753084975593694
+      32    0.9327759052183688
+      33    0.9953073615712860
+***************END LINE***************

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/Input__Flag_Interference

@@ -0,0 +1,12 @@
+# Level          QP   Density   PhaseLap   OnlyAtExtrema
+      0        0.03         0        1.0               0
+      1        0.03         0        1.0               0
+      2        0.03         0        1.0               0
+      3        0.03         0        1.0               0
+      4        0.03         0        1.0               0
+      5        0.03         0        1.0               0
+      6        0.03         0        1.0               0
+      7        0.03         0        1.0               0
+      8        0.03         0        1.0               0
+      9        0.03         0        1.0               0
+     10        0.03         0        1.0               0

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/Input__Flag_Rho

@@ -0,0 +1,15 @@
+# Level                         Density
+      0                             8.0
+      1                            32.0
+      2                            64.0
+      3                           256.0
+      4                           330.0
+      5                           430.0
+      6                           512.0
+      7                        262144.0
+      8                       2097152.0
+      9                      16777216.0
+     10                     134217728.0
+     11                    1073741824.0
+     12                    8589934592.0
+     13                   68719476736.0

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/Input__Flag_Spectral

@@ -0,0 +1,13 @@
+# Level   Refinement   Derefinement (derefinement currently not functional)
+      0          1e0             -1
+      1          1e0             -1
+      2          1e0             -1
+      3          1e0             -1
+      4          1e0             -1
+      5          1e0             -1
+      6          1e0             -1
+      7          1e0             -1
+      8          1e0             -1
+      9          1e0             -1
+     10          1e0             -1
+     11          1e0             -1

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/Input__TestProb

@@ -0,0 +1,3 @@
+# problem-specific runtime parameters
+LSS_InitMode         2       # initialization mode: (1=density-only with constant phase, 2=real and imaginary parts or density and phase) [1]
+ZoomIn_MaxLvOutside  3       # maximum refinement level outside of the zoom-in box

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/Input__ZoominBox

@@ -0,0 +1,8 @@
+#ScaleFactor    Lx(UNIT_L)    Ly(UNIT_L)    Lz(UNIT_L)   CenterX(UNIT_L)   CenterY(UNIT_L)   CenterZ(UNIT_L)
+        0.77          1.18          1.13          1.26           0.29491           9.52254           8.27318
+	0.58          1.80          1.66          1.94           0.29491           9.52254           8.27318
+	0.43          2.30          2.00          2.46           0.29491           9.52254           8.27318
+	0.31          2.70          2.31          2.87           0.29491           9.52254           8.27318
+	0.22          3.20          2.53          3.22           0.29491           9.52254           8.27318
+	0.14          3.60          2.70          3.50           0.29491           9.52254           8.27318
+	0.05          4.00          2.90          3.89           0.29491           9.52254           8.27318

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/README

@@ -0,0 +1,51 @@
+Compilation flags:
+========================================
+Enable : MODEL=ELBDM, ELBDM_SCHEME=ELBDM_HYBRID, GRAVITY, COMOVING
+Disable: PARTICLE
+
+
+Default setup:
+========================================
+1. ELBDM_MASS              5.0e-24 (eV/c^2)
+   A_INIT                  9.900990099e-3          # initial scale factor
+   OMEGA_M0                0.3158230904284232      # omega matter at the present time
+   HUBBLE0                 0.6732117               # dimensionless Hubble parameter (currently only for converting ELBDM_MASS to code units)
+   BOX_SIZE                10. (Mpc/h)
+
+
+2. MAX_LEVEL               5
+   OPT__FLAG_SPECTRAL      1
+   OPT__FLAG_INTERFERENCE  1
+   --> Input__Flag_Rho          = example/input/Input__Flag_Rho
+       Input__Flag_Spectral     = example/input/Input__Flag_Spectral
+       Input__Flag_Interference = example/input/Input__Flag_Interference
+
+3. OPT__FLAG_REGION        1			   # Enable zoom-in simulation
+
+
+Note:
+========================================
+1. Cosmological ZOOM-IN large-scale structure simulations using hybrid scheme
+
+2. Quickstart:
+   1. Download the IC file "sh download_IC_lowres.sh" for testing.
+   2. Compile GAMER with the "Makefile" generated by "generate_make.sh" and run simulation to redshift 0 with default settings.
+   3. Total data size is 74G. If less snapshots are preferred for smaller data size, please remove some rows in Input__DumpTable
+
+3. details on Zoom-in Simulation
+   1. For details on running a hybrid simulation, please see "example/test_problem/ELBDM/LSS_Hybrid/README"
+   2. Input__ZoomInBox controls the zoom-in volume of the time dependent zoom-in box, such as scale factor, length in x,y and z-axis.
+      Note that we read the scale factor in descending order. 
+      The UNIT_L in cosmological simulation refers to comoving Mpc/h. 
+   3. Input__TestProb takes the center of the zoom-in volume (ZoomIn_Center_x,y,z), and the maximum level outside of the volume (ZoomIn_Lvlim).
+   4. It is required to set ELBDM_FIRST_WAVE_LEVEL <= ZoomIn_MaxLvOutside, to avoid problems of the fluid-wave interface at the zoom-in boundary (see paper)
+
+4. Some yt visualization scripts are put in "plot_script"
+
+5. Simulation results in (Chan+24, submitted to MNRAS)
+   1. "download_IC_highres.sh" downloads the IC file
+   2. NX0_TOT_X   256
+      NX0_TOT_Y   256
+      NX0_TOT_Z   256
+      ELBDM_MASS  2.0e-23 
+      MAX_LEVEL   7

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/clean.sh

@@ -0,0 +1,5 @@
+rm -f Record__Note Record__Timing Record__TimeStep Record__PatchCount Record__Dump Record__MemInfo Record__L1Err \
+      Record__Conservation Data* stderr stdout log XYslice* YZslice* XZslice* Xline* Yline* Zline* \
+      Diag* BaseXYslice* BaseYZslice* BaseXZslice* BaseXline* BaseYline* BaseZline* BaseDiag* \
+      PowerSpec_* Particle_* nohup.out Record__Performance Record__TimingMPI_* \
+      Record__ParticleCount Record__User Patch_* Record__NCorrUnphy FailedPatchGroup* *.pyc Record__LoadBalance Record__Hybrid

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/download_IC_highres.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+API_URL="https://girder.hub.yt/api/v1"
+FILE_ID="66ea79a6999605c485c8d620"
+LOCAL_FILE="Zoomin_IC"
+
+# download
+girder-cli --api-url ${API_URL} download --parent-type item ${FILE_ID} ${LOCAL_FILE}
+
+# unzip
+tar zxvf ${LOCAL_FILE}/Zoomin_IC_highres.tar.gz
+rm -r ${LOCAL_FILE}
+ln -fs UM_IC_hybrid_N1024 UM_IC 

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/download_IC_lowres.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+API_URL="https://girder.hub.yt/api/v1"
+FILE_ID="66ea79b4999605c485c8d623"
+LOCAL_FILE="Zoomin_IC"
+
+# download
+girder-cli --api-url ${API_URL} download --parent-type item ${FILE_ID} ${LOCAL_FILE}
+
+# unzip
+tar zxvf ${LOCAL_FILE}/Zoomin_IC_lowres.tar.gz
+rm -r ${LOCAL_FILE}
+ln -fs UM_IC_hybrid_N256 UM_IC 

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/generate_make.sh

@@ -0,0 +1,7 @@
+# This script should run in the same directory as configure.py
+
+PYTHON=python3
+
+${PYTHON} configure.py --machine=eureka_intel --mpi=true --hdf5=true --fftw=FFTW3 --gpu=true \
+                       --model=ELBDM --elbdm_scheme=ELBDM_HYBRID --wave_scheme=WAVE_GRAMFE --gramfe_scheme=GRAMFE_MATMUL \
+                       --gravity=true --comoving=true --gsl=true --spectral_interpolation=true

example/test_problem/ELBDM/LSS_Hybrid_Zoomin/plot_script/plot_projection_grid.py

@@ -0,0 +1,77 @@
+import argparse
+import sys
+import yt
+
+# global parameter settings
+field         = ('gas', 'density')
+axes          = ["x", "y", "z"]                            # directions
+lv            = 10                                         # maximum level for sampling AMR grid
+dpi           = 300
+colormap_dens = 'algae'
+center_mode   = 'c'
+zooms         = [1, 7]
+# load the command-line parameters
+parser = argparse.ArgumentParser( description='Projection of mass density' )
+
+parser.add_argument( '-i', action='store', required=False, type=str, dest='prefix',
+                     help='path prefix [%(default)s]', default='../' )
+parser.add_argument( '-s', action='store', required=True,  type=int, dest='idx_start',
+                     help='first data index' )
+parser.add_argument( '-e', action='store', required=True,  type=int, dest='idx_end',
+                     help='last data index' )
+parser.add_argument( '-d', action='store', required=False, type=int, dest='didx',
+                     help='delta data index [%(default)d]', default=1 )
+
+args=parser.parse_args()
+
+# take note
+print( '\nCommand-line arguments:' )
+print( '-------------------------------------------------------------------' )
+for t in range( len(sys.argv) ):
+   print(str(sys.argv[t]))
+print( '' )
+print( '-------------------------------------------------------------------\n' )
+
+
+idx_start = args.idx_start
+idx_end   = args.idx_end
+didx      = args.didx
+prefix    = args.prefix
+
+center = [0.295,9.522,8.27] # center of the targeted halo in the zoom-in simulation
+
+
+yt.enable_parallelism()
+ts = yt.DatasetSeries( [ prefix+'/Data_%06d'%idx for idx in range(idx_start, idx_end+1, didx) ] )
+
+for ds in ts.piter():
+   num = '%s'%ds
+   num = int( num[5:11] )
+   for ax in axes:
+
+      # create thin slice
+      left_corner  = np.array(ds.domain_left_edge)
+      right_corner = np.array(ds.domain_right_edge)
+      depth        = 20.0
+      if ax == "x":
+         left_corner[0]  = center[0] - 0.5 * depth
+         right_corner[0] = center[0] + 0.5 * depth
+      if ax == "y":
+         left_corner[1]  = center[1] - 0.5 * depth
+         right_corner[1] = center[1] + 0.5 * depth
+      if ax == "z":
+         left_corner[2]  = center[2] - 0.5 * depth
+         right_corner[2] = center[2] + 0.5 * depth
+      region           = ds.box( left_corner, right_corner )
+      region.max_level = lv
+      pz_dens = yt.ProjectionPlot( ds, ax, field, center=center, data_source=region )
+      for zoom in zooms:
+         pz_dens.set_zlim( field, 1.0e-5, 2.0e-2 )
+         pz_dens.set_cmap( field, colormap_dens )
+         pz_dens.annotate_timestamp( time_unit='Gyr', redshift=True, corner='upper_right' )
+         pz_dens.set_axes_unit( 'Mpc' )
+         pz_dens.zoom( zoom )
+         pz_dens.save('Data_%06d_Proj_%s_%s_x%d.png'%( num, ax, field[1],zoom), mpl_kwargs={"dpi":dpi} )
+         pz_dens.annotate_grids()
+         pz_dens.save('Data_%06d_Proj_%s_%s_x%d_grid.png'%( num, ax, field[1],zoom), mpl_kwargs={"dpi":dpi} )
+         pz_dens.zoom( 1./zoom )