From f9fb687b92e84a4286197225b4db50e5d556a30c Mon Sep 17 00:00:00 2001 From: Yu Feng Date: Tue, 19 May 2015 15:23:17 -0700 Subject: [PATCH 1/4] Update software.rst --- pages/software.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pages/software.rst b/pages/software.rst index f27ca67..708f27a 100644 --- a/pages/software.rst +++ b/pages/software.rst @@ -69,6 +69,21 @@ to measure power spectrum, and construct Friend-of-Friend halso from n-body simu pfft-python is a thin python extension layer of the massively parallel faster fourier transformation library, PFFT. Fast Fourier Transform is the core component of Particle-Mesh based gravity solvers, which simulate the formation of large scale structure, galaxies and dark matter halos. +==== + +`BoxRemap`_ +-------------- + +a new technique to remap the cubical domain of a cosmological simulation into simple non-cubical shapes. + +==== + +`Copter`_ +-------------- + +a C++ class library written by Jordan Carlson to compute predictions for different cosmological perturbation theory schemes. + + .. _`Dierckx.jl`: http://github.com/kbarbary/Dierckx.jl .. _`SEP`: http://sep.readthedocs.org .. _`SEP on GitHub`: http://github.com/kbarbary/sep @@ -78,4 +93,5 @@ pfft-python is a thin python extension layer of the massively parallel faster fo .. _`kdcount`: https://github.com/rainwoodman/kdcount .. _`pypm`: https://github.com/rainwoodman/pypm .. _`pfft-python`: https://github.com/rainwoodman/pfft-python - +.. _`BoxRemap`: http://mwhite.berkeley.edu/BoxRemap/ +.. _`Copter`: http://mwhite.berkeley.edu/Copter/ From 8972d8df689d69e64272f65039776edc8900bd3e Mon Sep 17 00:00:00 2001 From: Yu Feng Date: Wed, 20 May 2015 17:44:36 -0700 Subject: [PATCH 2/4] Add other codes from Martin since the licenses are clean. --- pages/software.rst | 71 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/pages/software.rst b/pages/software.rst index 708f27a..9deaf88 100644 --- a/pages/software.rst +++ b/pages/software.rst @@ -18,6 +18,9 @@ adapted from the SourceExtractor code base. It makes the SourceExtractor background and detection algorithms available directly from Python and (eventually) Julia. `SEP on GitHub`_. +.. _`SEP`: http://sep.readthedocs.org +.. _`SEP on GitHub`: http://github.com/kbarbary/sep + ==== .. image:: /images/sncosmo.png @@ -30,6 +33,9 @@ from Python and (eventually) Julia. `SEP on GitHub`_. SNCosmo is a Python package for supernova cosmology data analysis and simulation. `SNCosmo on GitHub`_. +.. _`SNCosmo`: http://sncosmo.github.io +.. _`SNCosmo on GitHub`: http://github.com/sncosmo/sncosmo + ==== @@ -40,6 +46,8 @@ Dierckx.jl is a package for 1-d and 2-d splines in Julia. It's a wrapper of the dierckx Fortran library available from NETLIB, the same library underlying the spline classes in scipy.interpolate. +.. _`Dierckx.jl`: http://github.com/kbarbary/Dierckx.jl + ==== `cosmoxi2d`_ @@ -47,6 +55,8 @@ library underlying the spline classes in scipy.interpolate. cosmoxi2d provides theoretical predictions for the two-dimensional galaxy correlation function. This code was used to analyze `DR9 `_, `DR10, and DR11 `_ the SDSS-III BOSS CMASS galaxy sample and derive cosmological constraints on the geometry of the universe and the growth rate of cosmic structure. +.. _`cosmoxi2d`: https://github.com/bareid/cosmoxi2d + ==== `kdcount`_ @@ -54,6 +64,8 @@ cosmoxi2d provides theoretical predictions for the two-dimensional galaxy correl kdcount is a simple python extension for brute force pair-counting of point data sets. Pair-counting is the fundamental of estimating correlation functions from imaging and spectroscopy survey catalogues of quasars, galaxies, and Lyman-alpha forest. +.. _`kdcount`: https://github.com/rainwoodman/kdcount + ==== `pypm`_ @@ -62,6 +74,8 @@ kdcount is a simple python extension for brute force pair-counting of point data pypm is a set of python modules for particle mesh calculation with MPI parallel python. It also provides tools to measure power spectrum, and construct Friend-of-Friend halso from n-body simulations. +.. _`pypm`: https://github.com/rainwoodman/pypm + ==== `pfft-python`_ @@ -69,6 +83,18 @@ to measure power spectrum, and construct Friend-of-Friend halso from n-body simu pfft-python is a thin python extension layer of the massively parallel faster fourier transformation library, PFFT. Fast Fourier Transform is the core component of Particle-Mesh based gravity solvers, which simulate the formation of large scale structure, galaxies and dark matter halos. +.. _`pfft-python`: https://github.com/rainwoodman/pfft-python + +==== + +`Copter`_ +-------------- + +a C++ class library written by Jordan Carlson to compute predictions for different cosmological perturbation theory schemes. + + +.. _`Copter`: http://mwhite.berkeley.edu/Copter/ + ==== `BoxRemap`_ @@ -76,22 +102,41 @@ pfft-python is a thin python extension layer of the massively parallel faster fo a new technique to remap the cubical domain of a cosmological simulation into simple non-cubical shapes. +.. _`BoxRemap`: http://mwhite.berkeley.edu/BoxRemap/ + ==== -`Copter`_ +`ImagingLSS`_ -------------- -a C++ class library written by Jordan Carlson to compute predictions for different cosmological perturbation theory schemes. +.. _`ImagingLSS`: http://github.com/desihub/imaginglss/ -.. _`Dierckx.jl`: http://github.com/kbarbary/Dierckx.jl -.. _`SEP`: http://sep.readthedocs.org -.. _`SEP on GitHub`: http://github.com/kbarbary/sep -.. _`SNCosmo`: http://sncosmo.github.io -.. _`SNCosmo on GitHub`: http://github.com/sncosmo/sncosmo -.. _`cosmoxi2d`: https://github.com/bareid/cosmoxi2d -.. _`kdcount`: https://github.com/rainwoodman/kdcount -.. _`pypm`: https://github.com/rainwoodman/pypm -.. _`pfft-python`: https://github.com/rainwoodman/pfft-python -.. _`BoxRemap`: http://mwhite.berkeley.edu/BoxRemap/ -.. _`Copter`: http://mwhite.berkeley.edu/Copter/ +==== + +`LiteMangle`_ +-------------- + +.. _`LiteMangle`: http://githbu.com/martinjameswhite/litemangle + + +==== + +`mockFactory`_ +-------------- + +.. _`mockFactory`: https://github.com/mockFactory + +==== + +`CLPT_GSRSD`_ +-------------- + +.. _`CLPT_GSRSD`: https://github.com/wll745881210/CLPT_GSRSD + +==== + +`recon_code`_ +------------- + +.. _`recon_code`: https://github.com/martinjameswhite/recon_code From 8cc0aec0842a820a25bdaf6e779fcd08195f5000 Mon Sep 17 00:00:00 2001 From: Yu Feng Date: Wed, 20 May 2015 18:02:06 -0700 Subject: [PATCH 3/4] typo. --- pages/software.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/software.rst b/pages/software.rst index 9deaf88..1d319bc 100644 --- a/pages/software.rst +++ b/pages/software.rst @@ -117,7 +117,7 @@ a new technique to remap the cubical domain of a cosmological simulation into si `LiteMangle`_ -------------- -.. _`LiteMangle`: http://githbu.com/martinjameswhite/litemangle +.. _`LiteMangle`: http://github.com/martinjameswhite/litemangle ==== From 682d728af6fa20fb02ce9def973609ac0462f89a Mon Sep 17 00:00:00 2001 From: Yu Feng Date: Wed, 20 May 2015 18:40:21 -0700 Subject: [PATCH 4/4] add script to parse software.rst and json files. Also pulls description from github if it wasn't listed in the rst file. The logic flow here is broken! --- pages/software.rst | 99 ++++++++++++++++++++------------- software_to_json.py | 131 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 37 deletions(-) create mode 100644 software_to_json.py diff --git a/pages/software.rst b/pages/software.rst index 1d319bc..9174630 100644 --- a/pages/software.rst +++ b/pages/software.rst @@ -1,3 +1,4 @@ + .. link: .. description: .. tags: @@ -13,40 +14,35 @@ of our projects: `SEP`_ ------ -SEP is a Python and C library for source detection and photometry, -adapted from the SourceExtractor code base. It makes the -SourceExtractor background and detection algorithms available directly -from Python and (eventually) Julia. `SEP on GitHub`_. +SEP is a Python and C library for source detection and photometry, adapted from the SourceExtractor code base. It makes the SourceExtractor background and detection algorithms available directly from Python and (eventually) Julia. -.. _`SEP`: http://sep.readthedocs.org -.. _`SEP on GitHub`: http://github.com/kbarbary/sep +Links: -==== +- http://sep.readthedocs.org +- http://github.com/kbarbary/sep -.. image:: /images/sncosmo.png - :height: 40px - :align: left +==== `SNCosmo`_ ---------- -SNCosmo is a Python package for supernova cosmology data analysis and -simulation. `SNCosmo on GitHub`_. +SNCosmo is a Python package for supernova cosmology data analysis and simulation. -.. _`SNCosmo`: http://sncosmo.github.io -.. _`SNCosmo on GitHub`: http://github.com/sncosmo/sncosmo +Links: -==== +- http://sncosmo.github.io +- http://github.com/sncosmo/sncosmo +==== `Dierckx.jl`_ ------------- -Dierckx.jl is a package for 1-d and 2-d splines in Julia. It's a -wrapper of the dierckx Fortran library available from NETLIB, the same -library underlying the spline classes in scipy.interpolate. +Dierckx.jl is a package for 1-d and 2-d splines in Julia. It's a wrapper of the dierckx Fortran library available from NETLIB, the same library underlying the spline classes in scipy.interpolate. + +Links: -.. _`Dierckx.jl`: http://github.com/kbarbary/Dierckx.jl +- http://github.com/kbarbary/Dierckx.jl ==== @@ -55,7 +51,9 @@ library underlying the spline classes in scipy.interpolate. cosmoxi2d provides theoretical predictions for the two-dimensional galaxy correlation function. This code was used to analyze `DR9 `_, `DR10, and DR11 `_ the SDSS-III BOSS CMASS galaxy sample and derive cosmological constraints on the geometry of the universe and the growth rate of cosmic structure. -.. _`cosmoxi2d`: https://github.com/bareid/cosmoxi2d +Links: + +- https://github.com/bareid/cosmoxi2d ==== @@ -64,17 +62,20 @@ cosmoxi2d provides theoretical predictions for the two-dimensional galaxy correl kdcount is a simple python extension for brute force pair-counting of point data sets. Pair-counting is the fundamental of estimating correlation functions from imaging and spectroscopy survey catalogues of quasars, galaxies, and Lyman-alpha forest. -.. _`kdcount`: https://github.com/rainwoodman/kdcount +Links: + +- https://github.com/rainwoodman/kdcount ==== `pypm`_ ------- -pypm is a set of python modules for particle mesh calculation with MPI parallel python. It also provides tools -to measure power spectrum, and construct Friend-of-Friend halso from n-body simulations. +pypm is a set of python modules for particle mesh calculation with MPI parallel python. It also provides tools to measure power spectrum, and construct Friend-of-Friend halso from n-body simulations. -.. _`pypm`: https://github.com/rainwoodman/pypm +Links: + +- https://github.com/rainwoodman/pypm ==== @@ -83,60 +84,84 @@ to measure power spectrum, and construct Friend-of-Friend halso from n-body simu pfft-python is a thin python extension layer of the massively parallel faster fourier transformation library, PFFT. Fast Fourier Transform is the core component of Particle-Mesh based gravity solvers, which simulate the formation of large scale structure, galaxies and dark matter halos. -.. _`pfft-python`: https://github.com/rainwoodman/pfft-python +Links: + +- https://github.com/rainwoodman/pfft-python ==== `Copter`_ --------------- +--------- a C++ class library written by Jordan Carlson to compute predictions for different cosmological perturbation theory schemes. +Links: -.. _`Copter`: http://mwhite.berkeley.edu/Copter/ +- http://mwhite.berkeley.edu/Copter/ ==== `BoxRemap`_ --------------- +----------- a new technique to remap the cubical domain of a cosmological simulation into simple non-cubical shapes. -.. _`BoxRemap`: http://mwhite.berkeley.edu/BoxRemap/ +Links: + +- http://mwhite.berkeley.edu/BoxRemap/ ==== `ImagingLSS`_ --------------- +------------- +Code for doing large-scale structure studies with the imaging data. -.. _`ImagingLSS`: http://github.com/desihub/imaginglss/ +Links: + +- http://github.com/desihub/imaginglss/ ==== `LiteMangle`_ --------------- +------------- + +Light-weight library for some Mangle routines. -.. _`LiteMangle`: http://github.com/martinjameswhite/litemangle +Links: +- http://github.com/martinjameswhite/litemangle ==== `mockFactory`_ -------------- -.. _`mockFactory`: https://github.com/mockFactory + + +Links: + +- https://github.com/mockFactory ==== `CLPT_GSRSD`_ --------------- +------------- + +Convolution Lagrangian Perturbation Theory (CLPT) and Gaussian Streaming Redshift-Space Distortions (GSRSD) + +Links: -.. _`CLPT_GSRSD`: https://github.com/wll745881210/CLPT_GSRSD +- https://github.com/wll745881210/CLPT_GSRSD ==== `recon_code`_ ------------- -.. _`recon_code`: https://github.com/martinjameswhite/recon_code +Code to perform density field reconstruction for baryon acoustic oscillation (BAO) experiments + +Links: + +- https://github.com/martinjameswhite/recon_code + diff --git a/software_to_json.py b/software_to_json.py new file mode 100644 index 0000000..19c3a1b --- /dev/null +++ b/software_to_json.py @@ -0,0 +1,131 @@ +""" + This simple script converts the current software.rst page to a json file. +""" +import json +import urllib2 +import os +TOKEN = os.environ["GITHUB_OAUTH2_TOKEN"] + +def buildjson(): + with file('pages/software.rst', 'r') as ff: + lines = ff.readlines() + + def parse(lines): + STARTED = 1 + state = 0 + for line in lines: + line = line.strip() + if len(line) == 0: + continue + if state < STARTED: + if line == '====': + state = STARTED + entry = {} + L = [] + continue + else: + if line.startswith('---'): + entry['ProjectName'] = L[-1][1:-2] + L = [] + continue + elif line.startswith('.. _'): + if 'Links' not in entry: + entry['Links'] = [] + entry['Links'].append(line[line.index(':')+1:].strip()) + continue + elif line == 'Links:': + continue + elif line.startswith('- http'): + if 'Links' not in entry: + entry['Links'] = [] + entry['Links'].append(line[2:].strip()) + continue + elif line == '====': + entry['Description'] = ' '.join(L) + yield entry + entry = {} + L = [] + else: + L.append(line) + entry['Description'] = ' '.join(L) + yield entry + + entries = [entry for entry in parse(lines) ] + with file('software.json', 'w') as ff: + json.dump(entries, ff, sort_keys=True, + indent=4, separators=(',', ': ')) + +def fixjson(): + with file('software.json', 'r') as ff: + entries = json.load(ff) + class NotGithubURL(Exception): pass + def parse_github_url(url): + """ returns the :owner and :repo field """ + if 'github.com' in url: + url = url.strip('/') + slashes = url.split('/') + if slashes[-2].endswith('github.com'): + # this is a org + return slashes[-1], + else: + return slashes[-2], slashes[-1] + else: + raise NotGithubURL("Not a Github URL"); + for entry in entries: + if len(entry['Description'].strip()) == 0: + for url in entry['Links']: + try: + result = parse_github_url(url) + if len(result) == 2: + req = urllib2.Request("https://api.github.com/repos/%s/%s" % result) + elif len(result) == 1: + req = urllib2.Request("https://api.github.com/orgs/%s" % result) + else: + raise RuntimeError + + req.add_header('Authorization', 'token %s' % TOKEN) + ff = urllib2.urlopen(req) + data = json.load(ff) + if data['description']: + entry['Description'] = data['description'] + else: + pass + except NotGithubURL: + continue + with file('software.json', 'w') as ff: + json.dump(entries, ff, sort_keys=True, + indent=4, separators=(',', ': ')) + +def buildrst(): + header=""" +.. link: +.. description: +.. tags: +.. date: 2014/02/08 12:19:51 +.. title: Software +.. slug: software + +BCCP develops open-source software for use in cosmology. Here are some +of our projects: + +""" + with file('software.json', 'r') as ff: + entries = json.load(ff) + with file('software.rst', 'w') as ff: + ff.write(header) + for entry in entries: + ff.write('====\n\n') + ff.write('`%s`_\n' % entry['ProjectName']) + ff.write('-' * (len(entry['ProjectName']) + 3)) + ff.write('\n\n') + ff.write('%s\n\n' % entry['Description']) + if len(entry['Links']) > 0: + ff.write('Links:\n\n') + for link in entry['Links']: + ff.write('- %s\n' % link) + ff.write('\n') + +# these really should be subcommands! +buildjson() +fixjson() +buildrst()