mv repo

Author: ajjacobs

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 ajjacobs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Project.toml

@@ -0,0 +1,17 @@
+name = "Config"
+uuid = "9d05fc2c-525b-11e9-1a36-8f70a3e496dc"
+authors = ["AJ <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[compat]
+julia = "1.0"

README.md

@@ -0,0 +1,111 @@
+# Config.jl
+
+A configuration manager for Julia.
+
+## Quick Start
+
+### Initialize a ConfigManager
+
+Suppose you have a configuration file called "tdlambda.json", specifying 
+a set of parameter settings for an experiment
+
+```json
+{
+    "save_path": "RandomWalk19/tdlambda",
+
+    "experiment": {
+        "class": "MarkovRewardProcess",
+        "episodes": 10
+    },
+
+    "environment": {
+        "class": "RandomWalk",
+        "nstates": 19
+    },
+
+    "agent": {
+        "class": "TDLambda",
+        "gamma": 1.0,
+        "metastep": [0.025, 0.05, 0.075, 0.1],
+        "lambda": [0.0, 0.4, 0.8, 0.9]
+    }
+}
+```
+
+Note: any config file *must* have the "save_path" parameter. This 
+specifies the directory in `data/output` which the data will be saved to. In this 
+example, data will be saved to `data/output/RandomWalk19/tdlambda`
+
+Initialize a manager to manage all the details of this config file
+
+```julia
+cfg = ConfigManager("tdlambda.json", @__DIR__)
+```
+
+The second argument specifies where the data directory should be 
+setup. In this case, a directory `data/` will be setup in the same directory 
+as the experiment which `ConfigManager` was instantiated in.
+
+### Parsing a config
+
+Any lists of parameters in the lowest-level of the config can be swept over 
+(in this case, `cfg["agent"]["metastep"]` and `cfg["agent"]["lambda"]`). 
+The different parameter settings are linearized. In order to sweep all the 
+parameters of this config, we can first check how many different parameters 
+there are:
+
+```julia
+N = total_combinations(cfg)
+```
+
+Then, we need to parse each of the individual settings into a concrete 
+parameterization:
+
+```julia
+for idx=1:N
+    parse!(cfg, idx)
+    
+    run_some_experiment(cfg)
+end
+```
+
+parse sets up the settings of a particular parameterization. After parsing the 
+config, individual parameters can be accessed by indexing. For example, to this 
+parameterization's "metastep" parameter, we can call `cfg["agent"]["metastep"]`. 
+If you will be referencing certain nested parameters quite often, you can 
+get the subconfiguration instead: `subcfg = get_subconfig(cfg, "agent")`. 
+Then access parameters of the subconfig in the same way: `subcfg["metastep"]`.
+
+Note that `parse!` has a third argument -- the run number -- which defaults to 1.
+To do multiple runs of an experiment, you can therefore do:
+
+```julia
+for run=1:100
+    for idx=1:N
+        parse!(cfg, idx, run)
+        
+        run_some_experiment(cfg)
+    end
+end
+```
+
+### Saving data
+
+The `ConfigManager` also takes care of saving data to the right place. 
+Just collect whatever data you want during your experiment in a `Dict()` 
+and pass it to the ConfigManager. 
+
+```julia
+function experiment(cfg::ConfigManager)
+    data = Dict()
+    data["ValueError"] = Float64[]
+    for i=1:1000
+       push!(data["ValueError"], rand()) 
+    end
+    
+    save(cfg, data)
+end
+```
+
+Then load the data later using `load(cfg)` (where cfg is a parse!'d config).
+

src/Checklist.jl

@@ -0,0 +1,39 @@
+# Checklists which prevent already-ran
+# experiments from being re-ran. Useful when
+# ComputeCanada experiments timeout
+
+function mark_config(self::ConfigManager)
+    chk_lst = get_checklist(self)
+    chk_lst[self["param_setting"]] = 1
+end
+
+function check_config(self::ConfigManager)
+    chk_lst = get_checklist(self)
+    if chk_lst[self["param_setting"]] == 1
+        println(string("Already ran param setting ", self["param_setting"], " ", self["run"]))
+        exit(0)
+    end
+end
+
+function get_checklist(self::ConfigManager)
+    filename = join(split(self[ "save_path"],"/"), "_")
+    path = joinpath(_checklists(self), string(filename ,".bin"))
+
+    if !isfile(path)
+        touch(path)
+    end
+
+    f = open(path,"r+")
+    return Mmap.mmap(f, Vector{UInt8}, self.total_combinations, (self["run"] - 1) * self.total_combinations * sizeof(UInt8))
+end
+
+function unmarked_configs(self::ConfigManager)
+    chkLst = get_checklist(self)
+    indices = Int[]
+    for i=1:total_combinations(self)
+        if chkLst[i] != 1
+            push!(indices, i)
+        end
+    end
+    return indices
+end

src/Config.jl

@@ -0,0 +1,37 @@
+module Config
+
+using JSON
+using Dates
+using Mmap
+using Logging
+using HDF5
+using Statistics
+
+# utilities
+include("Utils.jl")
+
+# data managers
+include("Save.jl")
+
+# Base Class
+include("Manager.jl")
+export ConfigManager, total_combinations, add!, parse!,
+    get_sweep_params, param_setting_from_id, params_with, param_values,
+    get_datafile, get_stepslog, get_expdir, get_logdir,
+    log_config, get_subconfig, get_attrs,
+    getindex, setindex!, haskey
+
+# Saving/Loading/Parsing data
+include("Data.jl")
+export save, save!, load, update!, clear!,
+     get_run_data, get_data, get_summary, get_best,
+    sensitivity, get_max_runs,
+    Summary
+
+# Checklist to prevent overwriting results or redundantly re-running
+# already-ran experiments
+include("Checklist.jl")
+export dump_config, mark_config, check_config, unmarked_configs
+
+
+end # module

src/Data.jl

@@ -0,0 +1,102 @@
+struct Summary
+    μ
+    σ
+    n
+
+    Summary(data::Matrix{Float64}) = new(
+        mean(data,dims=2),
+        std(data,corrected=true,dims=2) / √(size(data)[2]-1),
+        size(data)[2]
+    )
+
+    Summary(data::Vector{Float64}) = new(
+        data,
+        zeros(length(data)),
+        1
+    )
+end
+
+# Saving/Loading data
+save!(self::ConfigManager, data::Dict) = save!(self.dataManager, get_datafile(self), data)
+save(self::ConfigManager, data::Dict) = save(self.dataManager, get_datafile(self), data)
+load(self::ConfigManager) = load(self.dataManager, get_datafile(self))
+load(self::ConfigManager, idx::Int) = load(parse!(self, idx))
+
+# Parsing saved data
+
+function get_run_data(self::ConfigManager, idx, run)
+    parse!(self, idx, run)
+    return load(self)
+end
+
+function get_data(self::ConfigManager, idx)
+    runs = get_max_runs(self,idx)
+    all_data = Dict()
+    for r in 1:runs
+        data = get_run_data(self, idx, r)
+        for k in keys(data)
+            if haskey(all_data, k)
+                all_data[k] = hcat(all_data[k], data[k])
+            else
+                all_data[k] = data[k]
+            end
+        end
+    end
+    return all_data
+end
+
+function get_fields(self::ConfigManager)
+    return keys(get_run_data(copy(self),1,1))
+end
+
+function get_summary(self::ConfigManager, idx::Int, key::String)
+    data = get_data(self,idx)
+    return Summary(data[key])
+end
+
+function get_best(self::ConfigManager, key; metric=mean, selector=argmin)
+    return get_best(self, key, 1:self.total_combinations, metric=metric, selector=selector)
+end
+
+function get_best(self::ConfigManager, key, indices; metric=mean, selector=argmin)
+    data = []
+    parsed_indices = []
+    for idx in indices
+        d = get_summary(self, idx, key).μ
+        m = metric(d)
+        if isfinite(m)
+            push!(data, metric(d))
+            push!(parsed_indices, idx)
+        end
+    end
+    if length(data) == 0
+        return nothing # all configs diverged
+    end
+    best = selector(data)
+    return parsed_indices[best]
+end
+
+function sensitivity(self::ConfigManager, key; metric=mean)
+    best_idx = get_best(self,key)
+    cfg = parse!(self, best_idx)
+
+    keys = get_sweep_params(self)
+    d = Dict( k=> self[k] for k in keys )
+
+    # delete the key we're looking for sensitivity over
+    delete!(d,key)
+
+    indices = configs_with(self, d)
+    sens_data = []
+    for idx in indices
+        value = metric(get_summary(self,idx).μ)
+        push!(sens_data, value)
+    end
+    return sens_data
+end
+
+function get_max_runs(self::ConfigManager, idx::Int)
+    cfg = parse!(self,idx,1)
+    runs = readdir(get_expdir(self))
+    return length(runs)
+end