diff --git a/ambrosia/tools/_lib/_bin_ci_aide.py b/ambrosia/tools/_lib/_bin_ci_aide.py index 2a21046..20816f2 100644 --- a/ambrosia/tools/_lib/_bin_ci_aide.py +++ b/ambrosia/tools/_lib/_bin_ci_aide.py @@ -42,7 +42,7 @@ def __helper_calc_empirical_power(conf_interval: types.ManyIntervalType) -> np.n def __helper_bin_search_for_size( - interval_type: str, confidence_level: float, p_a: float, p_b: float, amount: int, power: float + interval_type: str, confidence_level: float, p_a: float, p_b: float, amount: int, power: float, **kwargs ) -> int: """ Make binary search for size to gain given power. @@ -81,6 +81,7 @@ def power_helper(trials: int) -> float: "a_trials": trials, "b_trials": trials, "confidence_level": confidence_level, + **kwargs, } conf_interval: types.IntervalType = bi.BinomTwoSampleCI.confidence_interval(**binom_kwargs) return __helper_calc_empirical_power(conf_interval) @@ -112,6 +113,7 @@ def __helper_bin_search_for_delta( amount: int, power: float, epsilon: float = 0.0001, + **kwargs, ) -> Optional[float]: """ Make binary search for delta to gain given power for @@ -154,6 +156,7 @@ def power_helper(delta: float) -> float: "a_trials": trials, "b_trials": trials, "confidence_level": confidence_level, + **kwargs, } conf_interval: types.IntervalType = bi.BinomTwoSampleCI.confidence_interval(**binom_kwargs) return __helper_calc_empirical_power(conf_interval) diff --git a/ambrosia/tools/bin_intervals.py b/ambrosia/tools/bin_intervals.py index d1de482..450017f 100644 --- a/ambrosia/tools/bin_intervals.py +++ b/ambrosia/tools/bin_intervals.py @@ -390,6 +390,7 @@ def get_table_power_on_size_and_conversions( sample_sizes: Iterable[int] = (100,), amount: int = 10000, confidence_level: float = 0.95, + **kwargs, ) -> pd.DataFrame: """ Table with power / empirical 1 type error = 1 - coverage, for fixed size and conversions. @@ -432,6 +433,7 @@ def get_table_power_on_size_and_conversions( "a_trials": trials, "b_trials": trials, "confidence_level": confidence_level, + **kwargs, } conf_interval: types.ManyIntervalType = BinomTwoSampleCI.confidence_interval(**binom_kwargs) power: np.ndarray = helper_dir.__helper_calc_empirical_power(conf_interval) @@ -456,9 +458,9 @@ def get_table_power_on_size_and_delta( delta_values: Iterable[float] = None, delta_relative_values: Iterable[float] = None, interval_type: str = "wald", - # alternative: str = "two-sided", amount: int = 10000, as_numeric: bool = False, + **kwargs, ) -> pd.DataFrame: """ Table with power / empirical 1 type error = 1 - coverage for fixed size and effect. @@ -522,6 +524,7 @@ def get_table_power_on_size_and_delta( "b_trials": trials, "confidence_level": 1 - alpha, # "alternative": alternative, + **kwargs, } conf_interval: types.ManyIntervalType = BinomTwoSampleCI.confidence_interval(**binom_kwargs) power: np.ndarray = helper_dir.__helper_calc_empirical_power(conf_interval) @@ -541,6 +544,7 @@ def iterate_for_sample_size( p_b_values: Iterable[float], grid_delta: Iterable[float], amount: int, + **kwargs, ) -> pd.DataFrame: """ Iterate over params for different sample size @@ -567,6 +571,7 @@ def iterate_for_sample_size( p_b=p_b, amount=amount, power=power, + **kwargs, ) table.loc[delta, f"({alpha}; {beta})"] = trials return table @@ -580,6 +585,7 @@ def get_table_sample_size_on_effect( delta_values: Iterable[float] = None, delta_relative_values: Iterable[float] = None, amount: int = 10000, + **kwargs, ) -> pd.DataFrame: """ Table for sample sizes with given effect and errors. @@ -628,7 +634,9 @@ def get_table_sample_size_on_effect( grid_delta: np.ndarray = [f"{np.round((x - 1) * 100, ROUND_DIGITS_PERCENT)}%" for x in delta_relative_values] if not np.all((p_b_values >= 0) & (p_b_values <= 1)): raise ValueError(f"Probability of success in group B must be positive, not {p_b_values}") - table = iterate_for_sample_size(interval_type, first_errors, second_errors, p_a, p_b_values, grid_delta, amount) + table = iterate_for_sample_size( + interval_type, first_errors, second_errors, p_a, p_b_values, grid_delta, amount, **kwargs + ) return table @@ -641,6 +649,7 @@ def iterate_for_delta( amount: int, delta_type: str, as_numeric: bool = False, + **kwargs, ) -> pd.DataFrame: """ Helps to find effect for different params. @@ -662,6 +671,7 @@ def iterate_for_delta( trials=trials, amount=amount, power=power, + **kwargs, ) if delta is not None and delta_type == "relative": if as_numeric: @@ -681,6 +691,7 @@ def get_table_effect_on_sample_size( amount: int = 10000, delta_type: str = "relative", as_numeric: bool = False, + **kwargs, ) -> pd.DataFrame: """ Table for effects with given sample sizes and erros. @@ -723,6 +734,6 @@ def get_table_effect_on_sample_size( if delta_type not in delta_types: raise ValueError(f"Delta type must be absolute relative, not {delta_type}") table: pd.DataFrame = iterate_for_delta( - interval_type, first_errors, second_errors, sample_sizes, p_a, amount, delta_type, as_numeric + interval_type, first_errors, second_errors, sample_sizes, p_a, amount, delta_type, as_numeric, **kwargs ) return table diff --git a/tests/test_designer.py b/tests/test_designer.py index 2450366..62d13b7 100644 --- a/tests/test_designer.py +++ b/tests/test_designer.py @@ -6,7 +6,15 @@ import pytest import yaml -from ambrosia.designer import Designer, design, design_binary, load_from_config +from ambrosia.designer import ( + Designer, + design, + design_binary, + design_binary_effect, + design_binary_power, + design_binary_size, + load_from_config, +) store_path: str = "tests/configs/dumped_designer.yaml" @@ -327,3 +335,57 @@ def test_groups_ratio_parameter(to_design, method, effects, sizes, designer_ltv) ) results_list.append((1.0 + groups_ratio) * res) assert np.all(results_list[0].values < results_list[1].values < results_list[2].values) + + +@pytest.mark.smoke +def test_prior_designing_binary(): + # Correct prior + n_correct: int = design_binary_size( + 0.01, + effects=[1.01], + method="binary", + interval_type="bayes_beta", + n_success_conjugate=1, + n_failure_conjugate=1000002, + ).iloc[0, 0] + # Incorrect prior + n_incorrect: int = design_binary_size( + 0.99, + effects=[1.01], + methdo="binary", + interval_type="bayes_beta", + n_success_conjugate=1, + n_failure_conjugate=1000002, + ).iloc[0, 0] + assert n_correct > n_incorrect + + # Test effect + effect_correct = design_binary_effect( + 0.1, sizes=[5000], interval_type="bayes_beta", n_success_conjugate=1, n_failure_conjugate=10, as_numeric=True + ).iloc[0, 0] + effect_incorrect = design_binary_effect( + 0.9, sizes=[5000], interval_type="bayes_beta", n_success_conjugate=1, n_failure_conjugate=10, as_numeric=True + ).iloc[0, 0] + assert effect_correct > effect_incorrect + + # Test power + power_correct = design_binary_power( + 0.1, + sizes=5000, + effects=1.05, + interval_type="bayes_beta", + n_success_conjugate=1, + n_failure_conjugate=10, + as_numeric=True, + ).iloc[0, 0] + power_incorrect = design_binary_power( + 0.9, + sizes=5000, + effects=1.05, + interval_type="bayes_beta", + n_success_conjugate=1, + n_failure_conjugate=10, + as_numeric=True, + ).iloc[0, 0] + + assert power_correct < power_incorrect