diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec59c452..08c7f0d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,4 +23,4 @@ For docstrings and comments, we use [Google Style](http://google.github.io/style **[black](https://github.com/psf/black)**: Automatic code formatting for Python. You can run black manually from the console using `black .` in the top directory of the repository, which will format all files. -**[isort](https://github.com/timothycrosley/isort)**: Used to consistently order imports. You can run isort manually from the console using `isort -y` in the top directory. +**[isort](https://github.com/timothycrosley/isort)**: Used to consistently order imports. You can run isort manually from the console using `isort .` in the top directory. diff --git a/sbibm/__init__.py b/sbibm/__init__.py index d091d85d..dfcb711c 100644 --- a/sbibm/__init__.py +++ b/sbibm/__init__.py @@ -1,4 +1,4 @@ from sbibm.__version__ import __version__ from sbibm.tasks import get_available_tasks, get_task, get_task_name_display -from sbibm.utils.logging import get_logger from sbibm.utils.io import get_results +from sbibm.utils.logging import get_logger diff --git a/sbibm/algorithms/__init__.py b/sbibm/algorithms/__init__.py index f13b3a7d..821f3fea 100644 --- a/sbibm/algorithms/__init__.py +++ b/sbibm/algorithms/__init__.py @@ -1,7 +1,7 @@ from sbibm.algorithms.sbi.mcabc import run as mcabc -from sbibm.algorithms.sbi.snpe import run as snpe from sbibm.algorithms.sbi.smcabc import run as smcabc from sbibm.algorithms.sbi.snle import run as snle +from sbibm.algorithms.sbi.snpe import run as snpe from sbibm.algorithms.sbi.snre import run as snre rej_abc = mcabc diff --git a/sbibm/algorithms/elfi/bolfi.py b/sbibm/algorithms/elfi/bolfi.py index bca5491c..bf5a8b33 100644 --- a/sbibm/algorithms/elfi/bolfi.py +++ b/sbibm/algorithms/elfi/bolfi.py @@ -22,7 +22,7 @@ def run( num_warmup: int = 1000, ) -> (torch.Tensor, int, Optional[torch.Tensor]): """Runs BOLFI from elfi package - + Args: task: Task instance num_samples: Number of samples to generate from posterior diff --git a/sbibm/algorithms/elfi/utils/prior.py b/sbibm/algorithms/elfi/utils/prior.py index 842e1f50..7b3abdee 100644 --- a/sbibm/algorithms/elfi/utils/prior.py +++ b/sbibm/algorithms/elfi/utils/prior.py @@ -31,7 +31,11 @@ def build_prior(task: Task, model: elfi.ElfiModel): scale = np.sqrt(prior_params["C"][dim, dim]) elfi.Prior( - "norm", loc, scale, model=model, name=f"parameter_{dim}", + "norm", + loc, + scale, + model=model, + name=f"parameter_{dim}", ) bounds[f"parameter_{dim}"] = ( @@ -48,7 +52,11 @@ def build_prior(task: Task, model: elfi.ElfiModel): scale = prior_params["high"][dim] - loc elfi.Prior( - "uniform", loc, scale, model=model, name=f"parameter_{dim}", + "uniform", + loc, + scale, + model=model, + name=f"parameter_{dim}", ) bounds[f"parameter_{dim}"] = ( diff --git a/sbibm/algorithms/pyabc/pyabc_utils.py b/sbibm/algorithms/pyabc/pyabc_utils.py index 2c064533..a4f57353 100644 --- a/sbibm/algorithms/pyabc/pyabc_utils.py +++ b/sbibm/algorithms/pyabc/pyabc_utils.py @@ -1,22 +1,22 @@ import logging -from typing import Callable, Dict, Tuple, Optional +from typing import Callable, Dict, Optional, Tuple import numpy as np import pyabc import torch -import sbibm - from sbi.inference import MCABC + +import sbibm from sbibm.tasks.task import Task class PyAbcSimulator: - """Wrapper from sbibm task to pyABC. + """Wrapper from sbibm task to pyABC. pyABC defines its own priors and they are sampled without batch dimension. This wrapper defines a call method that takes a single parameter set from a pyABC prior and uses the sbibm task simulator to generate the corresponding data and to return - it in pyABC format. + it in pyABC format. """ def __init__(self, task): @@ -151,7 +151,7 @@ def run_pyabc( use_last_pop_samples: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: """Run pyabc SMC with fixed budget and return particles and weights. - + Return previous population or prior samples if budget is exceeded. """ log = sbibm.get_logger(__name__) diff --git a/sbibm/algorithms/pyabc/smcabc.py b/sbibm/algorithms/pyabc/smcabc.py index 13b99e19..d78cf6a5 100644 --- a/sbibm/algorithms/pyabc/smcabc.py +++ b/sbibm/algorithms/pyabc/smcabc.py @@ -9,7 +9,7 @@ import torch import sbibm -from sbibm.algorithms.sbi.utils import get_sass_transform, run_lra, clip_int +from sbibm.algorithms.sbi.utils import clip_int, get_sass_transform, run_lra from sbibm.tasks.task import Task from sbibm.utils.kde import get_kde from sbibm.utils.torch import sample_with_weights @@ -18,8 +18,8 @@ PyAbcSimulator, get_distance, run_pyabc, - wrap_prior, run_rejection_abc, + wrap_prior, ) diff --git a/sbibm/algorithms/pyro/mcmc.py b/sbibm/algorithms/pyro/mcmc.py index a77d8730..76b4bed6 100644 --- a/sbibm/algorithms/pyro/mcmc.py +++ b/sbibm/algorithms/pyro/mcmc.py @@ -43,7 +43,7 @@ def run( Produces `num_samples` while accounting for warmup (burn-in) and thinning. Note that the actual number of simulations is not controlled for with MCMC since - algorithms are only used as a reference method in the benchmark. + algorithms are only used as a reference method in the benchmark. MCMC is run on the potential function, which returns the unnormalized negative log posterior probability. Note that this requires a tractable likelihood. diff --git a/sbibm/algorithms/pyro/utils/tensorboard.py b/sbibm/algorithms/pyro/utils/tensorboard.py index 288e3eb4..8c844887 100644 --- a/sbibm/algorithms/pyro/utils/tensorboard.py +++ b/sbibm/algorithms/pyro/utils/tensorboard.py @@ -22,7 +22,9 @@ def tb_acf(writer, mcmc, site_name="parameters", num_samples=1000, maxlags=50): fig = plt.figure() plt.gca().acorr(samples[c, :].squeeze()[:, p].numpy(), maxlags=maxlags) writer.add_figure( - f"acf/chain {c+1}/parameter {p+1}", fig, close=True, + f"acf/chain {c+1}/parameter {p+1}", + fig, + close=True, ) @@ -39,13 +41,14 @@ def tb_marginals(writer, mcmc, site_name="parameters", num_samples=1000): for c in range(samples.shape[0]): for p in range(samples.shape[-1]): writer.add_histogram( - f"marginal/{site_name}/{p+1}", samples[c, :].squeeze()[:, p], c, + f"marginal/{site_name}/{p+1}", + samples[c, :].squeeze()[:, p], + c, ) def tb_make_hook_fn(writer, site_name="parameters"): - """Builds hook function for runtime logging - """ + """Builds hook function for runtime logging""" def hook_fn(kernel, samples, stage, i): """Logging during run diff --git a/sbibm/algorithms/pytorch/baseline_prior.py b/sbibm/algorithms/pytorch/baseline_prior.py index 5c6fbf16..beba8479 100644 --- a/sbibm/algorithms/pytorch/baseline_prior.py +++ b/sbibm/algorithms/pytorch/baseline_prior.py @@ -6,7 +6,11 @@ from sbibm.tasks.task import Task -def run(task: Task, num_samples: int, **kwargs: Any,) -> torch.Tensor: +def run( + task: Task, + num_samples: int, + **kwargs: Any, +) -> torch.Tensor: """Random samples from prior as baseline Args: diff --git a/sbibm/algorithms/sbi/mcabc.py b/sbibm/algorithms/sbi/mcabc.py index 31873444..59dc6565 100644 --- a/sbibm/algorithms/sbi/mcabc.py +++ b/sbibm/algorithms/sbi/mcabc.py @@ -7,6 +7,7 @@ from sbibm.tasks.task import Task from sbibm.utils.io import save_tensor_to_csv from sbibm.utils.kde import get_kde + from .utils import get_sass_transform, run_lra diff --git a/sbibm/algorithms/sbi/utils.py b/sbibm/algorithms/sbi/utils.py index 93e79c35..9af5ab9f 100644 --- a/sbibm/algorithms/sbi/utils.py +++ b/sbibm/algorithms/sbi/utils.py @@ -1,8 +1,7 @@ import numpy as np import torch - -from sklearn.preprocessing import PolynomialFeatures from sklearn.linear_model import LinearRegression +from sklearn.preprocessing import PolynomialFeatures from torch.distributions.transformed_distribution import TransformedDistribution from sbibm.utils.nflows import FlowWrapper @@ -47,10 +46,10 @@ def clip_int(value, minimum, maximum): def get_sass_transform(theta, x, expansion_degree=1, sample_weight=None): """Return semi-automatic summary statitics function. - Running weighted linear regressin as in + Running weighted linear regressin as in Fearnhead & Prandle 2012: https://arxiv.org/abs/1004.1112 - - Following implementation in + + Following implementation in https://abcpy.readthedocs.io/en/latest/_modules/abcpy/statistics.html#Identity and https://pythonhosted.org/abcpy/_modules/abcpy/summaryselections.html#Semiautomatic @@ -89,7 +88,9 @@ def run_lra( for parameter_idx in range(theta.shape[1]): regression_model = LinearRegression(fit_intercept=True) regression_model.fit( - X=x, y=theta[:, parameter_idx], sample_weight=sample_weight, + X=x, + y=theta[:, parameter_idx], + sample_weight=sample_weight, ) theta_adjusted[:, parameter_idx] += regression_model.predict( observation.reshape(1, -1) diff --git a/sbibm/metrics/c2st.py b/sbibm/metrics/c2st.py index d345680e..f6f255d6 100644 --- a/sbibm/metrics/c2st.py +++ b/sbibm/metrics/c2st.py @@ -28,7 +28,7 @@ def c2st( n_folds: Number of folds z_score: Z-scoring using X noise_scale: If passed, will add Gaussian noise with std noise_scale to samples - + References: [1]: https://scikit-learn.org/stable/modules/cross_validation.html """ @@ -56,7 +56,12 @@ def c2st( ) data = np.concatenate((X, Y)) - target = np.concatenate((np.zeros((X.shape[0],)), np.ones((Y.shape[0],)),)) + target = np.concatenate( + ( + np.zeros((X.shape[0],)), + np.ones((Y.shape[0],)), + ) + ) shuffle = KFold(n_splits=n_folds, shuffle=True, random_state=seed) scores = cross_val_score(clf, data, target, cv=shuffle, scoring=scoring) @@ -84,7 +89,7 @@ def c2st_auc( n_folds: Number of folds z_score: Z-scoring using X noise_scale: If passed, will add Gaussian noise with std noise_scale to samples - + Returns: Metric """ diff --git a/sbibm/metrics/ksd.py b/sbibm/metrics/ksd.py index 4628f88c..630ae899 100644 --- a/sbibm/metrics/ksd.py +++ b/sbibm/metrics/ksd.py @@ -4,11 +4,11 @@ import numpy as np import torch -from sbibm.third_party.kgof.kernel import KGauss -from sbibm.third_party.kgof.goftest import KernelSteinTest, bootstrapper_rademacher -from sbibm.third_party.kgof.util import meddistance from sbibm.tasks.task import Task +from sbibm.third_party.kgof.goftest import KernelSteinTest, bootstrapper_rademacher +from sbibm.third_party.kgof.kernel import KGauss +from sbibm.third_party.kgof.util import meddistance from sbibm.utils.torch import get_default_device log = logging.getLogger(__name__) @@ -114,8 +114,7 @@ def ksd_gaussian_kernel( class DataWrapped: def __init__(self, data): - """Wraps `data` such that it can be used with `kgof` - """ + """Wraps `data` such that it can be used with `kgof`""" self._data = data def data(self): @@ -124,8 +123,7 @@ def data(self): class UnnormalizedDensityWrapped: def __init__(self, log_prob_grad: Callable): - """Wraps `log_prob_grad` function such that it can be used with `kgof` - """ + """Wraps `log_prob_grad` function such that it can be used with `kgof`""" self.log_prob_grad = log_prob_grad def grad_log(self, X: np.ndarray) -> np.ndarray: diff --git a/sbibm/metrics/mvn_kl.py b/sbibm/metrics/mvn_kl.py index 73a1ee34..a7a7483a 100644 --- a/sbibm/metrics/mvn_kl.py +++ b/sbibm/metrics/mvn_kl.py @@ -1,7 +1,11 @@ import torch -def mvn_kl_pq(X: torch.Tensor, Y: torch.Tensor, z_score: bool = True,) -> torch.Tensor: +def mvn_kl_pq( + X: torch.Tensor, + Y: torch.Tensor, + z_score: bool = True, +) -> torch.Tensor: """KL(p||q) between Multivariate Normal distributions X and Y are both sets of samples, the mean and covariance of which is estimated diff --git a/sbibm/metrics/ppc.py b/sbibm/metrics/ppc.py index 4bb4e0fd..add0a464 100644 --- a/sbibm/metrics/ppc.py +++ b/sbibm/metrics/ppc.py @@ -3,7 +3,8 @@ def median_distance( - predictive_samples: torch.Tensor, observation: torch.Tensor, + predictive_samples: torch.Tensor, + observation: torch.Tensor, ) -> torch.Tensor: """Compute median distance diff --git a/sbibm/tasks/bernoulli_glm/task.py b/sbibm/tasks/bernoulli_glm/task.py index c470a543..9a04b8c9 100644 --- a/sbibm/tasks/bernoulli_glm/task.py +++ b/sbibm/tasks/bernoulli_glm/task.py @@ -14,8 +14,7 @@ class BernoulliGLM(Task): def __init__(self, summary="sufficient"): - """Bernoulli GLM - """ + """Bernoulli GLM""" self.summary = summary if self.summary == "sufficient": dim_data = 10 @@ -67,8 +66,8 @@ def get_simulator(self, max_calls: Optional[int] = None) -> Simulator: """Get function returning samples from simulator given parameters Args: - max_calls: Maximum number of function calls. Additional calls will - result in SimulationBudgetExceeded exceptions. Defaults to None + max_calls: Maximum number of function calls. Additional calls will + result in SimulationBudgetExceeded exceptions. Defaults to None for infinite budget Return: @@ -116,8 +115,7 @@ def simulator( return Simulator(task=self, simulator=simulator, max_calls=max_calls) def get_observation(self, num_observation: int) -> torch.Tensor: - """Get observed data for a given observation number - """ + """Get observed data for a given observation number""" if not self.raw: path = ( self.path @@ -146,7 +144,9 @@ def flatten_data(self, data: torch.Tensor) -> torch.Tensor: return data.reshape(-1, self.dim_data) def _sample_reference_posterior( - self, num_samples: int, num_observation: Optional[int] = None, + self, + num_samples: int, + num_observation: Optional[int] = None, ) -> torch.Tensor: from pypolyagamma import PyPolyaGamma from tqdm import tqdm diff --git a/sbibm/tasks/gaussian_linear/task.py b/sbibm/tasks/gaussian_linear/task.py index 47d045a9..9f57e02a 100644 --- a/sbibm/tasks/gaussian_linear/task.py +++ b/sbibm/tasks/gaussian_linear/task.py @@ -112,7 +112,8 @@ def _get_reference_posterior( self.simulator_params["precision_matrix"], observation.reshape(-1) ) + torch.matmul( - self.prior_params["precision_matrix"], self.prior_params["loc"], + self.prior_params["precision_matrix"], + self.prior_params["loc"], ) ), ) @@ -138,12 +139,13 @@ def _sample_reference_posterior( num_observation: Observation number observation: Instead of passing an observation number, an observation may be passed directly - + Returns: Samples from reference posterior """ posterior = self._get_reference_posterior( - num_observation=num_observation, observation=observation, + num_observation=num_observation, + observation=observation, ) return posterior.sample((num_samples,)) diff --git a/sbibm/tasks/gaussian_linear_uniform/task.py b/sbibm/tasks/gaussian_linear_uniform/task.py index f8ee713e..03c42fcd 100644 --- a/sbibm/tasks/gaussian_linear_uniform/task.py +++ b/sbibm/tasks/gaussian_linear_uniform/task.py @@ -59,8 +59,8 @@ def get_simulator(self, max_calls: Optional[int] = None) -> Simulator: """Get function returning samples from simulator given parameters Args: - max_calls: Maximum number of function calls. Additional calls will - result in SimulationBudgetExceeded exceptions. Defaults to None + max_calls: Maximum number of function calls. Additional calls will + result in SimulationBudgetExceeded exceptions. Defaults to None for infinite budget Return: @@ -93,7 +93,7 @@ def _sample_reference_posterior( num_observation: Observation number observation: Instead of passing an observation number, an observation may be passed directly - + Returns: Samples from reference posterior """ @@ -108,7 +108,8 @@ def _sample_reference_posterior( reference_posterior_samples = [] sampling_dist = pdist.MultivariateNormal( - loc=observation, precision_matrix=self.simulator_params["precision_matrix"], + loc=observation, + precision_matrix=self.simulator_params["precision_matrix"], ) # Reject samples outside of prior bounds diff --git a/sbibm/tasks/gaussian_mixture/task.py b/sbibm/tasks/gaussian_mixture/task.py index ddd7d0a1..349a4c85 100644 --- a/sbibm/tasks/gaussian_mixture/task.py +++ b/sbibm/tasks/gaussian_mixture/task.py @@ -12,7 +12,9 @@ class GaussianMixture(Task): def __init__( - self, dim: int = 2, prior_bound: float = 10.0, + self, + dim: int = 2, + prior_bound: float = 10.0, ): """Gaussian Mixture. @@ -95,7 +97,7 @@ def _sample_reference_posterior( num_observation: Observation number observation: Instead of passing an observation number, an observation may be passed directly - + Returns: Samples from reference posterior """ diff --git a/sbibm/tasks/lotka_volterra/task.py b/sbibm/tasks/lotka_volterra/task.py index d4f25076..c3d1dd46 100644 --- a/sbibm/tasks/lotka_volterra/task.py +++ b/sbibm/tasks/lotka_volterra/task.py @@ -106,8 +106,7 @@ def de(self): ) def get_labels_parameters(self) -> List[str]: - """Get list containing parameter labels - """ + """Get list containing parameter labels""" return [r"$\alpha$", r"$\beta$", r"$\gamma$", r"$\delta$"] def get_prior(self) -> Callable: @@ -116,7 +115,10 @@ def prior(num_samples=1): return prior - def get_simulator(self, max_calls: Optional[int] = None,) -> Simulator: + def get_simulator( + self, + max_calls: Optional[int] = None, + ) -> Simulator: """Get function returning samples from simulator given parameters Args: @@ -181,8 +183,7 @@ def simulator(parameters): return Simulator(task=self, simulator=simulator, max_calls=max_calls) def unflatten_data(self, data: torch.Tensor) -> torch.Tensor: - """Unflattens data into multiple observations - """ + """Unflattens data into multiple observations""" if self.summary is None: return data.reshape(-1, 2, int(self.dim_data / 2)) else: diff --git a/sbibm/tasks/simulator.py b/sbibm/tasks/simulator.py index afc11b54..08e69867 100644 --- a/sbibm/tasks/simulator.py +++ b/sbibm/tasks/simulator.py @@ -8,14 +8,17 @@ class Simulator: def __init__( - self, task: Task, simulator: Callable, max_calls: Optional[int] = None, + self, + task: Task, + simulator: Callable, + max_calls: Optional[int] = None, ): """Simulator Each task defines a simulator and passes it into this class, which wraps it. When a simulator is called with parameters, the `__call__` method of this - class is invoked. - + class is invoked. + `__call__` simply forwards the parameters to the simulator function, while checking parameter dimensions and increasing an internal counter. The internal counter ensures that a simulator can only be called a certain maximum number diff --git a/sbibm/tasks/sir/task.py b/sbibm/tasks/sir/task.py index 9a92b837..aca4c323 100644 --- a/sbibm/tasks/sir/task.py +++ b/sbibm/tasks/sir/task.py @@ -110,8 +110,7 @@ def de(self): ) def get_labels_parameters(self) -> List[str]: - """Get list containing parameter labels - """ + """Get list containing parameter labels""" return [r"$\beta$", r"$\gamma$"] def get_prior(self) -> Callable: @@ -120,7 +119,10 @@ def prior(num_samples=1): return prior - def get_simulator(self, max_calls: Optional[int] = None,) -> Simulator: + def get_simulator( + self, + max_calls: Optional[int] = None, + ) -> Simulator: """Get function returning samples from simulator given parameters Args: @@ -181,8 +183,7 @@ def simulator(parameters): return Simulator(task=self, simulator=simulator, max_calls=max_calls) def unflatten_data(self, data: torch.Tensor) -> torch.Tensor: - """Unflattens data into multiple observations - """ + """Unflattens data into multiple observations""" if self.summary is None: return data.reshape(-1, 3, int(self.dim_data / 3)) else: diff --git a/sbibm/tasks/slcp/task.py b/sbibm/tasks/slcp/task.py index fa1b946d..cf33d0f9 100644 --- a/sbibm/tasks/slcp/task.py +++ b/sbibm/tasks/slcp/task.py @@ -12,8 +12,7 @@ class SLCP(Task): def __init__(self, distractors: bool = False): - """SLCP - """ + """SLCP""" self.num_data = 4 self.distractors = distractors @@ -102,7 +101,12 @@ def simulator(parameters): data_dist = pdist.MultivariateNormal( m.unsqueeze(1).float(), S.unsqueeze(1).float() - ).expand((num_samples, self.num_data,)) + ).expand( + ( + num_samples, + self.num_data, + ) + ) if not self.distractors: return pyro.sample("data", data_dist) @@ -123,8 +127,7 @@ def simulator(parameters): return Simulator(task=self, simulator=simulator, max_calls=max_calls) def get_observation(self, num_observation: int) -> torch.Tensor: - """Get observed data for a given observation number - """ + """Get observed data for a given observation number""" if not self.distractors: path = ( self.path @@ -178,8 +181,7 @@ def _get_transforms( ) def unflatten_data(self, data: torch.Tensor) -> torch.Tensor: - """Unflattens data into multiple observations - """ + """Unflattens data into multiple observations""" if not self.distractors: return data.reshape(-1, self.num_data, 2) else: @@ -255,9 +257,14 @@ def _generate_noise_dist_parameters(self): ] scale_tril = torch.from_numpy(3 * np.array(cholesky_factors)) - mix = pdist.Categorical(torch.ones(n_noise_comps,)) + mix = pdist.Categorical( + torch.ones( + n_noise_comps, + ) + ) comp = pdist.Independent( - pdist.MultivariateStudentT(df=2, loc=loc, scale_tril=scale_tril), 0, + pdist.MultivariateStudentT(df=2, loc=loc, scale_tril=scale_tril), + 0, ) gmm = pdist.MixtureSameFamily(mix, comp) torch.save(gmm, "files/gmm.torch") diff --git a/sbibm/tasks/task.py b/sbibm/tasks/task.py index aa123b6c..adbfbb83 100644 --- a/sbibm/tasks/task.py +++ b/sbibm/tasks/task.py @@ -67,33 +67,27 @@ def __init__( @abstractmethod def get_prior(self) -> Callable: - """Get function returning parameters from prior - """ + """Get function returning parameters from prior""" raise NotImplementedError def get_prior_dist(self) -> torch.distributions.Distribution: - """Get prior distribution - """ + """Get prior distribution""" return self.prior_dist def get_prior_params(self) -> Dict[str, torch.Tensor]: - """Get parameters of prior distribution - """ + """Get parameters of prior distribution""" return self.prior_params def get_labels_data(self) -> List[str]: - """Get list containing parameter labels - """ + """Get list containing parameter labels""" return [f"data_{i+1}" for i in range(self.dim_data)] def get_labels_parameters(self) -> List[str]: - """Get list containing parameter labels - """ + """Get list containing parameter labels""" return [f"parameter_{i+1}" for i in range(self.dim_parameters)] def get_observation(self, num_observation: int) -> torch.Tensor: - """Get observed data for a given observation number - """ + """Get observed data for a given observation number""" path = ( self.path / "files" @@ -103,8 +97,7 @@ def get_observation(self, num_observation: int) -> torch.Tensor: return get_tensor_from_csv(path) def get_reference_posterior_samples(self, num_observation: int) -> torch.Tensor: - """Get reference posterior samples for a given observation number - """ + """Get reference posterior samples for a given observation number""" path = ( self.path / "files" @@ -115,13 +108,11 @@ def get_reference_posterior_samples(self, num_observation: int) -> torch.Tensor: @abstractmethod def get_simulator(self) -> Callable: - """Get function returning parameters from prior - """ + """Get function returning parameters from prior""" raise NotImplementedError def get_true_parameters(self, num_observation: int) -> torch.Tensor: - """Get true parameters (parameters that generated the data) for a given observation number - """ + """Get true parameters (parameters that generated the data) for a given observation number""" path = ( self.path / "files" @@ -131,13 +122,11 @@ def get_true_parameters(self, num_observation: int) -> torch.Tensor: return get_tensor_from_csv(path) def save_data(self, path: Union[str, Path], data: torch.Tensor): - """Save data to a given path - """ + """Save data to a given path""" save_tensor_to_csv(path, data, self.get_labels_data()) def save_parameters(self, path: Union[str, Path], parameters: torch.Tensor): - """Save parameters to a given path - """ + """Save parameters to a given path""" save_tensor_to_csv(path, parameters, self.get_labels_parameters()) def flatten_data(self, data: torch.Tensor) -> torch.Tensor: @@ -188,7 +177,9 @@ def _get_log_prob_fn( ) log_prob_fn, _ = get_log_prob_fn( - conditioned_model, implementation=implementation, **kwargs, + conditioned_model, + implementation=implementation, + **kwargs, ) def log_prob_pyro(parameters): @@ -234,7 +225,7 @@ def _get_log_prob_grad_fn( kwargs: Passed to `sbibm.utils.pyro.get_log_prob_grad_fn` Returns: - `log_prob_grad_fn` that returns gradients as `batch_size` x + `log_prob_grad_fn` that returns gradients as `batch_size` x `dim_parameter` """ assert not (num_observation is None and observation is None) @@ -248,7 +239,9 @@ def _get_log_prob_grad_fn( posterior=posterior, ) log_prob_grad_fn, _ = get_log_prob_grad_fn( - conditioned_model, implementation=implementation, **kwargs, + conditioned_model, + implementation=implementation, + **kwargs, ) def log_prob_grad_pyro(parameters): @@ -289,7 +282,7 @@ def _get_transforms( num_observation: Observation number observation: Instead of passing an observation number, an observation may be passed directly - automatic_transforms_enabled: If True, will automatically construct + automatic_transforms_enabled: If True, will automatically construct transforms to unconstrained space Returns: @@ -300,14 +293,14 @@ def _get_transforms( ) _, transforms = get_log_prob_fn( - conditioned_model, automatic_transform_enabled=automatic_transforms_enabled, + conditioned_model, + automatic_transform_enabled=automatic_transforms_enabled, ) return transforms def _get_observation_seed(self, num_observation: int) -> int: - """Get observation seed for a given observation number - """ + """Get observation seed for a given observation number""" path = ( self.path / "files" @@ -372,8 +365,7 @@ def _sample_reference_posterior( raise NotImplementedError def _save_observation_seed(self, num_observation: int, observation_seed: int): - """Save observation seed for a given observation number - """ + """Save observation seed for a given observation number""" path = ( self.path / "files" @@ -387,8 +379,7 @@ def _save_observation_seed(self, num_observation: int, observation_seed: int): ).to_csv(path, index=False) def _save_observation(self, num_observation: int, observation: torch.Tensor): - """Save observed data for a given observation number - """ + """Save observed data for a given observation number""" path = ( self.path / "files" @@ -401,8 +392,7 @@ def _save_observation(self, num_observation: int, observation: torch.Tensor): def _save_reference_posterior_samples( self, num_observation: int, reference_posterior_samples: torch.Tensor ): - """Save reference posterior samples for a given observation number - """ + """Save reference posterior samples for a given observation number""" path = ( self.path / "files" @@ -415,8 +405,7 @@ def _save_reference_posterior_samples( def _save_true_parameters( self, num_observation: int, true_parameters: torch.Tensor ): - """Save true parameters (parameters that generated the data) for a given observation number - """ + """Save true parameters (parameters that generated the data) for a given observation number""" path = ( self.path / "files" @@ -461,7 +450,8 @@ def run(num_observation, observation_seed, **kwargs): num_unique = torch.unique(reference_posterior_samples, dim=0).shape[0] assert num_unique == self.num_reference_posterior_samples self._save_reference_posterior_samples( - num_observation, reference_posterior_samples, + num_observation, + reference_posterior_samples, ) Parallel(n_jobs=n_jobs, verbose=50, backend="loky")( diff --git a/sbibm/tasks/two_moons/task.py b/sbibm/tasks/two_moons/task.py index 8c21ebc1..6635c293 100644 --- a/sbibm/tasks/two_moons/task.py +++ b/sbibm/tasks/two_moons/task.py @@ -15,8 +15,7 @@ class TwoMoons(Task): def __init__(self): - """Two Moons - """ + """Two Moons""" # Observation seeds to use when generating ground truth observation_seeds = [ @@ -70,8 +69,8 @@ def get_simulator(self, max_calls: Optional[int] = None) -> Simulator: """Get function returning samples from simulator given parameters Args: - max_calls: Maximum number of function calls. Additional calls will - result in SimulationBudgetExceeded exceptions. Defaults to None + max_calls: Maximum number of function calls. Additional calls will + result in SimulationBudgetExceeded exceptions. Defaults to None for infinite budget Return: @@ -131,7 +130,10 @@ def _map_fun_inv(parameters: torch.Tensor, x: torch.Tensor) -> torch.Tensor: return x - torch.cat((-torch.abs(z0), z1), dim=1) def _likelihood( - self, parameters: torch.Tensor, data: torch.Tensor, log: bool = True, + self, + parameters: torch.Tensor, + data: torch.Tensor, + log: bool = True, ) -> torch.Tensor: if parameters.ndim == 1: parameters = parameters.reshape(1, -1) @@ -157,7 +159,11 @@ def _likelihood( return L if log else torch.exp(L) - def _get_transforms(self, *args, **kwargs: Any,) -> Dict[str, Any]: + def _get_transforms( + self, + *args, + **kwargs: Any, + ) -> Dict[str, Any]: return {"parameters": torch.distributions.transforms.identity_transform} def _get_log_prob_fn( @@ -170,8 +176,8 @@ def _get_log_prob_fn( The potential function returns the unnormalized negative log posterior probability, and is useful to establish and verify - the reference posterior. - + the reference posterior. + Args: num_observation: Observation number observation: Instead of passing an observation number, an observation may be @@ -238,7 +244,7 @@ def _sample_reference_posterior( num_samples: Number of samples to generate num_observation: Observation number observation: Observed data, if None, will be loaded using `num_observation` - + Returns: Samples from reference posterior """ diff --git a/sbibm/third_party/kgof/config.py b/sbibm/third_party/kgof/config.py index 3bbc58a1..c982dcdc 100644 --- a/sbibm/third_party/kgof/config.py +++ b/sbibm/third_party/kgof/config.py @@ -1,21 +1,18 @@ - """ This file defines global configuration of the project. Casual usage of the package should not need to change this. """ -import sbibm.third_party.kgof.glo as glo import os +import sbibm.third_party.kgof.glo as glo + expr_configs = { # Full path to the directory to store temporary files when running # experiments. - 'scratch_path': '/nfs/data3/wittawat/tmp/', - + "scratch_path": "/nfs/data3/wittawat/tmp/", # Full path to the directory to store experimental results. - 'expr_results_path': '/nfs/data3/wittawat/kgof/results/', - + "expr_results_path": "/nfs/data3/wittawat/kgof/results/", # Full path to the data directory - 'data_path': os.path.join(os.path.dirname(glo.get_root()), 'data') + "data_path": os.path.join(os.path.dirname(glo.get_root()), "data"), } - diff --git a/sbibm/third_party/kgof/data.py b/sbibm/third_party/kgof/data.py index 3785ae05..b7e9c982 100644 --- a/sbibm/third_party/kgof/data.py +++ b/sbibm/third_party/kgof/data.py @@ -1,23 +1,26 @@ """ Module containing data structures for representing datasets. """ -from __future__ import print_function -from __future__ import division +from __future__ import division, print_function + +from builtins import object, range -from builtins import range -from past.utils import old_div -from builtins import object from future.utils import with_metaclass -__author__ = 'wittawat' +from past.utils import old_div + +__author__ = "wittawat" from abc import ABCMeta, abstractmethod + import autograd.numpy as np -import sbibm.third_party.kgof.util as util import scipy.stats as stats +import sbibm.third_party.kgof.util as util + + class Data(object): """ - Class representing a dataset i.e., en encapsulation of a data matrix + Class representing a dataset i.e., en encapsulation of a data matrix whose rows are vectors drawn from a distribution. """ @@ -28,17 +31,17 @@ def __init__(self, X): self.X = X if not np.all(np.isfinite(X)): - print('X:') + print("X:") print(util.fullprint(X)) - raise ValueError('Not all elements in X are finite.') + raise ValueError("Not all elements in X are finite.") def __str__(self): mean_x = np.mean(self.X, 0) - std_x = np.std(self.X, 0) + std_x = np.std(self.X, 0) prec = 4 - desc = '' - desc += 'E[x] = %s \n'%(np.array_str(mean_x, precision=prec ) ) - desc += 'Std[x] = %s \n' %(np.array_str(std_x, precision=prec)) + desc = "" + desc += "E[x] = %s \n" % (np.array_str(mean_x, precision=prec)) + desc += "Std[x] = %s \n" % (np.array_str(std_x, precision=prec)) return desc def dim(self): @@ -56,8 +59,8 @@ def data(self): """Return the data matrix.""" return self.X - def split_tr_te(self, tr_proportion=0.5, seed=820, return_tr_ind = False): - """Split the dataset into training and test sets. + def split_tr_te(self, tr_proportion=0.5, seed=820, return_tr_ind=False): + """Split the dataset into training and test sets. Return (Data for tr, Data for te)""" X = self.X @@ -70,11 +73,11 @@ def split_tr_te(self, tr_proportion=0.5, seed=820, return_tr_ind = False): else: return (tr_data, te_data) - def subsample(self, n, seed=87, return_ind = False): - """Subsample without replacement. Return a new Data. """ + def subsample(self, n, seed=87, return_ind=False): + """Subsample without replacement. Return a new Data.""" if n > self.X.shape[0]: - raise ValueError('n should not be larger than sizes of X') - ind_x = util.subsample_ind( self.X.shape[0], n, seed ) + raise ValueError("n should not be larger than sizes of X") + ind_x = util.subsample_ind(self.X.shape[0], n, seed) if return_ind: return Data(self.X[ind_x, :]), ind_x else: @@ -82,7 +85,7 @@ def subsample(self, n, seed=87, return_ind = False): def clone(self): """ - Return a new Data object with a separate copy of each internal + Return a new Data object with a separate copy of each internal variable, and with the same content. """ nX = np.copy(self.X) @@ -98,44 +101,48 @@ def __add__(self, data2): nX = np.vstack((copy.X, copy2.X)) return Data(nX) -### end Data class + +### end Data class class DataSource(with_metaclass(ABCMeta, object)): """ - A source of data allowing resampling. Subclasses may prefix - class names with DS. + A source of data allowing resampling. Subclasses may prefix + class names with DS. """ @abstractmethod def sample(self, n, seed): - """Return a Data. Returned result should be deterministic given + """Return a Data. Returned result should be deterministic given the input (n, seed).""" raise NotImplementedError() def dim(self): - """ - Return the dimension of the data. If possible, subclasses should - override this. Determining the dimension by sampling may not be - efficient, especially if the sampling relies on MCMC. - """ - dat = self.sample(n=1, seed=3) - return dat.dim() + """ + Return the dimension of the data. If possible, subclasses should + override this. Determining the dimension by sampling may not be + efficient, especially if the sampling relies on MCMC. + """ + dat = self.sample(n=1, seed=3) + return dat.dim() + # end DataSource + class DSIsotropicNormal(DataSource): """ A DataSource providing samples from a mulivariate isotropic normal distribution. """ + def __init__(self, mean, variance): """ - mean: a numpy array of length d for the mean + mean: a numpy array of length d for the mean variance: a positive floating-point number for the variance. """ assert len(mean.shape) == 1 - self.mean = mean + self.mean = mean self.variance = variance def sample(self, n, seed=2): @@ -143,7 +150,7 @@ def sample(self, n, seed=2): d = len(self.mean) mean = self.mean variance = self.variance - X = np.random.randn(n, d)*np.sqrt(variance) + mean + X = np.random.randn(n, d) * np.sqrt(variance) + mean return Data(X) @@ -151,12 +158,13 @@ class DSNormal(DataSource): """ A DataSource implementing a multivariate Gaussian. """ + def __init__(self, mean, cov): """ mean: a numpy array of length d. cov: d x d numpy array for the covariance. """ - self.mean = mean + self.mean = mean self.cov = cov assert mean.shape[0] == cov.shape[0] assert cov.shape[0] == cov.shape[1] @@ -165,18 +173,20 @@ def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): mvn = stats.multivariate_normal(self.mean, self.cov) X = mvn.rvs(size=n) - if len(X.shape) ==1: + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + class DSIsoGaussianMixture(DataSource): """ - A DataSource implementing a Gaussian mixture in R^d where each component + A DataSource implementing a Gaussian mixture in R^d where each component is an isotropic multivariate normal distribution. Let k be the number of mixture components. """ + def __init__(self, means, variances, pmix=None): """ means: a k x d 2d array specifying the means. @@ -185,13 +195,15 @@ def __init__(self, means, variances, pmix=None): """ k, d = means.shape if k != len(variances): - raise ValueError('Number of components in means and variances do not match.') + raise ValueError( + "Number of components in means and variances do not match." + ) if pmix is None: - pmix = old_div(np.ones(k),float(k)) + pmix = old_div(np.ones(k), float(k)) if np.abs(np.sum(pmix) - 1) > 1e-8: - raise ValueError('Mixture weights do not sum to 1.') + raise ValueError("Mixture weights do not sum to 1.") self.pmix = pmix self.means = means @@ -204,31 +216,34 @@ def sample(self, n, seed=29): k, d = self.means.shape sam_list = [] with util.NumpySeedContext(seed=seed): - # counts for each mixture component + # counts for each mixture component counts = np.random.multinomial(n, pmix, size=1) # counts is a 2d array counts = counts[0] - # For each component, draw from its corresponding mixture component. + # For each component, draw from its corresponding mixture component. for i, nc in enumerate(counts): # Sample from ith component - sam_i = np.random.randn(nc, d)*np.sqrt(variances[i]) + means[i] + sam_i = np.random.randn(nc, d) * np.sqrt(variances[i]) + means[i] sam_list.append(sam_i) sample = np.vstack(sam_list) assert sample.shape[0] == n np.random.shuffle(sample) return Data(sample) + # end of class DSIsoGaussianMixture + class DSGaussianMixture(DataSource): """ - A DataSource implementing a Gaussian mixture in R^d where each component + A DataSource implementing a Gaussian mixture in R^d where each component is an arbitrary Gaussian distribution. Let k be the number of mixture components. """ + def __init__(self, means, variances, pmix=None): """ means: a k x d 2d array specifying the means. @@ -238,13 +253,15 @@ def __init__(self, means, variances, pmix=None): """ k, d = means.shape if k != variances.shape[0]: - raise ValueError('Number of components in means and variances do not match.') + raise ValueError( + "Number of components in means and variances do not match." + ) if pmix is None: - pmix = old_div(np.ones(k),float(k)) + pmix = old_div(np.ones(k), float(k)) if np.abs(np.sum(pmix) - 1) > 1e-8: - raise ValueError('Mixture weights do not sum to 1.') + raise ValueError("Mixture weights do not sum to 1.") self.pmix = pmix self.means = means @@ -257,13 +274,13 @@ def sample(self, n, seed=29): k, d = self.means.shape sam_list = [] with util.NumpySeedContext(seed=seed): - # counts for each mixture component + # counts for each mixture component counts = np.random.multinomial(n, pmix, size=1) # counts is a 2d array counts = counts[0] - # For each component, draw from its corresponding mixture component. + # For each component, draw from its corresponding mixture component. for i, nc in enumerate(counts): # construct the component # https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.multivariate_normal.html @@ -277,6 +294,7 @@ def sample(self, n, seed=29): np.random.shuffle(sample) return Data(sample) + # end of DSGaussianMixture @@ -284,9 +302,10 @@ class DSLaplace(DataSource): """ A DataSource for a multivariate Laplace distribution. """ + def __init__(self, d, loc=0, scale=1): """ - loc: location + loc: location scale: scale parameter. Described in https://docs.scipy.org/doc/numpy/reference/generated/numpy.random.laplace.html#numpy.random.laplace """ @@ -300,16 +319,18 @@ def sample(self, n, seed=4): X = np.random.laplace(loc=self.loc, scale=self.scale, size=(n, self.d)) return Data(X) + class DSTDistribution(DataSource): """ A DataSource for a univariate T-distribution. """ + def __init__(self, df): """ df: degrees of freedom """ assert df > 0 - self.df = df + self.df = df def sample(self, n, seed=5): with util.NumpySeedContext(seed=seed): @@ -317,6 +338,7 @@ def sample(self, n, seed=5): X = X[:, np.newaxis] return Data(X) + # end class DSTDistribution @@ -324,16 +346,17 @@ class DSGaussBernRBM(DataSource): """ A DataSource implementing a Gaussian-Bernoulli Restricted Boltzmann Machine. The probability of the latent vector h is controlled by the vector c. - The parameterization of the Gaussian-Bernoulli RBM is given in + The parameterization of the Gaussian-Bernoulli RBM is given in density.GaussBernRBM. - It turns out that this is equivalent to drawing a vector of {-1, 1} for h according to h ~ Discrete(sigmoid(2c)). - Draw x | h ~ N(B*h+b, I) """ + def __init__(self, B, b, c, burnin=2000): """ - B: a dx x dh matrix + B: a dx x dh matrix b: a numpy array of length dx c: a numpy array of length dh burnin: burn-in iterations when doing Gibbs sampling @@ -355,7 +378,7 @@ def sigmoid(x): """ x: a numpy array. """ - return old_div(1.0,(1+np.exp(-x))) + return old_div(1.0, (1 + np.exp(-x))) def _blocked_gibbs_next(self, X, H): """ @@ -367,15 +390,15 @@ def _blocked_gibbs_next(self, X, H): b = self.b # Draw H. - XB2C = np.dot(X, self.B) + 2.0*self.c + XB2C = np.dot(X, self.B) + 2.0 * self.c # Ph: n x dh matrix Ph = DSGaussBernRBM.sigmoid(XB2C) # H: n x dh - H = (np.random.rand(n, dh) <= Ph)*2 - 1.0 - assert np.all(np.abs(H) - 1 <= 1e-6 ) + H = (np.random.rand(n, dh) <= Ph) * 2 - 1.0 + assert np.all(np.abs(H) - 1 <= 1e-6) # Draw X. # mean: n x dx - mean = old_div(np.dot(H, B.T),2.0) + b + mean = old_div(np.dot(H, B.T), 2.0) + b X = np.random.randn(n, dx) + mean return X, H @@ -392,7 +415,7 @@ def sample(self, n, seed=3, return_latent=False): # Initialize the state of the Markov chain with util.NumpySeedContext(seed=seed): X = np.random.randn(n, dx) - H = np.random.randint(1, 2, (n, dh))*2 - 1.0 + H = np.random.randint(1, 2, (n, dh)) * 2 - 1.0 # burn-in for t in range(self.burnin): @@ -407,45 +430,53 @@ def sample(self, n, seed=3, return_latent=False): def dim(self): return self.B.shape[0] + # end class DSGaussBernRBM + class DSISIPoissonLinear(DataSource): """ A DataSource implementing non homogenous poisson process. """ + def __init__(self, b): """ b: slope in of the linear function lambda_X = 1 + bX """ if b < 0: - raise ValueError('Parameter b must be non-negative.') + raise ValueError("Parameter b must be non-negative.") self.b = b - - def nonhom_linear(self,size): + + def nonhom_linear(self, size): b = self.b u = np.random.rand(size) if np.abs(b) <= 1e-8: - F_l = -np.log(1-u) + F_l = -np.log(1 - u) else: - F_l = np.sqrt(-2.0/b*np.log(1-u)+old_div(1.0,(b**2)))-old_div(1.0,b) + F_l = np.sqrt(-2.0 / b * np.log(1 - u) + old_div(1.0, (b ** 2))) - old_div( + 1.0, b + ) return F_l - + def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): X = self.nonhom_linear(size=n) - if len(X.shape) ==1: + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSISIPoissonLinear + class DSISIPoissonSine(DataSource): """ A DataSource implementing non homogenous poisson process with sine intensity. lambda = b*(1+sin(w*X)) """ + def __init__(self, w, b, delta=0.001): """ w: the frequency of sine function @@ -454,11 +485,11 @@ def __init__(self, w, b, delta=0.001): self.b = b self.w = w self.delta = delta - - def func(self,t): - val = (t + old_div((1-np.cos(self.w*t)),self.w) )*self.b + + def func(self, t): + val = (t + old_div((1 - np.cos(self.w * t)), self.w)) * self.b return val - + # slow step-by-step increment by assigned delta def find_time(self, x): t = 0.0 @@ -475,7 +506,7 @@ def search_time(self, x): t_new = b val_old = self.func(t_old) val_new = self.func(t_new) - while np.abs(val_new-x) > delta: + while np.abs(val_new - x) > delta: if val_new < x and t_old < t_new: t_old = t_new t_new = t_new * 2.0 @@ -500,10 +531,10 @@ def search_time(self, x): val_new = self.func(t_new) t = t_new return t - - def nonhom_sine(self,size=1000): + + def nonhom_sine(self, size=1000): u = np.random.rand(size) - x = -np.log(1-u) + x = -np.log(1 - u) t = np.zeros(size) for i in range(size): t[i] = self.search_time(x[i]) @@ -512,11 +543,12 @@ def nonhom_sine(self,size=1000): def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): X = self.nonhom_sine(size=n) - if len(X.shape) ==1: + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSISIPoissonSine @@ -524,6 +556,7 @@ class DSGamma(DataSource): """ A DataSource implementing gamma distribution. """ + def __init__(self, alpha, beta=1.0): """ alpha: shape of parameter @@ -534,12 +567,13 @@ def __init__(self, alpha, beta=1.0): def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): - X = stats.gamma.rvs(self.alpha, size=n, scale = old_div(1.0,self.beta)) - if len(X.shape) ==1: + X = stats.gamma.rvs(self.alpha, size=n, scale=old_div(1.0, self.beta)) + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSGamma @@ -547,6 +581,7 @@ class DSLogGamma(DataSource): """ A DataSource implementing the transformed gamma distribution. """ + def __init__(self, alpha, beta=1.0): """ alpha: shape of parameter @@ -557,103 +592,115 @@ def __init__(self, alpha, beta=1.0): def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): - X = np.log(stats.gamma.rvs(self.alpha, size=n, scale = old_div(1.0,self.beta))) - if len(X.shape) ==1: + X = np.log( + stats.gamma.rvs(self.alpha, size=n, scale=old_div(1.0, self.beta)) + ) + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSLogGamma + class DSISILogPoissonLinear(DataSource): """ A DataSource implementing non homogenous poisson process. """ + def __init__(self, b): """ b: slope in of the linear function lambda_X = 1 + bX """ if b < 0: - raise ValueError('Parameter b must be non-negative.') + raise ValueError("Parameter b must be non-negative.") self.b = b - - def nonhom_linear(self,size): + + def nonhom_linear(self, size): b = self.b u = np.random.rand(size) if np.abs(b) <= 1e-8: - F_l = -np.log(1-u) + F_l = -np.log(1 - u) else: - F_l = np.sqrt(-2.0/b*np.log(1-u)+old_div(1.0,(b**2)))-old_div(1.0,b) + F_l = np.sqrt(-2.0 / b * np.log(1 - u) + old_div(1.0, (b ** 2))) - old_div( + 1.0, b + ) return F_l - + def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): X = np.log(self.nonhom_linear(size=n)) - if len(X.shape) ==1: + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSISILogPoissonLinear + class DSISIPoisson2D(DataSource): """ - A DataSource implementing non homogenous poisson process. + A DataSource implementing non homogenous poisson process. """ - def __init__(self, intensity = 'quadratic', w=1.0): + + def __init__(self, intensity="quadratic", w=1.0): """ lambda_(X,Y) = X^2 + Y^2 lamb_bar: upper-bound used in rejection sampling """ self.w = w - if intensity == 'quadratic': + if intensity == "quadratic": self.intensity = self.quadratic_intensity - elif intensity == 'sine': + elif intensity == "sine": self.intensity = self.sine_intensity - elif intensity == 'xsine': + elif intensity == "xsine": self.intensity = self.cross_sine_intensity else: - raise ValueError('Not intensity function found') - + raise ValueError("Not intensity function found") def quadratic_intensity(self, X): - intensity = self.lamb_bar*np.sum(X**2,1) + intensity = self.lamb_bar * np.sum(X ** 2, 1) return intensity def sine_intensity(self, X): - intensity = self.lamb_bar*np.sum(np.sin(self.w*X*np.pi),1) + intensity = self.lamb_bar * np.sum(np.sin(self.w * X * np.pi), 1) return intensity def cross_sine_intensity(self, X): - intensity = self.lamb_bar*np.prod(np.sin(self.w*X*np.pi),1) + intensity = self.lamb_bar * np.prod(np.sin(self.w * X * np.pi), 1) return intensity - def inh2d(self, lamb_bar = 100000): + def inh2d(self, lamb_bar=100000): self.lamb_bar = lamb_bar - N = np.random.poisson(2*self.lamb_bar) - X = np.random.rand(N,2) + N = np.random.poisson(2 * self.lamb_bar) + X = np.random.rand(N, 2) intensity = self.intensity(X) u = np.random.rand(N) - lamb_T = old_div(intensity,lamb_bar) + lamb_T = old_div(intensity, lamb_bar) X_acc = X[u < lamb_T] return X_acc def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): X = self.inh2d(lamb_bar=n) - if len(X.shape) ==1: + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSISIPoisson2D + class DSISISigmoidPoisson2D(DataSource): """ - A DataSource implementing non homogenous poisson process. + A DataSource implementing non homogenous poisson process. """ - def __init__(self, intensity = 'quadratic', w=1.0, a=1.0): + + def __init__(self, intensity="quadratic", w=1.0, a=1.0): """ lambda_(X,Y) = a*X^2 + Y^2 X = 1/(1+exp(s)) @@ -662,88 +709,93 @@ def __init__(self, intensity = 'quadratic', w=1.0, a=1.0): """ self.a = a self.w = w - if intensity == 'quadratic': + if intensity == "quadratic": self.intensity = self.quadratic_intensity - elif intensity == 'sine': + elif intensity == "sine": self.intensity = self.sine_intensity - elif intensity == 'xsine': + elif intensity == "xsine": self.intensity = self.cross_sine_intensity else: - raise ValueError('Not intensity function found') + raise ValueError("Not intensity function found") def quadratic_intensity(self, X): - intensity = self.lamb_bar*np.average(X**2, axis=1, weights=[self.a,1]) + intensity = self.lamb_bar * np.average(X ** 2, axis=1, weights=[self.a, 1]) return intensity def cross_sine_intensity(self, X): - intensity = self.lamb_bar*np.prod(np.sin(self.w*X*np.pi),1) + intensity = self.lamb_bar * np.prod(np.sin(self.w * X * np.pi), 1) return intensity def sine_intensity(self, X): - intensity = self.lamb_bar*np.sum(np.sin(self.w*X*np.pi),1) + intensity = self.lamb_bar * np.sum(np.sin(self.w * X * np.pi), 1) return intensity - - def inh2d(self, lamb_bar = 100000): + def inh2d(self, lamb_bar=100000): self.lamb_bar = lamb_bar - N = np.random.poisson(2*self.lamb_bar) - X = np.random.rand(N,2) + N = np.random.poisson(2 * self.lamb_bar) + X = np.random.rand(N, 2) intensity = self.intensity(X) u = np.random.rand(N) - lamb_T = old_div(intensity,lamb_bar) + lamb_T = old_div(intensity, lamb_bar) X_acc = X[u < lamb_T] return X_acc def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): - X = np.log(old_div(1,self.inh2d(lamb_bar=n))-1) - if len(X.shape) ==1: + X = np.log(old_div(1, self.inh2d(lamb_bar=n)) - 1) + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + + # end class DSISISigmoidPoisson2D + class DSPoisson2D(DataSource): """ - A DataSource implementing non homogenous poisson process. + A DataSource implementing non homogenous poisson process. """ - def __init__(self, w = 1.0): + + def __init__(self, w=1.0): """ 2D spatial poission process with default lambda_(X,Y) = sin(w*pi*X)+sin(w*pi*Y) """ self.w = w - def gmm_sample(self, mean=None, w=None, N=10000,n=10,d=2,seed=10): + def gmm_sample(self, mean=None, w=None, N=10000, n=10, d=2, seed=10): np.random.seed(seed) self.d = d if mean is None: - mean = np.random.randn(n,d)*10 + mean = np.random.randn(n, d) * 10 if w is None: w = np.random.rand(n) - w = old_div(w,sum(w)) - multi = np.random.multinomial(N,w) - X = np.zeros((N,d)) + w = old_div(w, sum(w)) + multi = np.random.multinomial(N, w) + X = np.zeros((N, d)) base = 0 for i in range(n): - X[base:base+multi[i],:] = np.random.multivariate_normal(mean[i,:], np.eye(self.d), multi[i]) + X[base : base + multi[i], :] = np.random.multivariate_normal( + mean[i, :], np.eye(self.d), multi[i] + ) base += multi[i] - + llh = np.zeros(N) for i in range(n): - llh += w[i] * stats.multivariate_normal.pdf(X, mean[i,:], np.eye(self.d)) - #llh = llh/sum(llh) + llh += w[i] * stats.multivariate_normal.pdf(X, mean[i, :], np.eye(self.d)) + # llh = llh/sum(llh) return X, llh def const(self, X): - return np.ones(len(X))*8 + return np.ones(len(X)) * 8 def lamb_sin(self, X): - return np.prod(np.sin(self.w*np.pi*X),1)*15 + return np.prod(np.sin(self.w * np.pi * X), 1) * 15 - def rej_sample(self, X, llh, func = None): + def rej_sample(self, X, llh, func=None): if func is None: self.func = self.lamb_sin - rate = old_div(self.func(X),llh) + rate = old_div(self.func(X), llh) u = np.random.rand(len(X)) X_acc = X[u < rate] return X_acc @@ -752,17 +804,18 @@ def sample(self, n, seed=3): with util.NumpySeedContext(seed=seed): X_gmm, llh = self.gmm_sample(N=n) X = X_gmm - if len(X.shape) ==1: + if len(X.shape) == 1: # This can happen if d=1 X = X[:, np.newaxis] return Data(X) + # end class DSPoisson2D class DSResample(DataSource): """ - A DataSource which subsamples without replacement from the specified + A DataSource which subsamples without replacement from the specified numpy array (n x d). """ @@ -772,35 +825,40 @@ def __init__(self, X): """ self.X = X - def sample(self, n, seed=900, return_ind = False): + def sample(self, n, seed=900, return_ind=False): X = self.X if n > X.shape[0]: # Sample more than what we have - raise ValueError('Cannot subsample n={0} from only {1} points.'.format(n, X.shape[0])) + raise ValueError( + "Cannot subsample n={0} from only {1} points.".format(n, X.shape[0]) + ) dat = Data(self.X) - return dat.subsample(n, seed=seed, return_ind = return_ind) + return dat.subsample(n, seed=seed, return_ind=return_ind) def dim(self): return self.X.shape[1] + # end class DSResample + class DSGaussCosFreqs(DataSource): """ - A DataSource to sample from the density + A DataSource to sample from the density p(x) \propto exp(-||x||^2/2sigma^2)*(1+ prod_{i=1}^d cos(w_i*x_i)) where w1,..wd are frequencies of each dimension. sigma^2 is the overall variance. """ + def __init__(self, sigma2, freqs): """ sigma2: overall scale of the distribution. A positive scalar. freqs: a 1-d array of length d for the frequencies. """ self.sigma2 = sigma2 - if sigma2 <= 0 : - raise ValueError('sigma2 must be > 0') + if sigma2 <= 0: + raise ValueError("sigma2 must be > 0") self.freqs = freqs def sample(self, n, seed=872): @@ -818,14 +876,14 @@ def sample(self, n, seed=872): from_ind = 0 while from_ind < n: # The proposal q is N(0, sigma2*I) - X = np.random.randn(block_size, d)*np.sqrt(sigma2) - q_un = np.exp(old_div(-np.sum(X**2, 1),(2.0*sigma2))) + X = np.random.randn(block_size, d) * np.sqrt(sigma2) + q_un = np.exp(old_div(-np.sum(X ** 2, 1), (2.0 * sigma2))) # unnormalized density p - p_un = q_un*(1+np.prod(np.cos(X*freqs), 1)) + p_un = q_un * (1 + np.prod(np.cos(X * freqs), 1)) c = 2.0 - I = stats.uniform.rvs(size=block_size) < old_div(p_un,(c*q_un)) + I = stats.uniform.rvs(size=block_size) < old_div(p_un, (c * q_un)) - # accept + # accept accepted_count = np.sum(I) to_take = min(n - from_ind, accepted_count) end_ind = from_ind + to_take @@ -838,8 +896,3 @@ def sample(self, n, seed=872): def dim(self): return len(self.freqs) - - - - - diff --git a/sbibm/third_party/kgof/density.py b/sbibm/third_party/kgof/density.py index 24224d77..45047600 100644 --- a/sbibm/third_party/kgof/density.py +++ b/sbibm/third_party/kgof/density.py @@ -4,32 +4,42 @@ """ from __future__ import division -from builtins import range -from past.utils import old_div -from builtins import object +from builtins import object, range + from future.utils import with_metaclass -__author__ = 'wittawat' +from past.utils import old_div +__author__ = "wittawat" + +# import warnings +import logging from abc import ABCMeta, abstractmethod + import autograd import autograd.numpy as np -import sbibm.third_party.kgof.data as data import scipy.stats as stats -#import warnings -import logging + +import sbibm.third_party.kgof.data as data + def warn_bounded_domain(self): - logging.warning('{} has a bounded domain. This may have an unintended effect to the test result of FSSD.'.format(self.__class__) ) + logging.warning( + "{} has a bounded domain. This may have an unintended effect to the test result of FSSD.".format( + self.__class__ + ) + ) + def from_log_den(d, f): """ - Construct an UnnormalizedDensity from the function f, implementing the log + Construct an UnnormalizedDensity from the function f, implementing the log of an unnormalized density. f: X -> den where X: n x d and den is a numpy array of length n. """ return UDFromCallable(d, flog_den=f) + def from_grad_log(d, g): """ Construct an UnnormalizedDensity from the function g, implementing the @@ -97,13 +107,16 @@ def dim(self): """ raise NotImplementedError() + # end UnnormalizedDensity + class UDFromCallable(UnnormalizedDensity): """ - UnnormalizedDensity constructed from the specified implementations of + UnnormalizedDensity constructed from the specified implementations of log_den() and grad_log() as callable objects. """ + def __init__(self, d, flog_den=None, fgrad_log=None): """ Only one of log_den and grad_log are required. @@ -115,7 +128,7 @@ def __init__(self, d, flog_den=None, fgrad_log=None): grad_log: a callable object (function) implementing the gradient of the log of an unnormalized density. """ if flog_den is None and fgrad_log is None: - raise ValueError('At least one of {log_den, grad_log} must be specified.') + raise ValueError("At least one of {log_den, grad_log} must be specified.") self.d = d self.flog_den = flog_den self.fgrad_log = fgrad_log @@ -123,7 +136,7 @@ def __init__(self, d, flog_den=None, fgrad_log=None): def log_den(self, X): flog_den = self.flog_den if flog_den is None: - raise ValueError('log_den callable object is None.') + raise ValueError("log_den callable object is None.") return flog_den(X) def grad_log(self, X): @@ -139,6 +152,7 @@ def grad_log(self, X): def dim(self): return self.d + # end UDFromCallable @@ -146,23 +160,26 @@ class IsotropicNormal(UnnormalizedDensity): """ Unnormalized density of an isotropic multivariate normal distribution. """ + def __init__(self, mean, variance): """ - mean: a numpy array of length d for the mean + mean: a numpy array of length d for the mean variance: a positive floating-point number for the variance. """ - self.mean = mean + self.mean = mean self.variance = variance def log_den(self, X): - mean = self.mean + mean = self.mean variance = self.variance - unden = old_div(-np.sum((X-mean)**2, 1),(2.0*variance)) + unden = old_div(-np.sum((X - mean) ** 2, 1), (2.0 * variance)) return unden def log_normalized_den(self, X): d = self.dim() - return stats.multivariate_normal.logpdf(X, mean=self.mean, cov=self.variance*np.eye(d)) + return stats.multivariate_normal.logpdf( + X, mean=self.mean, cov=self.variance * np.eye(d) + ) def get_datasource(self): return data.DSIsotropicNormal(self.mean, self.variance) @@ -171,32 +188,32 @@ def dim(self): return len(self.mean) - class Normal(UnnormalizedDensity): """ A multivariate normal distribution. """ + def __init__(self, mean, cov): """ mean: a numpy array of length d. cov: d x d numpy array for the covariance. """ - self.mean = mean + self.mean = mean self.cov = cov assert mean.shape[0] == cov.shape[0] assert cov.shape[0] == cov.shape[1] E, V = np.linalg.eigh(cov) if np.any(np.abs(E) <= 1e-7): - raise ValueError('covariance matrix is not full rank.') + raise ValueError("covariance matrix is not full rank.") # The precision matrix - self.prec = np.dot(np.dot(V, np.diag(old_div(1.0,E))), V.T) - #print self.prec + self.prec = np.dot(np.dot(V, np.diag(old_div(1.0, E))), V.T) + # print self.prec def log_den(self, X): - mean = self.mean + mean = self.mean X0 = X - mean X0prec = np.dot(X0, self.prec) - unden = old_div(-np.sum(X0prec*X0, 1),2.0) + unden = old_div(-np.sum(X0prec * X0, 1), 2.0) return unden def get_datasource(self): @@ -205,15 +222,18 @@ def get_datasource(self): def dim(self): return len(self.mean) + # end Normal + class IsoGaussianMixture(UnnormalizedDensity): """ - UnnormalizedDensity of a Gaussian mixture in R^d where each component + UnnormalizedDensity of a Gaussian mixture in R^d where each component is an isotropic multivariate normal distribution. Let k be the number of mixture components. """ + def __init__(self, means, variances, pmix=None): """ means: a k x d 2d array specifying the means. @@ -222,13 +242,15 @@ def __init__(self, means, variances, pmix=None): """ k, d = means.shape if k != len(variances): - raise ValueError('Number of components in means and variances do not match.') + raise ValueError( + "Number of components in means and variances do not match." + ) if pmix is None: - pmix = old_div(np.ones(k),float(k)) + pmix = old_div(np.ones(k), float(k)) if np.abs(np.sum(pmix) - 1) > 1e-8: - raise ValueError('Mixture weights do not sum to 1.') + raise ValueError("Mixture weights do not sum to 1.") self.pmix = pmix self.means = means @@ -245,13 +267,11 @@ def log_normalized_den(self, X): n = X.shape[0] den = np.zeros(n, dtype=float) for i in range(k): - norm_den_i = IsoGaussianMixture.normal_density(means[i], - variances[i], X) - den = den + norm_den_i*pmix[i] + norm_den_i = IsoGaussianMixture.normal_density(means[i], variances[i], X) + den = den + norm_den_i * pmix[i] return np.log(den) - - #def grad_log(self, X): + # def grad_log(self, X): # """ # Return an n x d numpy array of gradients. # """ @@ -265,7 +285,6 @@ def log_normalized_den(self, X): # norm_den_i = IsoGaussianMixture.normal_density(means[i], # variances[i], X) - @staticmethod def normal_density(mean, variance, X): """ @@ -274,9 +293,9 @@ def normal_density(mean, variance, X): variance: scalar variances X: n x d 2d-array """ - Z = np.sqrt(2.0*np.pi*variance) - unden = np.exp(old_div(-np.sum((X-mean)**2.0, 1),(2.0*variance)) ) - den = old_div(unden,Z) + Z = np.sqrt(2.0 * np.pi * variance) + unden = np.exp(old_div(-np.sum((X - mean) ** 2.0, 1), (2.0 * variance))) + den = old_div(unden, Z) assert len(den) == X.shape[0] return den @@ -287,15 +306,18 @@ def dim(self): k, d = self.means.shape return d + # end class IsoGaussianMixture + class GaussianMixture(UnnormalizedDensity): """ - UnnormalizedDensity of a Gaussian mixture in R^d where each component + UnnormalizedDensity of a Gaussian mixture in R^d where each component can be arbitrary. This is the most general form of a Gaussian mixture. Let k be the number of mixture components. """ + def __init__(self, means, variances, pmix=None): """ means: a k x d 2d array specifying the means. @@ -305,13 +327,15 @@ def __init__(self, means, variances, pmix=None): """ k, d = means.shape if k != variances.shape[0]: - raise ValueError('Number of components in means and variances do not match.') + raise ValueError( + "Number of components in means and variances do not match." + ) if pmix is None: - pmix = old_div(np.ones(k),float(k)) + pmix = old_div(np.ones(k), float(k)) if np.abs(np.sum(pmix) - 1) > 1e-8: - raise ValueError('Mixture weights do not sum to 1.') + raise ValueError("Mixture weights do not sum to 1.") self.pmix = pmix self.means = means @@ -329,9 +353,10 @@ def log_normalized_den(self, X): den = np.zeros(n, dtype=float) for i in range(k): - norm_den_i = GaussianMixture.multivariate_normal_density(means[i], - variances[i], X) - den = den + norm_den_i*pmix[i] + norm_den_i = GaussianMixture.multivariate_normal_density( + means[i], variances[i], X + ) + den = den + norm_den_i * pmix[i] return np.log(den) @staticmethod @@ -342,16 +367,16 @@ def multivariate_normal_density(mean, cov, X): cov: a dxd covariance matrix X: n x d 2d-array """ - + evals, evecs = np.linalg.eigh(cov) - cov_half_inv = evecs.dot(np.diag(evals**(-0.5))).dot(evecs.T) - # print(evals) - half_evals = np.dot(X-mean, cov_half_inv) - full_evals = np.sum(half_evals**2, 1) - unden = np.exp(-0.5*full_evals) - - Z = np.sqrt(np.linalg.det(2.0*np.pi*cov)) - den = unden/Z + cov_half_inv = evecs.dot(np.diag(evals ** (-0.5))).dot(evecs.T) + # print(evals) + half_evals = np.dot(X - mean, cov_half_inv) + full_evals = np.sum(half_evals ** 2, 1) + unden = np.exp(-0.5 * full_evals) + + Z = np.sqrt(np.linalg.det(2.0 * np.pi * cov)) + den = unden / Z assert len(den) == X.shape[0] return den @@ -362,8 +387,10 @@ def dim(self): k, d = self.means.shape return d + # end GaussianMixture + class GaussBernRBM(UnnormalizedDensity): """ Gaussian-Bernoulli Restricted Boltzmann Machine. @@ -371,9 +398,10 @@ class GaussBernRBM(UnnormalizedDensity): p(x, h) = Z^{-1} exp(0.5*x^T B h + b^T x + c^T h - 0.5||x||^2) where h is a vector of {-1, 1}. """ + def __init__(self, B, b, c): """ - B: a dx x dh matrix + B: a dx x dh matrix b: a numpy array of length dx c: a numpy array of length dh """ @@ -392,18 +420,21 @@ def log_den(self, X): b = self.b c = self.c - XBC = 0.5*np.dot(X, B) + c - unden = np.dot(X, b) - 0.5*np.sum(X**2, 1) + np.sum(np.log(np.exp(XBC) - + np.exp(-XBC)), 1) + XBC = 0.5 * np.dot(X, B) + c + unden = ( + np.dot(X, b) + - 0.5 * np.sum(X ** 2, 1) + + np.sum(np.log(np.exp(XBC) + np.exp(-XBC)), 1) + ) assert len(unden) == X.shape[0] return unden def grad_log(self, X): - # """ - # Evaluate the gradients (with respect to the input) of the log density at - # each of the n points in X. This is the score function. + # """ + # Evaluate the gradients (with respect to the input) of the log density at + # each of the n points in X. This is the score function. - # X: n x d numpy array. + # X: n x d numpy array. """ Evaluate the gradients (with respect to the input) of the log density at each of the n points in X. This is the score function. @@ -413,12 +444,12 @@ def grad_log(self, X): Return an n x d numpy array of gradients. """ XB = np.dot(X, self.B) - Y = 0.5*XB + self.c - E2y = np.exp(2*Y) + Y = 0.5 * XB + self.c + E2y = np.exp(2 * Y) # n x dh - Phi = old_div((E2y-1.0),(E2y+1)) + Phi = old_div((E2y - 1.0), (E2y + 1)) # n x dx - T = np.dot(Phi, 0.5*self.B.T) + T = np.dot(Phi, 0.5 * self.B.T) S = self.b - X + T return S @@ -428,36 +459,42 @@ def get_datasource(self, burnin=2000): def dim(self): return len(self.b) + # end GaussBernRBM + class ISIPoissonLinear(UnnormalizedDensity): """ Unnormalized density of inter-arrival times from nonhomogeneous poisson process with linear intensity function. lambda = 1 + bt """ + def __init__(self, b): """ - b: slope of the linear function + b: slope of the linear function """ warn_bounded_domain(self) - self.b = b - + self.b = b + def log_den(self, X): b = self.b - unden = -np.sum(0.5*b*X**2+X-np.log(1.0+b*X), 1) + unden = -np.sum(0.5 * b * X ** 2 + X - np.log(1.0 + b * X), 1) return unden def dim(self): return 1 + # end ISIPoissonLinear + class ISIPoissonSine(UnnormalizedDensity): """ Unnormalized density of inter-arrival times from nonhomogeneous poisson process with sine intensity function. lambda = b*(1+sin(w*X)) """ - def __init__(self, w=10.0,b=1.0): + + def __init__(self, w=10.0, b=1.0): """ w: the frequency of sine function b: amplitude of intensity function @@ -465,42 +502,48 @@ def __init__(self, w=10.0,b=1.0): warn_bounded_domain(self) self.b = b self.w = w - + def log_den(self, X): b = self.b w = self.w - unden = np.sum(b*(-X + old_div((np.cos(w*X)-1),w)) + np.log(b*(1+np.sin(w*X))),1) + unden = np.sum( + b * (-X + old_div((np.cos(w * X) - 1), w)) + + np.log(b * (1 + np.sin(w * X))), + 1, + ) return unden def dim(self): return 1 + # end ISIPoissonSine + class Gamma(UnnormalizedDensity): """ A gamma distribution. """ - def __init__(self, alpha, beta = 1.0): + + def __init__(self, alpha, beta=1.0): """ alpha: shape of parameter beta: scale """ warn_bounded_domain(self) - self.alpha = alpha + self.alpha = alpha self.beta = beta - + def log_den(self, X): alpha = self.alpha beta = self.beta - #unden = np.sum(stats.gamma.logpdf(X, alpha, scale = beta), 1) - unden = np.sum(-beta*X + (alpha-1)*np.log(X), 1) + # unden = np.sum(stats.gamma.logpdf(X, alpha, scale = beta), 1) + unden = np.sum(-beta * X + (alpha - 1) * np.log(X), 1) return unden def get_datasource(self): return data.DSNormal(self.mean, self.cov) - def dim(self): return 1 @@ -510,19 +553,20 @@ class LogGamma(UnnormalizedDensity): A gamma distribution with transformed domain. t = exp(x), t \in R+ x \in R """ - def __init__(self, alpha, beta = 1.0): + + def __init__(self, alpha, beta=1.0): """ alpha: shape of parameter beta: scale """ self.alpha = alpha self.beta = beta - + def log_den(self, X): alpha = self.alpha beta = self.beta - #unden = np.sum(stats.gamma.logpdf(X, alpha, scale = beta), 1) - unden = np.sum(-beta*np.exp(X) + (alpha-1)*X + X , 1) + # unden = np.sum(stats.gamma.logpdf(X, alpha, scale = beta), 1) + unden = np.sum(-beta * np.exp(X) + (alpha - 1) * X + X, 1) return unden def get_datasource(self): @@ -530,53 +574,61 @@ def get_datasource(self): def dim(self): return 1 -# end LogGamma +# end LogGamma + class ISILogPoissonLinear(UnnormalizedDensity): """ Unnormalized density of inter-arrival times from nonhomogeneous poisson process with linear intensity function. lambda = 1 + bt """ + def __init__(self, b): """ - b: slope of the linear function + b: slope of the linear function """ warn_bounded_domain(self) - self.b = b - + self.b = b + def log_den(self, X): b = self.b - unden = -np.sum(0.5*b*np.exp(X)**2 + np.exp(X) - np.log(1.0+b*np.exp(X))-X, 1) + unden = -np.sum( + 0.5 * b * np.exp(X) ** 2 + np.exp(X) - np.log(1.0 + b * np.exp(X)) - X, 1 + ) return unden def dim(self): return 1 + # end ISIPoissonLinear + class ISIPoisson2D(UnnormalizedDensity): """ Unnormalized density of nonhomogeneous spatial poisson process """ + def __init__(self): """ lambda_(X,Y) = X^2 + Y^2 """ warn_bounded_domain(self) - def quadratic_intensity(self,X,Y): - int_intensity = -(X**2+Y**2)*X*Y + 3*np.log(X**2+Y**2) + def quadratic_intensity(self, X, Y): + int_intensity = -(X ** 2 + Y ** 2) * X * Y + 3 * np.log(X ** 2 + Y ** 2) return int_intensity def log_den(self, X): - unden = self.quadratic_intensity(X[:,0],X[:,1]) + unden = self.quadratic_intensity(X[:, 0], X[:, 1]) return unden def dim(self): return 1 + # end class ISIPoisson2D @@ -584,7 +636,8 @@ class ISISigmoidPoisson2D(UnnormalizedDensity): """ Unnormalized density of nonhomogeneous spatial poisson process with sigmoid transformation """ - def __init__(self, intensity = 'quadratic', w = 1.0, a=1.0): + + def __init__(self, intensity="quadratic", w=1.0, a=1.0): """ lambda_(X,Y) = a* X^2 + Y^2 X = 1/(1+exp(s)) @@ -594,30 +647,33 @@ def __init__(self, intensity = 'quadratic', w = 1.0, a=1.0): warn_bounded_domain(self) self.a = a self.w = w - if intensity == 'quadratic': + if intensity == "quadratic": self.intensity = self.quadratic_intensity - elif intensity == 'sine': + elif intensity == "sine": self.intensity = self.sine_intensity else: - raise ValueError('Not intensity function found') + raise ValueError("Not intensity function found") - def sigmoid(self,x): - sig = old_div(1,(1+np.exp(x))) + def sigmoid(self, x): + sig = old_div(1, (1 + np.exp(x))) return sig - def quadratic_intensity(self,s,t): + def quadratic_intensity(self, s, t): X = self.sigmoid(s) Y = self.sigmoid(t) - int_intensity = -(self.a*X**2+Y**2)*X*Y + 3*(np.log(self.a*X**2+Y**2)+np.log((X*(X-1)*Y*(Y-1)))) + int_intensity = -(self.a * X ** 2 + Y ** 2) * X * Y + 3 * ( + np.log(self.a * X ** 2 + Y ** 2) + np.log((X * (X - 1) * Y * (Y - 1))) + ) return int_intensity def log_den(self, S): - unden = self.quadratic_intensity(S[:,0],S[:,1]) + unden = self.quadratic_intensity(S[:, 0], S[:, 1]) return unden def dim(self): return 1 + # end class ISISigmoidPoisson2D @@ -625,6 +681,7 @@ class Poisson2D(UnnormalizedDensity): """ Unnormalized density of nonhomogeneous spatial poisson process """ + def __init__(self, w=1.0): """ lambda_(X,Y) = sin(w*pi*X)+sin(w*pi*Y) @@ -632,7 +689,7 @@ def __init__(self, w=1.0): self.w = w def lamb_sin(self, X): - return np.prod(np.sin(self.w*np.pi*X),1) + return np.prod(np.sin(self.w * np.pi * X), 1) def log_den(self, X): unden = np.log(self.gmm_den(X)) @@ -640,12 +697,14 @@ def log_den(self, X): def dim(self): return 1 - + + class Resample(UnnormalizedDensity): """ Unnormalized Density of real dataset with estimated intensity function fit takes the function to evaluate the density of resampled data """ + def __init__(self, fit): self.fit = fit @@ -656,8 +715,10 @@ def log_den(self, X): def dim(self): return 1 + # end class SigmoidPoisson2D + class GaussCosFreqs(UnnormalizedDensity): """ p(x) \propto exp(-||x||^2/2sigma^2)*(1+ prod_{i=1}^d cos(w_i*x_i)) @@ -672,14 +733,18 @@ def __init__(self, sigma2, freqs): freqs: a 1-d array of length d for the frequencies. """ self.sigma2 = sigma2 - if sigma2 <= 0 : - raise ValueError('sigma2 must be > 0') + if sigma2 <= 0: + raise ValueError("sigma2 must be > 0") self.freqs = freqs def log_den(self, X): sigma2 = self.sigma2 freqs = self.freqs - log_unden = old_div(-np.sum(X**2, 1),(2.0*sigma2)) + 1+np.prod(np.cos(X*freqs), 1) + log_unden = ( + old_div(-np.sum(X ** 2, 1), (2.0 * sigma2)) + + 1 + + np.prod(np.cos(X * freqs), 1) + ) return log_unden def dim(self): @@ -687,4 +752,3 @@ def dim(self): def get_datasource(self): return data.DSGaussCosFreqs(self.sigma2, self.freqs) - diff --git a/sbibm/third_party/kgof/ex/ex1_vary_n.py b/sbibm/third_party/kgof/ex/ex1_vary_n.py index d6ff9974..1fd4776f 100644 --- a/sbibm/third_party/kgof/ex/ex1_vary_n.py +++ b/sbibm/third_party/kgof/ex/ex1_vary_n.py @@ -1,35 +1,37 @@ """Simulation to test the test power vs increasing sample size""" -__author__ = 'wittawat' +__author__ = "wittawat" -import sbibm.third_party.kgof as kgof -import sbibm.third_party.kgof.data as data -import sbibm.third_party.kgof.glo as glo -import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.goftest as gof -import sbibm.third_party.kgof.intertst as tgof -import sbibm.third_party.kgof.mmd as mgof -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel +import logging +import math +import os +import sys +import time -# need independent_jobs package +# import numpy as np +import autograd.numpy as np + +# need independent_jobs package # https://github.com/karlnapf/independent-jobs # The independent_jobs and kgof have to be in the global search path (.bashrc) import independent_jobs as inj -from independent_jobs.jobs.IndependentJob import IndependentJob -from independent_jobs.results.SingleResult import SingleResult from independent_jobs.aggregators.SingleResultAggregator import SingleResultAggregator from independent_jobs.engines.BatchClusterParameters import BatchClusterParameters from independent_jobs.engines.SerialComputationEngine import SerialComputationEngine from independent_jobs.engines.SlurmComputationEngine import SlurmComputationEngine +from independent_jobs.jobs.IndependentJob import IndependentJob +from independent_jobs.results.SingleResult import SingleResult from independent_jobs.tools.Log import logger -import logging -import math -#import numpy as np -import autograd.numpy as np -import os -import sys -import time + +import sbibm.third_party.kgof as kgof +import sbibm.third_party.kgof.data as data +import sbibm.third_party.kgof.density as density +import sbibm.third_party.kgof.glo as glo +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.intertst as tgof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.mmd as mgof +import sbibm.third_party.kgof.util as util """ All the job functions return a dictionary with the following keys: @@ -38,6 +40,7 @@ - time_secs: run time in seconds """ + def job_fssdJ1q_med(p, data_source, tr, te, r, J=1, null_sim=None): """ FSSD test with a Gaussian kernel, where the test locations are randomized, @@ -56,14 +59,15 @@ def job_fssdJ1q_med(p, data_source, tr, te, r, J=1, null_sim=None): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) - V = util.fit_gaussian_draw(X, J, seed=r+1) + k = kernel.KGauss(med ** 2) + V = util.fit_gaussian_draw(X, J, seed=r + 1) fssd_med = gof.FSSD(p, k, V, null_sim=null_sim, alpha=alpha) fssd_med_result = fssd_med.perform_test(data) - return { 'test_result': fssd_med_result, 'time_secs': t.secs} + return {"test_result": fssd_med_result, "time_secs": t.secs} + def job_fssdJ5q_med(p, data_source, tr, te, r): """ @@ -71,6 +75,7 @@ def job_fssdJ5q_med(p, data_source, tr, te, r): """ return job_fssdJ1q_med(p, data_source, tr, te, r, J=5) + def job_fssdJ1q_opt(p, data_source, tr, te, r, J=1, null_sim=None): """ FSSD with optimization on tr. Test on te. Use a Gaussian kernel. @@ -82,45 +87,52 @@ def job_fssdJ1q_opt(p, data_source, tr, te, r, J=1, null_sim=None): with util.ContextTimer() as t: # Use grid search to initialize the gwidth n_gwidth_cand = 5 - gwidth_factors = 2.0**np.linspace(-3, 3, n_gwidth_cand) - med2 = util.meddistance(Xtr, 1000)**2 + gwidth_factors = 2.0 ** np.linspace(-3, 3, n_gwidth_cand) + med2 = util.meddistance(Xtr, 1000) ** 2 k = kernel.KGauss(med2) # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) - list_gwidth = np.hstack( ( (med2)*gwidth_factors ) ) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) + list_gwidth = np.hstack(((med2) * gwidth_factors)) besti, objs = gof.GaussFSSD.grid_search_gwidth(p, tr, V0, list_gwidth) gwidth = list_gwidth[besti] - assert util.is_real_num(gwidth), 'gwidth not real. Was %s'%str(gwidth) - assert gwidth > 0, 'gwidth not positive. Was %.3g'%gwidth - logging.info('After grid search, gwidth=%.3g'%gwidth) - + assert util.is_real_num(gwidth), "gwidth not real. Was %s" % str(gwidth) + assert gwidth > 0, "gwidth not positive. Was %.3g" % gwidth + logging.info("After grid search, gwidth=%.3g" % gwidth) + ops = { - 'reg': 1e-2, - 'max_iter': 30, - 'tol_fun': 1e-5, - 'disp': True, - 'locs_bounds_frac':30.0, - 'gwidth_lb': 1e-1, - 'gwidth_ub': 1e4, - } - - V_opt, gwidth_opt, info = gof.GaussFSSD.optimize_locs_widths(p, tr, - gwidth, V0, **ops) + "reg": 1e-2, + "max_iter": 30, + "tol_fun": 1e-5, + "disp": True, + "locs_bounds_frac": 30.0, + "gwidth_lb": 1e-1, + "gwidth_ub": 1e4, + } + + V_opt, gwidth_opt, info = gof.GaussFSSD.optimize_locs_widths( + p, tr, gwidth, V0, **ops + ) # Use the optimized parameters to construct a test k_opt = kernel.KGauss(gwidth_opt) fssd_opt = gof.FSSD(p, k_opt, V_opt, null_sim=null_sim, alpha=alpha) fssd_opt_result = fssd_opt.perform_test(te) - return {'test_result': fssd_opt_result, 'time_secs': t.secs, - 'goftest': fssd_opt, 'opt_info': info, - } + return { + "test_result": fssd_opt_result, + "time_secs": t.secs, + "goftest": fssd_opt, + "opt_info": info, + } + def job_fssdJ5q_opt(p, data_source, tr, te, r): return job_fssdJ1q_opt(p, data_source, tr, te, r, J=5) + def job_fssdJ10q_opt(p, data_source, tr, te, r): return job_fssdJ1q_opt(p, data_source, tr, te, r, J=10) + def job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=1, null_sim=None): """ FSSD with optimization on tr. Test on te. Use an inverse multiquadric @@ -140,17 +152,17 @@ def job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=1, null_sim=None): c = 1.0 # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) ops = { - 'reg': 1e-2, - 'max_iter': 40, - 'tol_fun': 1e-4, - 'disp': True, - 'locs_bounds_frac':10.0, - } + "reg": 1e-2, + "max_iter": 40, + "tol_fun": 1e-4, + "disp": True, + "locs_bounds_frac": 10.0, + } - V_opt, info = gof.IMQFSSD.optimize_locs(p, tr, b, c, V0, **ops) + V_opt, info = gof.IMQFSSD.optimize_locs(p, tr, b, c, V0, **ops) k_imq = kernel.KIMQ(b=b, c=c) @@ -158,13 +170,18 @@ def job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=1, null_sim=None): fssd_imq = gof.FSSD(p, k_imq, V_opt, null_sim=null_sim, alpha=alpha) fssd_imq_result = fssd_imq.perform_test(te) - return {'test_result': fssd_imq_result, 'time_secs': t.secs, - 'goftest': fssd_imq, 'opt_info': info, - } + return { + "test_result": fssd_imq_result, + "time_secs": t.secs, + "goftest": fssd_imq, + "opt_info": info, + } + def job_fssdJ5q_imq_optv(p, data_source, tr, te, r): return job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=5) + def job_me_opt(p, data_source, tr, te, r, J=5): """ ME test of Jitkrittum et al., 2016 used as a goodness-of-fit test. @@ -173,22 +190,30 @@ def job_me_opt(p, data_source, tr, te, r, J=5): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic - #pds = p.get_datasource() - #datY = pds.sample(data.sample_size(), seed=r+294) - #Y = datY.data() - #XY = np.vstack((X, Y)) - #med = util.meddistance(XY, subsample=1000) - op = {'n_test_locs': J, 'seed': r+5, 'max_iter': 40, - 'batch_proportion': 1.0, 'locs_step_size': 1.0, - 'gwidth_step_size': 0.1, 'tol_fun': 1e-4, - 'reg': 1e-4} + # median heuristic + # pds = p.get_datasource() + # datY = pds.sample(data.sample_size(), seed=r+294) + # Y = datY.data() + # XY = np.vstack((X, Y)) + # med = util.meddistance(XY, subsample=1000) + op = { + "n_test_locs": J, + "seed": r + 5, + "max_iter": 40, + "batch_proportion": 1.0, + "locs_step_size": 1.0, + "gwidth_step_size": 0.1, + "tol_fun": 1e-4, + "reg": 1e-4, + } # optimize on the training set - me_opt = tgof.GaussMETestOpt(p, n_locs=J, tr_proportion=tr_proportion, - alpha=alpha, seed=r+111) + me_opt = tgof.GaussMETestOpt( + p, n_locs=J, tr_proportion=tr_proportion, alpha=alpha, seed=r + 111 + ) me_result = me_opt.perform_test(data, op) - return { 'test_result': me_result, 'time_secs': t.secs} + return {"test_result": me_result, "time_secs": t.secs} + def job_kstein_med(p, data_source, tr, te, r): """ @@ -199,24 +224,25 @@ def job_kstein_med(p, data_source, tr, te, r): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) + k = kernel.KGauss(med ** 2) kstein = gof.KernelSteinTest(p, k, alpha=alpha, n_simulate=1000, seed=r) kstein_result = kstein.perform_test(data) - return { 'test_result': kstein_result, 'time_secs': t.secs} + return {"test_result": kstein_result, "time_secs": t.secs} + def job_kstein_imq(p, data_source, tr, te, r): """ Kernel Stein discrepancy test of Liu et al., 2016 and Chwialkowski et al., - 2016. Use full sample. Use the inverse multiquadric kernel (IMQ) studied - in + 2016. Use full sample. Use the inverse multiquadric kernel (IMQ) studied + in Measuring Sample Quality with Kernels - Gorham and Mackey 2017. + Gorham and Mackey 2017. - Parameters are fixed to the recommented values: beta = b = -0.5, c = 1. + Parameters are fixed to the recommented values: beta = b = -0.5, c = 1. """ # full data data = tr + te @@ -226,7 +252,7 @@ def job_kstein_imq(p, data_source, tr, te, r): kstein = gof.KernelSteinTest(p, k, alpha=alpha, n_simulate=1000, seed=r) kstein_result = kstein.perform_test(data) - return { 'test_result': kstein_result, 'time_secs': t.secs} + return {"test_result": kstein_result, "time_secs": t.secs} def job_lin_kstein_med(p, data_source, tr, te, r): @@ -238,27 +264,28 @@ def job_lin_kstein_med(p, data_source, tr, te, r): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) + k = kernel.KGauss(med ** 2) lin_kstein = gof.LinearKernelSteinTest(p, k, alpha=alpha, seed=r) lin_kstein_result = lin_kstein.perform_test(data) - return { 'test_result': lin_kstein_result, 'time_secs': t.secs} + return {"test_result": lin_kstein_result, "time_secs": t.secs} + def job_mmd_med(p, data_source, tr, te, r): """ MMD test of Gretton et al., 2012 used as a goodness-of-fit test. - Require the ability to sample from p i.e., the UnnormalizedDensity p has + Require the ability to sample from p i.e., the UnnormalizedDensity p has to be able to return a non-None from get_datasource() """ # full data data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic pds = p.get_datasource() - datY = pds.sample(data.sample_size(), seed=r+294) + datY = pds.sample(data.sample_size(), seed=r + 294) Y = datY.data() XY = np.vstack((X, Y)) @@ -269,17 +296,18 @@ def job_mmd_med(p, data_source, tr, te, r): medx = util.meddistance(X, subsample=1000) medy = util.meddistance(Y, subsample=1000) medxy = util.meddistance(XY, subsample=1000) - med_avg = (medx+medy+medxy)/3.0 - k = kernel.KGauss(med_avg**2) + med_avg = (medx + medy + medxy) / 3.0 + k = kernel.KGauss(med_avg ** 2) mmd_test = mgof.QuadMMDGof(p, k, n_permute=400, alpha=alpha, seed=r) mmd_result = mmd_test.perform_test(data) - return { 'test_result': mmd_result, 'time_secs': t.secs} + return {"test_result": mmd_result, "time_secs": t.secs} + def job_mmd_opt(p, data_source, tr, te, r): """ MMD test of Gretton et al., 2012 used as a goodness-of-fit test. - Require the ability to sample from p i.e., the UnnormalizedDensity p has + Require the ability to sample from p i.e., the UnnormalizedDensity p has to be able to return a non-None from get_datasource() With optimization. Gaussian kernel. @@ -287,9 +315,9 @@ def job_mmd_opt(p, data_source, tr, te, r): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic pds = p.get_datasource() - datY = pds.sample(data.sample_size(), seed=r+294) + datY = pds.sample(data.sample_size(), seed=r + 294) Y = datY.data() XY = np.vstack((X, Y)) @@ -297,22 +325,26 @@ def job_mmd_opt(p, data_source, tr, te, r): # Construct a list of kernels to try based on multiples of the median # heuristic - #list_gwidth = np.hstack( (np.linspace(20, 40, 10), (med**2) + # list_gwidth = np.hstack( (np.linspace(20, 40, 10), (med**2) # *(2.0**np.linspace(-2, 2, 20) ) ) ) - list_gwidth = (med**2)*(2.0**np.linspace(-4, 4, 30) ) + list_gwidth = (med ** 2) * (2.0 ** np.linspace(-4, 4, 30)) list_gwidth.sort() candidate_kernels = [kernel.KGauss(gw2) for gw2 in list_gwidth] mmd_opt = mgof.QuadMMDGofOpt(p, n_permute=300, alpha=alpha, seed=r) - mmd_result = mmd_opt.perform_test(data, - candidate_kernels=candidate_kernels, - tr_proportion=tr_proportion, reg=1e-3) - return { 'test_result': mmd_result, 'time_secs': t.secs} + mmd_result = mmd_opt.perform_test( + data, + candidate_kernels=candidate_kernels, + tr_proportion=tr_proportion, + reg=1e-3, + ) + return {"test_result": mmd_result, "time_secs": t.secs} + def job_mmd_dgauss_opt(p, data_source, tr, te, r): """ MMD test of Gretton et al., 2012 used as a goodness-of-fit test. - Require the ability to sample from p i.e., the UnnormalizedDensity p has + Require the ability to sample from p i.e., the UnnormalizedDensity p has to be able to return a non-None from get_datasource() With optimization. Diagonal Gaussian kernel where there is one Gaussian width @@ -322,9 +354,9 @@ def job_mmd_dgauss_opt(p, data_source, tr, te, r): X = data.data() d = X.shape[1] with util.ContextTimer() as t: - # median heuristic + # median heuristic pds = p.get_datasource() - datY = pds.sample(data.sample_size(), seed=r+294) + datY = pds.sample(data.sample_size(), seed=r + 294) Y = datY.data() XY = np.vstack((X, Y)) @@ -336,28 +368,30 @@ def job_mmd_dgauss_opt(p, data_source, tr, te, r): # Construct a list of kernels to try based on multiples of the median # heuristic - med_factors = 2.0**np.linspace(-4, 4, 20) + med_factors = 2.0 ** np.linspace(-4, 4, 20) candidate_kernels = [] for i in range(len(med_factors)): - ki = kernel.KDiagGauss( (meds**2)*med_factors[i] ) + ki = kernel.KDiagGauss((meds ** 2) * med_factors[i]) candidate_kernels.append(ki) - mmd_opt = mgof.QuadMMDGofOpt(p, n_permute=300, alpha=alpha, seed=r+56) - mmd_result = mmd_opt.perform_test(data, - candidate_kernels=candidate_kernels, - tr_proportion=tr_proportion, reg=1e-3) - return { 'test_result': mmd_result, 'time_secs': t.secs} + mmd_opt = mgof.QuadMMDGofOpt(p, n_permute=300, alpha=alpha, seed=r + 56) + mmd_result = mmd_opt.perform_test( + data, + candidate_kernels=candidate_kernels, + tr_proportion=tr_proportion, + reg=1e-3, + ) + return {"test_result": mmd_result, "time_secs": t.secs} + # Define our custom Job, which inherits from base class IndependentJob class Ex1Job(IndependentJob): - def __init__(self, aggregator, p, data_source, prob_label, rep, job_func, n): - #walltime = 60*59*24 - walltime = 60*59 - memory = int(tr_proportion*n*1e-2) + 50 + # walltime = 60*59*24 + walltime = 60 * 59 + memory = int(tr_proportion * n * 1e-2) + 50 - IndependentJob.__init__(self, aggregator, walltime=walltime, - memory=memory) + IndependentJob.__init__(self, aggregator, walltime=walltime, memory=memory) # p: an UnnormalizedDensity self.p = p self.data_source = data_source @@ -371,16 +405,19 @@ def __init__(self, aggregator, p, data_source, prob_label, rep, job_func, n): def compute(self): p = self.p - data_source = self.data_source + data_source = self.data_source r = self.rep n = self.n job_func = self.job_func data = data_source.sample(n, seed=r) with util.ContextTimer() as t: - tr, te = data.split_tr_te(tr_proportion=tr_proportion, seed=r+21 ) + tr, te = data.split_tr_te(tr_proportion=tr_proportion, seed=r + 21) prob_label = self.prob_label - logger.info("computing. %s. prob=%s, r=%d,\ - n=%d"%(job_func.__name__, prob_label, r, n)) + logger.info( + "computing. %s. prob=%s, r=%d,\ + n=%d" + % (job_func.__name__, prob_label, r, n) + ) job_result = job_func(p, data_source, tr, te, r) @@ -389,34 +426,44 @@ def compute(self): # submit the result to my own aggregator self.aggregator.submit_result(result) func_name = job_func.__name__ - logger.info("done. ex2: %s, prob=%s, r=%d, n=%d. Took: %.3g s "%(func_name, - prob_label, r, n, t.secs)) + logger.info( + "done. ex2: %s, prob=%s, r=%d, n=%d. Took: %.3g s " + % (func_name, prob_label, r, n, t.secs) + ) # save result - fname = '%s-%s-n%d_r%d_a%.3f_trp%.2f.p' \ - %(prob_label, func_name, n, r, alpha, tr_proportion) + fname = "%s-%s-n%d_r%d_a%.3f_trp%.2f.p" % ( + prob_label, + func_name, + n, + r, + alpha, + tr_proportion, + ) glo.ex_save_result(ex, job_result, prob_label, fname) + # This import is needed so that pickle knows about the class Ex1Job. # pickle is used when collecting the results from the submitted jobs. -from sbibm.third_party.kgof.ex.ex1_vary_n import Ex1Job -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ1q_med -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ5q_med -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ1q_opt -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ5q_opt -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ10q_opt -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ1q_imq_optv -from sbibm.third_party.kgof.ex.ex1_vary_n import job_fssdJ5q_imq_optv -from sbibm.third_party.kgof.ex.ex1_vary_n import job_me_opt -from sbibm.third_party.kgof.ex.ex1_vary_n import job_kstein_med -from sbibm.third_party.kgof.ex.ex1_vary_n import job_kstein_imq -from sbibm.third_party.kgof.ex.ex1_vary_n import job_lin_kstein_med -from sbibm.third_party.kgof.ex.ex1_vary_n import job_mmd_med -from sbibm.third_party.kgof.ex.ex1_vary_n import job_mmd_opt -from sbibm.third_party.kgof.ex.ex1_vary_n import job_mmd_dgauss_opt - - -#--- experimental setting ----- +from sbibm.third_party.kgof.ex.ex1_vary_n import ( + Ex1Job, + job_fssdJ1q_imq_optv, + job_fssdJ1q_med, + job_fssdJ1q_opt, + job_fssdJ5q_imq_optv, + job_fssdJ5q_med, + job_fssdJ5q_opt, + job_fssdJ10q_opt, + job_kstein_imq, + job_kstein_med, + job_lin_kstein_med, + job_me_opt, + job_mmd_dgauss_opt, + job_mmd_med, + job_mmd_opt, +) + +# --- experimental setting ----- ex = 1 # significance level of the test @@ -425,30 +472,28 @@ def compute(self): # Proportion of training sample relative to the full sample size n tr_proportion = 0.2 -# repetitions for each sample size +# repetitions for each sample size reps = 200 # tests to try -method_job_funcs = [ - job_fssdJ5q_opt, - #job_fssdJ5q_med, - #job_me_opt, - #job_kstein_med, - #job_lin_kstein_med, +method_job_funcs = [ + job_fssdJ5q_opt, + # job_fssdJ5q_med, + # job_me_opt, + # job_kstein_med, + # job_lin_kstein_med, job_mmd_opt, - - #job_fssdJ5q_imq_optv, - #job_fssdJ10q_opt, - #job_kstein_imq, - #job_mmd_dgauss_opt, - - #job_mmd_med, - ] + # job_fssdJ5q_imq_optv, + # job_fssdJ10q_opt, + # job_kstein_imq, + # job_mmd_dgauss_opt, + # job_mmd_med, +] # If is_rerun==False, do not rerun the experiment if a result file for the current # setting of (ni, r) already exists. is_rerun = False -#--------------------------- +# --------------------------- def gbrbm_perturb(var_perturb_B, dx=50, dh=10): @@ -463,23 +508,25 @@ def gbrbm_perturb(var_perturb_B, dx=50, dh=10): Return p (density), data source """ with util.NumpySeedContext(seed=10): - B = np.random.randint(0, 2, (dx, dh))*2 - 1.0 + B = np.random.randint(0, 2, (dx, dh)) * 2 - 1.0 b = np.random.randn(dx) c = np.random.randn(dh) p = density.GaussBernRBM(B, b, c) B_perturb = np.copy(B) if var_perturb_B > 1e-7: - B_perturb[0, 0] = B_perturb[0, 0] + \ - np.random.randn(1)*np.sqrt(var_perturb_B) + B_perturb[0, 0] = B_perturb[0, 0] + np.random.randn(1) * np.sqrt( + var_perturb_B + ) ds = data.DSGaussBernRBM(B_perturb, b, c, burnin=2000) return p, ds + def get_ns_pqsource(prob_label): """ Return (ns, p, ds), a tuple of - where + where - ns: a list of sample sizes - p: a Density representing the distribution p - ds: a DataSource, each corresponding to one parameter setting. @@ -487,93 +534,90 @@ def get_ns_pqsource(prob_label): """ gmd_p01_d10_ns = [1000, 3000, 5000] - #gb_rbm_dx50_dh10_vars = [0, 1e-3, 2e-3, 3e-3] - prob2tuples = { - - # vary d. P = N(0, I), Q = N( (c,..0), I) - 'gmd_p03_d10_ns': (gmd_p01_d10_ns, - density.IsotropicNormal(np.zeros(10), 1), - data.DSIsotropicNormal(np.hstack((0.03, np.zeros(10-1))), 1) - ), - - # Gaussian Bernoulli RBM. dx=50, dh=10 - # Perturbation variance to B[0, 0] is 0.1 - 'gbrbm_dx50_dh10_vp1': - ([i*1000 for i in range(1, 4+1)], ) + - #([1000, 5000], ) + - gbrbm_perturb(var_perturb_B=0.1, dx=50, dh=10), - - # Gaussian Bernoulli RBM. dx=50, dh=40 - # Perturbation variance to B[0, 0] is 0.1 - 'gbrbm_dx50_dh40_vp1': - ([i*1000 for i in range(1, 4+1)], ) + - #([1000, 5000], ) + - gbrbm_perturb(var_perturb_B=0.1, dx=50, dh=40), - - # Gaussian Bernoulli RBM. dx=50, dh=10 - # No perturbation - 'gbrbm_dx50_dh10_h0': - ([i*1000 for i in range(1, 4+1)], ) + - #([1000, 5000], ) + - gbrbm_perturb(var_perturb_B=0, dx=50, dh=10), - - # Gaussian Bernoulli RBM. dx=50, dh=40 - # No perturbation - 'gbrbm_dx50_dh40_h0': - ([i*1000 for i in range(1, 4+1)], ) + - #([1000, 5000], ) + - gbrbm_perturb(var_perturb_B=0, dx=50, dh=40), - - # Gaussian Bernoulli RBM. dx=20, dh=10 - # Perturbation variance to B[0, 0] is 0.1 - 'gbrbm_dx20_dh10_vp1': - ([i*1000 for i in range(2, 5+1)], ) + - gbrbm_perturb(var_perturb_B=0.1, dx=20, dh=10), - - # Gaussian Bernoulli RBM. dx=20, dh=10 - # No perturbation - 'gbrbm_dx20_dh10_h0': - ([i*1000 for i in range(2, 5+1)], ) + - gbrbm_perturb(var_perturb_B=0, dx=20, dh=10), - - - } + # gb_rbm_dx50_dh10_vars = [0, 1e-3, 2e-3, 3e-3] + prob2tuples = { + # vary d. P = N(0, I), Q = N( (c,..0), I) + "gmd_p03_d10_ns": ( + gmd_p01_d10_ns, + density.IsotropicNormal(np.zeros(10), 1), + data.DSIsotropicNormal(np.hstack((0.03, np.zeros(10 - 1))), 1), + ), + # Gaussian Bernoulli RBM. dx=50, dh=10 + # Perturbation variance to B[0, 0] is 0.1 + "gbrbm_dx50_dh10_vp1": ([i * 1000 for i in range(1, 4 + 1)],) + + # ([1000, 5000], ) + + gbrbm_perturb(var_perturb_B=0.1, dx=50, dh=10), + # Gaussian Bernoulli RBM. dx=50, dh=40 + # Perturbation variance to B[0, 0] is 0.1 + "gbrbm_dx50_dh40_vp1": ([i * 1000 for i in range(1, 4 + 1)],) + + # ([1000, 5000], ) + + gbrbm_perturb(var_perturb_B=0.1, dx=50, dh=40), + # Gaussian Bernoulli RBM. dx=50, dh=10 + # No perturbation + "gbrbm_dx50_dh10_h0": ([i * 1000 for i in range(1, 4 + 1)],) + + # ([1000, 5000], ) + + gbrbm_perturb(var_perturb_B=0, dx=50, dh=10), + # Gaussian Bernoulli RBM. dx=50, dh=40 + # No perturbation + "gbrbm_dx50_dh40_h0": ([i * 1000 for i in range(1, 4 + 1)],) + + # ([1000, 5000], ) + + gbrbm_perturb(var_perturb_B=0, dx=50, dh=40), + # Gaussian Bernoulli RBM. dx=20, dh=10 + # Perturbation variance to B[0, 0] is 0.1 + "gbrbm_dx20_dh10_vp1": ([i * 1000 for i in range(2, 5 + 1)],) + + gbrbm_perturb(var_perturb_B=0.1, dx=20, dh=10), + # Gaussian Bernoulli RBM. dx=20, dh=10 + # No perturbation + "gbrbm_dx20_dh10_h0": ([i * 1000 for i in range(2, 5 + 1)],) + + gbrbm_perturb(var_perturb_B=0, dx=20, dh=10), + } if prob_label not in prob2tuples: - raise ValueError('Unknown problem label. Need to be one of %s'%str(prob2tuples.keys()) ) + raise ValueError( + "Unknown problem label. Need to be one of %s" % str(prob2tuples.keys()) + ) return prob2tuples[prob_label] + def run_problem(prob_label): """Run the experiment""" ns, p, ds = get_ns_pqsource(prob_label) # /////// submit jobs ////////// # create folder name string - #result_folder = glo.result_folder() + # result_folder = glo.result_folder() from sbibm.third_party.kgof.config import expr_configs - tmp_dir = expr_configs['scratch_path'] - foldername = os.path.join(tmp_dir, 'kgof_slurm', 'e%d'%ex) + + tmp_dir = expr_configs["scratch_path"] + foldername = os.path.join(tmp_dir, "kgof_slurm", "e%d" % ex) logger.info("Setting engine folder to %s" % foldername) # create parameter instance that is needed for any batch computation engine logger.info("Creating batch parameter instance") batch_parameters = BatchClusterParameters( - foldername=foldername, job_name_base="e%d_"%ex, parameter_prefix="") + foldername=foldername, job_name_base="e%d_" % ex, parameter_prefix="" + ) # Use the following line if Slurm queue is not used. - #engine = SerialComputationEngine() - #engine = SlurmComputationEngine(batch_parameters, partition='wrkstn,compute') + # engine = SerialComputationEngine() + # engine = SlurmComputationEngine(batch_parameters, partition='wrkstn,compute') engine = SlurmComputationEngine(batch_parameters) n_methods = len(method_job_funcs) # repetitions x len(ns) x #methods - aggregators = np.empty((reps, len(ns), n_methods ), dtype=object) + aggregators = np.empty((reps, len(ns), n_methods), dtype=object) for r in range(reps): for ni, n in enumerate(ns): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ - fname = '%s-%s-n%d_r%d_a%.3f_trp%.2f.p' \ - %(prob_label, func_name, n, r, alpha, tr_proportion) + fname = "%s-%s-n%d_r%d_a%.3f_trp%.2f.p" % ( + prob_label, + func_name, + n, + r, + alpha, + tr_proportion, + ) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): - logger.info('%s exists. Load and return.'%fname) + logger.info("%s exists. Load and return." % fname) job_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() @@ -583,8 +627,7 @@ def run_problem(prob_label): # result not exists or rerun # p: an UnnormalizedDensity object - job = Ex1Job(SingleResultAggregator(), p, ds, prob_label, - r, f, n) + job = Ex1Job(SingleResultAggregator(), p, ds, prob_label, r, f, n) agg = engine.submit_job(job) aggregators[r, ni, mi] = agg @@ -598,8 +641,7 @@ def run_problem(prob_label): for r in range(reps): for ni, n in enumerate(ns): for mi, f in enumerate(method_job_funcs): - logger.info("Collecting result (%s, r=%d, n=%rd)" % - (f.__name__, r, n)) + logger.info("Collecting result (%s, r=%d, n=%rd)" % (f.__name__, r, n)) # let the aggregator finalize things aggregators[r, ni, mi].finalize() @@ -608,34 +650,46 @@ def run_problem(prob_label): job_result = aggregators[r, ni, mi].get_final_result().result job_results[r, ni, mi] = job_result - #func_names = [f.__name__ for f in method_job_funcs] - #func2labels = exglobal.get_func2label_map() - #method_labels = [func2labels[f] for f in func_names if f in func2labels] - - # save results - results = {'job_results': job_results, 'data_source': ds, - 'alpha': alpha, 'repeats': reps, 'ns': ns, - 'p': p, - 'tr_proportion': tr_proportion, - 'method_job_funcs': method_job_funcs, 'prob_label': prob_label, - } - - # class name - fname = 'ex%d-%s-me%d_rs%d_nmi%d_nma%d_a%.3f_trp%.2f.p' \ - %(ex, prob_label, n_methods, reps, min(ns), max(ns), alpha, - tr_proportion) + # func_names = [f.__name__ for f in method_job_funcs] + # func2labels = exglobal.get_func2label_map() + # method_labels = [func2labels[f] for f in func_names if f in func2labels] + + # save results + results = { + "job_results": job_results, + "data_source": ds, + "alpha": alpha, + "repeats": reps, + "ns": ns, + "p": p, + "tr_proportion": tr_proportion, + "method_job_funcs": method_job_funcs, + "prob_label": prob_label, + } + + # class name + fname = "ex%d-%s-me%d_rs%d_nmi%d_nma%d_a%.3f_trp%.2f.p" % ( + ex, + prob_label, + n_methods, + reps, + min(ns), + max(ns), + alpha, + tr_proportion, + ) glo.ex_save_result(ex, results, fname) - logger.info('Saved aggregated results to %s'%fname) + logger.info("Saved aggregated results to %s" % fname) def main(): if len(sys.argv) != 2: - print('Usage: %s problem_label'%sys.argv[0]) + print("Usage: %s problem_label" % sys.argv[0]) sys.exit(1) prob_label = sys.argv[1] run_problem(prob_label) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/sbibm/third_party/kgof/ex/ex2_prob_params.py b/sbibm/third_party/kgof/ex/ex2_prob_params.py index a98e081d..54a849b7 100644 --- a/sbibm/third_party/kgof/ex/ex2_prob_params.py +++ b/sbibm/third_party/kgof/ex/ex2_prob_params.py @@ -1,36 +1,38 @@ """Simulation to examine the P(reject) as the parameters for each problem are varied. What varies will depend on the problem.""" -__author__ = 'wittawat' +__author__ = "wittawat" -import sbibm.third_party.kgof as kgof -import sbibm.third_party.kgof.data as data -import sbibm.third_party.kgof.glo as glo -import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.goftest as gof -import sbibm.third_party.kgof.intertst as tgof -import sbibm.third_party.kgof.mmd as mgof -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel +import logging +import math +import os +import sys +import time + +# import numpy as np +import autograd.numpy as np -# need independent_jobs package +# need independent_jobs package # https://github.com/karlnapf/independent-jobs # The independent_jobs and kgof have to be in the global search path (.bashrc) import independent_jobs as inj -from independent_jobs.jobs.IndependentJob import IndependentJob -from independent_jobs.results.SingleResult import SingleResult from independent_jobs.aggregators.SingleResultAggregator import SingleResultAggregator from independent_jobs.engines.BatchClusterParameters import BatchClusterParameters from independent_jobs.engines.SerialComputationEngine import SerialComputationEngine from independent_jobs.engines.SlurmComputationEngine import SlurmComputationEngine +from independent_jobs.jobs.IndependentJob import IndependentJob +from independent_jobs.results.SingleResult import SingleResult from independent_jobs.tools.Log import logger -import logging -import math -#import numpy as np -import autograd.numpy as np -import os -import sys -import time + +import sbibm.third_party.kgof as kgof +import sbibm.third_party.kgof.data as data +import sbibm.third_party.kgof.density as density +import sbibm.third_party.kgof.glo as glo +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.intertst as tgof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.mmd as mgof +import sbibm.third_party.kgof.util as util """ All the job functions return a dictionary with the following keys: @@ -39,6 +41,7 @@ - time_secs: run time in seconds """ + def job_fssdJ1q_med(p, data_source, tr, te, r, J=1, null_sim=None): """ FSSD test with a Gaussian kernel, where the test locations are randomized, @@ -57,16 +60,14 @@ def job_fssdJ1q_med(p, data_source, tr, te, r, J=1, null_sim=None): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) - V = util.fit_gaussian_draw(X, J, seed=r+3) + k = kernel.KGauss(med ** 2) + V = util.fit_gaussian_draw(X, J, seed=r + 3) fssd_med = gof.FSSD(p, k, V, null_sim=null_sim, alpha=alpha) fssd_med_result = fssd_med.perform_test(data) - return { - 'goftest': fssd_med, - 'test_result': fssd_med_result, 'time_secs': t.secs} + return {"goftest": fssd_med, "test_result": fssd_med_result, "time_secs": t.secs} def job_fssdJ5q_med(p, data_source, tr, te, r): @@ -87,45 +88,52 @@ def job_fssdJ1q_opt(p, data_source, tr, te, r, J=1, null_sim=None): with util.ContextTimer() as t: # Use grid search to initialize the gwidth n_gwidth_cand = 5 - gwidth_factors = 2.0**np.linspace(-3, 3, n_gwidth_cand) - med2 = util.meddistance(Xtr, 1000)**2 + gwidth_factors = 2.0 ** np.linspace(-3, 3, n_gwidth_cand) + med2 = util.meddistance(Xtr, 1000) ** 2 - k = kernel.KGauss(med2*2) + k = kernel.KGauss(med2 * 2) # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) - list_gwidth = np.hstack( ( (med2)*gwidth_factors ) ) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) + list_gwidth = np.hstack(((med2) * gwidth_factors)) besti, objs = gof.GaussFSSD.grid_search_gwidth(p, tr, V0, list_gwidth) gwidth = list_gwidth[besti] - assert util.is_real_num(gwidth), 'gwidth not real. Was %s'%str(gwidth) - assert gwidth > 0, 'gwidth not positive. Was %.3g'%gwidth - logging.info('After grid search, gwidth=%.3g'%gwidth) - + assert util.is_real_num(gwidth), "gwidth not real. Was %s" % str(gwidth) + assert gwidth > 0, "gwidth not positive. Was %.3g" % gwidth + logging.info("After grid search, gwidth=%.3g" % gwidth) + ops = { - 'reg': 1e-2, - 'max_iter': 40, - 'tol_fun': 1e-4, - 'disp': True, - 'locs_bounds_frac':10.0, - 'gwidth_lb': 1e-1, - 'gwidth_ub': 1e4, - } - - V_opt, gwidth_opt, info = gof.GaussFSSD.optimize_locs_widths(p, tr, - gwidth, V0, **ops) + "reg": 1e-2, + "max_iter": 40, + "tol_fun": 1e-4, + "disp": True, + "locs_bounds_frac": 10.0, + "gwidth_lb": 1e-1, + "gwidth_ub": 1e4, + } + + V_opt, gwidth_opt, info = gof.GaussFSSD.optimize_locs_widths( + p, tr, gwidth, V0, **ops + ) # Use the optimized parameters to construct a test k_opt = kernel.KGauss(gwidth_opt) fssd_opt = gof.FSSD(p, k_opt, V_opt, null_sim=null_sim, alpha=alpha) fssd_opt_result = fssd_opt.perform_test(te) - return {'test_result': fssd_opt_result, 'time_secs': t.secs, - 'goftest': fssd_opt, 'opt_info': info, - } + return { + "test_result": fssd_opt_result, + "time_secs": t.secs, + "goftest": fssd_opt, + "opt_info": info, + } + def job_fssdJ5q_opt(p, data_source, tr, te, r): return job_fssdJ1q_opt(p, data_source, tr, te, r, J=5) + def job_fssdJ10q_opt(p, data_source, tr, te, r): return job_fssdJ1q_opt(p, data_source, tr, te, r, J=10) + def job_fssdJ5p_opt(p, data_source, tr, te, r): """ The suffix p means that p is sampled to get a sample for computing the @@ -134,6 +142,7 @@ def job_fssdJ5p_opt(p, data_source, tr, te, r): null_sim = gof.FSSDH0SimCovDraw(n_draw=2000, n_simulate=2000, seed=r) return job_fssdJ1q_opt(p, data_source, tr, te, r, J=5, null_sim=null_sim) + def job_fssdJ10p_opt(p, data_source, tr, te, r): """ The suffix p means that p is sampled to get a sample for computing the @@ -142,6 +151,7 @@ def job_fssdJ10p_opt(p, data_source, tr, te, r): null_sim = gof.FSSDH0SimCovDraw(n_draw=2000, n_simulate=2000, seed=r) return job_fssdJ1q_opt(p, data_source, tr, te, r, J=10, null_sim=null_sim) + def job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=1, b=-0.5, null_sim=None): """ FSSD with optimization on tr. Test on te. Use an inverse multiquadric @@ -160,17 +170,17 @@ def job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=1, b=-0.5, null_sim=None): c = 1.0 # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) ops = { - 'reg': 1e-5, - 'max_iter': 30, - 'tol_fun': 1e-6, - 'disp': True, - 'locs_bounds_frac':20.0, - } + "reg": 1e-5, + "max_iter": 30, + "tol_fun": 1e-6, + "disp": True, + "locs_bounds_frac": 20.0, + } - V_opt, info = gof.IMQFSSD.optimize_locs(p, tr, b, c, V0, **ops) + V_opt, info = gof.IMQFSSD.optimize_locs(p, tr, b, c, V0, **ops) k_imq = kernel.KIMQ(b=b, c=c) @@ -178,22 +188,30 @@ def job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=1, b=-0.5, null_sim=None): fssd_imq = gof.FSSD(p, k_imq, V_opt, null_sim=null_sim, alpha=alpha) fssd_imq_result = fssd_imq.perform_test(te) - return {'test_result': fssd_imq_result, 'time_secs': t.secs, - 'goftest': fssd_imq, 'opt_info': info, - } + return { + "test_result": fssd_imq_result, + "time_secs": t.secs, + "goftest": fssd_imq, + "opt_info": info, + } + def job_fssdJ5q_imq_optv(p, data_source, tr, te, r): return job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=5) + def job_fssdJ5q_imqb1_optv(p, data_source, tr, te, r): return job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=5, b=-1.0) + def job_fssdJ5q_imqb2_optv(p, data_source, tr, te, r): return job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=5, b=-2.0) + def job_fssdJ5q_imqb3_optv(p, data_source, tr, te, r): return job_fssdJ1q_imq_optv(p, data_source, tr, te, r, J=5, b=-3.0) + def job_fssdJ1q_imq_opt(p, data_source, tr, te, r, J=1, null_sim=None): """ FSSD with optimization on tr. Test on te. Use an inverse multiquadric @@ -210,23 +228,24 @@ def job_fssdJ1q_imq_opt(p, data_source, tr, te, r, J=1, null_sim=None): c0 = 1.0 # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) ops = { - 'reg': 1e-5, - 'max_iter': 50, - 'tol_fun': 1e-6, - 'disp': True, - 'locs_bounds_frac':20.0, + "reg": 1e-5, + "max_iter": 50, + "tol_fun": 1e-6, + "disp": True, + "locs_bounds_frac": 20.0, # IMQ kernel bounds - 'b_lb': -3, - 'b_ub': -0.5, - 'c_lb': 1e-1, - 'c_ub': np.sqrt(10), - } + "b_lb": -3, + "b_ub": -0.5, + "c_lb": 1e-1, + "c_ub": np.sqrt(10), + } - V_opt, b_opt, c_opt, info = gof.IMQFSSD.optimize_locs_params(p, tr, b0, - c0, V0, **ops) + V_opt, b_opt, c_opt, info = gof.IMQFSSD.optimize_locs_params( + p, tr, b0, c0, V0, **ops + ) k_imq = kernel.KIMQ(b=b_opt, c=c_opt) @@ -234,13 +253,18 @@ def job_fssdJ1q_imq_opt(p, data_source, tr, te, r, J=1, null_sim=None): fssd_imq = gof.FSSD(p, k_imq, V_opt, null_sim=null_sim, alpha=alpha) fssd_imq_result = fssd_imq.perform_test(te) - return {'test_result': fssd_imq_result, 'time_secs': t.secs, - 'goftest': fssd_imq, 'opt_info': info, - } + return { + "test_result": fssd_imq_result, + "time_secs": t.secs, + "goftest": fssd_imq, + "opt_info": info, + } + def job_fssdJ5q_imq_opt(p, data_source, tr, te, r, null_sim=None): return job_fssdJ1q_imq_opt(p, data_source, tr, te, r, J=5) + def job_fssdJ1q_imq_optbv(p, data_source, tr, te, r, J=1, null_sim=None): """ FSSD with optimization on tr. Test on te. Use an inverse multiquadric @@ -258,22 +282,23 @@ def job_fssdJ1q_imq_optbv(p, data_source, tr, te, r, J=1, null_sim=None): c0 = c # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) ops = { - 'reg': 1e-5, - 'max_iter': 40, - 'tol_fun': 1e-6, - 'disp': True, - 'locs_bounds_frac':20.0, + "reg": 1e-5, + "max_iter": 40, + "tol_fun": 1e-6, + "disp": True, + "locs_bounds_frac": 20.0, # IMQ kernel bounds - 'b_lb': -20, - 'c_lb': c, - 'c_ub': c, - } + "b_lb": -20, + "c_lb": c, + "c_ub": c, + } - V_opt, b_opt, c_opt, info = gof.IMQFSSD.optimize_locs_params(p, tr, b0, - c0, V0, **ops) + V_opt, b_opt, c_opt, info = gof.IMQFSSD.optimize_locs_params( + p, tr, b0, c0, V0, **ops + ) k_imq = kernel.KIMQ(b=b_opt, c=c_opt) @@ -281,9 +306,13 @@ def job_fssdJ1q_imq_optbv(p, data_source, tr, te, r, J=1, null_sim=None): fssd_imq = gof.FSSD(p, k_imq, V_opt, null_sim=null_sim, alpha=alpha) fssd_imq_result = fssd_imq.perform_test(te) - return {'test_result': fssd_imq_result, 'time_secs': t.secs, - 'goftest': fssd_imq, 'opt_info': info, - } + return { + "test_result": fssd_imq_result, + "time_secs": t.secs, + "goftest": fssd_imq, + "opt_info": info, + } + def job_fssdJ5q_imq_optbv(p, data_source, tr, te, r, null_sim=None): return job_fssdJ1q_imq_optbv(p, data_source, tr, te, r, J=5) @@ -297,22 +326,30 @@ def job_me_opt(p, data_source, tr, te, r, J=5): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic - #pds = p.get_datasource() - #datY = pds.sample(data.sample_size(), seed=r+294) - #Y = datY.data() - #XY = np.vstack((X, Y)) - #med = util.meddistance(XY, subsample=1000) - op = {'n_test_locs': J, 'seed': r+5, 'max_iter': 40, - 'batch_proportion': 1.0, 'locs_step_size': 1.0, - 'gwidth_step_size': 0.1, 'tol_fun': 1e-4, - 'reg': 1e-4} + # median heuristic + # pds = p.get_datasource() + # datY = pds.sample(data.sample_size(), seed=r+294) + # Y = datY.data() + # XY = np.vstack((X, Y)) + # med = util.meddistance(XY, subsample=1000) + op = { + "n_test_locs": J, + "seed": r + 5, + "max_iter": 40, + "batch_proportion": 1.0, + "locs_step_size": 1.0, + "gwidth_step_size": 0.1, + "tol_fun": 1e-4, + "reg": 1e-4, + } # optimize on the training set - me_opt = tgof.GaussMETestOpt(p, n_locs=J, tr_proportion=tr_proportion, - alpha=alpha, seed=r+111) + me_opt = tgof.GaussMETestOpt( + p, n_locs=J, tr_proportion=tr_proportion, alpha=alpha, seed=r + 111 + ) me_result = me_opt.perform_test(data, op) - return { 'test_result': me_result, 'time_secs': t.secs} + return {"test_result": me_result, "time_secs": t.secs} + def job_kstein_med(p, data_source, tr, te, r): """ @@ -323,24 +360,25 @@ def job_kstein_med(p, data_source, tr, te, r): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) + k = kernel.KGauss(med ** 2) kstein = gof.KernelSteinTest(p, k, alpha=alpha, n_simulate=1000, seed=r) kstein_result = kstein.perform_test(data) - return { 'test_result': kstein_result, 'time_secs': t.secs} + return {"test_result": kstein_result, "time_secs": t.secs} + def job_kstein_imq(p, data_source, tr, te, r): """ Kernel Stein discrepancy test of Liu et al., 2016 and Chwialkowski et al., - 2016. Use full sample. Use the inverse multiquadric kernel (IMQ) studied - in + 2016. Use full sample. Use the inverse multiquadric kernel (IMQ) studied + in Measuring Sample Quality with Kernels - Gorham and Mackey 2017. + Gorham and Mackey 2017. - Parameters are fixed to the recommented values: beta = b = -0.5, c = 1. + Parameters are fixed to the recommented values: beta = b = -0.5, c = 1. """ # full data data = tr + te @@ -350,7 +388,8 @@ def job_kstein_imq(p, data_source, tr, te, r): kstein = gof.KernelSteinTest(p, k, alpha=alpha, n_simulate=1000, seed=r) kstein_result = kstein.perform_test(data) - return { 'test_result': kstein_result, 'time_secs': t.secs} + return {"test_result": kstein_result, "time_secs": t.secs} + def job_lin_kstein_med(p, data_source, tr, te, r): """ @@ -361,27 +400,28 @@ def job_lin_kstein_med(p, data_source, tr, te, r): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) + k = kernel.KGauss(med ** 2) lin_kstein = gof.LinearKernelSteinTest(p, k, alpha=alpha, seed=r) lin_kstein_result = lin_kstein.perform_test(data) - return { 'test_result': lin_kstein_result, 'time_secs': t.secs} + return {"test_result": lin_kstein_result, "time_secs": t.secs} + def job_mmd_med(p, data_source, tr, te, r): """ MMD test of Gretton et al., 2012 used as a goodness-of-fit test. - Require the ability to sample from p i.e., the UnnormalizedDensity p has + Require the ability to sample from p i.e., the UnnormalizedDensity p has to be able to return a non-None from get_datasource() """ # full data data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic pds = p.get_datasource() - datY = pds.sample(data.sample_size(), seed=r+294) + datY = pds.sample(data.sample_size(), seed=r + 294) Y = datY.data() XY = np.vstack((X, Y)) @@ -390,17 +430,18 @@ def job_mmd_med(p, data_source, tr, te, r): medx = util.meddistance(X, subsample=1000) medy = util.meddistance(Y, subsample=1000) medxy = util.meddistance(XY, subsample=1000) - med_avg = (medx+medy+medxy)/3.0 - k = kernel.KGauss(med_avg**2) + med_avg = (medx + medy + medxy) / 3.0 + k = kernel.KGauss(med_avg ** 2) mmd_test = mgof.QuadMMDGof(p, k, n_permute=400, alpha=alpha, seed=r) mmd_result = mmd_test.perform_test(data) - return { 'test_result': mmd_result, 'time_secs': t.secs} + return {"test_result": mmd_result, "time_secs": t.secs} + def job_mmd_opt(p, data_source, tr, te, r): """ MMD test of Gretton et al., 2012 used as a goodness-of-fit test. - Require the ability to sample from p i.e., the UnnormalizedDensity p has + Require the ability to sample from p i.e., the UnnormalizedDensity p has to be able to return a non-None from get_datasource() With optimization. Gaussian kernel. @@ -408,9 +449,9 @@ def job_mmd_opt(p, data_source, tr, te, r): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic pds = p.get_datasource() - datY = pds.sample(data.sample_size(), seed=r+294) + datY = pds.sample(data.sample_size(), seed=r + 294) Y = datY.data() XY = np.vstack((X, Y)) @@ -418,30 +459,32 @@ def job_mmd_opt(p, data_source, tr, te, r): # Construct a list of kernels to try based on multiples of the median # heuristic - #list_gwidth = np.hstack( (np.linspace(20, 40, 10), (med**2) + # list_gwidth = np.hstack( (np.linspace(20, 40, 10), (med**2) # *(2.0**np.linspace(-2, 2, 20) ) ) ) - list_gwidth = (med**2)*(2.0**np.linspace(-3, 3, 30) ) + list_gwidth = (med ** 2) * (2.0 ** np.linspace(-3, 3, 30)) list_gwidth.sort() candidate_kernels = [kernel.KGauss(gw2) for gw2 in list_gwidth] - mmd_opt = mgof.QuadMMDGofOpt(p, n_permute=300, alpha=alpha, seed=r+56) - mmd_result = mmd_opt.perform_test(data, - candidate_kernels=candidate_kernels, - tr_proportion=tr_proportion, reg=1e-3) - return { 'test_result': mmd_result, 'time_secs': t.secs} + mmd_opt = mgof.QuadMMDGofOpt(p, n_permute=300, alpha=alpha, seed=r + 56) + mmd_result = mmd_opt.perform_test( + data, + candidate_kernels=candidate_kernels, + tr_proportion=tr_proportion, + reg=1e-3, + ) + return {"test_result": mmd_result, "time_secs": t.secs} # Define our custom Job, which inherits from base class IndependentJob class Ex2Job(IndependentJob): - - def __init__(self, aggregator, p, data_source, - prob_label, rep, job_func, prob_param): - #walltime = 60*59*24 - walltime = 60*59 - memory = int(tr_proportion*sample_size*1e-2) + 50 - - IndependentJob.__init__(self, aggregator, walltime=walltime, - memory=memory) + def __init__( + self, aggregator, p, data_source, prob_label, rep, job_func, prob_param + ): + # walltime = 60*59*24 + walltime = 60 * 59 + memory = int(tr_proportion * sample_size * 1e-2) + 50 + + IndependentJob.__init__(self, aggregator, walltime=walltime, memory=memory) # p: an UnnormalizedDensity self.p = p self.data_source = data_source @@ -455,18 +498,20 @@ def __init__(self, aggregator, p, data_source, def compute(self): p = self.p - data_source = self.data_source + data_source = self.data_source r = self.rep prob_param = self.prob_param job_func = self.job_func # sample_size is a global variable data = data_source.sample(sample_size, seed=r) with util.ContextTimer() as t: - tr, te = data.split_tr_te(tr_proportion=tr_proportion, seed=r+21 ) + tr, te = data.split_tr_te(tr_proportion=tr_proportion, seed=r + 21) prob_label = self.prob_label - logger.info("computing. %s. prob=%s, r=%d,\ - param=%.3g"%(job_func.__name__, prob_label, r, prob_param)) - + logger.info( + "computing. %s. prob=%s, r=%d,\ + param=%.3g" + % (job_func.__name__, prob_label, r, prob_param) + ) job_result = job_func(p, data_source, tr, te, r) @@ -475,43 +520,53 @@ def compute(self): # submit the result to my own aggregator self.aggregator.submit_result(result) func_name = job_func.__name__ - logger.info("done. ex2: %s, prob=%s, r=%d, param=%.3g. Took: %.3g s "%(func_name, - prob_label, r, prob_param, t.secs)) + logger.info( + "done. ex2: %s, prob=%s, r=%d, param=%.3g. Took: %.3g s " + % (func_name, prob_label, r, prob_param, t.secs) + ) # save result - fname = '%s-%s-n%d_r%d_p%g_a%.3f_trp%.2f.p' \ - %(prob_label, func_name, sample_size, r, prob_param, alpha, - tr_proportion) + fname = "%s-%s-n%d_r%d_p%g_a%.3f_trp%.2f.p" % ( + prob_label, + func_name, + sample_size, + r, + prob_param, + alpha, + tr_proportion, + ) glo.ex_save_result(ex, job_result, prob_label, fname) # This import is needed so that pickle knows about the class Ex2Job. # pickle is used when collecting the results from the submitted jobs. -from sbibm.third_party.kgof.ex.ex2_prob_params import Ex2Job -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ1q_med -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_med -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ1q_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ10q_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5p_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ10p_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ1q_imq_optv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_imq_optv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_imqb1_optv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_imqb2_optv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_imqb3_optv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ1q_imq_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_imq_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ1q_imq_optbv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_fssdJ5q_imq_optbv -from sbibm.third_party.kgof.ex.ex2_prob_params import job_me_opt -from sbibm.third_party.kgof.ex.ex2_prob_params import job_kstein_med -from sbibm.third_party.kgof.ex.ex2_prob_params import job_kstein_imq -from sbibm.third_party.kgof.ex.ex2_prob_params import job_lin_kstein_med -from sbibm.third_party.kgof.ex.ex2_prob_params import job_mmd_med -from sbibm.third_party.kgof.ex.ex2_prob_params import job_mmd_opt - -#--- experimental setting ----- +from sbibm.third_party.kgof.ex.ex2_prob_params import ( + Ex2Job, + job_fssdJ1q_imq_opt, + job_fssdJ1q_imq_optbv, + job_fssdJ1q_imq_optv, + job_fssdJ1q_med, + job_fssdJ1q_opt, + job_fssdJ5p_opt, + job_fssdJ5q_imq_opt, + job_fssdJ5q_imq_optbv, + job_fssdJ5q_imq_optv, + job_fssdJ5q_imqb1_optv, + job_fssdJ5q_imqb2_optv, + job_fssdJ5q_imqb3_optv, + job_fssdJ5q_med, + job_fssdJ5q_opt, + job_fssdJ10p_opt, + job_fssdJ10q_opt, + job_kstein_imq, + job_kstein_med, + job_lin_kstein_med, + job_me_opt, + job_mmd_med, + job_mmd_opt, +) + +# --- experimental setting ----- ex = 2 # sample size = n (the training and test sizes are n/2) @@ -525,31 +580,31 @@ def compute(self): # repetitions for each parameter setting reps = 200 -method_job_funcs = [ - job_fssdJ5q_opt, - job_fssdJ5q_med, - job_kstein_med, - job_lin_kstein_med, - job_mmd_opt, - job_me_opt, - - #job_fssdJ5q_imq_opt, - #job_fssdJ5q_imq_optv, - #job_fssdJ5q_imq_optbv, - #job_fssdJ5q_imqb1_optv, - #job_fssdJ5q_imqb2_optv, - #job_fssdJ5q_imqb3_optv, - #job_fssdJ10q_opt, - #job_fssdJ5p_opt, - #job_fssdJ10p_opt, - #job_kstein_imq, - #job_mmd_med, - ] +method_job_funcs = [ + job_fssdJ5q_opt, + job_fssdJ5q_med, + job_kstein_med, + job_lin_kstein_med, + job_mmd_opt, + job_me_opt, + # job_fssdJ5q_imq_opt, + # job_fssdJ5q_imq_optv, + # job_fssdJ5q_imq_optbv, + # job_fssdJ5q_imqb1_optv, + # job_fssdJ5q_imqb2_optv, + # job_fssdJ5q_imqb3_optv, + # job_fssdJ10q_opt, + # job_fssdJ5p_opt, + # job_fssdJ10p_opt, + # job_kstein_imq, + # job_mmd_med, +] # If is_rerun==False, do not rerun the experiment if a result file for the current # setting of (pi, r) already exists. is_rerun = False -#--------------------------- +# --------------------------- + def gaussbern_rbm_probs(stds_perturb_B, dx=50, dh=10, n=sample_size): """ @@ -563,8 +618,8 @@ def gaussbern_rbm_probs(stds_perturb_B, dx=50, dh=10, n=sample_size): """ probs = [] for i, std in enumerate(stds_perturb_B): - with util.NumpySeedContext(seed=i+1000): - B = np.random.randint(0, 2, (dx, dh))*2 - 1.0 + with util.NumpySeedContext(seed=i + 1000): + B = np.random.randint(0, 2, (dx, dh)) * 2 - 1.0 b = np.random.randn(dx) c = np.random.randn(dh) p = density.GaussBernRBM(B, b, c) @@ -572,16 +627,17 @@ def gaussbern_rbm_probs(stds_perturb_B, dx=50, dh=10, n=sample_size): if std <= 1e-8: B_perturb = B else: - B_perturb = B + np.random.randn(dx, dh)*std + B_perturb = B + np.random.randn(dx, dh) * std gb_rbm = data.DSGaussBernRBM(B_perturb, b, c, burnin=2000) probs.append((std, p, gb_rbm)) return probs + def get_pqsource_list(prob_label): """ Return [(prob_param, p, ds) for ... ], a list of tuples - where + where - prob_param: a problem parameters. Each parameter has to be a scalar (so that we can plot them later). Parameters are preferably positive integers. @@ -593,67 +649,104 @@ def get_pqsource_list(prob_label): gmd_ds = [5, 20, 40, 60] # vary the mean gmd_d10_ms = [0, 0.02, 0.04, 0.06] - gvinc_d1_vs = [1, 1.5, 2, 2.5] + gvinc_d1_vs = [1, 1.5, 2, 2.5] gvinc_d5_vs = [1, 1.5, 2, 2.5] gvsub1_d1_vs = [0.1, 0.3, 0.5, 0.7] gvd_ds = [1, 5, 10, 15] - #gb_rbm_dx50_dh10_stds = [0, 0.01, 0.02, 0.03] + # gb_rbm_dx50_dh10_stds = [0, 0.01, 0.02, 0.03] gb_rbm_dx50_dh10_stds = [0, 0.02, 0.04, 0.06] - #gb_rbm_dx50_dh10_stds = [0] + # gb_rbm_dx50_dh10_stds = [0] gb_rbm_dx50_dh40_stds = [0, 0.01, 0.02, 0.04, 0.06] glaplace_ds = [1, 5, 10, 15] - prob2tuples = { - # H0 is true. vary d. P = Q = N(0, I) - 'sg': [(d, density.IsotropicNormal(np.zeros(d), 1), - data.DSIsotropicNormal(np.zeros(d), 1) ) for d in sg_ds], - - # vary d. P = N(0, I), Q = N( (c,..0), I) - 'gmd': [(d, density.IsotropicNormal(np.zeros(d), 1), - data.DSIsotropicNormal(np.hstack((1, np.zeros(d-1))), 1) ) - for d in gmd_ds - ], - # P = N(0, I), Q = N( (m, ..0), I). Vary m - 'gmd_d10_ms': [(m, density.IsotropicNormal(np.zeros(10), 1), - data.DSIsotropicNormal(np.hstack((m, np.zeros(9))), 1) ) - for m in gmd_d10_ms - ], - # d=1. Increase the variance. P = N(0, I). Q = N(0, v*I) - 'gvinc_d1': [(var, density.IsotropicNormal(np.zeros(1), 1), - data.DSIsotropicNormal(np.zeros(1), var) ) - for var in gvinc_d1_vs - ], - # d=5. Increase the variance. P = N(0, I). Q = N(0, v*I) - 'gvinc_d5': [(var, density.IsotropicNormal(np.zeros(5), 1), - data.DSIsotropicNormal(np.zeros(5), var) ) - for var in gvinc_d5_vs - ], - # d=1. P=N(0,1), Q(0,v). Consider the variance below 1. - 'gvsub1_d1': [(var, density.IsotropicNormal(np.zeros(1), 1), - data.DSIsotropicNormal(np.zeros(1), var) ) - for var in gvsub1_d1_vs - ], - # Gaussian variance difference problem. Only the variance - # of the first dimenion differs. d varies. - 'gvd': [(d, density.Normal(np.zeros(d), np.eye(d) ), - data.DSNormal(np.zeros(d), np.diag(np.hstack((2, np.ones(d-1)))) )) - for d in gvd_ds], - - # Gaussian Bernoulli RBM. dx=50, dh=10 - 'gbrbm_dx50_dh10': gaussbern_rbm_probs(gb_rbm_dx50_dh10_stds, - dx=50, dh=10, n=sample_size), - - # Gaussian Bernoulli RBM. dx=50, dh=40 - 'gbrbm_dx50_dh40': gaussbern_rbm_probs(gb_rbm_dx50_dh40_stds, - dx=50, dh=40, n=sample_size), - - # p: N(0, I), q: standard Laplace. Vary d - 'glaplace': [(d, density.IsotropicNormal(np.zeros(d), 1), + prob2tuples = { + # H0 is true. vary d. P = Q = N(0, I) + "sg": [ + ( + d, + density.IsotropicNormal(np.zeros(d), 1), + data.DSIsotropicNormal(np.zeros(d), 1), + ) + for d in sg_ds + ], + # vary d. P = N(0, I), Q = N( (c,..0), I) + "gmd": [ + ( + d, + density.IsotropicNormal(np.zeros(d), 1), + data.DSIsotropicNormal(np.hstack((1, np.zeros(d - 1))), 1), + ) + for d in gmd_ds + ], + # P = N(0, I), Q = N( (m, ..0), I). Vary m + "gmd_d10_ms": [ + ( + m, + density.IsotropicNormal(np.zeros(10), 1), + data.DSIsotropicNormal(np.hstack((m, np.zeros(9))), 1), + ) + for m in gmd_d10_ms + ], + # d=1. Increase the variance. P = N(0, I). Q = N(0, v*I) + "gvinc_d1": [ + ( + var, + density.IsotropicNormal(np.zeros(1), 1), + data.DSIsotropicNormal(np.zeros(1), var), + ) + for var in gvinc_d1_vs + ], + # d=5. Increase the variance. P = N(0, I). Q = N(0, v*I) + "gvinc_d5": [ + ( + var, + density.IsotropicNormal(np.zeros(5), 1), + data.DSIsotropicNormal(np.zeros(5), var), + ) + for var in gvinc_d5_vs + ], + # d=1. P=N(0,1), Q(0,v). Consider the variance below 1. + "gvsub1_d1": [ + ( + var, + density.IsotropicNormal(np.zeros(1), 1), + data.DSIsotropicNormal(np.zeros(1), var), + ) + for var in gvsub1_d1_vs + ], + # Gaussian variance difference problem. Only the variance + # of the first dimenion differs. d varies. + "gvd": [ + ( + d, + density.Normal(np.zeros(d), np.eye(d)), + data.DSNormal(np.zeros(d), np.diag(np.hstack((2, np.ones(d - 1))))), + ) + for d in gvd_ds + ], + # Gaussian Bernoulli RBM. dx=50, dh=10 + "gbrbm_dx50_dh10": gaussbern_rbm_probs( + gb_rbm_dx50_dh10_stds, dx=50, dh=10, n=sample_size + ), + # Gaussian Bernoulli RBM. dx=50, dh=40 + "gbrbm_dx50_dh40": gaussbern_rbm_probs( + gb_rbm_dx50_dh40_stds, dx=50, dh=40, n=sample_size + ), + # p: N(0, I), q: standard Laplace. Vary d + "glaplace": [ + ( + d, + density.IsotropicNormal(np.zeros(d), 1), # Scaling of 1/sqrt(2) will make the variance 1. - data.DSLaplace(d=d, loc=0, scale=1.0/np.sqrt(2))) for d in glaplace_ds], - } + data.DSLaplace(d=d, loc=0, scale=1.0 / np.sqrt(2)), + ) + for d in glaplace_ds + ], + } if prob_label not in prob2tuples: - raise ValueError('Unknown problem label. Need to be one of %s'%str(prob2tuples.keys()) ) + raise ValueError( + "Unknown problem label. Need to be one of %s" % str(prob2tuples.keys()) + ) return prob2tuples[prob_label] @@ -661,41 +754,49 @@ def run_problem(prob_label): """Run the experiment""" L = get_pqsource_list(prob_label) prob_params, ps, data_sources = zip(*L) - # make them lists + # make them lists prob_params = list(prob_params) ps = list(ps) data_sources = list(data_sources) # /////// submit jobs ////////// # create folder name string - #result_folder = glo.result_folder() + # result_folder = glo.result_folder() from sbibm.third_party.kgof.config import expr_configs - tmp_dir = expr_configs['scratch_path'] - foldername = os.path.join(tmp_dir, 'kgof_slurm', 'e%d'%ex) + + tmp_dir = expr_configs["scratch_path"] + foldername = os.path.join(tmp_dir, "kgof_slurm", "e%d" % ex) logger.info("Setting engine folder to %s" % foldername) # create parameter instance that is needed for any batch computation engine logger.info("Creating batch parameter instance") batch_parameters = BatchClusterParameters( - foldername=foldername, job_name_base="e%d_"%ex, parameter_prefix="") + foldername=foldername, job_name_base="e%d_" % ex, parameter_prefix="" + ) # Use the following line if Slurm queue is not used. - #engine = SerialComputationEngine() + # engine = SerialComputationEngine() engine = SlurmComputationEngine(batch_parameters) - #engine = SlurmComputationEngine(batch_parameters, partition='wrkstn,compute') + # engine = SlurmComputationEngine(batch_parameters, partition='wrkstn,compute') n_methods = len(method_job_funcs) # repetitions x len(prob_params) x #methods - aggregators = np.empty((reps, len(prob_params), n_methods ), dtype=object) + aggregators = np.empty((reps, len(prob_params), n_methods), dtype=object) for r in range(reps): for pi, param in enumerate(prob_params): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ - fname = '%s-%s-n%d_r%d_p%g_a%.3f_trp%.2f.p' \ - %(prob_label, func_name, sample_size, r, param, alpha, - tr_proportion) + fname = "%s-%s-n%d_r%d_p%g_a%.3f_trp%.2f.p" % ( + prob_label, + func_name, + sample_size, + r, + param, + alpha, + tr_proportion, + ) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): - logger.info('%s exists. Load and return.'%fname) + logger.info("%s exists. Load and return." % fname) job_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() @@ -706,8 +807,15 @@ def run_problem(prob_label): # p: an UnnormalizedDensity object p = ps[pi] - job = Ex2Job(SingleResultAggregator(), p, data_sources[pi], - prob_label, r, f, param) + job = Ex2Job( + SingleResultAggregator(), + p, + data_sources[pi], + prob_label, + r, + f, + param, + ) agg = engine.submit_job(job) aggregators[r, pi, mi] = agg @@ -721,8 +829,9 @@ def run_problem(prob_label): for r in range(reps): for pi, param in enumerate(prob_params): for mi, f in enumerate(method_job_funcs): - logger.info("Collecting result (%s, r=%d, param=%.3g)" % - (f.__name__, r, param)) + logger.info( + "Collecting result (%s, r=%d, param=%.3g)" % (f.__name__, r, param) + ) # let the aggregator finalize things aggregators[r, pi, mi].finalize() @@ -731,37 +840,49 @@ def run_problem(prob_label): job_result = aggregators[r, pi, mi].get_final_result().result job_results[r, pi, mi] = job_result - #func_names = [f.__name__ for f in method_job_funcs] - #func2labels = exglobal.get_func2label_map() - #method_labels = [func2labels[f] for f in func_names if f in func2labels] - - # save results - results = {'job_results': job_results, 'prob_params': prob_params, - 'alpha': alpha, 'repeats': reps, - 'ps': ps, - 'list_data_source': data_sources, - 'tr_proportion': tr_proportion, - 'method_job_funcs': method_job_funcs, 'prob_label': prob_label, - 'sample_size': sample_size, - } - - # class name - fname = 'ex%d-%s-me%d_n%d_rs%d_pmi%g_pma%g_a%.3f_trp%.2f.p' \ - %(ex, prob_label, n_methods, sample_size, reps, min(prob_params), - max(prob_params), alpha, tr_proportion) + # func_names = [f.__name__ for f in method_job_funcs] + # func2labels = exglobal.get_func2label_map() + # method_labels = [func2labels[f] for f in func_names if f in func2labels] + + # save results + results = { + "job_results": job_results, + "prob_params": prob_params, + "alpha": alpha, + "repeats": reps, + "ps": ps, + "list_data_source": data_sources, + "tr_proportion": tr_proportion, + "method_job_funcs": method_job_funcs, + "prob_label": prob_label, + "sample_size": sample_size, + } + + # class name + fname = "ex%d-%s-me%d_n%d_rs%d_pmi%g_pma%g_a%.3f_trp%.2f.p" % ( + ex, + prob_label, + n_methods, + sample_size, + reps, + min(prob_params), + max(prob_params), + alpha, + tr_proportion, + ) glo.ex_save_result(ex, results, fname) - logger.info('Saved aggregated results to %s'%fname) + logger.info("Saved aggregated results to %s" % fname) def main(): if len(sys.argv) != 2: - print('Usage: %s problem_label'%sys.argv[0]) + print("Usage: %s problem_label" % sys.argv[0]) sys.exit(1) prob_label = sys.argv[1] run_problem(prob_label) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/sbibm/third_party/kgof/ex/ex3_vary_nlocs.py b/sbibm/third_party/kgof/ex/ex3_vary_nlocs.py index 3be6cd3a..9d31c3c3 100644 --- a/sbibm/third_party/kgof/ex/ex3_vary_nlocs.py +++ b/sbibm/third_party/kgof/ex/ex3_vary_nlocs.py @@ -2,34 +2,36 @@ Simulation to examine the P(reject) as the number of test locations increases. """ -__author__ = 'wittawat' +__author__ = "wittawat" -import sbibm.third_party.kgof as kgof -import sbibm.third_party.kgof.data as data -import sbibm.third_party.kgof.glo as glo -import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.goftest as gof -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel +import logging +import math +import os +import sys +import time -# need independent_jobs package +# import numpy as np +import autograd.numpy as np + +# need independent_jobs package # https://github.com/karlnapf/independent-jobs # The independent_jobs and kgof have to be in the global search path (.bashrc) import independent_jobs as inj -from independent_jobs.jobs.IndependentJob import IndependentJob -from independent_jobs.results.SingleResult import SingleResult from independent_jobs.aggregators.SingleResultAggregator import SingleResultAggregator from independent_jobs.engines.BatchClusterParameters import BatchClusterParameters from independent_jobs.engines.SerialComputationEngine import SerialComputationEngine from independent_jobs.engines.SlurmComputationEngine import SlurmComputationEngine +from independent_jobs.jobs.IndependentJob import IndependentJob +from independent_jobs.results.SingleResult import SingleResult from independent_jobs.tools.Log import logger -import logging -import math -#import numpy as np -import autograd.numpy as np -import os -import sys -import time + +import sbibm.third_party.kgof as kgof +import sbibm.third_party.kgof.data as data +import sbibm.third_party.kgof.density as density +import sbibm.third_party.kgof.glo as glo +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util """ All the job functions return a dictionary with the following keys: @@ -38,6 +40,7 @@ - time_secs: run time in seconds """ + def job_fssdq_med(p, data_source, tr, te, r, J, null_sim=None): """ FSSD test with a Gaussian kernel, where the test locations are randomized, @@ -56,15 +59,14 @@ def job_fssdq_med(p, data_source, tr, te, r, J, null_sim=None): data = tr + te X = data.data() with util.ContextTimer() as t: - # median heuristic + # median heuristic med = util.meddistance(X, subsample=1000) - k = kernel.KGauss(med**2) - V = util.fit_gaussian_draw(X, J, seed=r+1) + k = kernel.KGauss(med ** 2) + V = util.fit_gaussian_draw(X, J, seed=r + 1) fssd_med = gof.FSSD(p, k, V, null_sim=null_sim, alpha=alpha) fssd_med_result = fssd_med.perform_test(data) - return { 'test_result': fssd_med_result, 'time_secs': t.secs} - + return {"test_result": fssd_med_result, "time_secs": t.secs} def job_fssdq_opt(p, data_source, tr, te, r, J, null_sim=None): @@ -78,38 +80,43 @@ def job_fssdq_opt(p, data_source, tr, te, r, J, null_sim=None): with util.ContextTimer() as t: # Use grid search to initialize the gwidth n_gwidth_cand = 5 - gwidth_factors = 2.0**np.linspace(-3, 3, n_gwidth_cand) - med2 = util.meddistance(Xtr, 1000)**2 + gwidth_factors = 2.0 ** np.linspace(-3, 3, n_gwidth_cand) + med2 = util.meddistance(Xtr, 1000) ** 2 - k = kernel.KGauss(med2*2) + k = kernel.KGauss(med2 * 2) # fit a Gaussian to the data and draw to initialize V0 - V0 = util.fit_gaussian_draw(Xtr, J, seed=r+1, reg=1e-6) - list_gwidth = np.hstack( ( (med2)*gwidth_factors ) ) + V0 = util.fit_gaussian_draw(Xtr, J, seed=r + 1, reg=1e-6) + list_gwidth = np.hstack(((med2) * gwidth_factors)) besti, objs = gof.GaussFSSD.grid_search_gwidth(p, tr, V0, list_gwidth) gwidth = list_gwidth[besti] - assert util.is_real_num(gwidth), 'gwidth not real. Was %s'%str(gwidth) - assert gwidth > 0, 'gwidth not positive. Was %.3g'%gwidth - logging.info('After grid search, gwidth=%.3g'%gwidth) - + assert util.is_real_num(gwidth), "gwidth not real. Was %s" % str(gwidth) + assert gwidth > 0, "gwidth not positive. Was %.3g" % gwidth + logging.info("After grid search, gwidth=%.3g" % gwidth) + ops = { - 'reg': 1e-2, - 'max_iter': 50, - 'tol_fun': 1e-4, - 'disp': True, - 'locs_bounds_frac': 10.0, - 'gwidth_lb': 1e-1, - 'gwidth_ub': 1e3, - } - - V_opt, gwidth_opt, info = gof.GaussFSSD.optimize_locs_widths(p, tr, - gwidth, V0, **ops) + "reg": 1e-2, + "max_iter": 50, + "tol_fun": 1e-4, + "disp": True, + "locs_bounds_frac": 10.0, + "gwidth_lb": 1e-1, + "gwidth_ub": 1e3, + } + + V_opt, gwidth_opt, info = gof.GaussFSSD.optimize_locs_widths( + p, tr, gwidth, V0, **ops + ) # Use the optimized parameters to construct a test k_opt = kernel.KGauss(gwidth_opt) fssd_opt = gof.FSSD(p, k_opt, V_opt, null_sim=null_sim, alpha=alpha) fssd_opt_result = fssd_opt.perform_test(te) - return {'test_result': fssd_opt_result, 'time_secs': t.secs, - 'goftest': fssd_opt, 'opt_info': info, - } + return { + "test_result": fssd_opt_result, + "time_secs": t.secs, + "goftest": fssd_opt, + "opt_info": info, + } + def job_fssdp_opt(p, data_source, tr, te, r, J): """ @@ -122,15 +129,12 @@ def job_fssdp_opt(p, data_source, tr, te, r, J): # Define our custom Job, which inherits from base class IndependentJob class Ex3Job(IndependentJob): - - def __init__(self, aggregator, p, data_source, - prob_label, rep, job_func, n_locs): - #walltime = 60*59*24 - walltime = 60*59 - memory = int(tr_proportion*sample_size*1e-2) + 50 - - IndependentJob.__init__(self, aggregator, walltime=walltime, - memory=memory) + def __init__(self, aggregator, p, data_source, prob_label, rep, job_func, n_locs): + # walltime = 60*59*24 + walltime = 60 * 59 + memory = int(tr_proportion * sample_size * 1e-2) + 50 + + IndependentJob.__init__(self, aggregator, walltime=walltime, memory=memory) # p: an UnnormalizedDensity self.p = p self.data_source = data_source @@ -144,17 +148,20 @@ def __init__(self, aggregator, p, data_source, def compute(self): p = self.p - data_source = self.data_source + data_source = self.data_source r = self.rep n_locs = self.n_locs job_func = self.job_func # sample_size is a global variable data = data_source.sample(sample_size, seed=r) with util.ContextTimer() as t: - tr, te = data.split_tr_te(tr_proportion=tr_proportion, seed=r+21 ) + tr, te = data.split_tr_te(tr_proportion=tr_proportion, seed=r + 21) prob_label = self.prob_label - logger.info("computing. %s. prob=%s, r=%d,\ - J=%d"%(job_func.__name__, prob_label, r, n_locs)) + logger.info( + "computing. %s. prob=%s, r=%d,\ + J=%d" + % (job_func.__name__, prob_label, r, n_locs) + ) job_result = job_func(p, data_source, tr, te, r, n_locs) @@ -163,24 +170,34 @@ def compute(self): # submit the result to my own aggregator self.aggregator.submit_result(result) func_name = job_func.__name__ - logger.info("done. ex2: %s, prob=%s, r=%d, J=%d. Took: %.3g s "%(func_name, - prob_label, r, n_locs, t.secs)) + logger.info( + "done. ex2: %s, prob=%s, r=%d, J=%d. Took: %.3g s " + % (func_name, prob_label, r, n_locs, t.secs) + ) # save result - fname = '%s-%s-n%d_r%d_J%d_a%.3f_trp%.2f.p' \ - %(prob_label, func_name, sample_size, r, n_locs, alpha, - tr_proportion) + fname = "%s-%s-n%d_r%d_J%d_a%.3f_trp%.2f.p" % ( + prob_label, + func_name, + sample_size, + r, + n_locs, + alpha, + tr_proportion, + ) glo.ex_save_result(ex, job_result, prob_label, fname) # This import is needed so that pickle knows about the class Ex3Job. # pickle is used when collecting the results from the submitted jobs. -from sbibm.third_party.kgof.ex.ex3_vary_nlocs import Ex3Job -from sbibm.third_party.kgof.ex.ex3_vary_nlocs import job_fssdq_med -from sbibm.third_party.kgof.ex.ex3_vary_nlocs import job_fssdq_opt -from sbibm.third_party.kgof.ex.ex3_vary_nlocs import job_fssdp_opt - -#--- experimental setting ----- +from sbibm.third_party.kgof.ex.ex3_vary_nlocs import ( + Ex3Job, + job_fssdp_opt, + job_fssdq_med, + job_fssdq_opt, +) + +# --- experimental setting ----- ex = 3 # sample size = n (the training and test sizes are n/2) @@ -193,20 +210,23 @@ def compute(self): reps = 300 # list of number of test locations/frequencies -#Js = [5, 10, 15, 20, 25] -#Js = range(2, 6+1) -#Js = [2**x for x in range(5)] -Js = [2, 8, 32, 96, 384 ] -#Js = [2, 8, 32] - -method_job_funcs = [ job_fssdq_med, job_fssdq_opt, - #job_fssdp_opt, - ] +# Js = [5, 10, 15, 20, 25] +# Js = range(2, 6+1) +# Js = [2**x for x in range(5)] +Js = [2, 8, 32, 96, 384] +# Js = [2, 8, 32] + +method_job_funcs = [ + job_fssdq_med, + job_fssdq_opt, + # job_fssdp_opt, +] # If is_rerun==False, do not rerun the experiment if a result file for the current # setting already exists. is_rerun = False -#--------------------------- +# --------------------------- + def gaussbern_rbm_tuple(var, dx=50, dh=10, n=sample_size): """ @@ -221,16 +241,17 @@ def gaussbern_rbm_tuple(var, dx=50, dh=10, n=sample_size): Return p, a DataSource """ with util.NumpySeedContext(seed=1000): - B = np.random.randint(0, 2, (dx, dh))*2 - 1.0 + B = np.random.randint(0, 2, (dx, dh)) * 2 - 1.0 b = np.random.randn(dx) c = np.random.randn(dh) p = density.GaussBernRBM(B, b, c) - B_perturb = B + np.random.randn(dx, dh)*np.sqrt(var) + B_perturb = B + np.random.randn(dx, dh) * np.sqrt(var) gb_rbm = data.DSGaussBernRBM(B_perturb, b, c, burnin=50) return p, gb_rbm + def get_pqsource(prob_label): """ Return (p, ds), a tuple of @@ -238,77 +259,80 @@ def get_pqsource(prob_label): - ds: a DataSource, each corresponding to one parameter setting. The DataSource generates sample from q. """ - prob2tuples = { - # H0 is true. vary d. P = Q = N(0, I) - 'sg5': (density.IsotropicNormal(np.zeros(5), 1), - data.DSIsotropicNormal(np.zeros(5), 1) ), - - # P = N(0, I), Q = N( (0.2,..0), I) - 'gmd5': (density.IsotropicNormal(np.zeros(5), 1), - data.DSIsotropicNormal(np.hstack((0.2, np.zeros(4))), 1) ), - - 'gmd1': (density.IsotropicNormal(np.zeros(1), 1), - data.DSIsotropicNormal(np.ones(1)*0.2, 1) ), - - # P = N(0, I), Q = N( (1,..0), I) - 'gmd100': (density.IsotropicNormal(np.zeros(100), 1), - data.DSIsotropicNormal(np.hstack((1, np.zeros(99))), 1) ), - - # Gaussian variance difference problem. Only the variance - # of the first dimenion differs. d varies. - 'gvd5': (density.Normal(np.zeros(5), np.eye(5) ), - data.DSNormal(np.zeros(5), np.diag(np.hstack((2, np.ones(4)))) )), - - 'gvd10': (density.Normal(np.zeros(10), np.eye(10) ), - data.DSNormal(np.zeros(10), np.diag(np.hstack((2, np.ones(9)))) )), - - # Gaussian Bernoulli RBM. dx=50, dh=10. H0 is true - 'gbrbm_dx50_dh10_v0': gaussbern_rbm_tuple(0, - dx=50, dh=10, n=sample_size), - - # Gaussian Bernoulli RBM. dx=5, dh=3. H0 is true - 'gbrbm_dx5_dh3_v0': gaussbern_rbm_tuple(0, - dx=5, dh=3, n=sample_size), - - # Gaussian Bernoulli RBM. dx=50, dh=10. - 'gbrbm_dx50_dh10_v1em3': gaussbern_rbm_tuple(1e-3, - dx=50, dh=10, n=sample_size), - - # Gaussian Bernoulli RBM. dx=5, dh=3. Perturb with noise = 1e-2. - 'gbrbm_dx5_dh3_v5em3': gaussbern_rbm_tuple(5e-3, - dx=5, dh=3, n=sample_size), - - # Gaussian mixture of two components. Uniform mixture weights. - # p = 0.5*N(0, 1) + 0.5*N(3, 0.01) - # q = 0.5*N(-3, 0.01) + 0.5*N(0, 1) - 'gmm_d1': ( - density.IsoGaussianMixture(np.array([[0], [3.0]]), np.array([1, 0.01]) ), - data.DSIsoGaussianMixture(np.array([[-3.0], [0]]), np.array([0.01, 1]) ) - ), - - # p = N(0, 1) - # q = 0.1*N([-10, 0,..0], 0.001) + 0.9*N([0,0,..0], 1) - 'g_vs_gmm_d5': ( - density.IsotropicNormal(np.zeros(5), 1), - data.DSIsoGaussianMixture( - np.vstack(( np.hstack((0.0, np.zeros(4))), np.zeros(5) )), - np.array([0.0001, 1]), pmix=[0.1, 0.9] ) - ), - - 'g_vs_gmm_d2': ( - density.IsotropicNormal(np.zeros(2), 1), - data.DSIsoGaussianMixture( - np.vstack(( np.hstack((0.0, np.zeros(1))), np.zeros(2) )), - np.array([0.01, 1]), pmix=[0.1, 0.9] ) - ), - 'g_vs_gmm_d1': ( - density.IsotropicNormal(np.zeros(1), 1), - data.DSIsoGaussianMixture(np.array([[0.0], [0]]), - np.array([0.01, 1]), pmix=[0.1, 0.9] ) - ), - } + prob2tuples = { + # H0 is true. vary d. P = Q = N(0, I) + "sg5": ( + density.IsotropicNormal(np.zeros(5), 1), + data.DSIsotropicNormal(np.zeros(5), 1), + ), + # P = N(0, I), Q = N( (0.2,..0), I) + "gmd5": ( + density.IsotropicNormal(np.zeros(5), 1), + data.DSIsotropicNormal(np.hstack((0.2, np.zeros(4))), 1), + ), + "gmd1": ( + density.IsotropicNormal(np.zeros(1), 1), + data.DSIsotropicNormal(np.ones(1) * 0.2, 1), + ), + # P = N(0, I), Q = N( (1,..0), I) + "gmd100": ( + density.IsotropicNormal(np.zeros(100), 1), + data.DSIsotropicNormal(np.hstack((1, np.zeros(99))), 1), + ), + # Gaussian variance difference problem. Only the variance + # of the first dimenion differs. d varies. + "gvd5": ( + density.Normal(np.zeros(5), np.eye(5)), + data.DSNormal(np.zeros(5), np.diag(np.hstack((2, np.ones(4))))), + ), + "gvd10": ( + density.Normal(np.zeros(10), np.eye(10)), + data.DSNormal(np.zeros(10), np.diag(np.hstack((2, np.ones(9))))), + ), + # Gaussian Bernoulli RBM. dx=50, dh=10. H0 is true + "gbrbm_dx50_dh10_v0": gaussbern_rbm_tuple(0, dx=50, dh=10, n=sample_size), + # Gaussian Bernoulli RBM. dx=5, dh=3. H0 is true + "gbrbm_dx5_dh3_v0": gaussbern_rbm_tuple(0, dx=5, dh=3, n=sample_size), + # Gaussian Bernoulli RBM. dx=50, dh=10. + "gbrbm_dx50_dh10_v1em3": gaussbern_rbm_tuple(1e-3, dx=50, dh=10, n=sample_size), + # Gaussian Bernoulli RBM. dx=5, dh=3. Perturb with noise = 1e-2. + "gbrbm_dx5_dh3_v5em3": gaussbern_rbm_tuple(5e-3, dx=5, dh=3, n=sample_size), + # Gaussian mixture of two components. Uniform mixture weights. + # p = 0.5*N(0, 1) + 0.5*N(3, 0.01) + # q = 0.5*N(-3, 0.01) + 0.5*N(0, 1) + "gmm_d1": ( + density.IsoGaussianMixture(np.array([[0], [3.0]]), np.array([1, 0.01])), + data.DSIsoGaussianMixture(np.array([[-3.0], [0]]), np.array([0.01, 1])), + ), + # p = N(0, 1) + # q = 0.1*N([-10, 0,..0], 0.001) + 0.9*N([0,0,..0], 1) + "g_vs_gmm_d5": ( + density.IsotropicNormal(np.zeros(5), 1), + data.DSIsoGaussianMixture( + np.vstack((np.hstack((0.0, np.zeros(4))), np.zeros(5))), + np.array([0.0001, 1]), + pmix=[0.1, 0.9], + ), + ), + "g_vs_gmm_d2": ( + density.IsotropicNormal(np.zeros(2), 1), + data.DSIsoGaussianMixture( + np.vstack((np.hstack((0.0, np.zeros(1))), np.zeros(2))), + np.array([0.01, 1]), + pmix=[0.1, 0.9], + ), + ), + "g_vs_gmm_d1": ( + density.IsotropicNormal(np.zeros(1), 1), + data.DSIsoGaussianMixture( + np.array([[0.0], [0]]), np.array([0.01, 1]), pmix=[0.1, 0.9] + ), + ), + } if prob_label not in prob2tuples: - raise ValueError('Unknown problem label. Need to be one of %s'%str(prob2tuples.keys()) ) + raise ValueError( + "Unknown problem label. Need to be one of %s" % str(prob2tuples.keys()) + ) return prob2tuples[prob_label] @@ -318,33 +342,41 @@ def run_problem(prob_label): # /////// submit jobs ////////// # create folder name string - #result_folder = glo.result_folder() + # result_folder = glo.result_folder() from sbibm.third_party.kgof.config import expr_configs - tmp_dir = expr_configs['scratch_path'] - foldername = os.path.join(tmp_dir, 'kgof_slurm', 'e%d'%ex) + + tmp_dir = expr_configs["scratch_path"] + foldername = os.path.join(tmp_dir, "kgof_slurm", "e%d" % ex) logger.info("Setting engine folder to %s" % foldername) # create parameter instance that is needed for any batch computation engine logger.info("Creating batch parameter instance") batch_parameters = BatchClusterParameters( - foldername=foldername, job_name_base="e%d_"%ex, parameter_prefix="") + foldername=foldername, job_name_base="e%d_" % ex, parameter_prefix="" + ) # Use the following line if Slurm queue is not used. - #engine = SerialComputationEngine() + # engine = SerialComputationEngine() engine = SlurmComputationEngine(batch_parameters) n_methods = len(method_job_funcs) # repetitions x len(Js) x #methods - aggregators = np.empty((reps, len(Js), n_methods ), dtype=object) + aggregators = np.empty((reps, len(Js), n_methods), dtype=object) for r in range(reps): for ji, J in enumerate(Js): for mi, f in enumerate(method_job_funcs): # name used to save the result func_name = f.__name__ - fname = '%s-%s-n%d_r%d_J%d_a%.3f_trp%.2f.p' \ - %(prob_label, func_name, sample_size, r, J, alpha, - tr_proportion) + fname = "%s-%s-n%d_r%d_J%d_a%.3f_trp%.2f.p" % ( + prob_label, + func_name, + sample_size, + r, + J, + alpha, + tr_proportion, + ) if not is_rerun and glo.ex_file_exists(ex, prob_label, fname): - logger.info('%s exists. Load and return.'%fname) + logger.info("%s exists. Load and return." % fname) job_result = glo.ex_load_result(ex, prob_label, fname) sra = SingleResultAggregator() @@ -354,8 +386,7 @@ def run_problem(prob_label): # result not exists or rerun # p: an UnnormalizedDensity object - job = Ex3Job(SingleResultAggregator(), p, ds, prob_label, - r, f, J) + job = Ex3Job(SingleResultAggregator(), p, ds, prob_label, r, f, J) agg = engine.submit_job(job) aggregators[r, ji, mi] = agg @@ -369,8 +400,7 @@ def run_problem(prob_label): for r in range(reps): for ji, J in enumerate(Js): for mi, f in enumerate(method_job_funcs): - logger.info("Collecting result (%s, r=%d, J=%rd)" % - (f.__name__, r, J)) + logger.info("Collecting result (%s, r=%d, J=%rd)" % (f.__name__, r, J)) # let the aggregator finalize things aggregators[r, ji, mi].finalize() @@ -379,36 +409,49 @@ def run_problem(prob_label): job_result = aggregators[r, ji, mi].get_final_result().result job_results[r, ji, mi] = job_result - #func_names = [f.__name__ for f in method_job_funcs] - #func2labels = exglobal.get_func2label_map() - #method_labels = [func2labels[f] for f in func_names if f in func2labels] - - # save results - results = {'job_results': job_results, 'data_source': ds, - 'alpha': alpha, 'repeats': reps, 'Js': Js, - 'p': p, - 'tr_proportion': tr_proportion, - 'method_job_funcs': method_job_funcs, 'prob_label': prob_label, - 'sample_size': sample_size, - } - - # class name - fname = 'ex%d-%s-me%d_n%d_rs%d_Jmi%d_Jma%d_a%.3f_trp%.2f.p' \ - %(ex, prob_label, n_methods, sample_size, reps, min(Js), - max(Js), alpha, tr_proportion) + # func_names = [f.__name__ for f in method_job_funcs] + # func2labels = exglobal.get_func2label_map() + # method_labels = [func2labels[f] for f in func_names if f in func2labels] + + # save results + results = { + "job_results": job_results, + "data_source": ds, + "alpha": alpha, + "repeats": reps, + "Js": Js, + "p": p, + "tr_proportion": tr_proportion, + "method_job_funcs": method_job_funcs, + "prob_label": prob_label, + "sample_size": sample_size, + } + + # class name + fname = "ex%d-%s-me%d_n%d_rs%d_Jmi%d_Jma%d_a%.3f_trp%.2f.p" % ( + ex, + prob_label, + n_methods, + sample_size, + reps, + min(Js), + max(Js), + alpha, + tr_proportion, + ) glo.ex_save_result(ex, results, fname) - logger.info('Saved aggregated results to %s'%fname) + logger.info("Saved aggregated results to %s" % fname) def main(): if len(sys.argv) != 2: - print('Usage: %s problem_label'%sys.argv[0]) + print("Usage: %s problem_label" % sys.argv[0]) sys.exit(1) prob_label = sys.argv[1] run_problem(prob_label) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/sbibm/third_party/kgof/glo.py b/sbibm/third_party/kgof/glo.py index 52d4c90d..956369f2 100644 --- a/sbibm/third_party/kgof/glo.py +++ b/sbibm/third_party/kgof/glo.py @@ -1,100 +1,114 @@ """A global module containing functions for managing the project.""" from future import standard_library + standard_library.install_aliases() -__author__ = 'wittawat' +__author__ = "wittawat" -import sbibm.third_party.kgof as kgof import os import pickle +import sbibm.third_party.kgof as kgof + def get_root(): """Return the full path to the root of the package""" return os.path.abspath(os.path.dirname(kgof.__file__)) + def result_folder(): - """Return the full path to the result/ folder containing experimental result + """Return the full path to the result/ folder containing experimental result files""" import sbibm.third_party.kgof.config as config - results_path = config.expr_configs['expr_results_path'] + + results_path = config.expr_configs["expr_results_path"] return results_path - #return os.path.join(get_root(), 'result') + # return os.path.join(get_root(), 'result') + def data_folder(): """ - Return the full path to the data folder + Return the full path to the data folder """ import sbibm.third_party.kgof.config as config - data_path = config.expr_configs['data_path'] + + data_path = config.expr_configs["data_path"] return data_path - #return os.path.join(get_root(), 'data') + # return os.path.join(get_root(), 'data') + def data_file(*relative_path): """ - Access the file under the data folder. The path is relative to the + Access the file under the data folder. The path is relative to the data folder """ dfolder = data_folder() return os.path.join(dfolder, *relative_path) + def load_data_file(*relative_path): fpath = data_file(*relative_path) return pickle_load(fpath) + def ex_result_folder(ex): - """Return the full path to the folder containing result files of the specified - experiment. - ex: a positive integer. """ + """Return the full path to the folder containing result files of the specified + experiment. + ex: a positive integer.""" rp = result_folder() - fpath = os.path.join(rp, 'ex%d'%ex ) + fpath = os.path.join(rp, "ex%d" % ex) if not os.path.exists(fpath): create_dirs(fpath) return fpath + def create_dirs(full_path): - """Recursively create the directories along the specified path. - Assume that the path refers to a folder. """ + """Recursively create the directories along the specified path. + Assume that the path refers to a folder.""" if not os.path.exists(full_path): os.makedirs(full_path) -def ex_result_file(ex, *relative_path ): - """Return the full path to the file identified by the relative path as a list - of folders/files under the result folder of the experiment ex. """ + +def ex_result_file(ex, *relative_path): + """Return the full path to the file identified by the relative path as a list + of folders/files under the result folder of the experiment ex.""" rf = ex_result_folder(ex) return os.path.join(rf, *relative_path) + def ex_save_result(ex, result, *relative_path): - """Save a dictionary object result for the experiment ex. Serialization is - done with pickle. - EX: ex_save_result(1, result, 'data', 'result.p'). Save under result/ex1/data/result.p - EX: ex_save_result(1, result, 'result.p'). Save under result/ex1/result.p + """Save a dictionary object result for the experiment ex. Serialization is + done with pickle. + EX: ex_save_result(1, result, 'data', 'result.p'). Save under result/ex1/data/result.p + EX: ex_save_result(1, result, 'result.p'). Save under result/ex1/result.p """ fpath = ex_result_file(ex, *relative_path) dir_path = os.path.dirname(fpath) create_dirs(dir_path) - # - with open(fpath, 'wb') as f: + # + with open(fpath, "wb") as f: # expect result to be a dictionary pickle.dump(result, f) + def ex_load_result(ex, *relative_path): """Load a result identified by the path from the experiment ex""" fpath = ex_result_file(ex, *relative_path) return pickle_load(fpath) + def ex_file_exists(ex, *relative_path): """Return true if the result file in under the specified experiment folder exists""" fpath = ex_result_file(ex, *relative_path) return os.path.isfile(fpath) + def pickle_load(fpath): if not os.path.isfile(fpath): - raise ValueError('%s does not exist' % fpath) + raise ValueError("%s does not exist" % fpath) - with open(fpath, 'rb') as f: + with open(fpath, "rb") as f: # expect a dictionary result = pickle.load(f) return result - diff --git a/sbibm/third_party/kgof/goftest.py b/sbibm/third_party/kgof/goftest.py index 5f7295a3..32fb0cd4 100644 --- a/sbibm/third_party/kgof/goftest.py +++ b/sbibm/third_party/kgof/goftest.py @@ -3,26 +3,27 @@ """ from __future__ import division -from builtins import zip -from builtins import str -from builtins import range -from past.utils import old_div -from builtins import object +from builtins import object, range, str, zip + from future.utils import with_metaclass -__author__ = 'wittawat' +from past.utils import old_div +__author__ = "wittawat" + +import logging from abc import ABCMeta, abstractmethod + import autograd import autograd.numpy as np -import sbibm.third_party.kgof.data as data -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel -import logging import matplotlib.pyplot as plt - import scipy import scipy.stats as stats +import sbibm.third_party.kgof.data as data +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util + + class GofTest(with_metaclass(ABCMeta, object)): """ Abstract class for a goodness-of-fit test. @@ -40,10 +41,10 @@ def __init__(self, p, alpha): def perform_test(self, dat): """perform the goodness-of-fit test and return values computed in a dictionary: { - alpha: 0.01, - pvalue: 0.0002, - test_stat: 2.3, - h0_rejected: True, + alpha: 0.01, + pvalue: 0.0002, + test_stat: 2.3, + h0_rejected: True, time_secs: ... } @@ -56,12 +57,14 @@ def compute_stat(self, dat): """Compute the test statistic""" raise NotImplementedError() + # end of GofTest -#------------------------------------------------------ +# ------------------------------------------------------ + class H0Simulator(with_metaclass(ABCMeta, object)): """ - An abstract class representing a simulator to draw samples from the + An abstract class representing a simulator to draw samples from the null distribution. For some tests, these are needed to conduct the test. """ @@ -81,24 +84,27 @@ def simulate(self, gof, dat): gof: a GofTest dat: a Data (observed data) - Simulate from the null distribution and return a dictionary. - One of the item is + Simulate from the null distribution and return a dictionary. + One of the item is sim_stats: a numpy array of stats. """ raise NotImplementedError() + # end of H0Simulator -#------------------- +# ------------------- + class FSSDH0SimCovObs(H0Simulator): """ An asymptotic null distribution simulator for FSSD. Simulate from the asymptotic null distribution given by the weighted sum of chi-squares. The eigenvalues (weights) are computed from the covarince matrix wrt. the - observed sample. + observed sample. This is not the correct null distribution; but has the correct asymptotic types-1 error at alpha. """ + def __init__(self, n_simulate=3000, seed=10): super(FSSDH0SimCovObs, self).__init__(n_simulate, seed) @@ -120,24 +126,28 @@ def simulate(self, gof, dat, fea_tensor=None): # Make sure it is a matrix i.e, np.cov returns a scalar when Tau is # 1d. cov = np.cov(Tau.T) + np.zeros((1, 1)) - #cov = Tau.T.dot(Tau/n) + # cov = Tau.T.dot(Tau/n) + + arr_nfssd, eigs = FSSD.list_simulate_spectral( + cov, J, n_simulate, seed=self.seed + ) + return {"sim_stats": arr_nfssd} - arr_nfssd, eigs = FSSD.list_simulate_spectral(cov, J, n_simulate, - seed=self.seed) - return {'sim_stats': arr_nfssd} # end of FSSDH0SimCovObs -#----------------------- +# ----------------------- + class FSSDH0SimCovDraw(H0Simulator): """ An asymptotic null distribution simulator for FSSD. Simulate from the asymptotic null distribution given by the weighted sum of chi-squares. The eigenvalues (weights) are computed from the covarince matrix wrt. the - sample drawn from p (the density to test against). - + sample drawn from p (the density to test against). + - The UnnormalizedDensity p is required to implement get_datasource() method. """ + def __init__(self, n_draw=2000, n_simulate=3000, seed=10): """ n_draw: number of samples to draw from the UnnormalizedDensity p @@ -152,12 +162,12 @@ def simulate(self, gof, dat, fea_tensor=None): This method does not use dat. """ dat = None - #assert isinstance(gof, FSSD) + # assert isinstance(gof, FSSD) # p = an UnnormalizedDensity p = gof.p ds = p.get_datasource() if ds is None: - raise ValueError('DataSource associated with p must be available.') + raise ValueError("DataSource associated with p must be available.") Xdraw = ds.sample(n=self.n_draw, seed=self.seed) _, fea_tensor = gof.compute_stat(Xdraw, return_feature_tensor=True) @@ -168,21 +178,24 @@ def simulate(self, gof, dat, fea_tensor=None): Tau = fea_tensor.reshape(n, -1) # Make sure it is a matrix i.e, np.cov returns a scalar when Tau is # 1d. - #cov = np.cov(Tau.T) + np.zeros((1, 1)) - cov = old_div(Tau.T.dot(Tau),n) + np.zeros((1, 1)) + # cov = np.cov(Tau.T) + np.zeros((1, 1)) + cov = old_div(Tau.T.dot(Tau), n) + np.zeros((1, 1)) n_simulate = self.n_simulate - arr_nfssd, eigs = FSSD.list_simulate_spectral(cov, J, n_simulate, - seed=self.seed) - return {'sim_stats': arr_nfssd} + arr_nfssd, eigs = FSSD.list_simulate_spectral( + cov, J, n_simulate, seed=self.seed + ) + return {"sim_stats": arr_nfssd} + # end of FSSDH0SimCovDraw -#----------------------- +# ----------------------- + class FSSD(GofTest): """ Goodness-of-fit test using The Finite Set Stein Discrepancy statistic. - and a set of paired test locations. The statistic is n*FSSD^2. + and a set of paired test locations. The statistic is n*FSSD^2. The statistic can be negative because of the unbiased estimator. H0: the sample follows p @@ -191,27 +204,26 @@ class FSSD(GofTest): p is specified to the constructor in the form of an UnnormalizedDensity. """ - #NULLSIM_* are constants used to choose the way to simulate from the null - #distribution to do the test. + # NULLSIM_* are constants used to choose the way to simulate from the null + # distribution to do the test. - - # Same as NULLSIM_COVQ; but assume that sample can be drawn from p. + # Same as NULLSIM_COVQ; but assume that sample can be drawn from p. # Use the drawn sample to compute the covariance. NULLSIM_COVP = 1 - - def __init__(self, p, k, V, null_sim=FSSDH0SimCovObs(n_simulate=3000, - seed=101), alpha=0.01): + def __init__( + self, p, k, V, null_sim=FSSDH0SimCovObs(n_simulate=3000, seed=101), alpha=0.01 + ): """ p: an instance of UnnormalizedDensity k: a DifferentiableKernel object V: J x dx numpy array of J locations to test the difference null_sim: an instance of H0Simulator for simulating from the null distribution. - alpha: significance level + alpha: significance level """ super(FSSD, self).__init__(p, alpha) self.k = k - self.V = V + self.V = V self.null_sim = null_sim def perform_test(self, dat, return_simulated_stats=False): @@ -228,17 +240,21 @@ def perform_test(self, dat, return_simulated_stats=False): nfssd, fea_tensor = self.compute_stat(dat, return_feature_tensor=True) sim_results = null_sim.simulate(self, dat, fea_tensor) - arr_nfssd = sim_results['sim_stats'] + arr_nfssd = sim_results["sim_stats"] - # approximate p-value with the permutations + # approximate p-value with the permutations pvalue = np.mean(arr_nfssd > nfssd) - results = {'alpha': self.alpha, 'pvalue': pvalue, 'test_stat': nfssd, - 'h0_rejected': pvalue < alpha, 'n_simulate': n_simulate, - 'time_secs': t.secs, - } + results = { + "alpha": self.alpha, + "pvalue": pvalue, + "test_stat": nfssd, + "h0_rejected": pvalue < alpha, + "n_simulate": n_simulate, + "time_secs": t.secs, + } if return_simulated_stats: - results['sim_stats'] = arr_nfssd + results["sim_stats"] = arr_nfssd return results def compute_stat(self, dat, return_feature_tensor=False): @@ -251,13 +267,13 @@ def compute_stat(self, dat, return_feature_tensor=False): # n x d x J Xi = self.feature_tensor(X) unscaled_mean = FSSD.ustat_h1_mean_variance(Xi, return_variance=False) - stat = n*unscaled_mean + stat = n * unscaled_mean - #print 'Xi: {0}'.format(Xi) - #print 'Tau: {0}'.format(Tau) - #print 't1: {0}'.format(t1) - #print 't2: {0}'.format(t2) - #print 'stat: {0}'.format(stat) + # print 'Xi: {0}'.format(Xi) + # print 'Tau: {0}'.format(Tau) + # print 't1: {0}'.format(t1) + # print 't2: {0}'.format(t2) + # print 'stat: {0}'.format(stat) if return_feature_tensor: return stat, Xi else: @@ -288,41 +304,41 @@ def feature_tensor(self, X): n, d = X.shape # n x d matrix of gradients grad_logp = self.p.grad_log(X) - #assert np.all(util.is_real_num(grad_logp)) + # assert np.all(util.is_real_num(grad_logp)) # n x J matrix - #print 'V' - #print self.V + # print 'V' + # print self.V K = k.eval(X, self.V) - #assert np.all(util.is_real_num(K)) + # assert np.all(util.is_real_num(K)) list_grads = np.array([np.reshape(k.gradX_y(X, v), (1, n, d)) for v in self.V]) stack0 = np.concatenate(list_grads, axis=0) - #a numpy array G of size n x d x J such that G[:, :, J] + # a numpy array G of size n x d x J such that G[:, :, J] # is the derivative of k(X, V_j) with respect to X. dKdV = np.transpose(stack0, (1, 2, 0)) # n x d x J tensor grad_logp_K = util.outer_rows(grad_logp, K) - #print 'grad_logp' - #print grad_logp.dtype - #print grad_logp - #print 'K' - #print K - Xi = old_div((grad_logp_K + dKdV),np.sqrt(d*J)) - #Xi = (grad_logp_K + dKdV) + # print 'grad_logp' + # print grad_logp.dtype + # print grad_logp + # print 'K' + # print K + Xi = old_div((grad_logp_K + dKdV), np.sqrt(d * J)) + # Xi = (grad_logp_K + dKdV) return Xi - @staticmethod - def power_criterion(p, dat, k, test_locs, reg=1e-2, use_unbiased=True, - use_2terms=False): + def power_criterion( + p, dat, k, test_locs, reg=1e-2, use_unbiased=True, use_2terms=False + ): """ Compute the mean and standard deviation of the statistic under H1. Return mean/sd. - use_2terms: True if the objective should include the first term in the power - expression. This term carries the test threshold and is difficult to - compute (depends on the optimized test locations). If True, then - the objective will be -1/(n**0.5*sigma_H1) + n**0.5 FSSD^2/sigma_H1, + use_2terms: True if the objective should include the first term in the power + expression. This term carries the test threshold and is difficult to + compute (depends on the optimized test locations). If True, then + the objective will be -1/(n**0.5*sigma_H1) + n**0.5 FSSD^2/sigma_H1, which ignores the test threshold in the first term. """ X = dat.data() @@ -330,24 +346,24 @@ def power_criterion(p, dat, k, test_locs, reg=1e-2, use_unbiased=True, V = test_locs fssd = FSSD(p, k, V, null_sim=None) fea_tensor = fssd.feature_tensor(X) - u_mean, u_variance = FSSD.ustat_h1_mean_variance(fea_tensor, - return_variance=True, use_unbiased=use_unbiased) + u_mean, u_variance = FSSD.ustat_h1_mean_variance( + fea_tensor, return_variance=True, use_unbiased=use_unbiased + ) - # mean/sd criterion + # mean/sd criterion sigma_h1 = np.sqrt(u_variance + reg) - ratio = old_div(u_mean,sigma_h1) + ratio = old_div(u_mean, sigma_h1) if use_2terms: - obj = old_div(-1.0,(np.sqrt(n)*sigma_h1)) + np.sqrt(n)*ratio - #print obj + obj = old_div(-1.0, (np.sqrt(n) * sigma_h1)) + np.sqrt(n) * ratio + # print obj else: obj = ratio return obj @staticmethod - def ustat_h1_mean_variance(fea_tensor, return_variance=True, - use_unbiased=True): + def ustat_h1_mean_variance(fea_tensor, return_variance=True, use_unbiased=True): """ - Compute the mean and variance of the asymptotic normal distribution + Compute the mean and variance of the asymptotic normal distribution under H1 of the test statistic. fea_tensor: feature tensor obtained from feature_tensor() @@ -359,31 +375,31 @@ def ustat_h1_mean_variance(fea_tensor, return_variance=True, """ Xi = fea_tensor n, d, J = Xi.shape - #print 'Xi' - #print Xi - #assert np.all(util.is_real_num(Xi)) - assert n > 1, 'Need n > 1 to compute the mean of the statistic.' + # print 'Xi' + # print Xi + # assert np.all(util.is_real_num(Xi)) + assert n > 1, "Need n > 1 to compute the mean of the statistic." # n x d*J # Tau = Xi.reshape(n, d*J) - Tau = np.reshape(Xi, [n, d*J]) + Tau = np.reshape(Xi, [n, d * J]) if use_unbiased: - t1 = np.sum(np.mean(Tau, 0)**2)*(old_div(n,float(n-1))) - t2 = old_div(np.sum(np.mean(Tau**2, 0)),float(n-1)) + t1 = np.sum(np.mean(Tau, 0) ** 2) * (old_div(n, float(n - 1))) + t2 = old_div(np.sum(np.mean(Tau ** 2, 0)), float(n - 1)) # stat is the mean stat = t1 - t2 else: - stat = np.sum(np.mean(Tau, 0)**2) + stat = np.sum(np.mean(Tau, 0) ** 2) if not return_variance: return stat - # compute the variance + # compute the variance # mu: d*J vector mu = np.mean(Tau, 0) - variance = 4*np.mean(np.dot(Tau, mu)**2) - 4*np.sum(mu**2)**2 + variance = 4 * np.mean(np.dot(Tau, mu) ** 2) - 4 * np.sum(mu ** 2) ** 2 return stat, variance - @staticmethod + @staticmethod def list_simulate_spectral(cov, J, n_simulate=1000, seed=82): """ Simulate the null distribution using the spectrums of the covariance @@ -392,44 +408,43 @@ def list_simulate_spectral(cov, J, n_simulate=1000, seed=82): Return (a numpy array of simulated n*FSSD values, eigenvalues of cov) """ - # eigen decompose + # eigen decompose eigs, _ = np.linalg.eig(cov) eigs = np.real(eigs) - # sort in decreasing order + # sort in decreasing order eigs = -np.sort(-eigs) - sim_fssds = FSSD.simulate_null_dist(eigs, J, n_simulate=n_simulate, - seed=seed) + sim_fssds = FSSD.simulate_null_dist(eigs, J, n_simulate=n_simulate, seed=seed) return sim_fssds, eigs - @staticmethod + @staticmethod def simulate_null_dist(eigs, J, n_simulate=2000, seed=7): """ - Simulate the null distribution using the spectrums of the covariance + Simulate the null distribution using the spectrums of the covariance matrix of the U-statistic. The simulated statistic is n*FSSD^2 where FSSD is an unbiased estimator. - eigs: a numpy array of estimated eigenvalues of the covariance - matrix. eigs is of length d*J, where d is the input dimension, and + matrix. eigs is of length d*J, where d is the input dimension, and - J: the number of test locations. Return a numpy array of simulated statistics. """ - d = old_div(len(eigs),J) - assert d>0 + d = old_div(len(eigs), J) + assert d > 0 # draw at most d x J x block_size values at a time - block_size = max(20, int(old_div(1000.0,(d*J)))) + block_size = max(20, int(old_div(1000.0, (d * J)))) fssds = np.zeros(n_simulate) from_ind = 0 with util.NumpySeedContext(seed=seed): while from_ind < n_simulate: - to_draw = min(block_size, n_simulate-from_ind) - # draw chi^2 random variables. - chi2 = np.random.randn(d*J, to_draw)**2 - - # an array of length to_draw - sim_fssds = eigs.dot(chi2-1.0) - # store - end_ind = from_ind+to_draw + to_draw = min(block_size, n_simulate - from_ind) + # draw chi^2 random variables. + chi2 = np.random.randn(d * J, to_draw) ** 2 + + # an array of length to_draw + sim_fssds = eigs.dot(chi2 - 1.0) + # store + end_ind = from_ind + to_draw fssds[from_ind:end_ind] = sim_fssds from_ind = end_ind return fssds @@ -437,12 +452,12 @@ def simulate_null_dist(eigs, J, n_simulate=2000, seed=7): @staticmethod def fssd_grid_search_kernel(p, dat, test_locs, list_kernel): """ - Linear search for the best kernel in the list that maximizes + Linear search for the best kernel in the list that maximizes the test power criterion, fixing the test locations to V. - p: UnnormalizedDensity - dat: a Data object - - list_kernel: list of kernel candidates + - list_kernel: list of kernel candidates return: (best kernel index, array of test power criteria) """ @@ -453,33 +468,36 @@ def fssd_grid_search_kernel(p, dat, test_locs, list_kernel): for i in range(n_cand): ki = list_kernel[i] objs[i] = FSSD.power_criterion(p, dat, ki, test_locs) - logging.info('(%d), obj: %5.4g, k: %s' %(i, objs[i], str(ki))) + logging.info("(%d), obj: %5.4g, k: %s" % (i, objs[i], str(ki))) - #Widths that come early in the list + # Widths that come early in the list # are preferred if test powers are equal. - #bestij = np.unravel_index(objs.argmax(), objs.shape) + # bestij = np.unravel_index(objs.argmax(), objs.shape) besti = objs.argmax() return besti, objs + # end of FSSD # -------------------------------------- + class GaussFSSD(FSSD): """ FSSD using an isotropic Gaussian kernel. """ + def __init__(self, p, sigma2, V, alpha=0.01, n_simulate=3000, seed=10): k = kernel.KGauss(sigma2) null_sim = FSSDH0SimCovObs(n_simulate=n_simulate, seed=seed) super(GaussFSSD, self).__init__(p, k, V, null_sim, alpha) - @staticmethod + @staticmethod def power_criterion(p, dat, gwidth, test_locs, reg=1e-2, use_2terms=False): """ - use_2terms: True if the objective should include the first term in the power - expression. This term carries the test threshold and is difficult to - compute (depends on the optimized test locations). If True, then - the objective will be -1/(n**0.5*sigma_H1) + n**0.5 FSSD^2/sigma_H1, + use_2terms: True if the objective should include the first term in the power + expression. This term carries the test threshold and is difficult to + compute (depends on the optimized test locations). If True, then + the objective will be -1/(n**0.5*sigma_H1) + n**0.5 FSSD^2/sigma_H1, which ignores the test threshold in the first term. """ k = kernel.KGauss(gwidth) @@ -488,64 +506,73 @@ def power_criterion(p, dat, gwidth, test_locs, reg=1e-2, use_2terms=False): @staticmethod def optimize_auto_init(p, dat, J, **ops): """ - Optimize parameters by calling optimize_locs_widths(). Automatically + Optimize parameters by calling optimize_locs_widths(). Automatically initialize the test locations and the Gaussian width. Return optimized locations, Gaussian width, optimization info """ - assert J>0 + assert J > 0 # Use grid search to initialize the gwidth X = dat.data() n_gwidth_cand = 5 - gwidth_factors = 2.0**np.linspace(-3, 3, n_gwidth_cand) - med2 = util.meddistance(X, 1000)**2 + gwidth_factors = 2.0 ** np.linspace(-3, 3, n_gwidth_cand) + med2 = util.meddistance(X, 1000) ** 2 - k = kernel.KGauss(med2*2) + k = kernel.KGauss(med2 * 2) # fit a Gaussian to the data and draw to initialize V0 V0 = util.fit_gaussian_draw(X, J, seed=829, reg=1e-6) - list_gwidth = np.hstack( ( (med2)*gwidth_factors ) ) + list_gwidth = np.hstack(((med2) * gwidth_factors)) besti, objs = GaussFSSD.grid_search_gwidth(p, dat, V0, list_gwidth) gwidth = list_gwidth[besti] - assert util.is_real_num(gwidth), 'gwidth not real. Was %s'%str(gwidth) - assert gwidth > 0, 'gwidth not positive. Was %.3g'%gwidth - logging.info('After grid search, gwidth=%.3g'%gwidth) + assert util.is_real_num(gwidth), "gwidth not real. Was %s" % str(gwidth) + assert gwidth > 0, "gwidth not positive. Was %.3g" % gwidth + logging.info("After grid search, gwidth=%.3g" % gwidth) - - V_opt, gwidth_opt, info = GaussFSSD.optimize_locs_widths(p, dat, - gwidth, V0, **ops) + V_opt, gwidth_opt, info = GaussFSSD.optimize_locs_widths( + p, dat, gwidth, V0, **ops + ) # set the width bounds - #fac_min = 5e-2 - #fac_max = 5e3 - #gwidth_lb = fac_min*med2 - #gwidth_ub = fac_max*med2 - #gwidth_opt = max(gwidth_lb, min(gwidth_opt, gwidth_ub)) + # fac_min = 5e-2 + # fac_max = 5e3 + # gwidth_lb = fac_min*med2 + # gwidth_ub = fac_max*med2 + # gwidth_opt = max(gwidth_lb, min(gwidth_opt, gwidth_ub)) return V_opt, gwidth_opt, info @staticmethod def grid_search_gwidth(p, dat, test_locs, list_gwidth): """ - Linear search for the best Gaussian width in the list that maximizes - the test power criterion, fixing the test locations. + Linear search for the best Gaussian width in the list that maximizes + the test power criterion, fixing the test locations. - - V: a J x dx np-array for J test locations + - V: a J x dx np-array for J test locations return: (best width index, list of test power objectives) """ list_gauss_kernel = [kernel.KGauss(gw) for gw in list_gwidth] - besti, objs = FSSD.fssd_grid_search_kernel(p, dat, test_locs, - list_gauss_kernel) + besti, objs = FSSD.fssd_grid_search_kernel(p, dat, test_locs, list_gauss_kernel) return besti, objs @staticmethod - def optimize_locs_widths(p, dat, gwidth0, test_locs0, reg=1e-2, - max_iter=100, tol_fun=1e-5, disp=False, locs_bounds_frac=100, - gwidth_lb=None, gwidth_ub=None, use_2terms=False, - ): - """ - Optimize the test locations and the Gaussian kernel width by + def optimize_locs_widths( + p, + dat, + gwidth0, + test_locs0, + reg=1e-2, + max_iter=100, + tol_fun=1e-5, + disp=False, + locs_bounds_frac=100, + gwidth_lb=None, + gwidth_ub=None, + use_2terms=False, + ): + """ + Optimize the test locations and the Gaussian kernel width by maximizing a test power criterion. data should not be the same data as - used in the actual test (i.e., should be a held-out set). + used in the actual test (i.e., should be a held-out set). This function is deterministic. - data: a Data object @@ -565,9 +592,9 @@ def optimize_locs_widths(p, dat, gwidth0, test_locs0, reg=1e-2, criterion, the objective function will also include the first term that is dropped. - #- If the lb, ub bounds are None, use fraction of the median heuristics + #- If the lb, ub bounds are None, use fraction of the median heuristics # to automatically set the bounds. - + Return (V test_locs, gaussian width, optimization info log) """ J = test_locs0.shape[0] @@ -578,8 +605,11 @@ def optimize_locs_widths(p, dat, gwidth0, test_locs0, reg=1e-2, # to automatically enforce the positivity. def obj(sqrt_gwidth, V): return -GaussFSSD.power_criterion( - p, dat, sqrt_gwidth**2, V, reg=reg, use_2terms=use_2terms) + p, dat, sqrt_gwidth ** 2, V, reg=reg, use_2terms=use_2terms + ) + flatten = lambda gwidth, V: np.hstack((gwidth, V.reshape(-1))) + def unflatten(x): sqrt_gwidth = x[0] V = np.reshape(x[1:], (J, d)) @@ -588,19 +618,20 @@ def unflatten(x): def flat_obj(x): sqrt_gwidth, V = unflatten(x) return obj(sqrt_gwidth, V) + # gradient - #grad_obj = autograd.elementwise_grad(flat_obj) + # grad_obj = autograd.elementwise_grad(flat_obj) # Initial point x0 = flatten(np.sqrt(gwidth0), test_locs0) - - #make sure that the optimized gwidth is not too small or too large. - fac_min = 1e-2 + + # make sure that the optimized gwidth is not too small or too large. + fac_min = 1e-2 fac_max = 1e2 - med2 = util.meddistance(X, subsample=1000)**2 + med2 = util.meddistance(X, subsample=1000) ** 2 if gwidth_lb is None: - gwidth_lb = max(fac_min*med2, 1e-3) + gwidth_lb = max(fac_min * med2, 1e-3) if gwidth_ub is None: - gwidth_ub = min(fac_max*med2, 1e5) + gwidth_ub = min(fac_max * med2, 1e5) # Make a box to bound test locations X_std = np.std(X, axis=0) @@ -608,8 +639,8 @@ def flat_obj(x): X_min = np.min(X, axis=0) X_max = np.max(X, axis=0) # V_lb: J x d - V_lb = np.tile(X_min - locs_bounds_frac*X_std, (J, 1)) - V_ub = np.tile(X_max + locs_bounds_frac*X_std, (J, 1)) + V_lb = np.tile(X_min - locs_bounds_frac * X_std, (J, 1)) + V_ub = np.tile(X_max + locs_bounds_frac * X_std, (J, 1)) # (J*d+1) x 2. Take square root because we parameterize with the square # root x0_lb = np.hstack((np.sqrt(gwidth_lb), np.reshape(V_lb, -1))) @@ -621,26 +652,31 @@ def flat_obj(x): grad_obj = autograd.elementwise_grad(flat_obj) with util.ContextTimer() as timer: opt_result = scipy.optimize.minimize( - flat_obj, x0, method='L-BFGS-B', - bounds=x0_bounds, - tol=tol_fun, - options={ - 'maxiter': max_iter, 'ftol': tol_fun, 'disp': disp, - 'gtol': 1.0e-07, - }, - jac=grad_obj, + flat_obj, + x0, + method="L-BFGS-B", + bounds=x0_bounds, + tol=tol_fun, + options={ + "maxiter": max_iter, + "ftol": tol_fun, + "disp": disp, + "gtol": 1.0e-07, + }, + jac=grad_obj, ) opt_result = dict(opt_result) - opt_result['time_secs'] = timer.secs - x_opt = opt_result['x'] + opt_result["time_secs"] = timer.secs + x_opt = opt_result["x"] sq_gw_opt, V_opt = unflatten(x_opt) - gw_opt = sq_gw_opt**2 + gw_opt = sq_gw_opt ** 2 - assert util.is_real_num(gw_opt), 'gw_opt is not real. Was %s' % str(gw_opt) + assert util.is_real_num(gw_opt), "gw_opt is not real. Was %s" % str(gw_opt) return V_opt, gw_opt, opt_result + # end of class GaussFSSD @@ -649,7 +685,8 @@ def bootstrapper_rademacher(n): Produce a sequence of i.i.d {-1, 1} random variables. Suitable for boostrapping on an i.i.d. sample. """ - return 2.0*np.random.randint(0, 1+1, n)-1.0 + return 2.0 * np.random.randint(0, 1 + 1, n) - 1.0 + def bootstrapper_multinomial(n): """ @@ -657,19 +694,22 @@ def bootstrapper_multinomial(n): This is described on page 5 of Liu et al., 2016 (ICML 2016). """ import warnings - warnings.warn('Somehow bootstrapper_multinomial() does not give the right null distribution.') - M = np.random.multinomial(n, old_div(np.ones(n),float(n)), size=1) - return M.reshape(-1) - old_div(1.0,n) + warnings.warn( + "Somehow bootstrapper_multinomial() does not give the right null distribution." + ) + M = np.random.multinomial(n, old_div(np.ones(n), float(n)), size=1) + return M.reshape(-1) - old_div(1.0, n) class IMQFSSD(FSSD): """ FSSD using the inverse multiquadric kernel (IMQ). - k(x,y) = (c^2 + ||x-y||^2)^b - where c > 0 and b < 0. + k(x,y) = (c^2 + ||x-y||^2)^b + where c > 0 and b < 0. """ + def __init__(self, p, b, c, V, alpha=0.01, n_simulate=3000, seed=10): """ n_simulate: number of times to draw from the null distribution. @@ -678,15 +718,15 @@ def __init__(self, p, b, c, V, alpha=0.01, n_simulate=3000, seed=10): null_sim = FSSDH0SimCovObs(n_simulate=n_simulate, seed=seed) super(IMQFSSD, self).__init__(p, k, V, null_sim, alpha) - @staticmethod + @staticmethod def power_criterion(p, dat, b, c, test_locs, reg=1e-2): k = kernel.KIMQ(b=b, c=c) return FSSD.power_criterion(p, dat, k, test_locs, reg) - #@staticmethod - #def optimize_auto_init(p, dat, J, **ops): + # @staticmethod + # def optimize_auto_init(p, dat, J, **ops): # """ - # Optimize parameters by calling optimize_locs_widths(). Automatically + # Optimize parameters by calling optimize_locs_widths(). Automatically # initialize the test locations and the Gaussian width. # Return optimized locations, Gaussian width, optimization info @@ -695,7 +735,7 @@ def power_criterion(p, dat, b, c, test_locs, reg=1e-2): # # Use grid search to initialize the gwidth # X = dat.data() # n_gwidth_cand = 5 - # gwidth_factors = 2.0**np.linspace(-3, 3, n_gwidth_cand) + # gwidth_factors = 2.0**np.linspace(-3, 3, n_gwidth_cand) # med2 = util.meddistance(X, 1000)**2 # k = kernel.KGauss(med2*2) @@ -708,9 +748,8 @@ def power_criterion(p, dat, b, c, test_locs, reg=1e-2): # assert gwidth > 0, 'gwidth not positive. Was %.3g'%gwidth # logging.info('After grid search, gwidth=%.3g'%gwidth) - # V_opt, gwidth_opt, info = GaussFSSD.optimize_locs_widths(p, dat, - # gwidth, V0, **ops) + # gwidth, V0, **ops) # # set the width bounds # #fac_min = 5e-2 @@ -721,8 +760,18 @@ def power_criterion(p, dat, b, c, test_locs, reg=1e-2): # return V_opt, gwidth_opt, info @staticmethod - def optimize_locs(p, dat, b, c, test_locs0, reg=1e-5, max_iter=100, - tol_fun=1e-5, disp=False, locs_bounds_frac=100): + def optimize_locs( + p, + dat, + b, + c, + test_locs0, + reg=1e-5, + max_iter=100, + tol_fun=1e-5, + disp=False, + locs_bounds_frac=100, + ): """ Optimize just the test locations by maximizing a test power criterion, keeping the kernel parameters b, c fixed to the specified values. data @@ -741,7 +790,7 @@ def optimize_locs(p, dat, b, c, test_locs0, reg=1e-5, max_iter=100, - locs_bounds_frac: When making box bounds for the test_locs, extend the box defined by coordinate-wise min-max by std of each coordinate multiplied by this number. - + Return (V test_locs, optimization info log) """ J = test_locs0.shape[0] @@ -770,42 +819,60 @@ def flat_obj(x): X_min = np.min(X, axis=0) X_max = np.max(X, axis=0) # V_lb: J x d - V_lb = np.tile(X_min - locs_bounds_frac*X_std, (J, 1)) - V_ub = np.tile(X_max + locs_bounds_frac*X_std, (J, 1)) - # (J*d) x 2. - x0_bounds = list(zip(V_lb.reshape(-1)[:, np.newaxis], V_ub.reshape(-1)[:, np.newaxis])) + V_lb = np.tile(X_min - locs_bounds_frac * X_std, (J, 1)) + V_ub = np.tile(X_max + locs_bounds_frac * X_std, (J, 1)) + # (J*d) x 2. + x0_bounds = list( + zip(V_lb.reshape(-1)[:, np.newaxis], V_ub.reshape(-1)[:, np.newaxis]) + ) # optimize. Time the optimization as well. # https://docs.scipy.org/doc/scipy/reference/optimize.minimize-lbfgsb.html grad_obj = autograd.elementwise_grad(flat_obj) with util.ContextTimer() as timer: opt_result = scipy.optimize.minimize( - flat_obj, x0, method='L-BFGS-B', - bounds=x0_bounds, - tol=tol_fun, - options={ - 'maxiter': max_iter, 'ftol': tol_fun, 'disp': disp, - 'gtol': 1.0e-06, - }, - jac=grad_obj, + flat_obj, + x0, + method="L-BFGS-B", + bounds=x0_bounds, + tol=tol_fun, + options={ + "maxiter": max_iter, + "ftol": tol_fun, + "disp": disp, + "gtol": 1.0e-06, + }, + jac=grad_obj, ) opt_result = dict(opt_result) - opt_result['time_secs'] = timer.secs - x_opt = opt_result['x'] + opt_result["time_secs"] = timer.secs + x_opt = opt_result["x"] V_opt = unflatten(x_opt) return V_opt, opt_result @staticmethod - def optimize_locs_params(p, dat, b0, c0, test_locs0, reg=1e-2, - max_iter=100, tol_fun=1e-5, disp=False, locs_bounds_frac=100, - b_lb= -20.0, b_ub= -1e-4, c_lb=1e-6, c_ub=1e3, - ): + def optimize_locs_params( + p, + dat, + b0, + c0, + test_locs0, + reg=1e-2, + max_iter=100, + tol_fun=1e-5, + disp=False, + locs_bounds_frac=100, + b_lb=-20.0, + b_ub=-1e-4, + c_lb=1e-6, + c_ub=1e3, + ): """ Optimize the test locations and the the two parameters (b and c) of the - IMQ kernel by maximizing the test power criterion. - k(x,y) = (c^2 + ||x-y||^2)^b - where c > 0 and b < 0. + IMQ kernel by maximizing the test power criterion. + k(x,y) = (c^2 + ||x-y||^2)^b + where c > 0 and b < 0. data should not be the same data as used in the actual test (i.e., should be a held-out set). This function is deterministic. @@ -827,8 +894,8 @@ def optimize_locs_params(p, dat, b0, c0, test_locs0, reg=1e-2, - c_lb: absolute lower bound on c. c is always > 0. - c_ub: absolute upper bound on c - #- If the lb, ub bounds are None - + #- If the lb, ub bounds are None + Return (V test_locs, b, c, optimization info log) """ @@ -843,10 +910,11 @@ def optimize_locs_params(p, dat, b0, c0, test_locs0, reg=1e-2, n, d = X.shape def obj(sqrt_neg_b, c, V): - b = -sqrt_neg_b**2 + b = -(sqrt_neg_b ** 2) return -IMQFSSD.power_criterion(p, dat, b, c, V, reg=reg) flatten = lambda sqrt_neg_b, c, V: np.hstack((sqrt_neg_b, c, V.reshape(-1))) + def unflatten(x): sqrt_neg_b = x[0] c = x[1] @@ -858,11 +926,11 @@ def flat_obj(x): return obj(sqrt_neg_b, c, V) # gradient - #grad_obj = autograd.elementwise_grad(flat_obj) + # grad_obj = autograd.elementwise_grad(flat_obj) # Initial point b02 = np.sqrt(-b0) x0 = flatten(b02, c0, test_locs0) - + # Make a box to bound test locations X_std = np.std(X, axis=0) # X_min: length-d array @@ -870,8 +938,8 @@ def flat_obj(x): X_max = np.max(X, axis=0) # V_lb: J x d - V_lb = np.tile(X_min - locs_bounds_frac*X_std, (J, 1)) - V_ub = np.tile(X_max + locs_bounds_frac*X_std, (J, 1)) + V_lb = np.tile(X_min - locs_bounds_frac * X_std, (J, 1)) + V_ub = np.tile(X_max + locs_bounds_frac * X_std, (J, 1)) # (J*d+2) x 2. Make sure to bound the reparamterized values (not the original) """ @@ -890,24 +958,28 @@ def flat_obj(x): grad_obj = autograd.elementwise_grad(flat_obj) with util.ContextTimer() as timer: opt_result = scipy.optimize.minimize( - flat_obj, x0, method='L-BFGS-B', - bounds=x0_bounds, - tol=tol_fun, - options={ - 'maxiter': max_iter, 'ftol': tol_fun, 'disp': disp, - 'gtol': 1.0e-06, - }, - jac=grad_obj, + flat_obj, + x0, + method="L-BFGS-B", + bounds=x0_bounds, + tol=tol_fun, + options={ + "maxiter": max_iter, + "ftol": tol_fun, + "disp": disp, + "gtol": 1.0e-06, + }, + jac=grad_obj, ) opt_result = dict(opt_result) - opt_result['time_secs'] = timer.secs - x_opt = opt_result['x'] + opt_result["time_secs"] = timer.secs + x_opt = opt_result["x"] sqrt_neg_b, c, V_opt = unflatten(x_opt) - b = -sqrt_neg_b**2 - assert util.is_real_num(b), 'b is not real. Was {}'.format(b) + b = -(sqrt_neg_b ** 2) + assert util.is_real_num(b), "b is not real. Was {}".format(b) assert b < 0 - assert util.is_real_num(c), 'c is not real. Was {}'.format(c) + assert util.is_real_num(c), "c is not real. Was {}".format(c) assert c > 0 return V_opt, b, c, opt_result @@ -915,9 +987,10 @@ def flat_obj(x): # end of class IMQFSSD + class KernelSteinTest(GofTest): """ - Goodness-of-fit test using kernelized Stein discrepancy test of + Goodness-of-fit test using kernelized Stein discrepancy test of Chwialkowski et al., 2016 and Liu et al., 2016 in ICML 2016. Mainly follow the details in Chwialkowski et al., 2016. The test statistic is n*V_n where V_n is a V-statistic. @@ -930,15 +1003,22 @@ class KernelSteinTest(GofTest): p is specified to the constructor in the form of an UnnormalizedDensity. """ - def __init__(self, p, k, bootstrapper=bootstrapper_rademacher, alpha=0.01, - n_simulate=500, seed=11): + def __init__( + self, + p, + k, + bootstrapper=bootstrapper_rademacher, + alpha=0.01, + n_simulate=500, + seed=11, + ): """ p: an instance of UnnormalizedDensity k: a KSTKernel object - bootstrapper: a function: (n) |-> numpy array of n weights - to be multiplied in the double sum of the test statistic for generating + bootstrapper: a function: (n) |-> numpy array of n weights + to be multiplied in the double sum of the test statistic for generating bootstrap samples from the null distribution. - alpha: significance level + alpha: significance level n_simulate: The number of times to simulate from the null distribution by bootstrapping. Must be a positive integer. """ @@ -959,31 +1039,34 @@ def perform_test(self, dat, return_simulated_stats=False, return_ustat_gram=Fals n = X.shape[0] _, H = self.compute_stat(dat, return_ustat_gram=True) - test_stat = n*np.mean(H) + test_stat = n * np.mean(H) # bootrapping sim_stats = np.zeros(n_simulate) with util.NumpySeedContext(seed=self.seed): for i in range(n_simulate): - W = self.bootstrapper(n) - # n * [ (1/n^2) * \sum_i \sum_j h(x_i, x_j) w_i w_j ] - boot_stat = W.dot(H.dot(old_div(W,float(n)))) - # This is a bootstrap version of n*V_n - sim_stats[i] = boot_stat - - # approximate p-value with the permutations + W = self.bootstrapper(n) + # n * [ (1/n^2) * \sum_i \sum_j h(x_i, x_j) w_i w_j ] + boot_stat = W.dot(H.dot(old_div(W, float(n)))) + # This is a bootstrap version of n*V_n + sim_stats[i] = boot_stat + + # approximate p-value with the permutations pvalue = np.mean(sim_stats > test_stat) - - results = {'alpha': self.alpha, 'pvalue': pvalue, 'test_stat': test_stat, - 'h0_rejected': pvalue < alpha, 'n_simulate': n_simulate, - 'time_secs': t.secs, - } + + results = { + "alpha": self.alpha, + "pvalue": pvalue, + "test_stat": test_stat, + "h0_rejected": pvalue < alpha, + "n_simulate": n_simulate, + "time_secs": t.secs, + } if return_simulated_stats: - results['sim_stats'] = sim_stats + results["sim_stats"] = sim_stats if return_ustat_gram: - results['H'] = H - - return results + results["H"] = H + return results def compute_stat(self, dat, return_ustat_gram=False): """ @@ -1005,29 +1088,31 @@ def compute_stat(self, dat, return_ustat_gram=False): C = np.zeros((n, n)) for i in range(d): grad_logp_i = grad_logp[:, i] - B += k.gradX_Y(X, X, i)*grad_logp_i + B += k.gradX_Y(X, X, i) * grad_logp_i C += (k.gradY_X(X, X, i).T * grad_logp_i).T - H = K*gram_glogp + B + C + k.gradXY_sum(X, X) + H = K * gram_glogp + B + C + k.gradXY_sum(X, X) # V-statistic - stat = n*np.mean(H) + stat = n * np.mean(H) if return_ustat_gram: return stat, H else: return stat - #print 't1: {0}'.format(t1) - #print 't2: {0}'.format(t2) - #print 't3: {0}'.format(t3) - #print 't4: {0}'.format(t4) + # print 't1: {0}'.format(t1) + # print 't2: {0}'.format(t2) + # print 't3: {0}'.format(t3) + # print 't4: {0}'.format(t4) + # end KernelSteinTest + class LinearKernelSteinTest(GofTest): """ Goodness-of-fit test using the linear-version of kernelized Stein discrepancy test of Liu et al., 2016 in ICML 2016. Described in Liu et al., - 2016. + 2016. - This test runs in O(n d^2) time. - test stat = sqrt(n_half)*linear-time Stein discrepancy - Asymptotically normal under both H0 and H1. @@ -1042,7 +1127,7 @@ def __init__(self, p, k, alpha=0.01, seed=11): """ p: an instance of UnnormalizedDensity k: a LinearKSTKernel object - alpha: significance level + alpha: significance level n_simulate: The number of times to simulate from the null distribution by bootstrapping. Must be a positive integer. """ @@ -1061,16 +1146,19 @@ def perform_test(self, dat): # H: length-n vector _, H = self.compute_stat(dat, return_pointwise_stats=True) - test_stat = np.sqrt(old_div(n,2))*np.mean(H) - stat_var = np.mean(H**2) - pvalue = stats.norm.sf(test_stat, loc=0, scale=np.sqrt(stat_var) ) - - results = {'alpha': self.alpha, 'pvalue': pvalue, 'test_stat': test_stat, - 'h0_rejected': pvalue < alpha, 'time_secs': t.secs, - } + test_stat = np.sqrt(old_div(n, 2)) * np.mean(H) + stat_var = np.mean(H ** 2) + pvalue = stats.norm.sf(test_stat, loc=0, scale=np.sqrt(stat_var)) + + results = { + "alpha": self.alpha, + "pvalue": pvalue, + "test_stat": test_stat, + "h0_rejected": pvalue < alpha, + "time_secs": t.secs, + } return results - def compute_stat(self, dat, return_pointwise_stats=False): """ Compute the linear-time statistic described in Eq. 17 of Liu et al., 2016 @@ -1078,11 +1166,11 @@ def compute_stat(self, dat, return_pointwise_stats=False): X = dat.data() n, d = X.shape k = self.k - # Divide the sample into two halves of equal size. - n_half = old_div(n,2) + # Divide the sample into two halves of equal size. + n_half = old_div(n, 2) X1 = X[:n_half, :] # May throw away last sample - X2 = X[n_half:(2*n_half), :] + X2 = X[n_half : (2 * n_half), :] assert X1.shape[0] == n_half assert X2.shape[0] == n_half # score vectors @@ -1091,9 +1179,9 @@ def compute_stat(self, dat, return_pointwise_stats=False): S2 = self.p.grad_log(X2) Kvec = k.pair_eval(X1, X2) - A = np.sum(S1*S2, 1)*Kvec - B = np.sum(S2*k.pair_gradX_Y(X1, X2), 1) - C = np.sum(S1*k.pair_gradY_X(X1, X2), 1) + A = np.sum(S1 * S2, 1) * Kvec + B = np.sum(S2 * k.pair_gradX_Y(X1, X2), 1) + C = np.sum(S1 * k.pair_gradY_X(X1, X2), 1) D = k.pair_gradXY_sum(X1, X2) H = A + B + C + D @@ -1104,12 +1192,14 @@ def compute_stat(self, dat, return_pointwise_stats=False): else: return stat + # end LinearKernelSteinTest + class SteinWitness(object): """ - Construct a callable object representing the Stein witness function. - The witness function g is defined as in Eq. 1 of + Construct a callable object representing the Stein witness function. + The witness function g is defined as in Eq. 1 of A Linear-Time Kernel Goodness-of-Fit Test Wittawat Jitkrittum, Wenkai Xu, Zoltan Szabo, Kenji Fukumizu, @@ -1119,12 +1209,13 @@ class SteinWitness(object): The witness function requires taking an expectation over the sample generating distribution. This is approximated by an empirical expectation using the sample in the input (dat). The witness function - is a d-variate (d = dimension of the data) function, which depends on + is a d-variate (d = dimension of the data) function, which depends on the kernel k and the model p. The constructed object can be called as if it is a function: (J x d) numpy - array |-> (J x d) outputs + array |-> (J x d) outputs """ + def __init__(self, p, k, dat): """ :params p: an UnnormalizedDensity object @@ -1150,21 +1241,18 @@ def __call__(self, V): # When X, V contain many points, this can use a lot of memory. # Process chunk by chunk. - block_rows = util.constrain(50000//(d*J), 10, 5000) + block_rows = util.constrain(50000 // (d * J), 10, 5000) avg_rows = [] for (f, t) in util.ChunkIterable(start=0, end=n, chunk_size=block_rows): - assert f= 1: - raise ValueError('tr_proportion must be between 0 and 1 (exclusive)') + raise ValueError("tr_proportion must be between 0 and 1 (exclusive)") self.n_locs = n_locs self.tr_proportion = tr_proportion self.seed = seed ds = p.get_datasource() if ds is None: - raise ValueError('%s test requires a density p which implements get_datasource(', str(GaussMETest)) + raise ValueError( + "%s test requires a density p which implements get_datasource(", + str(GaussMETest), + ) def perform_test(self, dat, op=None, return_metest=False): """ @@ -138,36 +148,47 @@ def perform_test(self, dat, op=None, return_metest=False): with util.ContextTimer() as t: metest, tr_tst_data, te_tst_data = self._get_metest_opt(dat, op) - # Run the two-sample test + # Run the two-sample test results = metest.perform_test(te_tst_data) - results['time_secs'] = t.secs + results["time_secs"] = t.secs if return_metest: - results['metest'] = metest + results["metest"] = metest return results def _get_metest_opt(self, dat, op=None): seed = self.seed if op is None: - op = {'n_test_locs': self.n_locs, 'seed': seed+5, 'max_iter': 100, - 'batch_proportion': 1.0, 'locs_step_size': 1.0, - 'gwidth_step_size': 0.1, 'tol_fun': 1e-4, 'reg':1e-6} + op = { + "n_test_locs": self.n_locs, + "seed": seed + 5, + "max_iter": 100, + "batch_proportion": 1.0, + "locs_step_size": 1.0, + "gwidth_step_size": 0.1, + "tol_fun": 1e-4, + "reg": 1e-6, + } seed = self.seed alpha = self.alpha p = self.p # Draw sample from p. #sample to draw is the same as that of dat ds = p.get_datasource() p_sample = ds.sample(dat.sample_size(), seed=seed) - xtr, xte = p_sample.split_tr_te(tr_proportion=self.tr_proportion, seed=seed+18) + xtr, xte = p_sample.split_tr_te( + tr_proportion=self.tr_proportion, seed=seed + 18 + ) # ytr, yte are of type data.Data - ytr, yte = dat.split_tr_te(tr_proportion=self.tr_proportion, seed=seed+12) + ytr, yte = dat.split_tr_te(tr_proportion=self.tr_proportion, seed=seed + 12) # training and test data tr_tst_data = fdata.TSTData(xtr.data(), ytr.data()) te_tst_data = fdata.TSTData(xte.data(), yte.data()) # Train the ME test - V_opt, gw2_opt, _ = tst.MeanEmbeddingTest.optimize_locs_width(tr_tst_data, alpha, **op) + V_opt, gw2_opt, _ = tst.MeanEmbeddingTest.optimize_locs_width( + tr_tst_data, alpha, **op + ) metest = tst.MeanEmbeddingTest(V_opt, gw2_opt, alpha) return metest, tr_tst_data, te_tst_data @@ -177,4 +198,3 @@ def compute_stat(self, dat, op=None): # Make a two-sample test data s = metest.compute_stat(te_tst_data) return s - diff --git a/sbibm/third_party/kgof/kernel.py b/sbibm/third_party/kgof/kernel.py index f4a80d10..c9807410 100644 --- a/sbibm/third_party/kgof/kernel.py +++ b/sbibm/third_party/kgof/kernel.py @@ -1,20 +1,23 @@ - """Module containing kernel related classes""" from __future__ import division -from builtins import str -from past.utils import old_div -from builtins import object +from builtins import object, str + from future.utils import with_metaclass -__author__ = 'wittawat' +from past.utils import old_div + +__author__ = "wittawat" from abc import ABCMeta, abstractmethod + import autograd import autograd.numpy as np -#import numpy as np + +# import numpy as np import sbibm.third_party.kgof.config as config import sbibm.third_party.kgof.util as util + class Kernel(with_metaclass(ABCMeta, object)): """Abstract class for kernels. Inputs to all methods are numpy arrays.""" @@ -48,28 +51,27 @@ class KSTKernel(with_metaclass(ABCMeta, Kernel)): @abstractmethod def gradX_Y(self, X, Y, dim): - """ - Compute the gradient with respect to the dimension dim of X in k(X, Y). + """ + Compute the gradient with respect to the dimension dim of X in k(X, Y). - X: nx x d - Y: ny x d + X: nx x d + Y: ny x d - Return a numpy array of size nx x ny. - """ - raise NotImplementedError() + Return a numpy array of size nx x ny. + """ + raise NotImplementedError() @abstractmethod def gradY_X(self, X, Y, dim): - """ - Compute the gradient with respect to the dimension dim of Y in k(X, Y). - - X: nx x d - Y: ny x d + """ + Compute the gradient with respect to the dimension dim of Y in k(X, Y). - Return a numpy array of size nx x ny. - """ - raise NotImplementedError() + X: nx x d + Y: ny x d + Return a numpy array of size nx x ny. + """ + raise NotImplementedError() @abstractmethod def gradXY_sum(self, X, Y): @@ -84,8 +86,10 @@ def gradXY_sum(self, X, Y): """ raise NotImplementedError() + # end KSTKernel + class LinearKSTKernel(with_metaclass(ABCMeta, Kernel)): """ Interface specifiying methods a kernel has to implement to be used with @@ -95,30 +99,29 @@ class LinearKSTKernel(with_metaclass(ABCMeta, Kernel)): @abstractmethod def pair_gradX_Y(self, X, Y): - """ - Compute the gradient with respect to X in k(X, Y), evaluated at the - specified X and Y. + """ + Compute the gradient with respect to X in k(X, Y), evaluated at the + specified X and Y. - X: n x d - Y: n x d + X: n x d + Y: n x d - Return a numpy array of size n x d - """ - raise NotImplementedError() + Return a numpy array of size n x d + """ + raise NotImplementedError() @abstractmethod def pair_gradY_X(self, X, Y): - """ - Compute the gradient with respect to Y in k(X, Y), evaluated at the - specified X and Y. - - X: n x d - Y: n x d + """ + Compute the gradient with respect to Y in k(X, Y), evaluated at the + specified X and Y. - Return a numpy array of size n x d - """ - raise NotImplementedError() + X: n x d + Y: n x d + Return a numpy array of size n x d + """ + raise NotImplementedError() @abstractmethod def pair_gradXY_sum(self, X, Y): @@ -133,6 +136,7 @@ def pair_gradXY_sum(self, X, Y): """ raise NotImplementedError() + class DifferentiableKernel(with_metaclass(ABCMeta, Kernel)): def gradX_y(self, X, y): """ @@ -154,13 +158,16 @@ def gradX_y(self, X, y): assert G.shape[1] == X.shape[1] return G + # end class KSTKernel + class KDiagGauss(Kernel): """ A Gaussian kernel with diagonal covariance structure i.e., one Gaussian width for each dimension. """ + def __init__(self, sigma2s): """ sigma2s: a one-dimensional array of length d containing one width @@ -174,21 +181,23 @@ def eval(self, X, Y): width^2) and using the standard Gaussian kernel. """ sigma2s = self.sigma2s - Xs = old_div(X,np.sqrt(sigma2s)) - Ys = old_div(Y,np.sqrt(sigma2s)) + Xs = old_div(X, np.sqrt(sigma2s)) + Ys = old_div(Y, np.sqrt(sigma2s)) k = KGauss(1.0) return k.eval(Xs, Ys) def pair_eval(self, X, Y): """Evaluate k(x1, y1), k(x2, y2), ...""" sigma2s = self.sigma2s - Xs = old_div(X,np.sqrt(sigma2s)) - Ys = old_div(Y,np.sqrt(sigma2s)) + Xs = old_div(X, np.sqrt(sigma2s)) + Ys = old_div(Y, np.sqrt(sigma2s)) k = KGauss(1.0) return k.pair_eval(Xs, Ys) + # end class KDiagGauss + class KIMQ(DifferentiableKernel, KSTKernel): """ The inverse multiquadric (IMQ) kernel studied in @@ -204,27 +213,26 @@ class KIMQ(DifferentiableKernel, KSTKernel): def __init__(self, b=-0.5, c=1.0): if not b < 0: - raise ValueError('b has to be negative. Was {}'.format(b)) + raise ValueError("b has to be negative. Was {}".format(b)) if not c > 0: - raise ValueError('c has to be positive. Was {}'.format(c)) + raise ValueError("c has to be positive. Was {}".format(c)) self.b = b self.c = c def eval(self, X, Y): - """Evalute the kernel on data X and Y """ + """Evalute the kernel on data X and Y""" b = self.b c = self.c D2 = util.dist2_matrix(X, Y) - K = (c**2 + D2)**b + K = (c ** 2 + D2) ** b return K def pair_eval(self, X, Y): - """Evaluate k(x1, y1), k(x2, y2), ... - """ + """Evaluate k(x1, y1), k(x2, y2), ...""" assert X.shape[0] == Y.shape[0] b = self.b c = self.c - return (c**2 + np.sum((X-Y)**2, 1))**b + return (c ** 2 + np.sum((X - Y) ** 2, 1)) ** b def gradX_Y(self, X, Y, dim): """ @@ -245,7 +253,7 @@ def gradX_Y(self, X, Y, dim): b = self.b c = self.c - Gdim = ( 2.0*b*(c**2 + D2)**(b-1) )*dim_diff + Gdim = (2.0 * b * (c ** 2 + D2) ** (b - 1)) * dim_diff assert Gdim.shape[0] == X.shape[0] assert Gdim.shape[1] == Y.shape[0] return Gdim @@ -278,13 +286,15 @@ def gradXY_sum(self, X, Y): # d = input dimension d = X.shape[1] - c2D2 = c**2 + D2 - T1 = -4.0*b*(b-1)*D2*(c2D2**(b-2) ) - T2 = -2.0*b*d*c2D2**(b-1) + c2D2 = c ** 2 + D2 + T1 = -4.0 * b * (b - 1) * D2 * (c2D2 ** (b - 2)) + T2 = -2.0 * b * d * c2D2 ** (b - 1) return T1 + T2 + # end class KIMQ + class KGauss(DifferentiableKernel, KSTKernel, LinearKSTKernel): """ The standard isotropic Gaussian kernel. @@ -293,7 +303,7 @@ class KGauss(DifferentiableKernel, KSTKernel, LinearKSTKernel): """ def __init__(self, sigma2): - assert sigma2 > 0, 'sigma2 must be > 0. Was %s'%str(sigma2) + assert sigma2 > 0, "sigma2 must be > 0. Was %s" % str(sigma2) self.sigma2 = sigma2 def eval(self, X, Y): @@ -309,13 +319,13 @@ def eval(self, X, Y): ------ K : a n1 x n2 Gram matrix. """ - #(n1, d1) = X.shape - #(n2, d2) = Y.shape - #assert d1==d2, 'Dimensions of the two inputs must be the same' - sumx2 = np.reshape(np.sum(X**2, 1), (-1, 1)) - sumy2 = np.reshape(np.sum(Y**2, 1), (1, -1)) - D2 = sumx2 - 2*np.dot(X, Y.T) + sumy2 - K = np.exp(old_div(-D2,(2.0*self.sigma2))) + # (n1, d1) = X.shape + # (n2, d2) = Y.shape + # assert d1==d2, 'Dimensions of the two inputs must be the same' + sumx2 = np.reshape(np.sum(X ** 2, 1), (-1, 1)) + sumy2 = np.reshape(np.sum(Y ** 2, 1), (1, -1)) + D2 = sumx2 - 2 * np.dot(X, Y.T) + sumy2 + K = np.exp(old_div(-D2, (2.0 * self.sigma2))) return K def gradX_Y(self, X, Y, dim): @@ -330,8 +340,8 @@ def gradX_Y(self, X, Y, dim): sigma2 = self.sigma2 K = self.eval(X, Y) Diff = X[:, [dim]] - Y[:, [dim]].T - #Diff = np.reshape(X[:, dim], (-1, 1)) - np.reshape(Y[:, dim], (1, -1)) - G = -K*Diff/sigma2 + # Diff = np.reshape(X[:, dim], (-1, 1)) - np.reshape(Y[:, dim], (1, -1)) + G = -K * Diff / sigma2 return G def pair_gradX_Y(self, X, Y): @@ -348,7 +358,7 @@ def pair_gradX_Y(self, X, Y): Kvec = self.pair_eval(X, Y) # n x d Diff = X - Y - G = -Kvec[:, np.newaxis]*Diff/sigma2 + G = -Kvec[:, np.newaxis] * Diff / sigma2 return G def gradY_X(self, X, Y, dim): @@ -363,17 +373,16 @@ def gradY_X(self, X, Y, dim): return -self.gradX_Y(X, Y, dim) def pair_gradY_X(self, X, Y): - """ - Compute the gradient with respect to Y in k(X, Y), evaluated at the - specified X and Y. - - X: n x d - Y: n x d + """ + Compute the gradient with respect to Y in k(X, Y), evaluated at the + specified X and Y. - Return a numpy array of size n x d - """ - return -self.pair_gradX_Y(X, Y) + X: n x d + Y: n x d + Return a numpy array of size n x d + """ + return -self.pair_gradX_Y(X, Y) def gradXY_sum(self, X, Y): r""" @@ -387,12 +396,12 @@ def gradXY_sum(self, X, Y): """ (n1, d1) = X.shape (n2, d2) = Y.shape - assert d1==d2, 'Dimensions of the two inputs must be the same' + assert d1 == d2, "Dimensions of the two inputs must be the same" d = d1 sigma2 = self.sigma2 - D2 = np.sum(X**2, 1)[:, np.newaxis] - 2*np.dot(X, Y.T) + np.sum(Y**2, 1) - K = np.exp(old_div(-D2,(2.0*sigma2))) - G = K/sigma2*(d - old_div(D2,sigma2)) + D2 = np.sum(X ** 2, 1)[:, np.newaxis] - 2 * np.dot(X, Y.T) + np.sum(Y ** 2, 1) + K = np.exp(old_div(-D2, (2.0 * sigma2))) + G = K / sigma2 * (d - old_div(D2, sigma2)) return G def pair_gradXY_sum(self, X, Y): @@ -407,12 +416,11 @@ def pair_gradXY_sum(self, X, Y): """ d = X.shape[1] sigma2 = self.sigma2 - D2 = np.sum( (X-Y)**2, 1) - Kvec = np.exp(old_div(-D2,(2.0*self.sigma2))) - G = Kvec/sigma2*(d - old_div(D2,sigma2)) + D2 = np.sum((X - Y) ** 2, 1) + Kvec = np.exp(old_div(-D2, (2.0 * self.sigma2))) + G = Kvec / sigma2 * (d - old_div(D2, sigma2)) return G - def pair_eval(self, X, Y): """ Evaluate k(x1, y1), k(x2, y2), ... @@ -427,14 +435,14 @@ def pair_eval(self, X, Y): """ (n1, d1) = X.shape (n2, d2) = Y.shape - assert n1==n2, 'Two inputs must have the same number of instances' - assert d1==d2, 'Two inputs must have the same dimension' - D2 = np.sum( (X-Y)**2, 1) - Kvec = np.exp(old_div(-D2,(2.0*self.sigma2))) + assert n1 == n2, "Two inputs must have the same number of instances" + assert d1 == d2, "Two inputs must have the same dimension" + D2 = np.sum((X - Y) ** 2, 1) + Kvec = np.exp(old_div(-D2, (2.0 * self.sigma2))) return Kvec def __str__(self): - return "KGauss(%.3f)"%self.sigma2 + return "KGauss(%.3f)" % self.sigma2 class KMixGauss(DifferentiableKernel, KSTKernel): @@ -450,7 +458,7 @@ def __init__(self, sigma2s, wts=None): assert len(sigma2s) > 0 if wts is None: - self.wts = wts = np.full(len(sigma2s), 1/len(sigma2s)) + self.wts = wts = np.full(len(sigma2s), 1 / len(sigma2s)) else: self.wts = wts = np.asarray(wts) assert len(wts) == len(sigma2s) @@ -463,15 +471,16 @@ def eval(self, X, Y): Y: ny x d return nx x ny Gram matrix """ - sumx2 = np.sum(X**2, axis=1)[:, np.newaxis] - sumy2 = np.sum(Y**2, axis=1)[np.newaxis, :] + sumx2 = np.sum(X ** 2, axis=1)[:, np.newaxis] + sumy2 = np.sum(Y ** 2, axis=1)[np.newaxis, :] D2 = sumx2 - 2 * np.dot(X, Y.T) + sumy2 return np.tensordot( self.wts, np.exp( - D2[np.newaxis, :, :] - / (-2 * self.sigma2s[:, np.newaxis, np.newaxis])), - 1) + D2[np.newaxis, :, :] / (-2 * self.sigma2s[:, np.newaxis, np.newaxis]) + ), + 1, + ) def pair_eval(self, X, Y): """Evaluate k(x1, y1), k(x2, y2), ... @@ -482,13 +491,12 @@ def pair_eval(self, X, Y): """ n1, d1 = X.shape n2, d2 = Y.shape - assert n1 == n2, 'Two inputs must have the same number of instances' - assert d1 == d2, 'Two inputs must have the same dimension' - D2 = np.sum((X - Y)**2, axis=1) + assert n1 == n2, "Two inputs must have the same number of instances" + assert d1 == d2, "Two inputs must have the same dimension" + D2 = np.sum((X - Y) ** 2, axis=1) return np.tensordot( - self.wts, - np.exp(D2[np.newaxis, :] / (-2 * self.sigma2s[:, np.newaxis])), - 1) + self.wts, np.exp(D2[np.newaxis, :] / (-2 * self.sigma2s[:, np.newaxis])), 1 + ) def gradX_Y(self, X, Y, dim): """ @@ -500,9 +508,11 @@ def gradX_Y(self, X, Y, dim): Return a numpy array of size nx x ny. """ diffs = -X[:, [dim]] + Y[:, [dim]].T - exps = np.exp(diffs[np.newaxis, :, :] ** 2 - / (-2 * self.sigma2s[:, np.newaxis, np.newaxis])) - return np.einsum('w,wij,ij->ij', self.wts / self.sigma2s, exps, diffs) + exps = np.exp( + diffs[np.newaxis, :, :] ** 2 + / (-2 * self.sigma2s[:, np.newaxis, np.newaxis]) + ) + return np.einsum("w,wij,ij->ij", self.wts / self.sigma2s, exps, diffs) def gradY_X(self, X, Y, dim): """ @@ -526,12 +536,13 @@ def gradXY_sum(self, X, Y): Return a nx x ny numpy array of the derivatives. """ d = X.shape[1] - sumx2 = np.sum(X**2, axis=1)[:, np.newaxis] - sumy2 = np.sum(Y**2, axis=1)[np.newaxis, :] + sumx2 = np.sum(X ** 2, axis=1)[:, np.newaxis] + sumy2 = np.sum(Y ** 2, axis=1)[np.newaxis, :] D2 = sumx2 - 2 * np.dot(X, Y.T) + sumy2 - s = (D2[np.newaxis, :, :] / self.sigma2s[:, np.newaxis, np.newaxis]) - return np.einsum('w,wij,wij->ij', - self.wts / self.sigma2s, np.exp(s / -2), d - s) + s = D2[np.newaxis, :, :] / self.sigma2s[:, np.newaxis, np.newaxis] + return np.einsum( + "w,wij,wij->ij", self.wts / self.sigma2s, np.exp(s / -2), d - s + ) class KPoly(DifferentiableKernel, KSTKernel): @@ -560,7 +571,7 @@ def eval(self, X, Y): return nx x ny Gram matrix """ dot = np.dot(X, Y.T) - gamma = 1/X.shape[1] if self.gamma is None else self.gamma + gamma = 1 / X.shape[1] if self.gamma is None else self.gamma return (gamma * dot + self.coef0) ** self.degree def pair_eval(self, X, Y): @@ -572,11 +583,11 @@ def pair_eval(self, X, Y): """ n1, d1 = X.shape n2, d2 = Y.shape - assert n1 == n2, 'Two inputs must have the same number of instances' - assert d1 == d2, 'Two inputs must have the same dimension' + assert n1 == n2, "Two inputs must have the same number of instances" + assert d1 == d2, "Two inputs must have the same dimension" - dot = np.einsum('id,id->i', X, Y) - gamma = 1/d1 if self.gamma is None else self.gamma + dot = np.einsum("id,id->i", X, Y) + gamma = 1 / d1 if self.gamma is None else self.gamma return (gamma * dot + self.coef0) ** self.degree def gradX_Y(self, X, Y, dim): @@ -588,15 +599,19 @@ def gradX_Y(self, X, Y, dim): Return a numpy array of size nx x ny. """ - gamma = 1/X.shape[1] if self.gamma is None else self.gamma + gamma = 1 / X.shape[1] if self.gamma is None else self.gamma if self.degree == 1: # optimization, other expression is valid too out = gamma * Y[np.newaxis, :, dim] # 1 x ny return np.repeat(out, X.shape[0], axis=0) dot = np.dot(X, Y.T) - return (self.degree * (gamma * dot + self.coef0) ** (self.degree - 1) - * gamma * Y[np.newaxis, :, dim]) + return ( + self.degree + * (gamma * dot + self.coef0) ** (self.degree - 1) + * gamma + * Y[np.newaxis, :, dim] + ) def gradY_X(self, X, Y, dim): """ @@ -607,15 +622,19 @@ def gradY_X(self, X, Y, dim): Return a numpy array of size nx x ny. """ - gamma = 1/X.shape[1] if self.gamma is None else self.gamma + gamma = 1 / X.shape[1] if self.gamma is None else self.gamma if self.degree == 1: # optimization, other expression is valid too out = gamma * X[:, dim, np.newaxis] # nx x 1 return np.repeat(out, Y.shape[0], axis=1) dot = np.dot(X, Y.T) - return (self.degree * (gamma * dot + self.coef0) ** (self.degree - 1) - * gamma * X[:, dim, np.newaxis]) + return ( + self.degree + * (gamma * dot + self.coef0) ** (self.degree - 1) + * gamma + * X[:, dim, np.newaxis] + ) def gradXY_sum(self, X, Y): r""" @@ -627,7 +646,7 @@ def gradXY_sum(self, X, Y): Return a nx x ny numpy array of the derivatives. """ - gamma = 1/X.shape[1] if self.gamma is None else self.gamma + gamma = 1 / X.shape[1] if self.gamma is None else self.gamma if self.degree == 1: # optimization, other expression is valid too return np.tile(gamma, (X.shape[0], X.shape[1])) @@ -636,17 +655,16 @@ def gradXY_sum(self, X, Y): inside = gamma * dot + self.coef0 to_dminus2 = inside ** (self.degree - 2) to_dminus1 = to_dminus2 * inside - return ( - (self.degree * (self.degree-1) * gamma**2) * to_dminus2 * dot - + (X.shape[1] * gamma * self.degree) * to_dminus1 - ) + return (self.degree * (self.degree - 1) * gamma ** 2) * to_dminus2 * dot + ( + X.shape[1] * gamma * self.degree + ) * to_dminus1 class KMixture(KSTKernel, LinearKSTKernel, DifferentiableKernel): def __init__(self, ks, wts=None): self.ks = ks if wts is None: - self.wts = np.full(len(ks), 1/len(ks)) + self.wts = np.full(len(ks), 1 / len(ks)) else: self.wts = np.asarray(wts) diff --git a/sbibm/third_party/kgof/mmd.py b/sbibm/third_party/kgof/mmd.py index adc54f18..d5791e5d 100644 --- a/sbibm/third_party/kgof/mmd.py +++ b/sbibm/third_party/kgof/mmd.py @@ -11,28 +11,32 @@ """ from builtins import str -__author__ = 'wittawat' +__author__ = "wittawat" + +import logging from abc import ABCMeta, abstractmethod + import autograd import autograd.numpy as np +import freqopttest.data as fdata + # Require freqopttest https://github.com/wittawatj/interpretable-test import freqopttest.tst as tst -import freqopttest.data as fdata +import matplotlib.pyplot as plt +import scipy +import scipy.stats as stats + import sbibm.third_party.kgof.data as data import sbibm.third_party.kgof.goftest as gof -import sbibm.third_party.kgof.util as util import sbibm.third_party.kgof.kernel as kernel -import logging -import matplotlib.pyplot as plt +import sbibm.third_party.kgof.util as util -import scipy -import scipy.stats as stats class QuadMMDGof(gof.GofTest): """ Goodness-of-fit test by drawing sample from the density p and test with - the MMD test of Gretton et al., 2012. + the MMD test of Gretton et al., 2012. H0: the sample follows p H1: the sample does not follow p @@ -44,9 +48,9 @@ def __init__(self, p, k, n_permute=400, alpha=0.01, seed=28): """ p: an instance of UnnormalizedDensity k: an instance of Kernel - n_permute: number of times to permute the samples to simulate from the + n_permute: number of times to permute the samples to simulate from the null distribution (permutation test) - alpha: significance level + alpha: significance level seed: random seed """ super(QuadMMDGof, self).__init__(p, alpha) @@ -56,8 +60,10 @@ def __init__(self, p, k, n_permute=400, alpha=0.01, seed=28): self.seed = seed ds = p.get_datasource() if ds is None: - raise ValueError('%s test requires a density p which implements get_datasource(', str(QuadMMDGof)) - + raise ValueError( + "%s test requires a density p which implements get_datasource(", + str(QuadMMDGof), + ) def perform_test(self, dat): """ @@ -70,15 +76,15 @@ def perform_test(self, dat): # Draw sample from p. #sample to draw is the same as that of dat ds = p.get_datasource() - p_sample = ds.sample(dat.sample_size(), seed=seed+12) + p_sample = ds.sample(dat.sample_size(), seed=seed + 12) # Run the two-sample test on p_sample and dat # Make a two-sample test data tst_data = fdata.TSTData(p_sample.data(), dat.data()) - # Test + # Test results = mmdtest.perform_test(tst_data) - results['time_secs'] = t.secs + results["time_secs"] = t.secs return results def compute_stat(self, dat): @@ -93,9 +99,10 @@ def compute_stat(self, dat): s = mmdtest.compute_stat(tst_data) return s - + # end QuadMMDGof + class QuadMMDGofOpt(gof.GofTest): """ Goodness-of-fit test by drawing sample from the density p and test with the @@ -113,9 +120,9 @@ def __init__(self, p, n_permute=400, alpha=0.01, seed=28): """ p: an instance of UnnormalizedDensity k: an instance of Kernel - n_permute: number of times to permute the samples to simulate from the + n_permute: number of times to permute the samples to simulate from the null distribution (permutation test) - alpha: significance level + alpha: significance level seed: random seed """ super(QuadMMDGofOpt, self).__init__(p, alpha) @@ -123,56 +130,65 @@ def __init__(self, p, n_permute=400, alpha=0.01, seed=28): self.seed = seed ds = p.get_datasource() if ds is None: - raise ValueError('%s test requires a density p which implements get_datasource(', str(QuadMMDGof)) - - - def perform_test(self, dat, candidate_kernels=None, return_mmdtest=False, - tr_proportion=0.2, reg=1e-3): + raise ValueError( + "%s test requires a density p which implements get_datasource(", + str(QuadMMDGof), + ) + + def perform_test( + self, + dat, + candidate_kernels=None, + return_mmdtest=False, + tr_proportion=0.2, + reg=1e-3, + ): """ dat: an instance of Data candidate_kernels: a list of Kernel's to choose from tr_proportion: proportion of sample to be used to choosing the best kernel - reg: regularization parameter for the test power criterion + reg: regularization parameter for the test power criterion """ with util.ContextTimer() as t: seed = self.seed p = self.p ds = p.get_datasource() - p_sample = ds.sample(dat.sample_size(), seed=seed+77) - xtr, xte = p_sample.split_tr_te(tr_proportion=tr_proportion, seed=seed+18) + p_sample = ds.sample(dat.sample_size(), seed=seed + 77) + xtr, xte = p_sample.split_tr_te(tr_proportion=tr_proportion, seed=seed + 18) # ytr, yte are of type data.Data - ytr, yte = dat.split_tr_te(tr_proportion=tr_proportion, seed=seed+12) + ytr, yte = dat.split_tr_te(tr_proportion=tr_proportion, seed=seed + 12) # training and test data tr_tst_data = fdata.TSTData(xtr.data(), ytr.data()) te_tst_data = fdata.TSTData(xte.data(), yte.data()) if candidate_kernels is None: - # Assume a Gaussian kernel. Construct a list of + # Assume a Gaussian kernel. Construct a list of # kernels to try based on multiples of the median heuristic med = util.meddistance(tr_tst_data.stack_xy(), 1000) - list_gwidth = np.hstack( ( (med**2) *(2.0**np.linspace(-4, 4, 10) ) ) ) + list_gwidth = np.hstack(((med ** 2) * (2.0 ** np.linspace(-4, 4, 10)))) list_gwidth.sort() candidate_kernels = [kernel.KGauss(gw2) for gw2 in list_gwidth] alpha = self.alpha # grid search to choose the best Gaussian width - besti, powers = tst.QuadMMDTest.grid_search_kernel(tr_tst_data, - candidate_kernels, alpha, reg=reg) - # perform test + besti, powers = tst.QuadMMDTest.grid_search_kernel( + tr_tst_data, candidate_kernels, alpha, reg=reg + ) + # perform test best_ker = candidate_kernels[besti] mmdtest = tst.QuadMMDTest(best_ker, self.n_permute, alpha=alpha) results = mmdtest.perform_test(te_tst_data) if return_mmdtest: - results['mmdtest'] = mmdtest + results["mmdtest"] = mmdtest - results['time_secs'] = t.secs + results["time_secs"] = t.secs return results def compute_stat(self, dat): - raise NotImplementedError('Not implemented yet.') + raise NotImplementedError("Not implemented yet.") + - # end QuadMMDGofOpt diff --git a/sbibm/third_party/kgof/plot.py b/sbibm/third_party/kgof/plot.py index ecf90fef..4c53fbfc 100644 --- a/sbibm/third_party/kgof/plot.py +++ b/sbibm/third_party/kgof/plot.py @@ -1,13 +1,14 @@ """Module containing convenient functions for plotting""" -from builtins import range -from builtins import object -__author__ = 'wittawat' +from builtins import object, range -import sbibm.third_party.kgof.glo as glo +__author__ = "wittawat" + +import autograd.numpy as np import matplotlib import matplotlib.pyplot as plt -import autograd.numpy as np + +import sbibm.third_party.kgof.glo as glo def get_func_tuples(): @@ -16,88 +17,86 @@ def get_func_tuples(): (func_name used in the experiments, label name, plot line style) """ func_tuples = [ - ('job_fssdJ1q_med', 'FSSD-rand J1', 'r--^'), - ('job_fssdJ5q_med', 'FSSD-rand', 'r--^'), - ('job_fssdq_med', 'FSSD-rand', 'r--^'), - - ('job_fssdJ1q_opt', 'FSSD-opt J1', 'r-s'), - ('job_fssdq_opt', 'FSSD-opt', 'r-s'), - ('job_fssdJ5q_opt', 'FSSD-opt', 'r-s'), - ('job_fssdJ5q_imq_optv', 'FSSD-IMQv', 'k-h'), - ('job_fssdJ5q_imqb1_optv', 'FSSD-IMQ-1', 'k--s'), - ('job_fssdJ5q_imqb2_optv', 'FSSD-IMQ-2', 'k-->'), - ('job_fssdJ5q_imqb3_optv', 'FSSD-IMQ-3', 'k-.*'), - ('job_fssdJ5q_imq_opt', 'FSSD-IMQ', 'y-x'), - ('job_fssdJ5q_imq_optbv', 'FSSD-IMQ-bv', 'y--d'), - ('job_fssdJ10q_opt', 'FSSD-opt', 'k-s'), - - ('job_fssdJ5p_opt', 'FSSD-opt J5', 'm-s'), - ('job_fssdp_opt', 'FSSDp-opt', 'm-s'), - ('job_fssdJ10p_opt', 'FSSDp-opt J10', 'k-s'), - - ('job_fssdJ1q_opt2', 'FSSD-opt2 J1', 'b-^'), - ('job_fssdJ5q_opt2', 'FSSD-opt2 J5', 'r-^'), - ('job_me_opt', 'ME-opt', 'b-d'), - - ('job_kstein_med', 'KSD', 'g-o'), - ('job_kstein_imq', 'KSD-IMQ', 'c-*'), - ('job_lin_kstein_med', 'LKS', 'g-.h'), - ('job_mmd_med', 'MMD', 'm--^'), - ('job_mmd_opt', 'MMD-opt', 'm-<'), - ('job_mmd_dgauss_opt', 'MMD-dopt', 'y-<'), - ] + ("job_fssdJ1q_med", "FSSD-rand J1", "r--^"), + ("job_fssdJ5q_med", "FSSD-rand", "r--^"), + ("job_fssdq_med", "FSSD-rand", "r--^"), + ("job_fssdJ1q_opt", "FSSD-opt J1", "r-s"), + ("job_fssdq_opt", "FSSD-opt", "r-s"), + ("job_fssdJ5q_opt", "FSSD-opt", "r-s"), + ("job_fssdJ5q_imq_optv", "FSSD-IMQv", "k-h"), + ("job_fssdJ5q_imqb1_optv", "FSSD-IMQ-1", "k--s"), + ("job_fssdJ5q_imqb2_optv", "FSSD-IMQ-2", "k-->"), + ("job_fssdJ5q_imqb3_optv", "FSSD-IMQ-3", "k-.*"), + ("job_fssdJ5q_imq_opt", "FSSD-IMQ", "y-x"), + ("job_fssdJ5q_imq_optbv", "FSSD-IMQ-bv", "y--d"), + ("job_fssdJ10q_opt", "FSSD-opt", "k-s"), + ("job_fssdJ5p_opt", "FSSD-opt J5", "m-s"), + ("job_fssdp_opt", "FSSDp-opt", "m-s"), + ("job_fssdJ10p_opt", "FSSDp-opt J10", "k-s"), + ("job_fssdJ1q_opt2", "FSSD-opt2 J1", "b-^"), + ("job_fssdJ5q_opt2", "FSSD-opt2 J5", "r-^"), + ("job_me_opt", "ME-opt", "b-d"), + ("job_kstein_med", "KSD", "g-o"), + ("job_kstein_imq", "KSD-IMQ", "c-*"), + ("job_lin_kstein_med", "LKS", "g-.h"), + ("job_mmd_med", "MMD", "m--^"), + ("job_mmd_opt", "MMD-opt", "m-<"), + ("job_mmd_dgauss_opt", "MMD-dopt", "y-<"), + ] return func_tuples + def set_default_matplotlib_options(): # font options font = { - # 'family' : 'normal', + # 'family' : 'normal', #'weight' : 'bold', - 'size' : 30 + "size": 30 } - matplotlib.rc('font', **{'family': 'serif', 'serif': ['Computer Modern']}) - + matplotlib.rc("font", **{"family": "serif", "serif": ["Computer Modern"]}) # matplotlib.use('cairo') - matplotlib.rc('text', usetex=True) - matplotlib.rcParams['text.usetex'] = True - plt.rc('font', **font) - plt.rc('lines', linewidth=3, markersize=10) + matplotlib.rc("text", usetex=True) + matplotlib.rcParams["text.usetex"] = True + plt.rc("font", **font) + plt.rc("lines", linewidth=3, markersize=10) # matplotlib.rcParams['ps.useafm'] = True # matplotlib.rcParams['pdf.use14corefonts'] = True - matplotlib.rcParams['pdf.fonttype'] = 42 - matplotlib.rcParams['ps.fonttype'] = 42 + matplotlib.rcParams["pdf.fonttype"] = 42 + matplotlib.rcParams["ps.fonttype"] = 42 + def get_func2label_map(): # map: job_func_name |-> plot label func_tuples = get_func_tuples() - #M = {k:v for (k,v) in zip(func_names, labels)} - M = {k:v for (k,v,_) in func_tuples} + # M = {k:v for (k,v) in zip(func_names, labels)} + M = {k: v for (k, v, _) in func_tuples} return M def func_plot_fmt_map(): """ - Return a map from job function names to matplotlib plot styles + Return a map from job function names to matplotlib plot styles """ - # line_styles = ['o-', 'x-', '*-', '-_', 'D-', 'h-', '+-', 's-', 'v-', + # line_styles = ['o-', 'x-', '*-', '-_', 'D-', 'h-', '+-', 's-', 'v-', # ',-', '1-'] func_tuples = get_func_tuples() - M = {k:v for (k, _, v) in func_tuples} + M = {k: v for (k, _, v) in func_tuples} return M class PlotValues(object): """ - An object encapsulating values of a plot where there are many curves, + An object encapsulating values of a plot where there are many curves, each corresponding to one method, with common x-axis values. """ + def __init__(self, xvalues, methods, plot_matrix): """ xvalues: 1d numpy array of x-axis values methods: a list of method names - plot_matrix: len(methods) x len(xvalues) 2d numpy array containing + plot_matrix: len(methods) x len(xvalues) 2d numpy array containing values that can be used to plot """ self.xvalues = xvalues @@ -115,72 +114,76 @@ def ascii_table(self, tablefmt="pipe"): plot_matrix = self.plot_matrix import tabulate + # https://pypi.python.org/pypi/tabulate aug_table = np.hstack((np.array(methods)[:, np.newaxis], plot_matrix)) return tabulate.tabulate(aug_table, xvalues, tablefmt=tablefmt) + # end of class PlotValues -def plot_prob_reject(ex, fname, func_xvalues, xlabel, func_title=None, - return_plot_values=False): + +def plot_prob_reject( + ex, fname, func_xvalues, xlabel, func_title=None, return_plot_values=False +): """ plot the empirical probability that the statistic is above the threshold. - This can be interpreted as type-1 error (when H0 is true) or test power + This can be interpreted as type-1 error (when H0 is true) or test power (when H1 is true). The plot is against the specified x-axis. - - ex: experiment number + - ex: experiment number - fname: file name of the aggregated result - - func_xvalues: function taking aggregated results dictionary and return the values - to be used for the x-axis values. - - xlabel: label of the x-axis. + - func_xvalues: function taking aggregated results dictionary and return the values + to be used for the x-axis values. + - xlabel: label of the x-axis. - func_title: a function: results dictionary -> title of the plot - return_plot_values: if true, also return a PlotValues as the second output value. Return loaded results """ - #from IPython.core.debugger import Tracer - #Tracer()() + # from IPython.core.debugger import Tracer + # Tracer()() results = glo.ex_load_result(ex, fname) def rej_accessor(jr): - rej = jr['test_result']['h0_rejected'] - # When used with vectorize(), making the value float will make the resulting + rej = jr["test_result"]["h0_rejected"] + # When used with vectorize(), making the value float will make the resulting # numpy array to be of float. nan values can be stored. return float(rej) - #value_accessor = lambda job_results: job_results['test_result']['h0_rejected'] + # value_accessor = lambda job_results: job_results['test_result']['h0_rejected'] vf_pval = np.vectorize(rej_accessor) - # results['job_results'] is a dictionary: + # results['job_results'] is a dictionary: # {'test_result': (dict from running perform_test(te) '...':..., } - rejs = vf_pval(results['job_results']) - repeats, _, n_methods = results['job_results'].shape + rejs = vf_pval(results["job_results"]) + repeats, _, n_methods = results["job_results"].shape # yvalues (corresponding to xvalues) x #methods mean_rejs = np.mean(rejs, axis=0) - #print mean_rejs - #std_pvals = np.std(rejs, axis=0) - #std_pvals = np.sqrt(mean_rejs*(1.0-mean_rejs)) + # print mean_rejs + # std_pvals = np.std(rejs, axis=0) + # std_pvals = np.sqrt(mean_rejs*(1.0-mean_rejs)) xvalues = func_xvalues(results) - #ns = np.array(results[xkey]) - #te_proportion = 1.0 - results['tr_proportion'] - #test_sizes = ns*te_proportion + # ns = np.array(results[xkey]) + # te_proportion = 1.0 - results['tr_proportion'] + # test_sizes = ns*te_proportion line_styles = func_plot_fmt_map() method_labels = get_func2label_map() - - func_names = [f.__name__ for f in results['method_job_funcs'] ] + + func_names = [f.__name__ for f in results["method_job_funcs"]] plotted_methods = [] - for i in range(n_methods): - te_proportion = 1.0 - results['tr_proportion'] + for i in range(n_methods): + te_proportion = 1.0 - results["tr_proportion"] fmt = line_styles[func_names[i]] - #plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) + # plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) method_label = method_labels[func_names[i]] plotted_methods.append(method_label) plt.plot(xvalues, mean_rejs[:, i], fmt, label=method_label) - ''' + """ else: # h0 is true z = stats.norm.isf( (1-confidence)/2.0) @@ -189,65 +192,73 @@ def rej_accessor(jr): conf_iv = z*(phat*(1-phat)/repeats)**0.5 #plt.errorbar(test_sizes, phat, conf_iv, fmt=line_styles[i], label=method_labels[i]) plt.plot(test_sizes, mean_rejs[:, i], line_styles[i], label=method_labels[i]) - ''' - - ylabel = 'Rejection rate' + """ + + ylabel = "Rejection rate" plt.ylabel(ylabel) plt.xlabel(xlabel) - plt.xticks(np.hstack((xvalues) )) - - alpha = results['alpha'] - plt.legend(loc='best') - title = '%s. %d trials. $\\alpha$ = %.2g.'%( results['prob_label'], - repeats, alpha) if func_title is None else func_title(results) + plt.xticks(np.hstack((xvalues))) + + alpha = results["alpha"] + plt.legend(loc="best") + title = ( + "%s. %d trials. $\\alpha$ = %.2g." % (results["prob_label"], repeats, alpha) + if func_title is None + else func_title(results) + ) plt.title(title) plt.grid() if return_plot_values: - return results, PlotValues(xvalues=xvalues, methods=plotted_methods, - plot_matrix=mean_rejs.T) + return results, PlotValues( + xvalues=xvalues, methods=plotted_methods, plot_matrix=mean_rejs.T + ) else: return results - + def plot_runtime(ex, fname, func_xvalues, xlabel, func_title=None): results = glo.ex_load_result(ex, fname) - value_accessor = lambda job_results: job_results['time_secs'] + value_accessor = lambda job_results: job_results["time_secs"] vf_pval = np.vectorize(value_accessor) - # results['job_results'] is a dictionary: + # results['job_results'] is a dictionary: # {'test_result': (dict from running perform_test(te) '...':..., } - times = vf_pval(results['job_results']) - repeats, _, n_methods = results['job_results'].shape + times = vf_pval(results["job_results"]) + repeats, _, n_methods = results["job_results"].shape time_avg = np.mean(times, axis=0) time_std = np.std(times, axis=0) xvalues = func_xvalues(results) - #ns = np.array(results[xkey]) - #te_proportion = 1.0 - results['tr_proportion'] - #test_sizes = ns*te_proportion + # ns = np.array(results[xkey]) + # te_proportion = 1.0 - results['tr_proportion'] + # test_sizes = ns*te_proportion line_styles = func_plot_fmt_map() method_labels = get_func2label_map() - - func_names = [f.__name__ for f in results['method_job_funcs'] ] - for i in range(n_methods): - te_proportion = 1.0 - results['tr_proportion'] + + func_names = [f.__name__ for f in results["method_job_funcs"]] + for i in range(n_methods): + te_proportion = 1.0 - results["tr_proportion"] fmt = line_styles[func_names[i]] - #plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) + # plt.errorbar(ns*te_proportion, mean_rejs[:, i], std_pvals[:, i]) method_label = method_labels[func_names[i]] - plt.errorbar(xvalues, time_avg[:, i], yerr=time_std[:,i], fmt=fmt, - label=method_label) - - ylabel = 'Time (s)' + plt.errorbar( + xvalues, time_avg[:, i], yerr=time_std[:, i], fmt=fmt, label=method_label + ) + + ylabel = "Time (s)" plt.ylabel(ylabel) plt.xlabel(xlabel) plt.xlim([np.min(xvalues), np.max(xvalues)]) - plt.xticks( xvalues, xvalues ) - plt.legend(loc='best') - plt.gca().set_yscale('log') - title = '%s. %d trials. '%( results['prob_label'], - repeats ) if func_title is None else func_title(results) + plt.xticks(xvalues, xvalues) + plt.legend(loc="best") + plt.gca().set_yscale("log") + title = ( + "%s. %d trials. " % (results["prob_label"], repeats) + if func_title is None + else func_title(results) + ) plt.title(title) - #plt.grid() + # plt.grid() return results @@ -255,15 +266,15 @@ def box_meshgrid(func, xbound, ybound, nx=50, ny=50): """ Form a meshed grid (to be used with a contour plot) on a box specified by xbound, ybound. Evaluate the grid with [func]: (n x 2) -> n. - + - xbound: a tuple (xmin, xmax) - ybound: a tuple (ymin, ymax) - nx: number of points to evluate in the x direction - + return XX, YY, ZZ where XX is a 2D nd-array of size nx x ny """ - - # form a test location grid to try + + # form a test location grid to try minx, maxx = xbound miny, maxy = ybound loc0_cands = np.linspace(minx, maxx, nx) @@ -272,26 +283,27 @@ def box_meshgrid(func, xbound, ybound, nx=50, ny=50): # nd1 x nd0 x 2 loc3d = np.dstack((lloc0, lloc1)) # #candidates x 2 - all_loc2s = np.reshape(loc3d, (-1, 2) ) + all_loc2s = np.reshape(loc3d, (-1, 2)) # evaluate the function func_grid = func(all_loc2s) func_grid = np.reshape(func_grid, (ny, nx)) - + assert lloc0.shape[0] == ny assert lloc0.shape[1] == nx assert np.all(lloc0.shape == lloc1.shape) - + return lloc0, lloc1, func_grid + def get_density_cmap(): """ Return a colormap for plotting the model density p. - Red = high density + Red = high density white = very low density. Varying from white (low) to red (high). """ # Add completely white color to Reds colormap in Matplotlib - list_colors = plt.cm.datad['Reds'] + list_colors = plt.cm.datad["Reds"] list_colors = list(list_colors) list_colors.insert(0, (1, 1, 1)) list_colors.insert(0, (1, 1, 1)) diff --git a/sbibm/third_party/kgof/test/test_density.py b/sbibm/third_party/kgof/test/test_density.py index 3f7cf057..cf135631 100644 --- a/sbibm/third_party/kgof/test/test_density.py +++ b/sbibm/third_party/kgof/test/test_density.py @@ -2,26 +2,26 @@ Module for testing density module. """ -__author__ = 'wittawat' +__author__ = "wittawat" + +import unittest -import numpy as np import matplotlib.pyplot as plt +import numpy as np +import scipy.stats as stats + import sbibm.third_party.kgof.data as data import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel -import sbibm.third_party.kgof.goftest as gof import sbibm.third_party.kgof.glo as glo -import scipy.stats as stats - -import unittest +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util class TestIsotropicNormal(unittest.TestCase): def setUp(self): pass - def test_log_den(self): n = 7 with util.NumpySeedContext(seed=16): @@ -32,9 +32,9 @@ def test_log_den(self): isonorm = density.IsotropicNormal(mean, variance) log_dens = isonorm.log_den(X) - my_log_dens = -np.sum((X-mean)**2, 1)/(2.0*variance) + my_log_dens = -np.sum((X - mean) ** 2, 1) / (2.0 * variance) - # check correctness + # check correctness np.testing.assert_almost_equal(log_dens, my_log_dens) def test_grad_log(self): @@ -43,13 +43,13 @@ def test_grad_log(self): for d in [4, 1]: variance = 1.2 mean = np.random.randn(d) + 1 - X = np.random.rand(n, d) - 2 + X = np.random.rand(n, d) - 2 isonorm = density.IsotropicNormal(mean, variance) grad_log = isonorm.grad_log(X) - my_grad_log = -(X-mean)/variance + my_grad_log = -(X - mean) / variance - # check correctness + # check correctness np.testing.assert_almost_equal(grad_log, my_grad_log) def tearDown(self): @@ -57,15 +57,16 @@ def tearDown(self): class TestGaussianMixture(unittest.TestCase): - def test_multivariate_normal_density(self): for i in range(4): - with util.NumpySeedContext(seed=i+8): + with util.NumpySeedContext(seed=i + 8): d = i + 2 - cov = stats.wishart(df=10+d, scale=np.eye(d)).rvs(size=1) + cov = stats.wishart(df=10 + d, scale=np.eye(d)).rvs(size=1) mean = np.random.randn(d) X = np.random.randn(11, d) - den_estimate = density.GaussianMixture.multivariate_normal_density(mean, cov, X) + den_estimate = density.GaussianMixture.multivariate_normal_density( + mean, cov, X + ) mnorm = stats.multivariate_normal(mean=mean, cov=cov) den_truth = mnorm.pdf(X) @@ -73,6 +74,5 @@ def test_multivariate_normal_density(self): np.testing.assert_almost_equal(den_estimate, den_truth) -if __name__ == '__main__': - unittest.main() - +if __name__ == "__main__": + unittest.main() diff --git a/sbibm/third_party/kgof/test/test_goftest.py b/sbibm/third_party/kgof/test/test_goftest.py index b5981615..790bad08 100644 --- a/sbibm/third_party/kgof/test/test_goftest.py +++ b/sbibm/third_party/kgof/test/test_goftest.py @@ -2,20 +2,21 @@ Module for testing goftest module. """ -__author__ = 'wittawat' +__author__ = "wittawat" +import unittest + +import matplotlib.pyplot as plt import numpy as np import numpy.testing as testing -import matplotlib.pyplot as plt +import scipy.stats as stats + import sbibm.third_party.kgof.data as data import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel -import sbibm.third_party.kgof.goftest as gof import sbibm.third_party.kgof.glo as glo -import scipy.stats as stats - -import unittest +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util class TestFSSD(unittest.TestCase): @@ -36,27 +37,27 @@ def test_basic(self): isonorm = density.IsotropicNormal(mean, variance) # only one dimension of the mean is shifted - #draw_mean = mean + np.hstack((1, np.zeros(d-1))) - draw_mean = mean +0 + # draw_mean = mean + np.hstack((1, np.zeros(d-1))) + draw_mean = mean + 0 draw_variance = variance + 1 - X = util.randn(n, d, seed=seed)*np.sqrt(draw_variance) + draw_mean + X = util.randn(n, d, seed=seed) * np.sqrt(draw_variance) + draw_mean dat = data.Data(X) # Test for J in [1, 3]: - sig2 = util.meddistance(X, subsample=1000)**2 + sig2 = util.meddistance(X, subsample=1000) ** 2 k = kernel.KGauss(sig2) # random test locations - V = util.fit_gaussian_draw(X, J, seed=seed+1) + V = util.fit_gaussian_draw(X, J, seed=seed + 1) null_sim = gof.FSSDH0SimCovObs(n_simulate=200, seed=3) fssd = gof.FSSD(isonorm, k, V, null_sim=null_sim, alpha=alpha) tresult = fssd.perform_test(dat, return_simulated_stats=True) # assertions - self.assertGreaterEqual(tresult['pvalue'], 0) - self.assertLessEqual(tresult['pvalue'], 1) + self.assertGreaterEqual(tresult["pvalue"], 0) + self.assertLessEqual(tresult["pvalue"], 1) def test_optimized_fssd(self): """ @@ -64,38 +65,34 @@ def test_optimized_fssd(self): """ seed = 4 # sample size - n = 179 + n = 179 alpha = 0.01 for d in [1, 3]: mean = np.zeros(d) variance = 1.0 p = density.IsotropicNormal(mean, variance) # Mean difference. obvious reject - ds = data.DSIsotropicNormal(mean+4, variance+0) + ds = data.DSIsotropicNormal(mean + 4, variance + 0) dat = ds.sample(n, seed=seed) - # test + # test for J in [1, 4]: - opts = { - 'reg': 1e-2, - 'max_iter': 10, - 'tol_fun':1e-3, - 'disp':False - } - tr, te = dat.split_tr_te(tr_proportion=0.3, seed=seed+1) + opts = {"reg": 1e-2, "max_iter": 10, "tol_fun": 1e-3, "disp": False} + tr, te = dat.split_tr_te(tr_proportion=0.3, seed=seed + 1) Xtr = tr.X - gwidth0 = util.meddistance(Xtr, subsample=1000)**2 + gwidth0 = util.meddistance(Xtr, subsample=1000) ** 2 # random test locations - V0 = util.fit_gaussian_draw(Xtr, J, seed=seed+1) - V_opt, gw_opt, opt_result = \ - gof.GaussFSSD.optimize_locs_widths(p, tr, gwidth0, V0, **opts) + V0 = util.fit_gaussian_draw(Xtr, J, seed=seed + 1) + V_opt, gw_opt, opt_result = gof.GaussFSSD.optimize_locs_widths( + p, tr, gwidth0, V0, **opts + ) # construct a test k_opt = kernel.KGauss(gw_opt) null_sim = gof.FSSDH0SimCovObs(n_simulate=2000, seed=10) fssd_opt = gof.FSSD(p, k_opt, V_opt, null_sim=null_sim, alpha=alpha) fssd_opt_result = fssd_opt.perform_test(te, return_simulated_stats=True) - assert fssd_opt_result['h0_rejected'] + assert fssd_opt_result["h0_rejected"] def test_auto_init_opt_fssd(self): """ @@ -103,34 +100,30 @@ def test_auto_init_opt_fssd(self): """ seed = 5 # sample size - n = 191 + n = 191 alpha = 0.01 for d in [1, 4]: mean = np.zeros(d) variance = 1.0 p = density.IsotropicNormal(mean, variance) # Mean difference. obvious reject - ds = data.DSIsotropicNormal(mean+4, variance+0) + ds = data.DSIsotropicNormal(mean + 4, variance + 0) dat = ds.sample(n, seed=seed) - # test + # test for J in [1, 3]: - opts = { - 'reg': 1e-2, - 'max_iter': 10, - 'tol_fun': 1e-3, - 'disp':False - } - tr, te = dat.split_tr_te(tr_proportion=0.3, seed=seed+1) + opts = {"reg": 1e-2, "max_iter": 10, "tol_fun": 1e-3, "disp": False} + tr, te = dat.split_tr_te(tr_proportion=0.3, seed=seed + 1) - V_opt, gw_opt, opt_result = \ - gof.GaussFSSD.optimize_auto_init(p, tr, J, **opts) + V_opt, gw_opt, opt_result = gof.GaussFSSD.optimize_auto_init( + p, tr, J, **opts + ) # construct a test k_opt = kernel.KGauss(gw_opt) null_sim = gof.FSSDH0SimCovObs(n_simulate=2000, seed=10) fssd_opt = gof.FSSD(p, k_opt, V_opt, null_sim=null_sim, alpha=alpha) fssd_opt_result = fssd_opt.perform_test(te, return_simulated_stats=True) - assert fssd_opt_result['h0_rejected'] + assert fssd_opt_result["h0_rejected"] def test_ustat_h1_mean_variance(self): seed = 20 @@ -144,16 +137,16 @@ def test_ustat_h1_mean_variance(self): draw_mean = mean + 2 draw_variance = variance + 1 - X = util.randn(n, d, seed=seed)*np.sqrt(draw_variance) + draw_mean + X = util.randn(n, d, seed=seed) * np.sqrt(draw_variance) + draw_mean dat = data.Data(X) # Test for J in [1, 3]: - sig2 = util.meddistance(X, subsample=1000)**2 + sig2 = util.meddistance(X, subsample=1000) ** 2 k = kernel.KGauss(sig2) # random test locations - V = util.fit_gaussian_draw(X, J, seed=seed+1) + V = util.fit_gaussian_draw(X, J, seed=seed + 1) null_sim = gof.FSSDH0SimCovObs(n_simulate=200, seed=3) fssd = gof.FSSD(isonorm, k, V, null_sim=null_sim, alpha=alpha) @@ -169,13 +162,15 @@ def test_ustat_h1_mean_variance(self): def tearDown(self): pass + # end class TestFSSD + class TestSteinWitness(unittest.TestCase): def test_basic(self): d = 3 p = density.IsotropicNormal(mean=np.zeros(d), variance=3.0) - q = density.IsotropicNormal(mean=np.zeros(d)+2, variance=3.0) + q = density.IsotropicNormal(mean=np.zeros(d) + 2, variance=3.0) k = kernel.KGauss(2.0) ds = q.get_datasource() @@ -185,14 +180,14 @@ def test_basic(self): witness = gof.SteinWitness(p, k, dat) # points to evaluate the witness J = 4 - V = np.random.randn(J, d)*2 + V = np.random.randn(J, d) * 2 evals = witness(V) testing.assert_equal(evals.shape, (J, d)) -# end class TestSteinWitness +# end class TestSteinWitness -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/sbibm/third_party/kgof/test/test_kernel.py b/sbibm/third_party/kgof/test/test_kernel.py index 455d4cbf..3c9fafed 100644 --- a/sbibm/third_party/kgof/test/test_kernel.py +++ b/sbibm/third_party/kgof/test/test_kernel.py @@ -2,21 +2,22 @@ Module for testing kernel module. """ -__author__ = 'wittawat' +__author__ = "wittawat" + +import unittest import autograd import autograd.numpy as np import matplotlib.pyplot as plt +import numpy.testing as testing +import scipy.stats as stats + import sbibm.third_party.kgof.data as data import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.util as util -import sbibm.third_party.kgof.kernel as kernel -import sbibm.third_party.kgof.goftest as gof import sbibm.third_party.kgof.glo as glo -import scipy.stats as stats -import numpy.testing as testing - -import unittest +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util class TestKGauss(unittest.TestCase): @@ -31,21 +32,21 @@ def test_basic(self): n = 10 d = 3 with util.NumpySeedContext(seed=29): - X = np.random.randn(n, d)*3 + X = np.random.randn(n, d) * 3 k = kernel.KGauss(sigma2=1) K = k.eval(X, X) self.assertEqual(K.shape, (n, n)) - self.assertTrue(np.all(K >= 0-1e-6)) - self.assertTrue(np.all(K <= 1+1e-6), 'K not bounded by 1') + self.assertTrue(np.all(K >= 0 - 1e-6)) + self.assertTrue(np.all(K <= 1 + 1e-6), "K not bounded by 1") def test_pair_gradX_Y(self): # sample n = 11 d = 3 with util.NumpySeedContext(seed=20): - X = np.random.randn(n, d)*4 - Y = np.random.randn(n, d)*2 + X = np.random.randn(n, d) * 4 + Y = np.random.randn(n, d) * 2 k = kernel.KGauss(sigma2=2.1) # n x d pair_grad = k.pair_gradX_Y(X, Y) @@ -56,26 +57,24 @@ def test_pair_gradX_Y(self): testing.assert_almost_equal(pair_grad, loop_grad) - def test_gradX_y(self): n = 10 with util.NumpySeedContext(seed=10): for d in [1, 3]: - y = np.random.randn(d)*2 - X = np.random.rand(n, d)*3 + y = np.random.randn(d) * 2 + X = np.random.rand(n, d) * 3 sigma2 = 1.3 k = kernel.KGauss(sigma2=sigma2) # n x d G = k.gradX_y(X, y) - # check correctness + # check correctness K = k.eval(X, y[np.newaxis, :]) - myG = -K/sigma2*(X-y) + myG = -K / sigma2 * (X - y) self.assertEqual(G.shape, myG.shape) testing.assert_almost_equal(G, myG) - def test_gradXY_sum(self): n = 11 with util.NumpySeedContext(seed=12): @@ -89,21 +88,19 @@ def test_gradXY_sum(self): K = k.eval(X, X) for i in range(n): for j in range(n): - diffi2 = np.sum( (X[i, :] - X[j, :])**2 ) - #myG[i, j] = -diffi2*K[i, j]/(sigma2**2)+ d*K[i, j]/sigma2 - myG[i, j] = K[i, j]/sigma2*(d - diffi2/sigma2) + diffi2 = np.sum((X[i, :] - X[j, :]) ** 2) + # myG[i, j] = -diffi2*K[i, j]/(sigma2**2)+ d*K[i, j]/sigma2 + myG[i, j] = K[i, j] / sigma2 * (d - diffi2 / sigma2) - # check correctness + # check correctness G = k.gradXY_sum(X, X) self.assertEqual(G.shape, myG.shape) testing.assert_almost_equal(G, myG) - def tearDown(self): pass -if __name__ == '__main__': - unittest.main() - +if __name__ == "__main__": + unittest.main() diff --git a/sbibm/third_party/kgof/util.py b/sbibm/third_party/kgof/util.py index 4420e834..278e5fcf 100644 --- a/sbibm/third_party/kgof/util.py +++ b/sbibm/third_party/kgof/util.py @@ -1,29 +1,30 @@ """A module containing convenient methods for general machine learning""" -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals -from __future__ import absolute_import - -from builtins import zip -from builtins import int -from builtins import range +from __future__ import absolute_import, division, print_function, unicode_literals + +from builtins import int, range, zip + from future import standard_library + standard_library.install_aliases() -from past.utils import old_div from builtins import object -__author__ = 'wittawat' + +from past.utils import old_div + +__author__ = "wittawat" + +import time import autograd.numpy as np -import time + class ContextTimer(object): """ - A class used to time an execution of a code snippet. + A class used to time an execution of a code snippet. Use it with with .... as ... - For example, + For example, with ContextTimer() as t: - # do something + # do something time_spent = t.secs From https://www.huyng.com/posts/python-performance-analysis @@ -38,19 +39,22 @@ def __enter__(self): def __exit__(self, *args): self.end = time.time() - self.secs = self.end - self.start + self.secs = self.end - self.start if self.verbose: - print('elapsed time: %f ms' % (self.secs*1000)) + print("elapsed time: %f ms" % (self.secs * 1000)) + # end class ContextTimer + class NumpySeedContext(object): """ A context manager to reset the random seed by numpy.random.seed(..). - Set the seed back at the end of the block. + Set the seed back at the end of the block. """ + def __init__(self, seed): - self.seed = seed + self.seed = seed def __enter__(self): rstate = np.random.get_state() @@ -61,8 +65,10 @@ def __enter__(self): def __exit__(self, *args): np.random.set_state(self.cur_state) + # end NumpySeedContext + class ChunkIterable(object): """ Construct an Iterable such that each call to its iterator returns a tuple @@ -70,11 +76,12 @@ class ChunkIterable(object): index of a chunk. f and t are (chunk_size) apart except for the last tuple which will always cover the rest. """ + def __init__(self, start, end, chunk_size): self.start = start self.end = end self.chunk_size = chunk_size - + def __iter__(self): s = self.start e = self.end @@ -84,33 +91,38 @@ def __iter__(self): L.append(e) return zip(L, L[1:]) + # end ChunkIterable + def constrain(val, min_val, max_val): return min(max_val, max(min_val, val)) + def dist_matrix(X, Y): """ Construct a pairwise Euclidean distance matrix of size X.shape[0] x Y.shape[0] """ - sx = np.sum(X**2, 1) - sy = np.sum(Y**2, 1) - D2 = sx[:, np.newaxis] - 2.0*X.dot(Y.T) + sy[np.newaxis, :] + sx = np.sum(X ** 2, 1) + sy = np.sum(Y ** 2, 1) + D2 = sx[:, np.newaxis] - 2.0 * X.dot(Y.T) + sy[np.newaxis, :] # to prevent numerical errors from taking sqrt of negative numbers D2[D2 < 0] = 0 D = np.sqrt(D2) return D + def dist2_matrix(X, Y): """ Construct a pairwise Euclidean distance **squared** matrix of size X.shape[0] x Y.shape[0] """ - sx = np.sum(X**2, 1) - sy = np.sum(Y**2, 1) - D2 = sx[:, np.newaxis] - 2.0*np.dot(X, Y.T) + sy[np.newaxis, :] + sx = np.sum(X ** 2, 1) + sy = np.sum(Y ** 2, 1) + D2 = sx[:, np.newaxis] - 2.0 * np.dot(X, Y.T) + sy[np.newaxis, :] return D2 + def meddistance(X, subsample=None, mean_on_fail=True): """ Compute the median of pairwise distances (not distance squared) of points @@ -120,7 +132,7 @@ def meddistance(X, subsample=None, mean_on_fail=True): ---------- X : n x d numpy array mean_on_fail: True/False. If True, use the mean when the median distance is 0. - This can happen especially, when the data are discrete e.g., 0/1, and + This can happen especially, when the data are discrete e.g., 0/1, and there are more slightly more 0 than 1. In this case, the m Return @@ -149,19 +161,21 @@ def meddistance(X, subsample=None, mean_on_fail=True): def is_real_num(X): - """return true if x is a real number. + """return true if x is a real number. Work for a numpy array as well. Return an array of the same dimension.""" + def each_elem_true(x): try: float(x) return not (np.isnan(x) or np.isinf(x)) except: return False + f = np.vectorize(each_elem_true) return f(X) - -def tr_te_indices(n, tr_proportion, seed=9282 ): + +def tr_te_indices(n, tr_proportion, seed=9282): """Get two logical vectors for indexing train/test points. Return (tr_ind, te_ind) @@ -170,13 +184,14 @@ def tr_te_indices(n, tr_proportion, seed=9282 ): np.random.seed(seed) Itr = np.zeros(n, dtype=bool) - tr_ind = np.random.choice(n, int(tr_proportion*n), replace=False) + tr_ind = np.random.choice(n, int(tr_proportion * n), replace=False) Itr[tr_ind] = True Ite = np.logical_not(Itr) np.random.set_state(rand_state) return (Itr, Ite) + def subsample_ind(n, k, seed=32): """ Return a list of indices to choose k out of n without replacement @@ -185,46 +200,48 @@ def subsample_ind(n, k, seed=32): ind = np.random.choice(n, k, replace=False) return ind + def subsample_rows(X, k, seed=29): """ Subsample k rows from the matrix X. """ n = X.shape[0] if k > n: - raise ValueError('k exceeds the number of rows.') + raise ValueError("k exceeds the number of rows.") ind = subsample_ind(n, k, seed=seed) return X[ind, :] - + def fit_gaussian_draw(X, J, seed=28, reg=1e-7, eig_pow=1.0): """ - Fit a multivariate normal to the data X (n x d) and draw J points - from the fit. + Fit a multivariate normal to the data X (n x d) and draw J points + from the fit. - reg: regularizer to use with the covariance matrix - - eig_pow: raise eigenvalues of the covariance matrix to this power to construct - a new covariance matrix before drawing samples. Useful to shrink the spread + - eig_pow: raise eigenvalues of the covariance matrix to this power to construct + a new covariance matrix before drawing samples. Useful to shrink the spread of the variance. """ with NumpySeedContext(seed=seed): d = X.shape[1] mean_x = np.mean(X, 0) cov_x = np.cov(X.T) - if d==1: + if d == 1: cov_x = np.array([[cov_x]]) [evals, evecs] = np.linalg.eig(cov_x) evals = np.maximum(0, np.real(evals)) assert np.all(np.isfinite(evals)) evecs = np.real(evecs) - shrunk_cov = evecs.dot(np.diag(evals**eig_pow)).dot(evecs.T) + reg*np.eye(d) + shrunk_cov = evecs.dot(np.diag(evals ** eig_pow)).dot(evecs.T) + reg * np.eye(d) V = np.random.multivariate_normal(mean_x, shrunk_cov, J) return V + def bound_by_data(Z, Data): """ - Determine lower and upper bound for each dimension from the Data, and project + Determine lower and upper bound for each dimension from the Data, and project Z so that all points in Z live in the bounds. - Z: m x d + Z: m x d Data: n x d Return a projected Z of size m x d. @@ -243,8 +260,8 @@ def bound_by_data(Z, Data): def one_of_K_code(arr): """ Make a one-of-K coding out of the numpy array. - For example, if arr = ([0, 1, 0, 2]), then return a 2d array of the form - [[1, 0, 0], + For example, if arr = ([0, 1, 0, 2]), then return a 2d array of the form + [[1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]] @@ -254,17 +271,20 @@ def one_of_K_code(arr): nu = len(U) X = np.zeros((n, nu)) for i, u in enumerate(U): - Ii = np.where( np.abs(arr - u) < 1e-8 ) - #ni = len(Ii) + Ii = np.where(np.abs(arr - u) < 1e-8) + # ni = len(Ii) X[Ii[0], i] = 1 return X + def fullprint(*args, **kwargs): "https://gist.github.com/ZGainsforth/3a306084013633c52881" from pprint import pprint + import numpy + opt = numpy.get_printoptions() - numpy.set_printoptions(threshold='nan') + numpy.set_printoptions(threshold="nan") pprint(*args, **kwargs) numpy.set_printoptions(**opt) @@ -273,10 +293,11 @@ def standardize(X): mx = np.mean(X, 0) stdx = np.std(X, axis=0) # Assume standard deviations are not 0 - Zx = old_div((X-mx),stdx) + Zx = old_div((X - mx), stdx) assert np.all(np.isfinite(Zx)) return Zx + def outer_rows(X, Y): """ Compute the outer product of each row in X, and Y. @@ -289,18 +310,20 @@ def outer_rows(X, Y): # Matlab way to do this. According to Jonathan Huggins, this is not # efficient. Use einsum instead. See below. - #n, dx = X.shape - #dy = Y.shape[1] - #X_col_rep = X[:, np.tile(range(dx), (dy, 1)).T.reshape(-1) ] - #Y_tile = np.tile(Y, (1, dx)) - #Z = X_col_rep*Y_tile - #return np.reshape(Z, (n, dx, dy)) - return np.einsum('ij,ik->ijk', X, Y) + # n, dx = X.shape + # dy = Y.shape[1] + # X_col_rep = X[:, np.tile(range(dx), (dy, 1)).T.reshape(-1) ] + # Y_tile = np.tile(Y, (1, dx)) + # Z = X_col_rep*Y_tile + # return np.reshape(Z, (n, dx, dy)) + return np.einsum("ij,ik->ijk", X, Y) + def randn(m, n, seed=3): with NumpySeedContext(seed=seed): return np.random.randn(m, n) + def matrix_inner_prod(A, B): """ Compute the matrix inner product = trace(A^T * B). @@ -309,14 +332,16 @@ def matrix_inner_prod(A, B): assert A.shape[1] == B.shape[1] return A.reshape(-1).dot(B.reshape(-1)) + def get_classpath(obj): """ - Return the full module and class path of the obj. For instance, + Return the full module and class path of the obj. For instance, kgof.density.IsotropicNormal Return a string. """ - return obj.__class__.__module__ + '.' + obj.__class__.__name__ + return obj.__class__.__module__ + "." + obj.__class__.__name__ + def merge_dicts(*dict_args): """ @@ -329,4 +354,3 @@ def merge_dicts(*dict_args): for dictionary in dict_args: result.update(dictionary) return result - diff --git a/sbibm/third_party/torch_two_sample/main.py b/sbibm/third_party/torch_two_sample/main.py index ba63b4cb..319f5c87 100644 --- a/sbibm/third_party/torch_two_sample/main.py +++ b/sbibm/third_party/torch_two_sample/main.py @@ -77,15 +77,15 @@ def pdist(sample_1, sample_2, norm=2, eps=1e-5): def logsumexp(x, dim): """Compute the log-sum-exp in a numerically stable way. - Arguments - --------- - x : :class:`torch:torch.Tensor` - dim : int - The dimension along wich the operation should be computed. - Returns - -------- - :class:`torch:torch.Tensor` - The dimension along which the sum is done is not squeezed. + Arguments + --------- + x : :class:`torch:torch.Tensor` + dim : int + The dimension along wich the operation should be computed. + Returns + -------- + :class:`torch:torch.Tensor` + The dimension along which the sum is done is not squeezed. """ x_max = torch.max(x, dim, keepdim=True)[0] return ( @@ -337,8 +337,7 @@ def forward(ctx, weights): class KSmallest(Function): - """Return an indicator vector holing the smallest k elements in each row. - """ + """Return an indicator vector holing the smallest k elements in each row.""" @staticmethod def forward(ctx, k, matrix): diff --git a/sbibm/utils/io.py b/sbibm/utils/io.py index 130b5495..d40b51df 100644 --- a/sbibm/utils/io.py +++ b/sbibm/utils/io.py @@ -9,10 +9,10 @@ def get_float_from_csv( - path: Union[str, Path], dtype: type = np.float32, + path: Union[str, Path], + dtype: type = np.float32, ): - """Get a single float from a csv file - """ + """Get a single float from a csv file""" with open(path, "r") as fh: return np.loadtxt(fh).astype(dtype) @@ -38,8 +38,7 @@ def get_results( def get_tensor_from_csv( path: Union[str, Path], dtype: type = np.float32, atleast_2d: bool = True ) -> torch.Tensor: - """Get `torch.Tensor` from csv at given path - """ + """Get `torch.Tensor` from csv at given path""" device = get_default_device() if atleast_2d: @@ -53,8 +52,7 @@ def get_tensor_from_csv( def get_ndarray_from_csv( path: Union[str, Path], dtype: type = np.float32, atleast_2d: bool = True ) -> np.ndarray: - """Get `np.ndarray` from csv at given path - """ + """Get `np.ndarray` from csv at given path""" if atleast_2d: return np.atleast_2d(pd.read_csv(path)).astype(dtype) else: @@ -62,12 +60,15 @@ def get_ndarray_from_csv( def save_float_to_csv( - path: Union[str, Path], data: float, dtype: type = np.float32, + path: Union[str, Path], + data: float, + dtype: type = np.float32, ): - """Save a single float to a csv file - """ + """Save a single float to a csv file""" np.savetxt( - path, np.asarray(data).reshape(-1).astype(np.float32), delimiter=",", + path, + np.asarray(data).reshape(-1).astype(np.float32), + delimiter=",", ) @@ -78,8 +79,8 @@ def save_tensor_to_csv( dtype: type = np.float32, index: bool = False, ): - """Save torch.Tensor to csv at given path - """ - pd.DataFrame(data.cpu().numpy().astype(dtype), columns=columns,).to_csv( - path, index=index - ) + """Save torch.Tensor to csv at given path""" + pd.DataFrame( + data.cpu().numpy().astype(dtype), + columns=columns, + ).to_csv(path, index=index) diff --git a/sbibm/utils/kde.py b/sbibm/utils/kde.py index 55695796..09f45ebb 100644 --- a/sbibm/utils/kde.py +++ b/sbibm/utils/kde.py @@ -5,6 +5,7 @@ from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KernelDensity from torch import distributions as dist + from sbibm.utils.torch import get_log_abs_det_jacobian transform_types = Optional[ diff --git a/sbibm/utils/logging.py b/sbibm/utils/logging.py index c19f90cd..f09d1c8c 100644 --- a/sbibm/utils/logging.py +++ b/sbibm/utils/logging.py @@ -6,7 +6,7 @@ def get_logger( name: str, level: Optional[int] = logging.INFO, console_logging: bool = True ) -> logging.Logger: """Gets logger with given name, while setting level and optionally adding handler - + Note: Logging to `sys.stdout` for Jupyter as done in this Gist https://gist.github.com/joshbode/58fac7ababc700f51e2a9ecdebe563ad @@ -14,7 +14,7 @@ def get_logger( name: Name of logger level: Log level console_logging: Whether or not to log to console - + Returns: Logger """ diff --git a/sbibm/utils/tensorboard.py b/sbibm/utils/tensorboard.py index b137fc68..d2392936 100644 --- a/sbibm/utils/tensorboard.py +++ b/sbibm/utils/tensorboard.py @@ -18,10 +18,10 @@ def tb_plot_posterior( def tb_make_writer( - logger: logging.Logger = None, basepath: str = "tensorboard", + logger: logging.Logger = None, + basepath: str = "tensorboard", ) -> (SummaryWriter, Callable): - """Builds tensorboard summary writers - """ + """Builds tensorboard summary writers""" log_dir = Path(f"{basepath}/summary") if log_dir.exists() and log_dir.is_dir(): shutil.rmtree(log_dir) @@ -41,7 +41,10 @@ def close_fn(): class TensorboardHandler(logging.Handler): def __init__( - self, writer: SummaryWriter, *args: Any, **kwargs: Any, + self, + writer: SummaryWriter, + *args: Any, + **kwargs: Any, ): super().__init__(*args, **kwargs) self.writer = writer diff --git a/sbibm/visualisation/correlation.py b/sbibm/visualisation/correlation.py index 7a2124e4..afc8f1a1 100644 --- a/sbibm/visualisation/correlation.py +++ b/sbibm/visualisation/correlation.py @@ -16,8 +16,7 @@ def fig_correlation( keywords: Dict[str, Any] = {}, style: Dict[str, Any] = {}, ): - """Plots correlation matrices - """ + """Plots correlation matrices""" keywords["sparse"] = True keywords["limits"] = [0.0, 1.0] keywords["font_size"] = 14 @@ -60,9 +59,11 @@ def fig_correlation( chart = den.correlation_matrix(df, metrics=metrics, **keywords) if title is not None: - chart = chart.properties(title={"text": [title],}).configure_title( - offset=10, orient="top", anchor="middle", dx=title_dx - ) + chart = chart.properties( + title={ + "text": [title], + } + ).configure_title(offset=10, orient="top", anchor="middle", dx=title_dx) if config == "manuscript": chart = chart.configure_text(font="Inter") diff --git a/sbibm/visualisation/metric.py b/sbibm/visualisation/metric.py index 9e5e1277..34558a26 100644 --- a/sbibm/visualisation/metric.py +++ b/sbibm/visualisation/metric.py @@ -23,7 +23,7 @@ def fig_metric( Args: df: Dataframe which at least has columns `algorithm`, `num_simulations` and - a column titled accordingly to `metric`. + a column titled accordingly to `metric`. metric: Metric to plot, should be a column in `df`. title: Title for plot title_dx: x-direction offset for title @@ -137,7 +137,10 @@ def fig_metric( if title is not None: chart = chart.properties(title={"text": [title],}).configure_title( - offset=10, orient="top", anchor="middle", dx=title_dx, + offset=10, + orient="top", + anchor="middle", + dx=title_dx, ) return chart diff --git a/sbibm/visualisation/posterior.py b/sbibm/visualisation/posterior.py index 6d7608ef..0a1e9b26 100644 --- a/sbibm/visualisation/posterior.py +++ b/sbibm/visualisation/posterior.py @@ -8,7 +8,6 @@ from sbibm.utils.io import get_ndarray_from_csv from sbibm.utils.torch import sample - _LIMITS_ = { "bernoulli_glm": [[-6.0, +6.0] for _ in range(10)], "bernoulli_glm_raw": [[-6.0, +6.0] for _ in range(10)], @@ -230,12 +229,18 @@ def fig_posterior( ) chart = den.pairplot( - df, field="sample", scatter_size=scatter_size, bar_opacity=0.4, **keywords, + df, + field="sample", + scatter_size=scatter_size, + bar_opacity=0.4, + **keywords, ) if title is not None: - chart = chart.properties(title={"text": [title],}).configure_title( - offset=10, orient="top", anchor="middle", dx=title_dx - ) + chart = chart.properties( + title={ + "text": [title], + } + ).configure_title(offset=10, orient="top", anchor="middle", dx=title_dx) return chart diff --git a/setup.py b/setup.py index f9918cdd..d42ef253 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ import sys from shutil import rmtree -from setuptools import find_packages, setup, Command +from setuptools import Command, find_packages, setup NAME = "sbibm" DESCRIPTION = "Simulation-based inference benchmark" diff --git a/tests/algorithms/test_baseline_posterior.py b/tests/algorithms/test_baseline_posterior.py index 5b1f730e..cba4f69f 100644 --- a/tests/algorithms/test_baseline_posterior.py +++ b/tests/algorithms/test_baseline_posterior.py @@ -10,17 +10,26 @@ "task_name,num_observation", [ (task_name, num_observation) - for task_name in ["gaussian_linear", "gaussian_linear_uniform", "slcp",] + for task_name in [ + "gaussian_linear", + "gaussian_linear_uniform", + "slcp", + ] for num_observation in range(1, 11) ], ) def test_posterior( - task_name, num_observation, num_samples=10_000, + task_name, + num_observation, + num_samples=10_000, ): task = sbibm.get_task(task_name) samples = run_posterior( - task=task, num_observation=num_observation, num_samples=num_samples, rerun=True, + task=task, + num_observation=num_observation, + num_samples=num_samples, + rerun=True, ) reference_samples = task.get_reference_posterior_samples( diff --git a/tests/metrics/test_c2st.py b/tests/metrics/test_c2st.py index 37077fbc..b0073012 100644 --- a/tests/metrics/test_c2st.py +++ b/tests/metrics/test_c2st.py @@ -1,6 +1,7 @@ import torch from sbibm.metrics import c2st + from .utils import sample_blobs_same diff --git a/tests/metrics/test_fssd.py b/tests/metrics/test_fssd.py index b3e92e91..440092f3 100644 --- a/tests/metrics/test_fssd.py +++ b/tests/metrics/test_fssd.py @@ -1,11 +1,5 @@ from pathlib import Path -import sbibm.third_party.kgof as kgof -import sbibm.third_party.kgof.data as data -import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.goftest as gof -import sbibm.third_party.kgof.kernel as kernel -import sbibm.third_party.kgof.util as util import matplotlib import matplotlib.pyplot as plt import numpy as np @@ -13,9 +7,15 @@ import scipy.stats as stats import torch import torch.distributions.transforms as transforms -from sbibm.third_party.kgof.density import UnnormalizedDensity import sbibm +import sbibm.third_party.kgof as kgof +import sbibm.third_party.kgof.data as data +import sbibm.third_party.kgof.density as density +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util +from sbibm.third_party.kgof.density import UnnormalizedDensity def test_fssd(): diff --git a/tests/metrics/test_ksd.py b/tests/metrics/test_ksd.py index c32d1ca5..d6bf84a8 100644 --- a/tests/metrics/test_ksd.py +++ b/tests/metrics/test_ksd.py @@ -1,11 +1,5 @@ from pathlib import Path -import sbibm.third_party.kgof -import sbibm.third_party.kgof.data as data -import sbibm.third_party.kgof.density as density -import sbibm.third_party.kgof.goftest as gof -import sbibm.third_party.kgof.kernel as kernel -import sbibm.third_party.kgof.util as util import matplotlib import matplotlib.pyplot as plt import numpy as np @@ -13,9 +7,15 @@ import scipy.stats as stats import torch import torch.distributions.transforms as transforms -from sbibm.third_party.kgof.density import UnnormalizedDensity import sbibm +import sbibm.third_party.kgof +import sbibm.third_party.kgof.data as data +import sbibm.third_party.kgof.density as density +import sbibm.third_party.kgof.goftest as gof +import sbibm.third_party.kgof.kernel as kernel +import sbibm.third_party.kgof.util as util +from sbibm.third_party.kgof.density import UnnormalizedDensity def test_ksd(): diff --git a/tests/metrics/test_mmd.py b/tests/metrics/test_mmd.py index 0264c49e..644d36b9 100644 --- a/tests/metrics/test_mmd.py +++ b/tests/metrics/test_mmd.py @@ -1,6 +1,7 @@ import torch from sbibm.metrics import mmd + from .utils import sample_blobs_same diff --git a/tests/utils/test_pyro.py b/tests/utils/test_pyro.py index fd86e7fe..add22b74 100644 --- a/tests/utils/test_pyro.py +++ b/tests/utils/test_pyro.py @@ -2,7 +2,6 @@ import torch import sbibm - from sbibm.utils.torch import get_log_abs_det_jacobian