validation set too large

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include requirements.txt

deepaugment/augmenter.py

@@ -0,0 +1,183 @@
+# (C) 2019 Baris Ozmen <hbaristr@gmail.com>
+
+import numpy as np
+from imgaug import augmenters as iaa
+
+
+def normalize(X):
+    return (X / 255.0).copy()
+
+
+def denormalize(X):
+    X_dn = (X * 255).copy()
+    return X_dn
+
+
+def transform(aug_type, magnitude, X):
+    if aug_type == "crop":
+        X_aug = iaa.Crop(px=(0, int(magnitude * 32))).augment_images(X)
+    elif aug_type == "gaussian-blur":
+        X_aug = iaa.GaussianBlur(sigma=(0, magnitude * 25.0)).augment_images(X)
+    elif aug_type == "rotate":
+        X_aug = iaa.Affine(rotate=(-180 * magnitude, 180 * magnitude)).augment_images(X)
+    elif aug_type == "shear":
+        X_aug = iaa.Affine(shear=(-90 * magnitude, 90 * magnitude)).augment_images(X)
+    elif aug_type == "translate-x":
+        X_aug = iaa.Affine(
+            translate_percent={"x": (-magnitude, magnitude), "y": (0, 0)}
+        ).augment_images(X)
+    elif aug_type == "translate-y":
+        X_aug = iaa.Affine(
+            translate_percent={"x": (0, 0), "y": (-magnitude, magnitude)}
+        ).augment_images(X)
+    elif aug_type == "horizontal-flip":
+        X_aug = iaa.Fliplr(magnitude).augment_images(X)
+    elif aug_type == "vertical-flip":
+        X_aug = iaa.Flipud(magnitude).augment_images(X)
+    elif aug_type == "sharpen":
+        X_aug = iaa.Sharpen(
+            alpha=(0, 1.0), lightness=(0.50, 5 * magnitude)
+        ).augment_images(X)
+    elif aug_type == "emboss":
+        X_aug = iaa.Emboss(
+            alpha=(0, 1.0), strength=(0.0, 20.0 * magnitude)
+        ).augment_images(X)
+    elif aug_type == "additive-gaussian-noise":
+        X_aug = iaa.AdditiveGaussianNoise(
+            loc=0, scale=(0.0, magnitude * 255), per_channel=0.5
+        ).augment_images(X)
+    elif aug_type == "dropout":
+        X_aug = iaa.Dropout(
+            (0.01, max(0.011, magnitude)), per_channel=0.5
+        ).augment_images(
+            X
+        )  # Dropout first argument should be smaller than second one
+    elif aug_type == "coarse-dropout":
+        X_aug = iaa.CoarseDropout(
+            (0.03, 0.15), size_percent=(0.30, np.log10(magnitude * 3)), per_channel=0.2
+        ).augment_images(X)
+    elif aug_type == "gamma-contrast":
+        X_norm = normalize(X)
+        X_aug_norm = iaa.GammaContrast(magnitude * 1.75).augment_images(
+            X_norm
+        )  # needs 0-1 values
+        X_aug = denormalize(X_aug_norm)
+    elif aug_type == "brighten":
+        X_aug = iaa.Add(
+            (int(-40 * magnitude), int(40 * magnitude)), per_channel=0.5
+        ).augment_images(
+            X
+        )  # brighten
+    elif aug_type == "invert":
+        X_aug = iaa.Invert(1.0).augment_images(X)  # magnitude not used
+    elif aug_type == "fog":
+        X_aug = iaa.Fog().augment_images(X)  # magnitude not used
+    elif aug_type == "clouds":
+        X_aug = iaa.Clouds().augment_images(X)  # magnitude not used
+    elif aug_type == "histogram-equalize":
+        X_aug = iaa.AllChannelsHistogramEqualization().augment_images(
+            X
+        )  # magnitude not used
+    elif aug_type == "super-pixels":  # deprecated
+        X_norm = normalize(X)
+        X_norm2 = (X_norm * 2) - 1
+        X_aug_norm2 = iaa.Superpixels(
+            p_replace=(0, magnitude), n_segments=(100, 100)
+        ).augment_images(X_norm2)
+        X_aug_norm = (X_aug_norm2 + 1) / 2
+        X_aug = denormalize(X_aug_norm)
+    elif aug_type == "perspective-transform":
+        X_norm = normalize(X)
+        X_aug_norm = iaa.PerspectiveTransform(
+            scale=(0.01, max(0.02, magnitude))
+        ).augment_images(
+            X_norm
+        )  # first scale param must be larger
+        np.clip(X_aug_norm, 0.0, 1.0, out=X_aug_norm)
+        X_aug = denormalize(X_aug_norm)
+    elif aug_type == "elastic-transform":  # deprecated
+        X_norm = normalize(X)
+        X_norm2 = (X_norm * 2) - 1
+        X_aug_norm2 = iaa.ElasticTransformation(
+            alpha=(0.0, max(0.5, magnitude * 300)), sigma=5.0
+        ).augment_images(X_norm2)
+        X_aug_norm = (X_aug_norm2 + 1) / 2
+        X_aug = denormalize(X_aug_norm)
+    elif aug_type == "add-to-hue-and-saturation":
+        X_aug = iaa.AddToHueAndSaturation(
+            (int(-45 * magnitude), int(45 * magnitude))
+        ).augment_images(X)
+    elif aug_type == "coarse-salt-pepper":
+        X_aug = iaa.CoarseSaltAndPepper(p=0.2, size_percent=magnitude).augment_images(X)
+    elif aug_type == "grayscale":
+        X_aug = iaa.Grayscale(alpha=(0.0, magnitude)).augment_images(X)
+    else:
+        raise ValueError
+    return X_aug
+
+
+def augment_by_policy(
+    X, y, *hyperparams
+):
+    """
+    """
+    portion = 1
+
+    assert (
+        portion >= 0.0 and portion <= 1.0
+    ), "portion argument value is out of accepted interval"
+
+    # convert data to 255 from normalized
+    _X = denormalize(X)
+
+    if portion == 1.0:
+        X_portion = _X
+        y_portion = y
+    else:
+        # get a portion of data
+        ix = np.random.choice(len(_X), int(len(_X) * portion), False)
+
+        X_portion = _X[ix].copy()
+        y_portion = y[ix].copy()
+
+    if X_portion.shape[0] == 0:
+        print("X_portion has zero size !!!")
+        nix = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        X_portion = _X[nix].copy()
+        y_portion = y[nix].copy()
+
+
+
+    all_X_portion_aug=None
+    all_y_portion = None
+    for i in range(0,len(hyperparams)-1,4):
+
+        # transform that portion
+        X_portion_aug = transform(hyperparams[i], hyperparams[i+1], X_portion)  # first transform
+
+        assert (
+            X_portion_aug.min() >= -0.1 and X_portion_aug.max() <= 255.1
+        ), "first transform is unvalid"
+        np.clip(X_portion_aug, 0, 255, out=X_portion_aug)
+
+        X_portion_aug = transform(
+            hyperparams[i+2], hyperparams[i+3], X_portion_aug
+        )  # second transform
+        assert (
+            X_portion_aug.min() >= -0.1 and X_portion_aug.max() <= 255.1
+        ), "second transform is unvalid"
+        np.clip(X_portion_aug, 0, 255, out=X_portion_aug)
+
+        if all_X_portion_aug is None:
+            all_X_portion_aug = X_portion_aug
+            all_y_portion = y_portion
+        else:
+            all_X_portion_aug = np.concatenate([all_X_portion_aug, X_portion_aug])
+            all_y_portion = np.concatenate([all_y_portion, y_portion])
+
+    augmented_data = {
+        "X_train": all_X_portion_aug / 255.0,
+        "y_train": all_y_portion,
+    }  # back to normalization
+
+    return augmented_data  # augmenteed data is mostly smaller than whole data

deepaugment/build_features.py

@@ -0,0 +1,153 @@
+# (C) 2019 Baris Ozmen <hbaristr@gmail.com>
+
+import sys
+import numpy as np
+import keras
+
+
+class DataOp:
+    @staticmethod
+    def load(dataset_name):
+        """Loads dataset from keras and returns a sample out of it
+
+        Args:
+            dataset_name (str):
+            training_set_size (int):
+            validation_set_size (int):
+        Returns:
+            dict: data, with keys X_train, Y_train, X_val, Y_val
+            list: input shape
+        """
+        if hasattr(keras.datasets, dataset_name):
+            (x_train, y_train), (x_val, y_val) = getattr(
+                keras.datasets, dataset_name
+            ).load_data()
+        else:
+            sys.exit(f"Unknown dataset {dataset_name}")
+
+        X = np.concatenate([x_train, x_val])
+        y = np.concatenate([y_train, y_val])
+        input_shape = x_train.shape[1:]
+
+        return X, y, input_shape
+
+    @staticmethod
+    def preprocess_normal(data):
+        # normalize images
+        data["X_train"] = data["X_train"].astype("float32") / 255
+        data["X_val"] = data["X_val"].astype("float32") / 255
+
+        # convert labels to categorical
+        data["y_train"] = keras.utils.to_categorical(data["y_train"])
+        data["y_val"] = keras.utils.to_categorical(data["y_val"])
+        return data
+
+    @staticmethod
+    def split_train_val_sets(X, y, train_set_size, val_set_size):
+        """Splits given images randomly into `train` and `val_seed` groups
+
+        val_seed -> is validation seed dataset, from where validation sets are sampled
+
+        Args:
+            X (numpy.array):
+            y (numpy.array):
+            train_set_size (int):
+            val_set_size (int):
+        return:
+            dict: dict with keys `X_train`, `y_train`, `X_val_seed`, `y_val_seed`
+        """
+        if train_set_size == None:
+            print(f"Using all training images")
+            train_set_size = len(X) - val_set_size
+        else:
+            print(f"Using {train_set_size} training images")
+
+        # reduce training dataset
+        ix = np.random.choice(len(X), train_set_size, False)
+        X_train = X[ix]
+        y_train = y[ix]
+
+        other_ix = set(np.arange(len(X))).difference(set(ix))
+        other_ix = list(other_ix)
+        X_val_seed = X[other_ix]
+        y_val_seed = y[other_ix]
+
+        print("split_train_val_sets")
+        print("x train")
+        print(X_train)
+        print("y train")
+        print(y_train)
+        print("X_val_seed")
+        print(X_val_seed)
+        print("y_val_seed")
+        print(y_val_seed)        
+        data = {
+            "X_train": X_train,
+            "y_train": y_train,
+            "X_val_seed": X_val_seed,
+            "y_val_seed": y_val_seed,
+        }
+        return data
+
+    @staticmethod
+    def preprocess(X, y, train_set_size, val_set_size=1000, model=None):
+        """Preprocess images by:
+            1. normalize to 0-1 range (divide by 255)
+            2. convert labels to categorical)
+
+        Args:
+            X (numpy.array):
+            y (numpy.array):
+            train_set_size (int):
+            val_set_size (int):
+
+        Returns:
+            dict: preprocessed data
+        """
+
+        if ( val_set_size > (0.5*train_set_size)):
+            val_set_size = int(0.1*train_set_size)
+        print(f"Using {val_set_size} validation images")
+
+        model_regression = "basicregression"
+        is_regression = (model == model_regression)
+        print("is_regression:"+str(is_regression))
+
+        print("split_train_val_sets y")
+        print(y)
+        data = DataOp.split_train_val_sets(X, y, train_set_size, val_set_size)
+
+        # normalize images
+        data["X_train"] = data["X_train"].astype("float32") / 255
+        data["X_val_seed"] = data["X_val_seed"].astype("float32") / 255
+
+        # convert labels to categorical
+        # if not a regression model
+        if is_regression==False:
+            data["y_train"] = keras.utils.to_categorical(data["y_train"])
+            data["y_val_seed"] = keras.utils.to_categorical(data["y_val_seed"])
+            print("data is categorical")
+            print(data["y_train"])
+        else: 
+            print("data is regression")
+            print(data["y_train"])
+            data["y_train"] = data["y_train"]
+            data["y_val_seed"] = data["y_val_seed"]
+
+        print("y_train:")
+        print(data["y_train"])
+        print("len y_train"+str(len(data["y_train"])))
+        print("len y_val_seed"+str(len(data["y_val_seed"])))
+        return data
+
+    @staticmethod
+    def sample_validation_set(data):
+        val_seed_size = len(data["X_val_seed"])
+        ix = np.random.choice(range(val_seed_size), min(val_seed_size, 1000), False)
+        X_val = data["X_val_seed"][ix].copy()
+        y_val = data["y_val_seed"][ix].copy()
+        return X_val, y_val
+
+    @staticmethod
+    def find_num_classes(data):
+        return data["y_train"].shape[1]