{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\nUnsupervised clustering with GMVAE\n==================================\n\nCredit: A Grigis\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Imports\nimport os\nimport sys\nif \"CI_MODE\" in os.environ:\n    sys.exit()\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfrom matplotlib.ticker import NullFormatter\nfrom sklearn import manifold\nfrom sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler\nimport torch\nimport torch.nn as nn\nimport pynet\nfrom pynet import NetParameters\nfrom pynet.datasets import DataManager\nfrom pynet.interfaces import GMVAENetClassifier\nfrom pynet.utils import setup_logging\n\n\n# Global parameters\nn_samples = 100\nn_classes = 3\nn_feats = 4\ntrue_lat_dims = 2\nfit_lat_dims = 5\nsnr = 10\nbatch_size = 10\nadam_lr = 2e-3\nepochs = 50\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nlosses = pynet.get_tools(tool_name=\"losses\")\nmetrics = pynet.get_tools(tool_name=\"metrics\")\nsetup_logging(level=\"info\")\n\n\n# Create synthetic data\n\n\nclass GeneratorUniform(nn.Module):\n    \"\"\" Generate multiple sources (channels) of data through a linear\n    generative model:\n\n    z ~ N(mu,sigma)\n    for c_idx in n_channels:\n        x_ch = W_ch(c_idx)\n    where 'W_ch' is an arbitrary linear mapping z -> x_ch\n    \"\"\"\n    def __init__(self, lat_dim=2, n_channels=2, n_feats=5, seed=100):\n        super(GeneratorUniform, self).__init__()\n        self.lat_dim = lat_dim\n        self.n_channels = n_channels\n        self.n_feats = n_feats\n        self.seed = seed\n        np.random.seed(self.seed)\n        W = []\n        for c_idx in range(n_channels):\n            w_ = np.random.uniform(-1, 1, (self.n_feats, lat_dim))\n            u, s, vt = np.linalg.svd(w_, full_matrices=False)\n            w = (u if self.n_feats >= lat_dim else vt)\n            W.append(torch.nn.Linear(lat_dim, self.n_feats, bias=False))\n            W[c_idx].weight.data = torch.FloatTensor(w)\n        self.W = torch.nn.ModuleList(W)\n\n    def forward(self, z):\n        if isinstance(z, list):\n            return [self.forward(_) for _ in z]\n        if type(z) == np.ndarray:\n            z = torch.FloatTensor(z)\n        assert z.size(dim=1) == self.lat_dim\n        obs = []\n        for c_idx in range(self.n_channels):\n            x = self.W[c_idx](z)\n            obs.append(x.detach())\n        return obs\n\n\nclass SyntheticDataset(object):\n    def __init__(self, n_samples=500, lat_dim=2, n_feats=5, n_classes=2,\n                 generatorclass=GeneratorUniform, snr=1, train=True):\n        super(SyntheticDataset, self).__init__()\n        self.n_samples = n_samples\n        self.lat_dim = lat_dim\n        self.n_feats = n_feats\n        self.n_classes = n_classes\n        self.snr = snr\n        self.train = train\n        self.labels = []\n        self.z = []\n        self.x = []\n        seed = 7 if self.train else 14\n        np.random.seed(seed)\n        locs = np.random.uniform(-5, 5, (self.n_classes, ))\n        np.random.seed(seed)\n        scales = np.random.uniform(0, 2, (self.n_classes, ))\n        np.random.seed(seed)\n        for k_idx in range(self.n_classes):\n            self.z.append(\n                np.random.normal(loc=locs[k_idx], scale=scales[k_idx],\n                                 size=(self.n_samples, self.lat_dim)))\n            self.generator = generatorclass(\n                lat_dim=self.lat_dim, n_channels=1, n_feats=self.n_feats)\n            self.x.append(self.generator(self.z[-1])[0])\n            self.labels += [k_idx] * self.n_samples\n        self.data = np.concatenate(self.x, axis=0).astype(np.float32)\n        self.labels = np.asarray(self.labels)\n        _, self.data = preprocess_and_add_noise(self.data, snr=snr)\n\n\ndef preprocess_and_add_noise(x, snr, seed=0):\n    scalers = StandardScaler().fit(x)\n    x_std = scalers.transform(x)\n    np.random.seed(seed)\n    sigma_noise = np.sqrt(1. / snr)\n    x_std_noisy = x_std + sigma_noise * np.random.randn(*x_std.shape)\n    return x_std, x_std_noisy\n\n\n# Create dataset\nds_train = SyntheticDataset(\n    n_samples=n_samples,\n    lat_dim=true_lat_dims,\n    n_feats=n_feats,\n    n_classes=n_classes,\n    train=True,\n    snr=snr)\nds_val = SyntheticDataset(\n    n_samples=n_samples,\n    lat_dim=true_lat_dims,\n    n_feats=n_feats,\n    n_classes=n_classes,\n    train=False,\n    snr=snr)\nimage_datasets = {\n    \"train\": ds_train,\n    \"val\": ds_val}\nmanager = DataManager.from_numpy(\n    train_inputs=ds_train.data, train_outputs=None, train_labels=ds_train.labels,\n    validation_inputs=ds_val.data, validation_outputs=None,\n    validation_labels=ds_val.labels, batch_size=batch_size, sampler=\"random\",\n    add_input=True)\nprint(\"- datasets:\", image_datasets)\nprint(\"- shapes:\", ds_train.data.shape, ds_val.data.shape)\n\n\n# Display generated data\nmethod = manifold.TSNE(n_components=2, init=\"pca\", random_state=0)\ny = method.fit_transform(ds_train.data)\ncolors = ds_train.labels.astype(float)\ncolors /= colors.max()\nfig, ax = plt.subplots()\nax.scatter(y[:, 0], y[:, 1], c=colors, cmap=plt.cm.Spectral)\nax.xaxis.set_major_formatter(NullFormatter())\nax.yaxis.set_major_formatter(NullFormatter())\nax.axis(\"tight\")\n\n\n# Generate GT using k-means\nkmeans = KMeans(n_clusters=n_classes, random_state=0).fit(ds_train.data)\ntrain_labels = kmeans.labels_\nprint(\"-- K-Means ACC train\", losses[\"GMVAELoss\"].cluster_acc(\n    train_labels, ds_train.labels))\nval_labels = kmeans.predict(ds_val.data)\nprint(\"-- K-Means ACC val\", losses[\"GMVAELoss\"].cluster_acc(\n    val_labels, ds_val.labels))\n\n\n# Create models\ntorch.manual_seed(42)\nparams = NetParameters(\n    input_dim=n_feats,\n    latent_dim=fit_lat_dims,\n    n_mix_components=n_classes,\n    sigma_min=0.001,\n    raw_sigma_bias=0.25,\n    dropout=0,\n    temperature=1,\n    gen_bias_init=0.)\nmodel = GMVAENetClassifier(\n    params,\n    optimizer_name=\"Adam\",\n    learning_rate=adam_lr,\n    loss=losses[\"GMVAELoss\"](),\n    use_cuda=False)\nprint(\"- model:\", model)\n\n\n# Fit models\nprint(\"- training...\")\ntrain_history, valid_history = model.training(\n    manager=manager,\n    nb_epochs=epochs,\n    checkpointdir=None,\n    fold_index=0,\n    with_validation=True)\n\n\n# Display results\n\nplt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}