{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read CSV with TrackId and Genre Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "tracks_info_df = pd.read_csv('combined_dataset.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read processed song data from pkl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 100390 processed tracks\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import os\n",
    "\n",
    "\n",
    "results_file = 'audio_features_backup.pkl'\n",
    "\n",
    "if os.path.exists(results_file):\n",
    "    with open(results_file, 'rb') as file:\n",
    "        saved_data = pickle.load(file)\n",
    "    X = saved_data.get('X', [])\n",
    "    y_labels = saved_data.get('y_labels', [])\n",
    "    \n",
    "    processed_files = set(saved_data.get('processed_files', []))\n",
    "    print(f\"Loaded {len(processed_files)} processed tracks\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Turn X into an np array for easier processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of feature matrix X:  (100390, 55)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = np.array(X)\n",
    "print(\"shape of feature matrix X: \", X.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove all entries where the genre is defined as None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = X[np.array(y_labels) != None]\n",
    "y_labels = np.array(y_labels)[np.array(y_labels) != None]\n",
    "\n",
    "genre_mapping = {\n",
    "    'hip-hop': 'hip hop'\n",
    "}\n",
    "\n",
    "y_labels_normalized = [genre_mapping.get(label, label) for label in y_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_labels_normalized_df = pd.DataFrame(y_labels_normalized)\n",
    "\n",
    "# Count the frequency of each label in the \"label\" column\n",
    "label_counts = y_labels_normalized_df.value_counts().reset_index()\n",
    "\n",
    "# Rename the columns for clarity\n",
    "label_counts.columns = ['Label', 'Frequency']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remap genres"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replace sub-genres with their parent genre based on a predefined mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_mapping = {\n",
    "    'lo-fi beats': 'lo-fi',\n",
    "    'chillwave': 'lo-fi',\n",
    "\n",
    "    'nu metal': 'metal',\n",
    "    'folk metal': 'metal',\n",
    "    'heavy metal': 'metal',\n",
    "    'metalcore': 'metal',\n",
    "    'power metal': 'metal',\n",
    "    'industrial metal': 'metal',\n",
    "    'glam metal': 'metal',\n",
    "    'melodic death metal': 'metal',\n",
    "    'progressive metal': 'metal',\n",
    "    'thrash metal': 'metal',\n",
    "    'gothic metal': 'metal',\n",
    "    'groove metal': 'metal',\n",
    "    'death metal': 'metal',\n",
    "    'alternative metal': 'metal',\n",
    "    'black metal': 'metal',\n",
    "    'speed metal': 'metal',\n",
    "\n",
    "    'acid house': 'acid',\n",
    "    'acid techno': 'acid',\n",
    "\n",
    "    'afrobeat': 'afro',\n",
    "    'afropop': 'afro',\n",
    "\n",
    "    'art rock': 'rock',\n",
    "\n",
    "    'britpop': 'alternative rock',\n",
    "    \n",
    "    'art pop': 'pop',\n",
    "    'acoustic pop': 'pop',\n",
    "    'bedroom pop': 'pop',\n",
    "    'soft pop': 'pop', \n",
    "    'folk pop': 'pop',\n",
    "    'norwegian pop': 'pop',\n",
    "    'acoustic pop': 'pop',\n",
    "    'dream pop': 'pop',\n",
    "    'chamber pop': 'pop',\n",
    "    'german pop': 'pop',\n",
    "    'bedroom pop': 'pop',\n",
    "    'city pop': 'pop',\n",
    "    'dance pop': 'pop',\n",
    "    'art pop': 'pop',\n",
    "    'indie pop': 'pop',\n",
    "\n",
    "    'alt country': 'country',\n",
    "    'traditional country': 'country',\n",
    "\n",
    "    'blues rock': 'blues',\n",
    "\n",
    "    'brooklyn drill': 'drill',\n",
    "\n",
    "    'gangster rap': 'rap',\n",
    "    'memphis rap': 'rap',\n",
    "    'rock rap': 'rap',\n",
    "    'meme rap': 'rap',\n",
    "    'punk rap': 'rap',\n",
    "    'emo rap': 'rap',\n",
    "    'jazz rap': 'rap',\n",
    "    'melodic rap': 'rap',\n",
    "    'cloud rap': 'rap',\n",
    "    'k-rap': 'rap',\n",
    "    'rage rap': 'rap',\n",
    "\n",
    "    'edm trap': 'trap',\n",
    "    'italian trap': 'trap',\n",
    "    'dark trap': 'trap',\n",
    "\n",
    "    'hip-hop': 'hip hop',\n",
    "    'southern hip hop': 'hip hop',\n",
    "    'west coast hip hop': 'hip hop',\n",
    "    'east coast hip hop': 'hip hop',\n",
    "    'finnish hip hop': 'hip hop',\n",
    "    'german hip hop': 'hip hop',\n",
    "    'underground hip hop': 'hip hop',\n",
    "    'norwegian hip hop': 'hip hop',\n",
    "    'experimental hip hop': 'hip hop',\n",
    "    'alternative hip hop': 'hip hop',\n",
    "    'latin hip hop': 'hip hop',\n",
    "\n",
    "    'jazz blues': 'jazz',\n",
    "    'vocal jazz': 'jazz',\n",
    "    'french jazz': 'jazz',\n",
    "    'free jazz': 'jazz',\n",
    "    'soul jazz': 'jazz',\n",
    "    'jazz fusion': 'jazz',\n",
    "    'nu jazz': 'jazz',\n",
    "    'jazz funk': 'jazz',\n",
    "    'cool jazz': 'jazz',\n",
    "    'jazz beats': 'jazz',\n",
    "    'jazz house': 'jazz',\n",
    "    'indie jazz': 'jazz',\n",
    "    'smooth jazz': 'jazz',\n",
    "\n",
    "    'melodic techno': 'techno',\n",
    "    'minimal techno': 'techno',\n",
    "    'hypertechno': 'techno',\n",
    "    'hard techno': 'techno',\n",
    "    'hardcore techno': 'techno',\n",
    "    'dub techno': 'techno',\n",
    "\n",
    "    'future house': 'house',\n",
    "    'tech house': 'house',\n",
    "    'bass house': 'house',\n",
    "    'electro house': 'house',\n",
    "    'tropical house': 'house',\n",
    "    'french house': 'house',\n",
    "    'melodic house': 'house',\n",
    "    'organic house': 'house',\n",
    "    'progressive house': 'house',\n",
    "    'latin house': 'house',\n",
    "    'deep house': 'house',\n",
    "    'slap house': 'house',\n",
    "    'chicago house': 'house'\n",
    "    \n",
    "} \n",
    "\n",
    "y_labels_normalized = [genre_mapping.get(label, label) for label in y_labels]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map hip-hop to rap -> Models without this are too confused due to the genres being so alike"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_mapping = {\n",
    "    'hip hop': 'rap'\n",
    "} \n",
    "\n",
    "y_labels_normalized = [genre_mapping.get(label, label) for label in y_labels_normalized]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Or remove hip hop\n",
    "\n",
    "X = X[np.array(y_labels_normalized) != \"hip hop\"]\n",
    "y_labels_normalized = np.array(y_labels_normalized)[np.array(y_labels_normalized) != \"hip hop\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run this code in case the last two cells were not executed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_labels_normalized = y_labels\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extra: Keep only a popular type of genre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filtered dataset size: 15248 entries (from 98347 original)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "original_labels_list = y_labels_normalized\n",
    "y_labels_normalized = np.array(y_labels_normalized)\n",
    "\n",
    "popular_genres = {\n",
    "    'country', 'idm', 'black-metal', 'hardstyle', 'heavy-metal',\n",
    "    'blues', 'rap', 'hip hop', 'classical', 'folk', 'rock-n-roll',\n",
    "    'electronic', 'jazz', 'lo-fi', 'punk-rock', 'soul', 'punk',\n",
    "    'rock'\n",
    "}\n",
    "\n",
    "#popular_genres = {\n",
    "#    'pop', 'rock', 'rap', 'hip hop', 'metal', 'country', 'jazz',\n",
    "#    'blues', 'classical', 'reggae', 'punk', 'funk', 'lo-fi', 'acid'\n",
    "#    'r&b', 'soul', 'trap', 'techno', 'folk', 'indie', 'house'\n",
    "#}\n",
    "\n",
    "y_lower = np.array([genre.lower() for genre in y_labels_normalized])\n",
    "\n",
    "mask = np.array([genre in popular_genres for genre in y_lower])\n",
    "\n",
    "X = X[mask]\n",
    "y_labels_normalized = y_labels_normalized[mask]\n",
    "\n",
    "# Optional: print some info\n",
    "print(f\"Filtered dataset size: {len(y_labels_normalized)} entries (from {len(original_labels_list)} original)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chart the leftover genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58824 characters]",
      "text/plain": [
       "<Figure size 1500x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "y = y_labels_normalized\n",
    "\n",
    "freq_original = Counter(y)\n",
    "\n",
    "orig_genres = list(freq_original.keys())\n",
    "orig_counts = list(freq_original.values())\n",
    "\n",
    "fig, axs = plt.subplots(figsize=(15, 6))\n",
    "\n",
    "sns.barplot(x=orig_genres, y=orig_counts, ax=axs)\n",
    "axs.set_title(\"Original Genre Frequencies\")\n",
    "axs.set_xlabel(\"Genre\")\n",
    "axs.set_ylabel(\"Count\")\n",
    "axs.tick_params(axis='x', rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove all songs with genres which do not come up more than n times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class counts: Counter({'rap': 2100, 'country': 1117, 'idm': 989, 'black-metal': 989, 'hardstyle': 987, 'heavy-metal': 985, 'jazz': 948, 'blues': 947, 'lo-fi': 945, 'classical': 849, 'folk': 819, 'rock-n-roll': 813, 'electronic': 807, 'punk-rock': 642, 'rock': 494, 'soul': 457, 'punk': 360})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_24713/1765738642.py:49: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  axs.set_xticklabels(le.classes_)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56904 characters]",
      "text/plain": [
       "<Figure size 1500x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# --- Encoding/Standardizing X ---\n",
    "# Remove compex object variables\n",
    "if np.iscomplexobj(X):\n",
    "    X = np.abs(X)\n",
    "\n",
    "# Standardize the features\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "\n",
    "# --- Counting Labels ---\n",
    "# Check class counts.\n",
    "counter = Counter(y)\n",
    "print(\"Class counts:\", counter)\n",
    "\n",
    "# Keep only classes with at least n samples. At least 2  or any other % 2 == 0 number\n",
    "n = 600\n",
    "classes_to_keep = {cls for cls, count in counter.items() if count >= n}\n",
    "indices_to_keep = [i for i, label in enumerate(y) if label in classes_to_keep]\n",
    "\n",
    "y = np.array(y)\n",
    "\n",
    "# Filter the data.\n",
    "X_filtered = X_scaled[indices_to_keep]\n",
    "y_filtered = y[indices_to_keep]\n",
    "\n",
    "# Encode the Genres into numerical values\n",
    "le = LabelEncoder()\n",
    "y_encoded = le.fit_transform(y_filtered)\n",
    "\n",
    "freq_leftover = Counter(y_encoded)\n",
    "leftover_genres = list(freq_leftover.keys())\n",
    "leftover_counts = list(freq_leftover.values())\n",
    "\n",
    "fig, axs = plt.subplots(figsize=(15, 6))\n",
    "\n",
    "# Plot normalized label frequencies.\n",
    "sns.barplot(x=leftover_genres, y=leftover_counts, ax=axs)\n",
    "axs.set_title(\"Normalized Genre Frequencies\")\n",
    "axs.set_xlabel(\"Genre\")\n",
    "axs.set_ylabel(\"Count\")\n",
    "axs.tick_params(axis='x', rotation=45)\n",
    "axs.set_xticklabels(le.classes_)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove highly collerating data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropping columns: ['feature_5', 'feature_8', 'feature_9', 'feature_10', 'feature_12', 'feature_14', 'feature_16', 'feature_20', 'feature_21', 'feature_23', 'feature_25', 'feature_26', 'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_34', 'feature_37', 'feature_41', 'feature_43', 'feature_44', 'feature_48', 'feature_50', 'feature_51', 'feature_53']\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 108144 characters]",
      "text/plain": [
       "<Figure size 1200x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "CORRELATING_VALUE = 0.9\n",
    "\n",
    "X = X_filtered\n",
    "\n",
    "def remove_highly_correlated_features(X, threshold):\n",
    "    # If X is a NumPy array, convert it to a DataFrame.\n",
    "    if isinstance(X, np.ndarray):\n",
    "        # Create column names if not provided\n",
    "        X_df = pd.DataFrame(X, columns=[f'feature_{i}' for i in range(X.shape[1])])\n",
    "    else:\n",
    "        X_df = X.copy()\n",
    "    \n",
    "    # Compute the absolute correlation matrix\n",
    "    corr_matrix = X_df.corr().abs()\n",
    "    \n",
    "    # Create an upper triangle matrix of correlations\n",
    "    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n",
    "    \n",
    "    # Identify columns to drop: any feature with correlation greater than the threshold\n",
    "    to_drop = [column for column in upper.columns if any(upper[column] > threshold)]\n",
    "    print(\"Dropping columns:\", to_drop)\n",
    "    \n",
    "    # Drop these columns from the dataframe\n",
    "    X_df_reduced = X_df.drop(columns=to_drop)\n",
    "    \n",
    "    # If original X was a NumPy array, return as a NumPy array.\n",
    "    if isinstance(X, np.ndarray):\n",
    "        return X_df_reduced.values\n",
    "    else:\n",
    "        return X_df_reduced\n",
    "\n",
    "\n",
    "X_reduced = remove_highly_correlated_features(X, threshold=CORRELATING_VALUE)\n",
    "\n",
    "# -- Create a feature correlation matrix --\n",
    "df_features = pd.DataFrame(X_reduced, columns=[f'feature_{i}' for i in range(X_reduced.shape[1])])\n",
    "\n",
    "# Compute the correlation matrix.\n",
    "corr = df_features.corr()\n",
    "\n",
    "plt.figure(figsize=(12,10))\n",
    "sns.heatmap(corr, cmap='coolwarm', annot=False)\n",
    "plt.title('Feature Correlation Matrix')\n",
    "plt.show()\n",
    "\n",
    "X_filtered = X_reduced\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a Test-Train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set shape: (11149, 30) (11149,)\n",
      "Test set shape: (2788, 30) (2788,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X_filtered, y_encoded, test_size=0.2, random_state=42, stratify=y_encoded\n",
    ")\n",
    "\n",
    "print(\"Training set shape:\", X_train.shape, y_train.shape)\n",
    "print(\"Test set shape:\", X_test.shape, y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/agres/projects/predictify/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "[I 2025-04-25 14:57:48,684] A new study created in memory with name: no-name-fa4ae2a3-fd71-4c6e-a51d-840533da06ed\n",
      "/tmp/ipykernel_24713/521000553.py:95: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.\n",
      "Consider using tensor.detach() first. (Triggered internally at /pytorch/aten/src/ATen/native/Scalar.cpp:22.)\n",
      "  train_loss += loss.item()\n",
      "[I 2025-04-25 14:58:01,400] Trial 0 finished with value: 0.5778335724533716 and parameters: {'activation_1': 'tanh', 'units_1': 128, 'dropout_rate': 0.4, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 32, 'dropout_rate_2': 0.4, 'learning_rate': 0.0001}. Best is trial 0 with value: 0.5778335724533716.\n",
      "[I 2025-04-25 14:58:12,869] Trial 1 finished with value: 0.5871592539454806 and parameters: {'activation_1': 'sigmoid', 'units_1': 128, 'dropout_rate': 0.6000000000000001, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 128, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.001}. Best is trial 1 with value: 0.5871592539454806.\n",
      "[I 2025-04-25 14:58:22,631] Trial 2 finished with value: 0.6018651362984218 and parameters: {'activation_1': 'tanh', 'units_1': 32, 'dropout_rate': 0.4, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 112, 'dropout_rate_2': 0.30000000000000004, 'learning_rate': 0.001}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:58:35,532] Trial 3 finished with value: 0.6007890961262554 and parameters: {'activation_1': 'tanh', 'units_1': 64, 'dropout_rate': 0.0, 'second_layer': True, 'activation_2': 'sigmoid', 'units_2': 112, 'dropout_rate_2': 0.7, 'learning_rate': 0.001}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:58:48,336] Trial 4 finished with value: 0.5125538020086083 and parameters: {'activation_1': 'tanh', 'units_1': 128, 'dropout_rate': 0.5, 'second_layer': True, 'activation_2': 'sigmoid', 'units_2': 80, 'dropout_rate_2': 0.7, 'learning_rate': 0.0001}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:58:54,607] Trial 5 finished with value: 0.5875179340028694 and parameters: {'activation_1': 'sigmoid', 'units_1': 48, 'dropout_rate': 0.7, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 64, 'dropout_rate_2': 0.0, 'learning_rate': 0.01}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:59:07,107] Trial 6 finished with value: 0.5627690100430416 and parameters: {'activation_1': 'tanh', 'units_1': 32, 'dropout_rate': 0.30000000000000004, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 112, 'dropout_rate_2': 0.30000000000000004, 'learning_rate': 0.0001}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:59:19,540] Trial 7 finished with value: 0.5347919655667145 and parameters: {'activation_1': 'relu', 'units_1': 16, 'dropout_rate': 0.4, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 96, 'dropout_rate_2': 0.2, 'learning_rate': 0.0001}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:59:31,951] Trial 8 finished with value: 0.5477044476327116 and parameters: {'activation_1': 'tanh', 'units_1': 48, 'dropout_rate': 0.1, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 32, 'dropout_rate_2': 0.5, 'learning_rate': 0.0001}. Best is trial 2 with value: 0.6018651362984218.\n",
      "[I 2025-04-25 14:59:36,391] Trial 9 finished with value: 0.6355810616929699 and parameters: {'activation_1': 'relu', 'units_1': 96, 'dropout_rate': 0.30000000000000004, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 48, 'dropout_rate_2': 0.4, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 14:59:49,139] Trial 10 finished with value: 0.5649210903873745 and parameters: {'activation_1': 'tanh', 'units_1': 80, 'dropout_rate': 0.4, 'second_layer': True, 'activation_2': 'relu', 'units_2': 80, 'dropout_rate_2': 0.5, 'learning_rate': 0.0001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 14:59:55,158] Trial 11 finished with value: 0.5681492109038737 and parameters: {'activation_1': 'sigmoid', 'units_1': 48, 'dropout_rate': 0.0, 'second_layer': True, 'activation_2': 'sigmoid', 'units_2': 16, 'dropout_rate_2': 0.7, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:06,276] Trial 12 finished with value: 0.4171449067431851 and parameters: {'activation_1': 'sigmoid', 'units_1': 16, 'dropout_rate': 0.5, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 112, 'dropout_rate_2': 0.4, 'learning_rate': 0.0001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:11,744] Trial 13 finished with value: 0.5774748923959828 and parameters: {'activation_1': 'tanh', 'units_1': 16, 'dropout_rate': 0.30000000000000004, 'second_layer': True, 'activation_2': 'sigmoid', 'units_2': 64, 'dropout_rate_2': 0.5, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:26,015] Trial 14 finished with value: 0.5200860832137734 and parameters: {'activation_1': 'relu', 'units_1': 32, 'dropout_rate': 0.6000000000000001, 'second_layer': True, 'activation_2': 'relu', 'units_2': 112, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.0001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:36,622] Trial 15 finished with value: 0.5211621233859397 and parameters: {'activation_1': 'sigmoid', 'units_1': 80, 'dropout_rate': 0.30000000000000004, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 64, 'dropout_rate_2': 0.0, 'learning_rate': 0.0001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:45,593] Trial 16 finished with value: 0.5896700143472023 and parameters: {'activation_1': 'relu', 'units_1': 64, 'dropout_rate': 0.7, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 80, 'dropout_rate_2': 0.0, 'learning_rate': 0.001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:51,397] Trial 17 finished with value: 0.6323529411764706 and parameters: {'activation_1': 'sigmoid', 'units_1': 32, 'dropout_rate': 0.0, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 16, 'dropout_rate_2': 0.5, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:00:58,133] Trial 18 finished with value: 0.5900286944045912 and parameters: {'activation_1': 'tanh', 'units_1': 32, 'dropout_rate': 0.4, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 48, 'dropout_rate_2': 0.4, 'learning_rate': 0.001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:08,675] Trial 19 finished with value: 0.6119081779053085 and parameters: {'activation_1': 'sigmoid', 'units_1': 48, 'dropout_rate': 0.0, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 128, 'dropout_rate_2': 0.5, 'learning_rate': 0.001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:20,700] Trial 20 finished with value: 0.5749641319942611 and parameters: {'activation_1': 'tanh', 'units_1': 96, 'dropout_rate': 0.5, 'second_layer': True, 'activation_2': 'relu', 'units_2': 112, 'dropout_rate_2': 0.2, 'learning_rate': 0.0001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:26,036] Trial 21 finished with value: 0.6036585365853658 and parameters: {'activation_1': 'sigmoid', 'units_1': 128, 'dropout_rate': 0.2, 'second_layer': True, 'activation_2': 'relu', 'units_2': 112, 'dropout_rate_2': 0.5, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:36,134] Trial 22 finished with value: 0.5247489239598279 and parameters: {'activation_1': 'sigmoid', 'units_1': 32, 'dropout_rate': 0.7, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 32, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:39,917] Trial 23 finished with value: 0.6104734576757532 and parameters: {'activation_1': 'tanh', 'units_1': 128, 'dropout_rate': 0.6000000000000001, 'second_layer': False, 'activation_2': 'relu', 'units_2': 96, 'dropout_rate_2': 0.30000000000000004, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:44,252] Trial 24 finished with value: 0.6172883787661406 and parameters: {'activation_1': 'tanh', 'units_1': 96, 'dropout_rate': 0.5, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 96, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.01}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:01:51,314] Trial 25 finished with value: 0.5462697274031564 and parameters: {'activation_1': 'relu', 'units_1': 16, 'dropout_rate': 0.5, 'second_layer': False, 'activation_2': 'relu', 'units_2': 80, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:02:03,783] Trial 26 finished with value: 0.48708751793400284 and parameters: {'activation_1': 'sigmoid', 'units_1': 96, 'dropout_rate': 0.1, 'second_layer': True, 'activation_2': 'relu', 'units_2': 48, 'dropout_rate_2': 0.7, 'learning_rate': 0.0001}. Best is trial 9 with value: 0.6355810616929699.\n",
      "[I 2025-04-25 15:02:08,916] Trial 27 finished with value: 0.6359397417503587 and parameters: {'activation_1': 'tanh', 'units_1': 64, 'dropout_rate': 0.30000000000000004, 'second_layer': True, 'activation_2': 'sigmoid', 'units_2': 112, 'dropout_rate_2': 0.1, 'learning_rate': 0.01}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:02:20,956] Trial 28 finished with value: 0.5211621233859397 and parameters: {'activation_1': 'sigmoid', 'units_1': 48, 'dropout_rate': 0.4, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 128, 'dropout_rate_2': 0.4, 'learning_rate': 0.0001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:02:27,552] Trial 29 finished with value: 0.6219512195121951 and parameters: {'activation_1': 'tanh', 'units_1': 112, 'dropout_rate': 0.30000000000000004, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 48, 'dropout_rate_2': 0.4, 'learning_rate': 0.01}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:02:37,965] Trial 30 finished with value: 0.5175753228120517 and parameters: {'activation_1': 'relu', 'units_1': 32, 'dropout_rate': 0.6000000000000001, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 96, 'dropout_rate_2': 0.5, 'learning_rate': 0.0001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:02:50,197] Trial 31 finished with value: 0.6022238163558106 and parameters: {'activation_1': 'sigmoid', 'units_1': 96, 'dropout_rate': 0.1, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 48, 'dropout_rate_2': 0.4, 'learning_rate': 0.001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:02:59,593] Trial 32 finished with value: 0.6212338593974175 and parameters: {'activation_1': 'relu', 'units_1': 64, 'dropout_rate': 0.5, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 128, 'dropout_rate_2': 0.2, 'learning_rate': 0.001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:03:10,196] Trial 33 finished with value: 0.5179340028694405 and parameters: {'activation_1': 'sigmoid', 'units_1': 96, 'dropout_rate': 0.5, 'second_layer': False, 'activation_2': 'relu', 'units_2': 112, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.0001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:03:16,440] Trial 34 finished with value: 0.6190817790530847 and parameters: {'activation_1': 'tanh', 'units_1': 32, 'dropout_rate': 0.2, 'second_layer': False, 'activation_2': 'relu', 'units_2': 96, 'dropout_rate_2': 0.1, 'learning_rate': 0.01}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:03:23,990] Trial 35 finished with value: 0.6093974175035868 and parameters: {'activation_1': 'sigmoid', 'units_1': 64, 'dropout_rate': 0.6000000000000001, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 112, 'dropout_rate_2': 0.1, 'learning_rate': 0.01}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:03:34,030] Trial 36 finished with value: 0.6176470588235294 and parameters: {'activation_1': 'relu', 'units_1': 112, 'dropout_rate': 0.7, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 48, 'dropout_rate_2': 0.4, 'learning_rate': 0.001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:03:44,721] Trial 37 finished with value: 0.5100430416068866 and parameters: {'activation_1': 'sigmoid', 'units_1': 64, 'dropout_rate': 0.30000000000000004, 'second_layer': False, 'activation_2': 'relu', 'units_2': 128, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.0001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:03:55,297] Trial 38 finished with value: 0.6262553802008608 and parameters: {'activation_1': 'relu', 'units_1': 112, 'dropout_rate': 0.6000000000000001, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 112, 'dropout_rate_2': 0.1, 'learning_rate': 0.001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:04:07,443] Trial 39 finished with value: 0.5387374461979914 and parameters: {'activation_1': 'tanh', 'units_1': 80, 'dropout_rate': 0.7, 'second_layer': True, 'activation_2': 'relu', 'units_2': 48, 'dropout_rate_2': 0.4, 'learning_rate': 0.0001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:04:09,132] Trial 40 finished with value: 0.21987087517934004 and parameters: {'activation_1': 'sigmoid', 'units_1': 32, 'dropout_rate': 0.4, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 32, 'dropout_rate_2': 0.30000000000000004, 'learning_rate': 0.0001}. Best is trial 27 with value: 0.6359397417503587.\n",
      "[I 2025-04-25 15:04:13,086] Trial 41 finished with value: 0.6434720229555236 and parameters: {'activation_1': 'tanh', 'units_1': 128, 'dropout_rate': 0.0, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 96, 'dropout_rate_2': 0.0, 'learning_rate': 0.01}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[I 2025-04-25 15:04:20,875] Trial 42 finished with value: 0.6388091822094691 and parameters: {'activation_1': 'relu', 'units_1': 80, 'dropout_rate': 0.1, 'second_layer': True, 'activation_2': 'tanh', 'units_2': 32, 'dropout_rate_2': 0.5, 'learning_rate': 0.01}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[I 2025-04-25 15:04:27,171] Trial 43 finished with value: 0.6215925394548063 and parameters: {'activation_1': 'sigmoid', 'units_1': 64, 'dropout_rate': 0.4, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 112, 'dropout_rate_2': 0.6000000000000001, 'learning_rate': 0.01}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[I 2025-04-25 15:04:38,959] Trial 44 finished with value: 0.582855093256815 and parameters: {'activation_1': 'tanh', 'units_1': 96, 'dropout_rate': 0.7, 'second_layer': True, 'activation_2': 'sigmoid', 'units_2': 112, 'dropout_rate_2': 0.5, 'learning_rate': 0.001}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[I 2025-04-25 15:04:48,320] Trial 45 finished with value: 0.601506456241033 and parameters: {'activation_1': 'sigmoid', 'units_1': 64, 'dropout_rate': 0.4, 'second_layer': True, 'activation_2': 'relu', 'units_2': 16, 'dropout_rate_2': 0.2, 'learning_rate': 0.01}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[I 2025-04-25 15:04:57,108] Trial 46 finished with value: 0.5652797704447633 and parameters: {'activation_1': 'sigmoid', 'units_1': 96, 'dropout_rate': 0.7, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 80, 'dropout_rate_2': 0.0, 'learning_rate': 0.001}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[I 2025-04-25 15:05:06,448] Trial 47 finished with value: 0.6352223816355811 and parameters: {'activation_1': 'relu', 'units_1': 80, 'dropout_rate': 0.30000000000000004, 'second_layer': False, 'activation_2': 'tanh', 'units_2': 80, 'dropout_rate_2': 0.2, 'learning_rate': 0.001}. Best is trial 41 with value: 0.6434720229555236.\n",
      "[W 2025-04-25 15:05:10,469] Trial 48 failed with parameters: {'activation_1': 'tanh', 'units_1': 80, 'dropout_rate': 0.4, 'second_layer': False, 'activation_2': 'sigmoid', 'units_2': 16, 'dropout_rate_2': 0.30000000000000004, 'learning_rate': 0.0001} because of the following error: KeyboardInterrupt().\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/agres/projects/predictify/.venv/lib/python3.10/site-packages/optuna/study/_optimize.py\", line 197, in _run_trial\n",
      "    value_or_values = func(trial)\n",
      "  File \"/tmp/ipykernel_24713/521000553.py\", line 96, in objective\n",
      "    loss.backward()\n",
      "  File \"/home/agres/projects/predictify/.venv/lib/python3.10/site-packages/torch/_tensor.py\", line 648, in backward\n",
      "    torch.autograd.backward(\n",
      "  File \"/home/agres/projects/predictify/.venv/lib/python3.10/site-packages/torch/autograd/__init__.py\", line 353, in backward\n",
      "    _engine_run_backward(\n",
      "  File \"/home/agres/projects/predictify/.venv/lib/python3.10/site-packages/torch/autograd/graph.py\", line 824, in _engine_run_backward\n",
      "    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n",
      "KeyboardInterrupt\n",
      "[W 2025-04-25 15:05:10,477] Trial 48 failed with value None.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 140\u001b[0m\n\u001b[1;32m    138\u001b[0m \u001b[38;5;66;03m# Run Optuna random search\u001b[39;00m\n\u001b[1;32m    139\u001b[0m study \u001b[38;5;241m=\u001b[39m optuna\u001b[38;5;241m.\u001b[39mcreate_study(direction\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmaximize\u001b[39m\u001b[38;5;124m'\u001b[39m, sampler\u001b[38;5;241m=\u001b[39moptuna\u001b[38;5;241m.\u001b[39msamplers\u001b[38;5;241m.\u001b[39mRandomSampler())\n\u001b[0;32m--> 140\u001b[0m \u001b[43mstudy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobjective\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_trials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mTRIAL_COUNT\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    142\u001b[0m \u001b[38;5;66;03m# Show best trial\u001b[39;00m\n\u001b[1;32m    143\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBest Hyperparameters:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/optuna/study/study.py:475\u001b[0m, in \u001b[0;36mStudy.optimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    373\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21moptimize\u001b[39m(\n\u001b[1;32m    374\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    375\u001b[0m     func: ObjectiveFuncType,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    382\u001b[0m     show_progress_bar: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    383\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    384\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Optimize an objective function.\u001b[39;00m\n\u001b[1;32m    385\u001b[0m \n\u001b[1;32m    386\u001b[0m \u001b[38;5;124;03m    Optimization is done by choosing a suitable set of hyperparameter values from a given\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    473\u001b[0m \u001b[38;5;124;03m            If nested invocation of this method occurs.\u001b[39;00m\n\u001b[1;32m    474\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 475\u001b[0m     \u001b[43m_optimize\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    476\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstudy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    477\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfunc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    478\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_trials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_trials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    479\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    480\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_jobs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_jobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    481\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcatch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mtuple\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43misinstance\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mIterable\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    482\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    483\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgc_after_trial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgc_after_trial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    484\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_progress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_progress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    485\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/optuna/study/_optimize.py:63\u001b[0m, in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     62\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m n_jobs \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m---> 63\u001b[0m         \u001b[43m_optimize_sequential\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     64\u001b[0m \u001b[43m            \u001b[49m\u001b[43mstudy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     65\u001b[0m \u001b[43m            \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[43m            \u001b[49m\u001b[43mn_trials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     67\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     68\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     69\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     70\u001b[0m \u001b[43m            \u001b[49m\u001b[43mgc_after_trial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     71\u001b[0m \u001b[43m            \u001b[49m\u001b[43mreseed_sampler_rng\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m     72\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtime_start\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m     73\u001b[0m \u001b[43m            \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprogress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     74\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     75\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     76\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m n_jobs \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/optuna/study/_optimize.py:160\u001b[0m, in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    157\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m    159\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 160\u001b[0m     frozen_trial \u001b[38;5;241m=\u001b[39m \u001b[43m_run_trial\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstudy\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcatch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    161\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    162\u001b[0m     \u001b[38;5;66;03m# The following line mitigates memory problems that can be occurred in some\u001b[39;00m\n\u001b[1;32m    163\u001b[0m     \u001b[38;5;66;03m# environments (e.g., services that use computing containers such as GitHub Actions).\u001b[39;00m\n\u001b[1;32m    164\u001b[0m     \u001b[38;5;66;03m# Please refer to the following PR for further details:\u001b[39;00m\n\u001b[1;32m    165\u001b[0m     \u001b[38;5;66;03m# https://github.com/optuna/optuna/pull/325.\u001b[39;00m\n\u001b[1;32m    166\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m gc_after_trial:\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/optuna/study/_optimize.py:248\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    241\u001b[0m         \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShould not reach.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    243\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m    244\u001b[0m     frozen_trial\u001b[38;5;241m.\u001b[39mstate \u001b[38;5;241m==\u001b[39m TrialState\u001b[38;5;241m.\u001b[39mFAIL\n\u001b[1;32m    245\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m func_err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    246\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(func_err, catch)\n\u001b[1;32m    247\u001b[0m ):\n\u001b[0;32m--> 248\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m func_err\n\u001b[1;32m    249\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m frozen_trial\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/optuna/study/_optimize.py:197\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m get_heartbeat_thread(trial\u001b[38;5;241m.\u001b[39m_trial_id, study\u001b[38;5;241m.\u001b[39m_storage):\n\u001b[1;32m    196\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 197\u001b[0m         value_or_values \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    198\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m exceptions\u001b[38;5;241m.\u001b[39mTrialPruned \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    199\u001b[0m         \u001b[38;5;66;03m# TODO(mamu): Handle multi-objective cases.\u001b[39;00m\n\u001b[1;32m    200\u001b[0m         state \u001b[38;5;241m=\u001b[39m TrialState\u001b[38;5;241m.\u001b[39mPRUNED\n",
      "Cell \u001b[0;32mIn[13], line 96\u001b[0m, in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     94\u001b[0m     loss \u001b[38;5;241m=\u001b[39m criterion(output, y_batch)\n\u001b[1;32m     95\u001b[0m     train_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mitem()\n\u001b[0;32m---> 96\u001b[0m     \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     97\u001b[0m     optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m     99\u001b[0m train_loss \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(train_loader)\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/torch/_tensor.py:648\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    638\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    639\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    640\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    641\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    646\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m    647\u001b[0m     )\n\u001b[0;32m--> 648\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    649\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m    650\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/torch/autograd/__init__.py:353\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    348\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    350\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m    351\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 353\u001b[0m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    354\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    355\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    356\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    357\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    358\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    359\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    360\u001b[0m \u001b[43m    \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    361\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/projects/predictify/.venv/lib/python3.10/site-packages/torch/autograd/graph.py:824\u001b[0m, in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    822\u001b[0m     unregister_hooks \u001b[38;5;241m=\u001b[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[1;32m    823\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 824\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    825\u001b[0m \u001b[43m        \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m    826\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m    827\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    828\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import optuna\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "from torch.utils.data import DataLoader, TensorDataset, random_split\n",
    "\n",
    "# Ensure reproducibility\n",
    "torch.manual_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "#device = torch.device(\"cpu\")\n",
    "\n",
    "BATCH_SIZE = 128\n",
    "\n",
    "# Convert numpy data to PyTorch tensors\n",
    "X_train_tensor = torch.tensor(X_train, dtype=torch.float32)\n",
    "y_train_tensor = torch.tensor(y_train, dtype=torch.long)\n",
    "X_test_tensor = torch.tensor(X_test, dtype=torch.float32)\n",
    "y_test_tensor = torch.tensor(y_test, dtype=torch.long)\n",
    "\n",
    "train_dataset = TensorDataset(X_train_tensor, y_train_tensor)\n",
    "test_dataset = TensorDataset(X_test_tensor, y_test_tensor)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "val_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n",
    "\n",
    "num_features = X_train.shape[1]\n",
    "num_classes = len(np.unique(y_train))\n",
    "\n",
    "EPOCHS = 50\n",
    "EARLY_STOPPING_PATIENCE = 6\n",
    "\n",
    "# Define model class\n",
    "class CustomNet(nn.Module):\n",
    "    def __init__(self, input_size, output_size, hp):\n",
    "        super().__init__()\n",
    "        self.layers = nn.Sequential()\n",
    "        self.layers.add_module('fc1', nn.Linear(input_size, hp['units_1']))\n",
    "        self.layers.add_module('act1', self.get_activation(hp['activation_1']))\n",
    "        self.layers.add_module('drop1', nn.Dropout(hp['dropout_rate']))\n",
    "        \n",
    "        if hp['second_layer']:\n",
    "            self.layers.add_module('fc2', nn.Linear(hp['units_1'], hp['units_2']))\n",
    "            self.layers.add_module('act2', self.get_activation(hp['activation_2']))\n",
    "            self.layers.add_module('drop2', nn.Dropout(hp['dropout_rate_2']))\n",
    "            last_dim = hp['units_2']\n",
    "        else:\n",
    "            last_dim = hp['units_1']\n",
    "\n",
    "        self.output_layer = nn.Linear(last_dim, output_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return self.output_layer(x)\n",
    "\n",
    "    def get_activation(self, name):\n",
    "        return {\n",
    "            'relu': nn.ReLU(),\n",
    "            'tanh': nn.Tanh(),\n",
    "            'sigmoid': nn.Sigmoid()\n",
    "        }[name]\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "    hp = {\n",
    "        'activation_1': trial.suggest_categorical('activation_1', ['relu', 'tanh', 'sigmoid']),\n",
    "        'units_1': trial.suggest_int('units_1', 16, 128, step=16),\n",
    "        'dropout_rate': trial.suggest_float('dropout_rate', 0.0, 0.7, step=0.1),\n",
    "        'second_layer': trial.suggest_categorical('second_layer', [True, False]),\n",
    "        'activation_2': trial.suggest_categorical('activation_2', ['relu', 'tanh', 'sigmoid']),\n",
    "        'units_2': trial.suggest_int('units_2', 16, 128, step=16),\n",
    "        'dropout_rate_2': trial.suggest_float('dropout_rate_2', 0.0, 0.7, step=0.1),\n",
    "        'learning_rate': trial.suggest_categorical('learning_rate', [1e-4, 1e-3, 1e-2])\n",
    "    }\n",
    "\n",
    "    model = CustomNet(num_features, num_classes, hp).to(device)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=hp['learning_rate'])\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    best_val_acc = 0\n",
    "    patience, wait = EARLY_STOPPING_PATIENCE, 0\n",
    "\n",
    "    for epoch in range(EPOCHS):\n",
    "        model.train()\n",
    "        train_loss = 0.0\n",
    "        for X_batch, y_batch in train_loader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            output = model(X_batch)\n",
    "            loss = criterion(output, y_batch)\n",
    "            train_loss += loss.item()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        train_loss /= len(train_loader)\n",
    "        val_loss = 0.0\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            for X_batch, y_batch in val_loader:\n",
    "                X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "                output = model(X_batch)\n",
    "                loss = criterion(output, y_batch)\n",
    "                val_loss += loss.item()\n",
    "        val_loss /= len(val_loader)\n",
    "\n",
    "        val_acc = compute_accuracy(model, val_loader)\n",
    "\n",
    "        if val_acc > best_val_acc:\n",
    "            best_val_acc = val_acc\n",
    "            wait = 0\n",
    "        else:\n",
    "            wait += 1\n",
    "            if wait >= patience:\n",
    "                break\n",
    "\n",
    "    return best_val_acc\n",
    "\n",
    "\n",
    "def compute_accuracy(model, dataloader):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for X_batch, y_batch in dataloader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            outputs = model(X_batch)\n",
    "            preds = torch.argmax(outputs, dim=1)\n",
    "            correct += (preds == y_batch).sum().item()\n",
    "            total += y_batch.size(0)\n",
    "    return correct / total\n",
    "\n",
    "TRIAL_COUNT = 500\n",
    "\n",
    "# Run Optuna random search\n",
    "study = optuna.create_study(direction='maximize', sampler=optuna.samplers.RandomSampler())\n",
    "study.optimize(objective, n_trials=TRIAL_COUNT)\n",
    "\n",
    "# Show best trial\n",
    "print(\"Best Hyperparameters:\")\n",
    "print(study.best_params)\n",
    "\n",
    "# Retrain final model on full training set with early stopping\n",
    "best_hp = study.best_params\n",
    "final_model = CustomNet(num_features, num_classes, best_hp).to(device)\n",
    "optimizer = optim.Adam(final_model.parameters(), lr=best_hp['learning_rate'])\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "train_losses = []\n",
    "val_losses = []\n",
    "train_accuracies = []\n",
    "val_accuracies = []\n",
    "\n",
    "# Early stopping variables\n",
    "best_val_loss = float('inf')\n",
    "patience = EARLY_STOPPING_PATIENCE\n",
    "epochs_no_improve = 0\n",
    "best_model_state = None\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    # Training\n",
    "    final_model.train()\n",
    "    running_loss = 0\n",
    "    correct_train = 0\n",
    "    total_train = 0\n",
    "\n",
    "    for X_batch, y_batch in train_loader:\n",
    "        X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = final_model(X_batch)\n",
    "        loss = criterion(outputs, y_batch)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        preds = torch.argmax(outputs, dim=1)\n",
    "        correct_train += (preds == y_batch).sum().item()\n",
    "        total_train += y_batch.size(0)\n",
    "\n",
    "    avg_train_loss = running_loss / len(train_loader)\n",
    "    train_acc = correct_train / total_train\n",
    "\n",
    "    # Validation\n",
    "    final_model.eval()\n",
    "    val_loss = 0\n",
    "    correct_val = 0\n",
    "    total_val = 0\n",
    "    with torch.no_grad():\n",
    "        for X_batch, y_batch in val_loader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            outputs = final_model(X_batch)\n",
    "            loss = criterion(outputs, y_batch)\n",
    "            val_loss += loss.item()\n",
    "            preds = torch.argmax(outputs, dim=1)\n",
    "            correct_val += (preds == y_batch).sum().item()\n",
    "            total_val += y_batch.size(0)\n",
    "\n",
    "    avg_val_loss = val_loss / len(val_loader)\n",
    "    val_acc = correct_val / total_val\n",
    "\n",
    "    # Save history\n",
    "    train_losses.append(avg_train_loss)\n",
    "    val_losses.append(avg_val_loss)\n",
    "    train_accuracies.append(train_acc)\n",
    "    val_accuracies.append(val_acc)\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{EPOCHS} | \"\n",
    "          f\"Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f} | \"\n",
    "          f\"Train Acc: {train_acc:.4f} | Val Acc: {val_acc:.4f}\")\n",
    "\n",
    "    # Early stopping check\n",
    "    if avg_val_loss < best_val_loss:\n",
    "        best_val_loss = avg_val_loss\n",
    "        best_model_state = final_model.state_dict()\n",
    "        epochs_no_improve = 0\n",
    "    else:\n",
    "        epochs_no_improve += 1\n",
    "        if epochs_no_improve >= patience:\n",
    "            print(f\"Early stopping triggered after {epoch+1} epochs.\")\n",
    "            break\n",
    "\n",
    "# Load best model state after training\n",
    "if best_model_state is not None:\n",
    "    final_model.load_state_dict(best_model_state)\n",
    "    print(\"Restored best model weights based on validation loss.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation Charts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix - Shows what was the predicted value compared to the true value when evaluating the Model\n",
    "\n",
    "The more values in the diagonal line the better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 71980 characters]",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Put model in eval mode\n",
    "final_model.eval()\n",
    "\n",
    "# Get predictions on test data\n",
    "all_preds = []\n",
    "all_labels = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for X_batch, y_batch in val_loader:  # or test_loader if you named it that\n",
    "        X_batch = X_batch.to(device)\n",
    "        outputs = final_model(X_batch)\n",
    "        preds = torch.argmax(outputs, dim=1).cpu().numpy()\n",
    "        all_preds.extend(preds)\n",
    "        all_labels.extend(y_batch.numpy())\n",
    "\n",
    "# Optionally inverse transform labels (if using LabelEncoder)\n",
    "decoded_y_test = le.inverse_transform(all_labels)\n",
    "decoded_y_pred = le.inverse_transform(all_preds)\n",
    "\n",
    "# Create confusion matrix\n",
    "cm = confusion_matrix(decoded_y_test, decoded_y_pred)\n",
    "\n",
    "# Plot\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='viridis', \n",
    "            xticklabels=le.classes_, yticklabels=le.classes_)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss and Accuracy Plots - Shows how the models loss and accuracy values changed during the training\n",
    "\n",
    "The closer the lines to each other the better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 95160 characters]",
      "text/plain": [
       "<Figure size 1500x650 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(15, 6.5))\n",
    "\n",
    "# Loss Plot\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(train_losses, label='Training Loss')\n",
    "plt.plot(val_losses, label='Validation Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Training vs. Validation Loss')\n",
    "plt.legend()\n",
    "\n",
    "# Accuracy Plot\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(train_accuracies, label='Training Accuracy')\n",
    "plt.plot(val_accuracies, label='Validation Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('Training vs. Validation Accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Corrleation Matrix - How much the features correlate to each other \n",
    "\n",
    "Less correlating features is better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 101964 characters]",
      "text/plain": [
       "<Figure size 1200x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a DataFrame for features. If X has many features, consider selecting a subset.\n",
    "df_features = pd.DataFrame(X_train, columns=[f'feature_{i}' for i in range(X_train.shape[1])])\n",
    "\n",
    "# Compute the correlation matrix.\n",
    "corr = df_features.corr()\n",
    "\n",
    "plt.figure(figsize=(12,10))\n",
    "sns.heatmap(corr, cmap='coolwarm', annot=False)\n",
    "plt.title('Feature Correlation Matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature permutation importance - shows how much each feature was important in making a decision in the model\n",
    "\n",
    "Potential: remove the non-important features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 39724 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator, ClassifierMixin\n",
    "from sklearn.inspection import permutation_importance\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "class TorchClassifierWrapper(BaseEstimator, ClassifierMixin):\n",
    "    def __init__(self, model, device):\n",
    "        self.model = model.to(device)\n",
    "        self.device = device\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        # Model is already trained\n",
    "        return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        self.model.eval()\n",
    "        with torch.no_grad():\n",
    "            X_tensor = torch.tensor(X, dtype=torch.float32).to(self.device)\n",
    "            outputs = self.model(X_tensor)\n",
    "            preds = torch.argmax(outputs, dim=1).cpu().numpy()\n",
    "        return preds\n",
    "\n",
    "    def score(self, X, y):\n",
    "        preds = self.predict(X)\n",
    "        return np.mean(preds == y)\n",
    "    \n",
    "# Wrap the PyTorch model\n",
    "wrapper = TorchClassifierWrapper(final_model, device=device)\n",
    "\n",
    "# Compute permutation importance\n",
    "result = permutation_importance(wrapper, X_test, y_test, n_repeats=10,\n",
    "                                random_state=42, scoring='accuracy')\n",
    "\n",
    "importance_means = result.importances_mean\n",
    "\n",
    "features = [f'feature_{i}' for i in range(X_test.shape[1])]\n",
    "\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.bar(features, importance_means)\n",
    "plt.xticks(rotation=90)\n",
    "plt.xlabel('Feature')\n",
    "plt.ylabel('Mean Importance')\n",
    "plt.title('Permutation Feature Importance')\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove unimportant features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Important feature indices: [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 18 22 23 24 25 28 29]\n",
      "Important features: ['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5', 'feature_6', 'feature_7', 'feature_8', 'feature_9', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_18', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_28', 'feature_29']\n",
      "Original training shape: (5012, 31)\n",
      "Reduced training shape: (5012, 23)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Assume importance_means and importance_threshold are already defined\n",
    "importance_threshold = 0.01  # adjust this threshold as needed\n",
    "\n",
    "# Check if X_train is a DataFrame, if not, convert it and assign column names.\n",
    "if isinstance(X_train, np.ndarray):\n",
    "    X_train_df = pd.DataFrame(X_train, columns=[f'feature_{i}' for i in range(X_train.shape[1])])\n",
    "else:\n",
    "    X_train_df = X_train.copy()\n",
    "\n",
    "# Do the same for X_test.\n",
    "if isinstance(X_test, np.ndarray):\n",
    "    X_test_df = pd.DataFrame(X_test, columns=[f'feature_{i}' for i in range(X_test.shape[1])])\n",
    "else:\n",
    "    X_test_df = X_test.copy()\n",
    "\n",
    "# Identify the indices of features with importance greater than or equal to the threshold.\n",
    "important_feature_indices = np.where(importance_means >= importance_threshold)[0]\n",
    "print(\"Important feature indices:\", important_feature_indices)\n",
    "\n",
    "# Retrieve the names of the important features.\n",
    "important_features = X_train_df.columns[important_feature_indices]\n",
    "print(\"Important features:\", important_features.tolist())\n",
    "\n",
    "# Filter the training and test DataFrames to keep only the important features.\n",
    "X_train_reduced = X_train_df[important_features]\n",
    "X_test_reduced = X_test_df[important_features]\n",
    "\n",
    "print(\"Original training shape:\", X_train_df.shape)\n",
    "print(\"Reduced training shape:\", X_train_reduced.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Re-run Hyperparameter search"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
