{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read CSV with TrackId and Genre Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "tracks_info_df = pd.read_csv('combined_dataset.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read processed song data from pkl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 100390 processed tracks\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import os\n",
    "\n",
    "\n",
    "results_file = 'audio_features_backup.pkl'\n",
    "\n",
    "if os.path.exists(results_file):\n",
    "    with open(results_file, 'rb') as file:\n",
    "        saved_data = pickle.load(file)\n",
    "    X = saved_data.get('X', [])\n",
    "    y_labels = saved_data.get('y_labels', [])\n",
    "    \n",
    "    processed_files = set(saved_data.get('processed_files', []))\n",
    "    print(f\"Loaded {len(processed_files)} processed tracks\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Turn X into an np array for easier processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of feature matrix X:  (100390, 55)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = np.array(X)\n",
    "print(\"shape of feature matrix X: \", X.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove all entries where the genre is defined as None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = X[np.array(y_labels) != None]\n",
    "y_labels = np.array(y_labels)[np.array(y_labels) != None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remap genres"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replace sub-genres with their parent genre based on a predefined mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_mapping = {\n",
    "    'lo-fi beats': 'lo-fi',\n",
    "    'chillwave': 'lo-fi',\n",
    "\n",
    "    'nu metal': 'metal',\n",
    "    'folk metal': 'metal',\n",
    "    'heavy metal': 'metal',\n",
    "    'metalcore': 'metal',\n",
    "    'power metal': 'metal',\n",
    "    'industrial metal': 'metal',\n",
    "    'glam metal': 'metal',\n",
    "    'melodic death metal': 'metal',\n",
    "    'progressive metal': 'metal',\n",
    "    'thrash metal': 'metal',\n",
    "    'gothic metal': 'metal',\n",
    "    'groove metal': 'metal',\n",
    "    'death metal': 'metal',\n",
    "    'alternative metal': 'metal',\n",
    "    'black metal': 'metal',\n",
    "    'speed metal': 'metal',\n",
    "\n",
    "    'acid house': 'acid',\n",
    "    'acid techno': 'acid',\n",
    "\n",
    "    'afrobeat': 'afro',\n",
    "    'afropop': 'afro',\n",
    "\n",
    "    'art rock': 'rock',\n",
    "\n",
    "    'britpop': 'alternative rock',\n",
    "    \n",
    "    'art pop': 'pop',\n",
    "    'acoustic pop': 'pop',\n",
    "    'bedroom pop': 'pop',\n",
    "    'soft pop': 'pop', \n",
    "    'folk pop': 'pop',\n",
    "    'norwegian pop': 'pop',\n",
    "    'acoustic pop': 'pop',\n",
    "    'dream pop': 'pop',\n",
    "    'chamber pop': 'pop',\n",
    "    'german pop': 'pop',\n",
    "    'bedroom pop': 'pop',\n",
    "    'city pop': 'pop',\n",
    "    'dance pop': 'pop',\n",
    "    'art pop': 'pop',\n",
    "    'indie pop': 'pop',\n",
    "\n",
    "    'alt country': 'country',\n",
    "    'traditional country': 'country',\n",
    "\n",
    "    'blues rock': 'blues',\n",
    "\n",
    "    'brooklyn drill': 'drill',\n",
    "\n",
    "    'gangster rap': 'rap',\n",
    "    'memphis rap': 'rap',\n",
    "    'rock rap': 'rap',\n",
    "    'meme rap': 'rap',\n",
    "    'punk rap': 'rap',\n",
    "    'emo rap': 'rap',\n",
    "    'jazz rap': 'rap',\n",
    "    'melodic rap': 'rap',\n",
    "    'cloud rap': 'rap',\n",
    "    'k-rap': 'rap',\n",
    "    'rage rap': 'rap',\n",
    "\n",
    "    'edm trap': 'trap',\n",
    "    'italian trap': 'trap',\n",
    "    'dark trap': 'trap',\n",
    "\n",
    "    'hip-hop': 'hip hop',\n",
    "    'southern hip hop': 'hip hop',\n",
    "    'west coast hip hop': 'hip hop',\n",
    "    'east coast hip hop': 'hip hop',\n",
    "    'finnish hip hop': 'hip hop',\n",
    "    'german hip hop': 'hip hop',\n",
    "    'underground hip hop': 'hip hop',\n",
    "    'norwegian hip hop': 'hip hop',\n",
    "    'experimental hip hop': 'hip hop',\n",
    "    'alternative hip hop': 'hip hop',\n",
    "    'latin hip hop': 'hip hop',\n",
    "\n",
    "    'jazz blues': 'jazz',\n",
    "    'vocal jazz': 'jazz',\n",
    "    'french jazz': 'jazz',\n",
    "    'free jazz': 'jazz',\n",
    "    'soul jazz': 'jazz',\n",
    "    'jazz fusion': 'jazz',\n",
    "    'nu jazz': 'jazz',\n",
    "    'jazz funk': 'jazz',\n",
    "    'cool jazz': 'jazz',\n",
    "    'jazz beats': 'jazz',\n",
    "    'jazz house': 'jazz',\n",
    "    'indie jazz': 'jazz',\n",
    "    'smooth jazz': 'jazz',\n",
    "\n",
    "    'melodic techno': 'techno',\n",
    "    'minimal techno': 'techno',\n",
    "    'hypertechno': 'techno',\n",
    "    'hard techno': 'techno',\n",
    "    'hardcore techno': 'techno',\n",
    "    'dub techno': 'techno',\n",
    "\n",
    "    'future house': 'house',\n",
    "    'tech house': 'house',\n",
    "    'bass house': 'house',\n",
    "    'electro house': 'house',\n",
    "    'tropical house': 'house',\n",
    "    'french house': 'house',\n",
    "    'melodic house': 'house',\n",
    "    'organic house': 'house',\n",
    "    'progressive house': 'house',\n",
    "    'latin house': 'house',\n",
    "    'deep house': 'house',\n",
    "    'slap house': 'house',\n",
    "    'chicago house': 'house'\n",
    "    \n",
    "} \n",
    "\n",
    "y_labels_normalized = [genre_mapping.get(label, label) for label in y_labels]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map hip-hop to rap -> Models without this are too confused due to the genres being so alike"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_mapping = {\n",
    "    'hip hop': 'rap'\n",
    "} \n",
    "\n",
    "y_labels_normalized = [genre_mapping.get(label, label) for label in y_labels_normalized]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Or remove hip hop\n",
    "\n",
    "X = X[np.array(y_labels_normalized) != \"hip hop\"]\n",
    "y_labels_normalized = np.array(y_labels_normalized)[np.array(y_labels_normalized) != \"hip hop\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run this code in case the last two cells were not executed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_labels_normalized = y_labels\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extra: Keep only a popular type of genre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filtered dataset size: 13254 entries (from 98347 original)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "original_labels_list = y_labels_normalized\n",
    "y_labels_normalized = np.array(y_labels_normalized)\n",
    "\n",
    "\n",
    "popular_genres = {\n",
    "    'pop', 'rock', 'rap', 'hip hop', 'metal', 'country', 'jazz',\n",
    "    'blues', 'classical', 'reggae', 'punk', 'funk',\n",
    "    'r&b', 'soul', 'trap', 'techno', 'folk', 'indie'\n",
    "}\n",
    "\n",
    "y_lower = np.array([genre.lower() for genre in y_labels_normalized])\n",
    "\n",
    "mask = np.array([genre in popular_genres for genre in y_lower])\n",
    "\n",
    "X = X[mask]\n",
    "y_labels_normalized = y_labels_normalized[mask]\n",
    "\n",
    "# Optional: print some info\n",
    "print(f\"Filtered dataset size: {len(y_labels_normalized)} entries (from {len(original_labels_list)} original)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chart the leftover genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50268 characters]",
      "text/plain": [
       "<Figure size 1500x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "y = y_labels_normalized\n",
    "\n",
    "freq_original = Counter(y)\n",
    "\n",
    "orig_genres = list(freq_original.keys())\n",
    "orig_counts = list(freq_original.values())\n",
    "\n",
    "fig, axs = plt.subplots(figsize=(15, 6))\n",
    "\n",
    "sns.barplot(x=orig_genres, y=orig_counts, ax=axs)\n",
    "axs.set_title(\"Original Genre Frequencies\")\n",
    "axs.set_xlabel(\"Genre\")\n",
    "axs.set_ylabel(\"Count\")\n",
    "axs.tick_params(axis='x', rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove all songs with genres which do not come up more than n times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class counts: Counter({'rap': 2100, 'metal': 1721, 'country': 1117, 'pop': 975, 'jazz': 948, 'blues': 947, 'classical': 849, 'folk': 819, 'funk': 738, 'techno': 626, 'indie': 525, 'rock': 494, 'soul': 457, 'punk': 360, 'reggae': 291, 'trap': 258, 'r&b': 29})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_216855/1765738642.py:49: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  axs.set_xticklabels(le.classes_)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 42972 characters]",
      "text/plain": [
       "<Figure size 1500x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# --- Encoding/Standardizing X ---\n",
    "# Remove compex object variables\n",
    "if np.iscomplexobj(X):\n",
    "    X = np.abs(X)\n",
    "\n",
    "# Standardize the features\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "\n",
    "# --- Counting Labels ---\n",
    "# Check class counts.\n",
    "counter = Counter(y)\n",
    "print(\"Class counts:\", counter)\n",
    "\n",
    "# Keep only classes with at least n samples. At least 2  or any other % 2 == 0 number\n",
    "n = 600\n",
    "classes_to_keep = {cls for cls, count in counter.items() if count >= n}\n",
    "indices_to_keep = [i for i, label in enumerate(y) if label in classes_to_keep]\n",
    "\n",
    "y = np.array(y)\n",
    "\n",
    "# Filter the data.\n",
    "X_filtered = X_scaled[indices_to_keep]\n",
    "y_filtered = y[indices_to_keep]\n",
    "\n",
    "# Encode the Genres into numerical values\n",
    "le = LabelEncoder()\n",
    "y_encoded = le.fit_transform(y_filtered)\n",
    "\n",
    "freq_leftover = Counter(y_encoded)\n",
    "leftover_genres = list(freq_leftover.keys())\n",
    "leftover_counts = list(freq_leftover.values())\n",
    "\n",
    "fig, axs = plt.subplots(figsize=(15, 6))\n",
    "\n",
    "# Plot normalized label frequencies.\n",
    "sns.barplot(x=leftover_genres, y=leftover_counts, ax=axs)\n",
    "axs.set_title(\"Normalized Genre Frequencies\")\n",
    "axs.set_xlabel(\"Genre\")\n",
    "axs.set_ylabel(\"Count\")\n",
    "axs.tick_params(axis='x', rotation=45)\n",
    "axs.set_xticklabels(le.classes_)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove highly collerating data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropping columns: ['feature_5', 'feature_8', 'feature_9', 'feature_10', 'feature_12', 'feature_14', 'feature_16', 'feature_20', 'feature_21', 'feature_23', 'feature_25', 'feature_26', 'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_34', 'feature_37', 'feature_38', 'feature_41', 'feature_43', 'feature_44', 'feature_46', 'feature_48', 'feature_50', 'feature_51', 'feature_53']\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 101800 characters]",
      "text/plain": [
       "<Figure size 1200x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "CORRELATING_VALUE = 0.9\n",
    "\n",
    "X = X_filtered\n",
    "\n",
    "def remove_highly_correlated_features(X, threshold):\n",
    "    # If X is a NumPy array, convert it to a DataFrame.\n",
    "    if isinstance(X, np.ndarray):\n",
    "        # Create column names if not provided\n",
    "        X_df = pd.DataFrame(X, columns=[f'feature_{i}' for i in range(X.shape[1])])\n",
    "    else:\n",
    "        X_df = X.copy()\n",
    "    \n",
    "    # Compute the absolute correlation matrix\n",
    "    corr_matrix = X_df.corr().abs()\n",
    "    \n",
    "    # Create an upper triangle matrix of correlations\n",
    "    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n",
    "    \n",
    "    # Identify columns to drop: any feature with correlation greater than the threshold\n",
    "    to_drop = [column for column in upper.columns if any(upper[column] > threshold)]\n",
    "    print(\"Dropping columns:\", to_drop)\n",
    "    \n",
    "    # Drop these columns from the dataframe\n",
    "    X_df_reduced = X_df.drop(columns=to_drop)\n",
    "    \n",
    "    # If original X was a NumPy array, return as a NumPy array.\n",
    "    if isinstance(X, np.ndarray):\n",
    "        return X_df_reduced.values\n",
    "    else:\n",
    "        return X_df_reduced\n",
    "\n",
    "\n",
    "X_reduced = remove_highly_correlated_features(X, threshold=CORRELATING_VALUE)\n",
    "\n",
    "# -- Create a feature correlation matrix --\n",
    "df_features = pd.DataFrame(X_reduced, columns=[f'feature_{i}' for i in range(X_reduced.shape[1])])\n",
    "\n",
    "# Compute the correlation matrix.\n",
    "corr = df_features.corr()\n",
    "\n",
    "plt.figure(figsize=(12,10))\n",
    "sns.heatmap(corr, cmap='coolwarm', annot=False)\n",
    "plt.title('Feature Correlation Matrix')\n",
    "plt.show()\n",
    "\n",
    "X_filtered = X_reduced\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a Test-Train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set shape: (8672, 28) (8672,)\n",
      "Test set shape: (2168, 28) (2168,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X_filtered, y_encoded, test_size=0.2, random_state=42, stratify=y_encoded\n",
    ")\n",
    "\n",
    "print(\"Training set shape:\", X_train.shape, y_train.shape)\n",
    "print(\"Test set shape:\", X_test.shape, y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best RF parameters: {'max_depth': None, 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "Best RF accuracy: 0.7062\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# Parameter grid\n",
    "param_grid_rf = {\n",
    "    'n_estimators': [50, 100, 200, 250, 300, 350],\n",
    "    'max_depth': [None, 10, 20],\n",
    "    'min_samples_split': [2, 5],\n",
    "    'min_samples_leaf': [1, 2]\n",
    "}\n",
    "\n",
    "# Grid search\n",
    "rf = RandomForestClassifier(random_state=42)\n",
    "grid_search_rf = GridSearchCV(rf,\n",
    "                              param_grid_rf,\n",
    "                              cv=5,\n",
    "                              n_jobs=-1,\n",
    "                              scoring='accuracy'\n",
    "                              )\n",
    "grid_search_rf.fit(X_train, y_train)\n",
    "\n",
    "# Best model\n",
    "final_model = grid_search_rf.best_estimator_\n",
    "print(\"Best RF parameters:\", grid_search_rf.best_params_)\n",
    "print(f\"Best RF accuracy: {grid_search_rf.best_score_:.4f}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation Charts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix - Shows what was the predicted value compared to the true value when evaluating the Model\n",
    "\n",
    "The more values in the diagonal line the better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 67224 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "# Predict on the test set\n",
    "y_pred = final_model.predict(X_test)\n",
    "\n",
    "# Compute confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "# Display it nicely\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=le.classes_)\n",
    "disp.plot(cmap='Blues')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.title(\"Confusion Matrix\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
