media_ml/spotify.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 397,
   "id": "e9bf1d26",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession, DataFrame\n",
    "from pyspark.sql import functions as F\n",
    "from pathlib import Path\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "id": "11bb5dc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "                            ## Extract ##\n",
    "\n",
    "BASE = Path.cwd() / \"ingest\"\n",
    "\n",
    "def start_spark(app_name):\n",
    "    spark = SparkSession.builder.appName(app_name).getOrCreate()\n",
    "    spark.sparkContext.setLogLevel(\"ERROR\")\n",
    "    return spark\n",
    "\n",
    "\n",
    "def music_source_paths():\n",
    "    return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n",
    "\n",
    "def video_source_paths():\n",
    "    return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n",
    "\n",
    "def read_json(spark, paths):\n",
    "    if not paths:\n",
    "        raise FileNotFoundError(\"No JSON files matched the pattern\")\n",
    "    return (\n",
    "        spark.read\n",
    "        .format(\"json\")\n",
    "        .option(\"multiLine\", True)\n",
    "        .json(paths)                                  # list of paths, one read\n",
    "        # .withColumn(\"source_file\", F.input_file_name())\n",
    "        # .withColumn(\n",
    "        #     \"source_folder\",\n",
    "        #     F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n",
    "        # )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a898326e",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Str' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[399]\u001b[39m\u001b[32m, line 33\u001b[39m\n\u001b[32m     29\u001b[39m     \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;28;01min\u001b[39;00m columns:\n\u001b[32m     30\u001b[39m         df = df.withColumn(column, F.to_timestamp(column))\n\u001b[32m     31\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m     32\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m33\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_minutes_played(df: DataFrame, src: Str, dest: Str, decimals: Int) -> DataFrame:\n\u001b[32m     34\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m (df.withColumn(dest, F.round(F.col(src) / \u001b[32m60000\u001b[39m, \u001b[32m2\u001b[39m)))\n\u001b[32m     35\u001b[39m \n\u001b[32m     36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_time_parts(df: DataFrame, ts_col=\u001b[33m\"ts\"\u001b[39m) -> DataFrame:\n",
      "\u001b[31mNameError\u001b[39m: name 'Str' is not defined"
     ]
    }
   ],
   "source": [
    "                                ## Transform ##\n",
    "def null_counts_by_column(df):\n",
    "    null_counts = df.select([\n",
    "        F.count(F.when(F.col(c).isNull(), c)).alias(c)\n",
    "        for c in df.columns\n",
    "    ])\n",
    "    return null_counts\n",
    "\n",
    "def metadata_df(df):\n",
    "    return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
    "\n",
    "def rename_columns(df, rename_map, drop_columns):\n",
    "    missing_rename = set(rename_map) - set(df.columns)\n",
    "    missing_dropped = set(drop_columns) - set(df.columns)\n",
    "\n",
    "    if missing_rename:\n",
    "        print(f\"Warning: columns not found: {missing_rename}\")\n",
    "    if missing_dropped:\n",
    "        print(f\"Warning: columns not found: {missing_dropped}\")\n",
    "\n",
    "    df = df.drop(*drop_columns)\n",
    "    \n",
    "    return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n",
    "\n",
    "def track_df(df):\n",
    "    return df.where(df[\"track_name\"].isNotNull())\n",
    "\n",
    "def parse_timestamps(df: DataFrame, columns) -> DataFrame:\n",
    "    for column in columns:\n",
    "        df = df.withColumn(column, F.to_timestamp(column))\n",
    "    return df\n",
    "\n",
    "def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n",
    "    return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n",
    "\n",
    "def add_time_parts(df: DataFrame, ts_col=\"ts\") -> DataFrame:\n",
    "    return (df\n",
    "            .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
    "            .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n",
    "            )\n",
    "\n",
    "def clean_track_df(df: DataFrame) -> DataFrame:\n",
    "    return(df\n",
    "            .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
    "            .transform(track_df)\n",
    "            .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n",
    "            .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n",
    "            .transform(add_time_parts)\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "id": "c609291c",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Config \n",
    "SPOTIFY_RENAMES = {\n",
    "    \"master_metadata_album_album_name\": \"album_name\",\n",
    "    \"master_metadata_album_artist_name\": \"artist_name\",\n",
    "    \"master_metadata_track_name\": \"track_name\",\n",
    "    \"spotify_episode_uri\" : \"episode_uri\",\n",
    "    \"spotify_track_uri\" : \"track_uri\"\n",
    "}\n",
    "\n",
    "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n",
    "\n",
    "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\", \"conn_country\", \"spotify_episode_uri\", \"episode_name\", \"episode_show_name\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "id": "4cc14bd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
      "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform|          reason_end|reason_start|shuffle|skipped|spotify_episode_uri|   spotify_track_uri|                  ts|user_agent_decrypted|username|\n",
      "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
      "|          GB|        NULL|             NULL|         false|     31.94.22.158|              Last Day Of Summer|                    Summer Walker|           Girls Need Love|    80505|  false|       1680818230|     ios|unexpected-exit-w...|   trackdone|   true|  false|               NULL|spotify:track:644...|2023-04-06T21:58:31Z|             unknown| csm3-us|\n",
      "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
      "only showing top 1 row\n",
      "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
      "|        album_name|   artist_name|          track_name|ms_played|offline|  offline_timestamp|platform|          reason_end|reason_start|shuffle|skipped|           track_uri|                 ts|username|\n",
      "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
      "|Last Day Of Summer| Summer Walker|     Girls Need Love|     1.34|  false|2023-04-06 22:57:10|     ios|unexpected-exit-w...|   trackdone|   true|  false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n",
      "|Last Day Of Summer| Summer Walker|     Girls Need Love|     0.17|  false|2023-04-07 12:39:00|     ios|             endplay|     appload|   true|   true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n",
      "|            Famouz|        JHAYCO|No Me Conoce - Remix|     5.15|  false|2023-04-07 12:39:15|     ios|           trackdone|    clickrow|   true|  false|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n",
      "|                4K|       El Alfa|                  4K|     3.36|  false|2023-04-07 12:45:51|     ios|           trackdone|   trackdone|   true|  false|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n",
      "|      Sauce Boyz 2|Eladio Carrion|  No Te Deseo el Mal|      0.3|  false|2023-04-07 12:49:13|     ios|              fwdbtn|   trackdone|   true|   true|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n",
      "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
      "only showing top 5 rows\n",
      "+--------------------+-----+\n",
      "|            username|count|\n",
      "+--------------------+-----+\n",
      "|             csm3-us|64556|\n",
      "|fm30ajetkgrrkbvae...|38070|\n",
      "+--------------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "                                    ## MAIN ## \n",
    "\n",
    "app_name = \"ml_media\"\n",
    "spark = start_spark(app_name)\n",
    "\n",
    "df = read_json(spark, music_source_paths())\n",
    "df.show(1)\n",
    "\n",
    "track_df = clean_track_df(df)\n",
    "\n",
    "# rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
    "# track_df = track_df(rd_df)\n",
    "\n",
    "# track_df = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\")).withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\")).withColumn(\"ms_played\", F.round(F.col(\"ms_played\") / 60000, 2)).withColumn('time', F.date_format('ts', 'HH:mm:ss')).withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n",
    "track_df.show(5)\n",
    "\n",
    "track_df.groupBy(\"username\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "faa6a8cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "21\n",
      "-RECORD 0----------------------\n",
      " conn_country         | 0      \n",
      " episode_name         | 105093 \n",
      " episode_show_name    | 105093 \n",
      " incognito_mode       | 0      \n",
      " ip_addr_decrypted    | 4492   \n",
      " album_name           | 2851   \n",
      " artist_name          | 2851   \n",
      " track_name           | 2851   \n",
      " ms_played            | 0      \n",
      " offline              | 279    \n",
      " offline_timestamp    | 2      \n",
      " platform             | 0      \n",
      " reason_end           | 4490   \n",
      " reason_start         | 0      \n",
      " shuffle              | 0      \n",
      " skipped              | 44457  \n",
      " spotify_episode_uri  | 105093 \n",
      " spotify_track_uri    | 2851   \n",
      " ts                   | 0      \n",
      " user_agent_decrypted | 24700  \n",
      " username             | 0      \n",
      "\n",
      "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
      "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform|          reason_end|reason_start|shuffle|skipped|spotify_episode_uri|   spotify_track_uri|                  ts|user_agent_decrypted|username|\n",
      "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
      "|          GB|        NULL|             NULL|         false|     31.94.22.158|              Last Day Of Summer|                    Summer Walker|           Girls Need Love|    80505|  false|       1680818230|     ios|unexpected-exit-w...|   trackdone|   true|  false|               NULL|spotify:track:644...|2023-04-06T21:58:31Z|             unknown| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|     31.94.22.158|              Last Day Of Summer|                    Summer Walker|           Girls Need Love|    10100|  false|       1680867540|     ios|             endplay|     appload|   true|   true|               NULL|spotify:track:644...|2023-04-07T11:39:14Z|             unknown| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|     31.94.22.158|                          Famouz|                           JHAYCO|      No Me Conoce - Remix|   309120|  false|       1680867555|     ios|           trackdone|    clickrow|   true|  false|               NULL|spotify:track:2be...|2023-04-07T11:45:50Z|             unknown| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|     31.94.22.158|                              4K|                          El Alfa|                        4K|   201500|  false|       1680867951|     ios|           trackdone|   trackdone|   true|  false|               NULL|spotify:track:4xI...|2023-04-07T11:49:12Z|             unknown| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|     31.94.22.158|                    Sauce Boyz 2|                   Eladio Carrion|        No Te Deseo el Mal|    18282|  false|       1680868153|     ios|              fwdbtn|   trackdone|   true|   true|               NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z|             unknown| csm3-us|\n",
      "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
      "only showing top 5 rows\n",
      "Original Rows 105477\n",
      "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
      "|conn_country|episode_name|episode_show_name|incognito_mode|        album_name|   artist_name|          track_name|ms_played|offline|offline_timestamp|platform|          reason_end|reason_start|shuffle|skipped|episode_uri|   spotify_track_uri|                  ts|username|\n",
      "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
      "|          GB|        NULL|             NULL|         false|Last Day Of Summer| Summer Walker|     Girls Need Love|    80505|  false|       1680818230|     ios|unexpected-exit-w...|   trackdone|   true|  false|       NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|Last Day Of Summer| Summer Walker|     Girls Need Love|    10100|  false|       1680867540|     ios|             endplay|     appload|   true|   true|       NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|            Famouz|        JHAYCO|No Me Conoce - Remix|   309120|  false|       1680867555|     ios|           trackdone|    clickrow|   true|  false|       NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|                4K|       El Alfa|                  4K|   201500|  false|       1680867951|     ios|           trackdone|   trackdone|   true|  false|       NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n",
      "|          GB|        NULL|             NULL|         false|      Sauce Boyz 2|Eladio Carrion|  No Te Deseo el Mal|    18282|  false|       1680868153|     ios|              fwdbtn|   trackdone|   true|   true|       NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n",
      "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
      "only showing top 5 rows\n",
      "Track Rows 102626\n"
     ]
    }
   ],
   "source": [
    "## Potential Test Help\n",
    "print(metadata_df(music_df).rdd.getNumPartitions())  # Partitions for Pyspark\n",
    "print(len(metadata_df(music_df).columns)) # # of columns in JSON\n",
    "null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n",
    "null_counts.show(vertical=True) # Print Null counts by column_df\n",
    "total_rows = music_df.count\n",
    "\n",
    "df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n",
    "                            ## same can be done for podcasts\n",
    "\n",
    "df.show(5)\n",
    "print(f\"Original Rows {df.count()}\")\n",
    "\n",
    "track_df.show(5)\n",
    "print(f\"Track Rows {track_df.count()}\")\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}