diff --git a/spotify.ipynb b/spotify.ipynb index e584990..39180a6 100644 --- a/spotify.ipynb +++ b/spotify.ipynb @@ -2,12 +2,12 @@ "cells": [ { "cell_type": "code", - "execution_count": 110, + "execution_count": 397, "id": "e9bf1d26", "metadata": {}, "outputs": [], "source": [ - "from pyspark.sql import SparkSession\n", + "from pyspark.sql import SparkSession, DataFrame\n", "from pyspark.sql import functions as F\n", "from pathlib import Path\n", "import glob" @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 275, + "execution_count": 398, "id": "11bb5dc2", "metadata": {}, "outputs": [], @@ -54,10 +54,22 @@ }, { "cell_type": "code", - "execution_count": 289, + "execution_count": null, "id": "a898326e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'Str' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[399]\u001b[39m\u001b[32m, line 33\u001b[39m\n\u001b[32m 29\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;28;01min\u001b[39;00m columns:\n\u001b[32m 30\u001b[39m df = df.withColumn(column, F.to_timestamp(column))\n\u001b[32m 31\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m 32\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m33\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_minutes_played(df: DataFrame, src: Str, dest: Str, decimals: Int) -> DataFrame:\n\u001b[32m 34\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m (df.withColumn(dest, F.round(F.col(src) / \u001b[32m60000\u001b[39m, \u001b[32m2\u001b[39m)))\n\u001b[32m 35\u001b[39m \n\u001b[32m 36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_time_parts(df: DataFrame, ts_col=\u001b[33m\"ts\"\u001b[39m) -> DataFrame:\n", + "\u001b[31mNameError\u001b[39m: name 'Str' is not defined" + ] + } + ], "source": [ " ## Transform ##\n", "def null_counts_by_column(df):\n", @@ -85,12 +97,34 @@ "\n", "def track_df(df):\n", " return df.where(df[\"track_name\"].isNotNull())\n", - "\n" + "\n", + "def parse_timestamps(df: DataFrame, columns) -> DataFrame:\n", + " for column in columns:\n", + " df = df.withColumn(column, F.to_timestamp(column))\n", + " return df\n", + "\n", + "def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n", + " return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n", + "\n", + "def add_time_parts(df: DataFrame, ts_col=\"ts\") -> DataFrame:\n", + " return (df\n", + " .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n", + " .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n", + " )\n", + "\n", + "def clean_track_df(df: DataFrame) -> DataFrame:\n", + " return(df\n", + " .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", + " .transform(track_df)\n", + " .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n", + " .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n", + " .transform(add_time_parts)\n", + " )" ] }, { "cell_type": "code", - "execution_count": 290, + "execution_count": 395, "id": "c609291c", "metadata": {}, "outputs": [], @@ -101,18 +135,18 @@ " \"master_metadata_album_artist_name\": \"artist_name\",\n", " \"master_metadata_track_name\": \"track_name\",\n", " \"spotify_episode_uri\" : \"episode_uri\",\n", - " \"spotify_track_uri\" : \"spotify_track_uri\"\n", + " \"spotify_track_uri\" : \"track_uri\"\n", "}\n", "\n", "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n", "\n", - "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\"]\n", + "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\", \"conn_country\", \"spotify_episode_uri\", \"episode_name\", \"episode_show_name\"]\n", "\n" ] }, { "cell_type": "code", - "execution_count": 291, + "execution_count": 400, "id": "4cc14bd0", "metadata": {}, "outputs": [ @@ -126,35 +160,22 @@ "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "only showing top 1 row\n", - "+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", - "|conn_country|episode_name|episode_show_name| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", - "+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", - "| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n", - "| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n", - "| GB| NULL| NULL| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n", - "| GB| NULL| NULL| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n", - "| GB| NULL| NULL| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n", - "+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", + "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", + "| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username|\n", + "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", + "|Last Day Of Summer| Summer Walker| Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n", + "|Last Day Of Summer| Summer Walker| Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n", + "| Famouz| JHAYCO|No Me Conoce - Remix| 5.15| false|2023-04-07 12:39:15| ios| trackdone| clickrow| true| false|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n", + "| 4K| El Alfa| 4K| 3.36| false|2023-04-07 12:45:51| ios| trackdone| trackdone| true| false|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n", + "| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 0.3| false|2023-04-07 12:49:13| ios| fwdbtn| trackdone| true| true|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n", + "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", "only showing top 5 rows\n", - "root\n", - " |-- conn_country: string (nullable = true)\n", - " |-- episode_name: string (nullable = true)\n", - " |-- episode_show_name: string (nullable = true)\n", - " |-- album_name: string (nullable = true)\n", - " |-- artist_name: string (nullable = true)\n", - " |-- track_name: string (nullable = true)\n", - " |-- ms_played: long (nullable = true)\n", - " |-- offline: boolean (nullable = true)\n", - " |-- offline_timestamp: long (nullable = true)\n", - " |-- platform: string (nullable = true)\n", - " |-- reason_end: string (nullable = true)\n", - " |-- reason_start: string (nullable = true)\n", - " |-- shuffle: boolean (nullable = true)\n", - " |-- skipped: boolean (nullable = true)\n", - " |-- episode_uri: string (nullable = true)\n", - " |-- spotify_track_uri: string (nullable = true)\n", - " |-- ts: timestamp (nullable = true)\n", - " |-- username: string (nullable = true)\n", + "+--------------------+-----+\n", + "| username|count|\n", + "+--------------------+-----+\n", + "| csm3-us|64556|\n", + "|fm30ajetkgrrkbvae...|38070|\n", + "+--------------------+-----+\n", "\n" ] } @@ -168,13 +189,15 @@ "df = read_json(spark, music_source_paths())\n", "df.show(1)\n", "\n", - "rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", - "track_df = track_df(rd_df)\n", + "track_df = clean_track_df(df)\n", "\n", + "# rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", + "# track_df = track_df(rd_df)\n", "\n", - "timestamp = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\"))\n", - "timestamp.show(5)\n", - "timestamp.printSchema()" + "# track_df = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\")).withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\")).withColumn(\"ms_played\", F.round(F.col(\"ms_played\") / 60000, 2)).withColumn('time', F.date_format('ts', 'HH:mm:ss')).withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n", + "track_df.show(5)\n", + "\n", + "track_df.groupBy(\"username\").count().show()" ] }, {