diff --git a/spotify.ipynb b/spotify.ipynb index 0f7bf8e..014d6a3 100644 --- a/spotify.ipynb +++ b/spotify.ipynb @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 212, "id": "a898326e", "metadata": {}, "outputs": [], @@ -70,22 +70,27 @@ "def metadata_df(df):\n", " return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n", "\n", - "def rename_columns(df, rename_map):\n", - " missing = set(rename_map) - set(df.columns)\n", - " if missing:\n", - " print(f\"Warning: columns not found: {missing}\")\n", + "def rename_columns(df, rename_map, drop_columns):\n", + " missing_rename = set(rename_map) - set(df.columns)\n", + " missing_dropped = set(drop_columns) - set(df.columns)\n", + "\n", + " if missing_rename:\n", + " print(f\"Warning: columns not found: {missing_rename}\")\n", + " if missing_dropped:\n", + " print(f\"Warning: columns not found: {missing_dropped}\")\n", + "\n", + " df = df.drop(*drop_columns)\n", + " \n", " return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n", "\n", "def track_df(df):\n", " return df.where(df[\"track_name\"].isNotNull())\n", - "\n", - "def drop_pii_columns(df):\n", - " return df.drop(\"ip_addr_decrypted\", \"user_agent_decrypted\")\n" + "\n" ] }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 213, "id": "c609291c", "metadata": {}, "outputs": [], @@ -95,14 +100,18 @@ " \"master_metadata_album_album_name\": \"album_name\",\n", " \"master_metadata_album_artist_name\": \"artist_name\",\n", " \"master_metadata_track_name\": \"track_name\",\n", + " \"spotify_episode_uri\" : \"episode_uri\",\n", + " \"spotify_track_uri\" : \"spotify_track_uri\"\n", "}\n", "\n", - "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n" + "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n", + "\n", + "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\"]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 214, "id": "4cc14bd0", "metadata": {}, "outputs": [ @@ -120,29 +129,16 @@ "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "only showing top 5 rows\n", - "Original Rows 105477\n", - "+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", - "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", - "+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", - "| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", - "+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", - "only showing top 5 rows\n", - "Track Rows 102626\n", - "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", - "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|username|\n", - "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", - "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", - "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", - "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", - "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", - "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", - "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", - "only showing top 5 rows\n", - "Dropped Rows 102626\n" + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", + "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", + "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", + "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", + "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", + "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", + "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", + "only showing top 5 rows\n" ] } ], @@ -153,15 +149,16 @@ "spark = start_spark(app_name)\n", "\n", "df = read_json(spark, music_source_paths())\n", + "df.show(5)\n", "\n", - "track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES))\n", + "track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS))\n", "\n", - "drop_df = drop_pii_columns(track_df) \n" + "track_df.show(5)\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 215, "id": "faa6a8cb", "metadata": {}, "outputs": [ @@ -193,18 +190,30 @@ " ts | 0 \n", " user_agent_decrypted | 24700 \n", " username | 0 \n", - "\n" + "\n", + "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", + "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", + "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "only showing top 5 rows\n", + "Original Rows 105477\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", + "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", + "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", + "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", + "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", + "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", + "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", + "only showing top 5 rows\n", + "Track Rows 102626\n" ] - }, - { - "data": { - "text/plain": [ - "2851" - ] - }, - "execution_count": 155, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -223,9 +232,7 @@ "\n", "track_df.show(5)\n", "print(f\"Track Rows {track_df.count()}\")\n", - "\n", - "drop_df.show(5)\n", - "drop_df.count()\n" + "\n" ] }, {