diff --git a/spotify.ipynb b/spotify.ipynb index 39180a6..c980c80 100644 --- a/spotify.ipynb +++ b/spotify.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 397, + "execution_count": 459, "id": "e9bf1d26", "metadata": {}, "outputs": [], @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 398, + "execution_count": 460, "id": "11bb5dc2", "metadata": {}, "outputs": [], @@ -54,22 +54,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 461, "id": "a898326e", "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'Str' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[399]\u001b[39m\u001b[32m, line 33\u001b[39m\n\u001b[32m 29\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;28;01min\u001b[39;00m columns:\n\u001b[32m 30\u001b[39m df = df.withColumn(column, F.to_timestamp(column))\n\u001b[32m 31\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m 32\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m33\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_minutes_played(df: DataFrame, src: Str, dest: Str, decimals: Int) -> DataFrame:\n\u001b[32m 34\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m (df.withColumn(dest, F.round(F.col(src) / \u001b[32m60000\u001b[39m, \u001b[32m2\u001b[39m)))\n\u001b[32m 35\u001b[39m \n\u001b[32m 36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_time_parts(df: DataFrame, ts_col=\u001b[33m\"ts\"\u001b[39m) -> DataFrame:\n", - "\u001b[31mNameError\u001b[39m: name 'Str' is not defined" - ] - } - ], + "outputs": [], "source": [ " ## Transform ##\n", "def null_counts_by_column(df):\n", @@ -103,13 +91,21 @@ " df = df.withColumn(column, F.to_timestamp(column))\n", " return df\n", "\n", + "def parse_booleans(df: DataFrame, columns) -> DataFrame:\n", + " df = df.select([F.col(c).cast(\"string\") for c in columns])\n", + " return df\n", + "\n", "def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n", " return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n", "\n", - "def add_time_parts(df: DataFrame, ts_col=\"ts\") -> DataFrame:\n", + "def add_time_parts(df: DataFrame, ts_col) -> DataFrame:\n", " return (df\n", " .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n", " .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n", + " .withColumn('time', F.date_format('ts', 'HH:mm:ss'))\n", + " .withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n", + " .withColumn('offline_time', F.date_format('offline_timestamp', 'HH:mm:ss'))\n", + " .withColumn('offline_date', F.date_format('offline_timestamp', 'yyyy-MM-dd')) \n", " )\n", "\n", "def clean_track_df(df: DataFrame) -> DataFrame:\n", @@ -117,14 +113,15 @@ " .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", " .transform(track_df)\n", " .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n", + " # .transform(parse_booleans, [\"offline\", \"shuffle\", \"skipped\"]) ##returns only the selected columns\n", " .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n", - " .transform(add_time_parts)\n", + " .transform(add_time_parts, \"ts\")\n", " )" ] }, { "cell_type": "code", - "execution_count": 395, + "execution_count": 462, "id": "c609291c", "metadata": {}, "outputs": [], @@ -146,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 400, + "execution_count": 463, "id": "4cc14bd0", "metadata": {}, "outputs": [ @@ -160,23 +157,36 @@ "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "only showing top 1 row\n", - "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", - "| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username|\n", - "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", - "|Last Day Of Summer| Summer Walker| Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n", - "|Last Day Of Summer| Summer Walker| Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n", - "| Famouz| JHAYCO|No Me Conoce - Remix| 5.15| false|2023-04-07 12:39:15| ios| trackdone| clickrow| true| false|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n", - "| 4K| El Alfa| 4K| 3.36| false|2023-04-07 12:45:51| ios| trackdone| trackdone| true| false|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n", - "| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 0.3| false|2023-04-07 12:49:13| ios| fwdbtn| trackdone| true| true|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n", - "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", - "only showing top 5 rows\n", - "+--------------------+-----+\n", - "| username|count|\n", - "+--------------------+-----+\n", - "| csm3-us|64556|\n", - "|fm30ajetkgrrkbvae...|38070|\n", - "+--------------------+-----+\n", - "\n" + "+------------------+-------------+---------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+--------+----------+------------+------------+\n", + "| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username| time| date|offline_time|offline_date|\n", + "+------------------+-------------+---------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+--------+----------+------------+------------+\n", + "|Last Day Of Summer|Summer Walker|Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|22:58:31|2023-04-06| 22:57:10| 2023-04-06|\n", + "|Last Day Of Summer|Summer Walker|Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|12:39:14|2023-04-07| 12:39:00| 2023-04-07|\n", + "+------------------+-------------+---------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+--------+----------+------------+------------+\n", + "only showing top 2 rows\n", + "root\n", + " |-- album_name: string (nullable = true)\n", + " |-- artist_name: string (nullable = true)\n", + " |-- track_name: string (nullable = true)\n", + " |-- ms_played: double (nullable = true)\n", + " |-- offline: boolean (nullable = true)\n", + " |-- offline_timestamp: timestamp (nullable = true)\n", + " |-- platform: string (nullable = true)\n", + " |-- reason_end: string (nullable = true)\n", + " |-- reason_start: string (nullable = true)\n", + " |-- shuffle: boolean (nullable = true)\n", + " |-- skipped: boolean (nullable = true)\n", + " |-- track_uri: string (nullable = true)\n", + " |-- ts: timestamp (nullable = true)\n", + " |-- username: string (nullable = true)\n", + " |-- time: string (nullable = true)\n", + " |-- date: string (nullable = true)\n", + " |-- offline_time: string (nullable = true)\n", + " |-- offline_date: string (nullable = true)\n", + "\n", + " Raw DF Rows: 105477\n", + " Transformed DF Rows: 102626\n", + " Row difference is 2851\n" ] } ], @@ -186,18 +196,21 @@ "app_name = \"ml_media\"\n", "spark = start_spark(app_name)\n", "\n", - "df = read_json(spark, music_source_paths())\n", - "df.show(1)\n", + "raw_df = read_json(spark, music_source_paths())\n", + "raw_df.show(1)\n", "\n", - "track_df = clean_track_df(df)\n", + "transformed_df = clean_track_df(raw_df)\n", + "transformed_df.show(2)\n", + "transformed_df.printSchema()\n", "\n", - "# rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", - "# track_df = track_df(rd_df)\n", + "print(f\" Raw DF Rows: {raw_df.count()}\")\n", + "print(f\" Transformed DF Rows: {transformed_df.count()}\")\n", + "print(f\" Row difference is {raw_df.count() - transformed_df.count()}\") \n", "\n", - "# track_df = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\")).withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\")).withColumn(\"ms_played\", F.round(F.col(\"ms_played\") / 60000, 2)).withColumn('time', F.date_format('ts', 'HH:mm:ss')).withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n", - "track_df.show(5)\n", + "spark.stop()\n", + " \n", "\n", - "track_df.groupBy(\"username\").count().show()" + "# transformed_df.groupBy(\"username\").count().show()" ] }, {