diff --git a/spotify_bronze_silver.ipynb b/spotify_bronze_silver.ipynb index 3dbcca2..45610f2 100644 --- a/spotify_bronze_silver.ipynb +++ b/spotify_bronze_silver.ipynb @@ -187,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 91, "id": "4cc14bd0", "metadata": {}, "outputs": [ @@ -201,46 +201,21 @@ "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "only showing top 1 row\n", - "+-------+--------------------+-----+\n", - "|skipped| reason_end|count|\n", - "+-------+--------------------+-----+\n", - "| false| unknown| 99|\n", - "| NULL| unknown| 375|\n", - "| false|unexpected-exit-w...| 1243|\n", - "| NULL|unexpected-exit-w...| 432|\n", - "| false| unexpected-exit| 36|\n", - "| NULL| unexpected-exit| 51|\n", - "| false| trackerror| 69|\n", - "| NULL| trackerror| 33|\n", - "| false| trackdone|36102|\n", - "| NULL| trackdone|24077|\n", - "| false| remote| 121|\n", - "| NULL| remote| 859|\n", - "| false| logout| 491|\n", - "| NULL| logout| 745|\n", - "| true| fwdbtn|16571|\n", - "| NULL| fwdbtn|11445|\n", - "| true| endplay| 2943|\n", - "| NULL| endplay| 1447|\n", - "| true| backbtn| 650|\n", - "| NULL| backbtn| 402|\n", - "+-------+--------------------+-----+\n", - "only showing top 20 rows\n", - "+----------------------------+-----+\n", - "|reason_end |count|\n", - "+----------------------------+-----+\n", - "|trackdone |60179|\n", - "|fwdbtn |28016|\n", - "|NULL |4435 |\n", - "|endplay |4390 |\n", - "|unexpected-exit-while-paused|1675 |\n", - "|logout |1236 |\n", - "|backbtn |1052 |\n", - "|remote |980 |\n", - "|unknown |474 |\n", - "|trackerror |102 |\n", - "|unexpected-exit |87 |\n", - "+----------------------------+-----+\n", + "+----------------------------+--------------+\n", + "|reason_end |skipped_values|\n", + "+----------------------------+--------------+\n", + "|NULL |[] |\n", + "|backbtn |[true] |\n", + "|endplay |[true] |\n", + "|fwdbtn |[true] |\n", + "|logout |[false] |\n", + "|remote |[false] |\n", + "|trackdone |[false] |\n", + "|trackerror |[false] |\n", + "|unexpected-exit |[false] |\n", + "|unexpected-exit-while-paused|[false] |\n", + "|unknown |[false] |\n", + "+----------------------------+--------------+\n", "\n", " Raw DF Rows: 105477\n", " Transformed DF Rows: 102626\n", @@ -260,10 +235,14 @@ " transformed_df = clean_track_df(raw_df)\n", " #load_data(transformed_df, table_name)\n", " # transformed_df.show(5)\n", - " skip_reason_df = transformed_df.groupBy(\"skipped\", \"reason_end\").count().orderBy(F.desc(\"reason_end\")).show()\n", + " # skip_reason_df = transformed_df.groupBy(\"skipped\", \"reason_end\").count().orderBy(F.desc(\"reason_end\")).show()\n", " # reason_df = transformed_df.groupBy(\"reason_end\").orderBy(F.desc(\"count\")).show(5)\n", " # skip_df = transformed_df.groupBy(\"skipped\").count().orderBy(F.desc(\"count\")).show(5)\n", - " skip_join_bool = transformed_df.select(\"reason_end\", \"was_skipped\").distinct().orderBy(\"reason_end\").show(truncate=False))\n", + " # skip_join_bool = transformed_df.select(\"reason_end\", \"skipped\").distinct().orderBy(\"reason_end\").show(100)\n", + " skip_reason_bool = (transformed_df.groupBy(\"reason_end\")\n", + " .agg(F.collect_set(\"skipped\").alias(\"skipped_values\"))\n", + " .orderBy(\"reason_end\")\n", + " .show(truncate=False))\n", " #join(transformed_df.F.col(\"reason_end\"))\n", "\n", " # missing_boolean = set(rename_map) - set(df.columns)\n", diff --git a/spotify_silver_gold.ipynb b/spotify_silver_gold.ipynb index dc0696a..d4086f2 100644 --- a/spotify_silver_gold.ipynb +++ b/spotify_silver_gold.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "f7673c72", "metadata": {}, "outputs": [ @@ -87,16 +87,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n", - "|album_name| artist_name| track_name|ms_played|offline|offline_timestamp| platform|reason_end|reason_start|shuffle|skipped| track_uri| ts| username| offline_timestamped| time| date|offline_time|offline_date|\n", - "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n", - "|Elton John| Elton John| Your Song| 0.42| false| 1665740300104|iOS 16.0 (iPhone1...| endplay| clickrow| false| NULL|spotify:track:38z...|2022-10-14 10:38:46|fm30ajetkgrrkbvae...|+54755-03-18 14:5...|10:38:46|2022-10-14| 14:55:04|+54755-03-18|\n", - "| Lights 10|Ellie Goulding| Your Song| 3.11| false| 1665740325724|iOS 16.0 (iPhone1...| endplay| clickrow| true| NULL|spotify:track:6XK...|2022-10-14 10:42:32|fm30ajetkgrrkbvae...|+54755-03-18 22:0...|10:42:32|2022-10-14| 22:02:04|+54755-03-18|\n", - "| 19| Adele|Make You Feel My ...| 3.53| false| 1665740551741|iOS 16.0 (iPhone1...| trackdone| clickrow| false| NULL|spotify:track:273...|2022-10-14 10:46:27|fm30ajetkgrrkbvae...|+54755-03-21 12:4...|10:46:27|2022-10-14| 12:49:01|+54755-03-21|\n", - "| May It Be| Enya| May It Be| 0.07| false| 1665740786443|iOS 16.0 (iPhone1...| backbtn| trackdone| true| NULL|spotify:track:1wI...|2022-10-14 10:46:32|fm30ajetkgrrkbvae...|+54755-03-24 06:0...|10:46:32|2022-10-14| 06:00:43|+54755-03-24|\n", - "| 19| Adele|Make You Feel My ...| 0.13| false| 1665740791987|iOS 16.0 (iPhone1...| fwdbtn| backbtn| true| NULL|spotify:track:273...|2022-10-14 10:46:40|fm30ajetkgrrkbvae...|+54755-03-24 07:3...|10:46:40|2022-10-14| 07:33:07|+54755-03-24|\n", - "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n", - "only showing top 5 rows\n" + "+--------------------+--------------------+-----+\n", + "| track_name| username|count|\n", + "+--------------------+--------------------+-----+\n", + "| Mujeriego| csm3-us| 93|\n", + "| Angelito| csm3-us| 16|\n", + "|Navidad en Mi Barrio| csm3-us| 1|\n", + "| Como Perro Y Gato| csm3-us| 1|\n", + "| The Taste of Ink|fm30ajetkgrrkbvae...| 2|\n", + "| Hasta el Amanecer| csm3-us| 4|\n", + "| Tu Foto| csm3-us| 1|\n", + "|Love Is Not Over ...| csm3-us| 3|\n", + "|Like I'm Gonna Lo...|fm30ajetkgrrkbvae...| 2|\n", + "| Lo Siento BB:/| csm3-us| 5|\n", + "| Superbeast| csm3-us| 1|\n", + "| Amor de Hospital| csm3-us| 11|\n", + "| All Right Now|fm30ajetkgrrkbvae...| 2|\n", + "|Dancing On The Ce...|fm30ajetkgrrkbvae...| 4|\n", + "|Prologue / The Da...|fm30ajetkgrrkbvae...| 4|\n", + "|That Way (feat. J...| csm3-us| 11|\n", + "| Blame| csm3-us| 29|\n", + "| Speak Now|fm30ajetkgrrkbvae...| 2|\n", + "|A Little Respect ...|fm30ajetkgrrkbvae...| 3|\n", + "| Rockstar|fm30ajetkgrrkbvae...| 4|\n", + "+--------------------+--------------------+-----+\n", + "only showing top 20 rows\n" ] }, { @@ -115,9 +130,10 @@ "\n", " table_name = \"spotify_data\"\n", " silver_df = read_jdbc(spark, table_name)\n", - " silver_df.show(1)\n", "\n", - " spark.stop()\n", + " silver_df.groupBy(\"track_name\", \"username\").count().show()\n", + "\n", + " #spark.stop()\n", "if __name__ == \"__main__\":\n", " main()" ]