diff --git a/spotify_bronze_silver.ipynb b/spotify_bronze_silver.ipynb index 6a2bfe2..df0fc71 100644 --- a/spotify_bronze_silver.ipynb +++ b/spotify_bronze_silver.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 68, + "execution_count": 14, "id": "e9bf1d26", "metadata": {}, "outputs": [], @@ -17,7 +17,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "11bb5dc2", "metadata": {}, "outputs": [], @@ -29,7 +29,6 @@ "def start_spark(app_name):\n", " jar_path = str(Path.cwd() / \"postgresql.jar\")\n", " spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n", - " spark.log.level(\"ERROR\")\n", " return spark\n", "\n", "def music_source_paths():\n", @@ -56,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "a898326e", "metadata": {}, "outputs": [], @@ -125,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "817475cf", "metadata": {}, "outputs": [], @@ -171,7 +170,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 18, "id": "c609291c", "metadata": {}, "outputs": [], @@ -198,34 +197,10 @@ "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", - "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", - "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", - "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", - "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", - "only showing top 1 row\n", - "+----------------------------+--------------+\n", - "|reason_end |skipped_values|\n", - "+----------------------------+--------------+\n", - "|NULL |[] |\n", - "|backbtn |[true] |\n", - "|endplay |[true] |\n", - "|fwdbtn |[true] |\n", - "|logout |[false] |\n", - "|remote |[false] |\n", - "|trackdone |[false] |\n", - "|trackerror |[false] |\n", - "|unexpected-exit |[false] |\n", - "|unexpected-exit-while-paused|[false] |\n", - "|unknown |[false] |\n", - "+----------------------------+--------------+\n", - "\n", - " Raw DF Rows: 105477\n", - " Transformed DF Rows: 102626\n", - " Row difference is 2851\n" + "[Stage 2:> (0 + 6) / 7]\r" ] } ], @@ -234,17 +209,19 @@ "def main():\n", " app_name = \"ml_media\"\n", " spark = start_spark(app_name)\n", + " spark.sparkContext.setLogLevel(\"ERROR\")\n", "\n", " raw_table_name = \"raw_spotify_data\"\n", " bronze_table_name = \"bronze_spotify_data\"\n", " silver_table_name = \"silver_spotify_data\"\n", + " \n", " raw_df = read_json(spark, music_source_paths())\n", "\n", - " load_data(raw_df, raw_table_name)\n", + " jdbc_write(raw_df, raw_table_name)\n", "\n", " transformed_df = clean_track_df(raw_df)\n", " \n", - " load_data(transformed_df, bronze_table_name)\n", + " jdbc_write(transformed_df, bronze_table_name)\n", " # transformed_df.show(5)\n", " # skip_reason_df = transformed_df.groupBy(\"skipped\", \"reason_end\").count().orderBy(F.desc(\"reason_end\")).show()\n", " # reason_df = transformed_df.groupBy(\"reason_end\").orderBy(F.desc(\"count\")).show(5)\n", @@ -289,7 +266,7 @@ " ## MAIN ## \n", "\n", "# app_name = \"ml_media\"\n", - "\n", + "# spark.log.level(\"ERROR\")\n", "\n", "# raw_table_name = \"raw_spotify_data\"\n", "# bronze_table_name = \"bronze_spotify_data\"\n",