CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

databricks

Browse Source
This commit is contained in:
CC
2026-10-18 22:59:56 +01:00
parent 7fb281f9db
commit 8fc40a9015
1 changed files with 12 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
spotify_bronze_silver.ipynb
View File

@@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 68, "execution_count": 14,
"id": "e9bf1d26", "id": "e9bf1d26",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -17,7 +17,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 15,
"id": "11bb5dc2", "id": "11bb5dc2",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -29,7 +29,6 @@
"def start_spark(app_name):\n", "def start_spark(app_name):\n",
" jar_path = str(Path.cwd() / \"postgresql.jar\")\n", " jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
" spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n", " spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
" spark.log.level(\"ERROR\")\n",
" return spark\n", " return spark\n",
"\n", "\n",
"def music_source_paths():\n", "def music_source_paths():\n",
@@ -56,7 +55,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 16,
"id": "a898326e", "id": "a898326e",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -125,7 +124,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 17,
"id": "817475cf", "id": "817475cf",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -171,7 +170,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 72, "execution_count": 18,
"id": "c609291c", "id": "c609291c",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -198,34 +197,10 @@
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "[Stage 2:> (0 + 6) / 7]\r"
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 1 row\n",
"+----------------------------+--------------+\n",
"|reason_end |skipped_values|\n",
"+----------------------------+--------------+\n",
"|NULL |[] |\n",
"|backbtn |[true] |\n",
"|endplay |[true] |\n",
"|fwdbtn |[true] |\n",
"|logout |[false] |\n",
"|remote |[false] |\n",
"|trackdone |[false] |\n",
"|trackerror |[false] |\n",
"|unexpected-exit |[false] |\n",
"|unexpected-exit-while-paused|[false] |\n",
"|unknown |[false] |\n",
"+----------------------------+--------------+\n",
"\n",
" Raw DF Rows: 105477\n",
" Transformed DF Rows: 102626\n",
" Row difference is 2851\n"
] ]
} }
], ],
@@ -234,17 +209,19 @@
"def main():\n", "def main():\n",
" app_name = \"ml_media\"\n", " app_name = \"ml_media\"\n",
" spark = start_spark(app_name)\n", " spark = start_spark(app_name)\n",
" spark.sparkContext.setLogLevel(\"ERROR\")\n",
"\n", "\n",
" raw_table_name = \"raw_spotify_data\"\n", " raw_table_name = \"raw_spotify_data\"\n",
" bronze_table_name = \"bronze_spotify_data\"\n", " bronze_table_name = \"bronze_spotify_data\"\n",
" silver_table_name = \"silver_spotify_data\"\n", " silver_table_name = \"silver_spotify_data\"\n",
" \n",
" raw_df = read_json(spark, music_source_paths())\n", " raw_df = read_json(spark, music_source_paths())\n",
"\n", "\n",
" load_data(raw_df, raw_table_name)\n", " jdbc_write(raw_df, raw_table_name)\n",
"\n", "\n",
" transformed_df = clean_track_df(raw_df)\n", " transformed_df = clean_track_df(raw_df)\n",
" \n", " \n",
" load_data(transformed_df, bronze_table_name)\n", " jdbc_write(transformed_df, bronze_table_name)\n",
" # transformed_df.show(5)\n", " # transformed_df.show(5)\n",
" # skip_reason_df = transformed_df.groupBy(\"skipped\", \"reason_end\").count().orderBy(F.desc(\"reason_end\")).show()\n", " # skip_reason_df = transformed_df.groupBy(\"skipped\", \"reason_end\").count().orderBy(F.desc(\"reason_end\")).show()\n",
" # reason_df = transformed_df.groupBy(\"reason_end\").orderBy(F.desc(\"count\")).show(5)\n", " # reason_df = transformed_df.groupBy(\"reason_end\").orderBy(F.desc(\"count\")).show(5)\n",
@@ -289,7 +266,7 @@
" ## MAIN ## \n", " ## MAIN ## \n",
"\n", "\n",
"# app_name = \"ml_media\"\n", "# app_name = \"ml_media\"\n",
"\n", "# spark.log.level(\"ERROR\")\n",
"\n", "\n",
"# raw_table_name = \"raw_spotify_data\"\n", "# raw_table_name = \"raw_spotify_data\"\n",
"# bronze_table_name = \"bronze_spotify_data\"\n", "# bronze_table_name = \"bronze_spotify_data\"\n",