CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

siv databricks

Browse Source
This commit is contained in:
CC
2026-10-18 22:00:35 +01:00
parent f03814ab93
commit f409cefa18
1 changed files with 86 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
spotify_bronze_silver.ipynb
View File

@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 68,
"id": "e9bf1d26",
"metadata": {},
"outputs": [],
@@ -17,7 +17,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 69,
"id": "11bb5dc2",
"metadata": {},
"outputs": [],
@@ -56,7 +56,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 70,
"id": "a898326e",
"metadata": {},
"outputs": [],
@@ -106,6 +106,8 @@
" .withColumn(\"offline_timestamped\", F.to_timestamp(\"offline_timestamp\"))\n",
" .withColumn('time', F.date_format('ts', 'HH:mm:ss'))\n",
" .withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n",
" .withColumn('day of week', F.day('ts'))\n",
" .withColumn('hour', F.hour('ts'))\n",
" .withColumn('offline_time', F.date_format('offline_timestamped', 'HH:mm:ss'))\n",
" .withColumn('offline_date', F.date_format('offline_timestamped', 'yyyy-MM-dd')) \n",
" )\n",
@@ -123,7 +125,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 71,
"id": "817475cf",
"metadata": {},
"outputs": [],
@@ -163,7 +165,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 72,
"id": "c609291c",
"metadata": {},
"outputs": [],
@@ -189,25 +191,57 @@
"id": "4cc14bd0",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING: Using incubator modules: jdk.incubator.vector\n",
"Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n",
"26/10/18 17:57:09 WARN Utils: Your hostname, MacBook-Neo-232.local, resolves to a loopback address: 127.0.0.1; using 192.168.10.130 instead (on interface en0)\n",
"26/10/18 17:57:09 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
"26/10/18 17:57:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
"Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n",
"Setting default log level to \"WARN\".\n",
"To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
" \r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 1 row\n",
"+-------+--------------------+-----+\n",
"|skipped| reason_end|count|\n",
"+-------+--------------------+-----+\n",
"| false| unknown| 99|\n",
"| NULL| unknown| 375|\n",
"| false|unexpected-exit-w...| 1243|\n",
"| NULL|unexpected-exit-w...| 432|\n",
"| false| unexpected-exit| 36|\n",
"| NULL| unexpected-exit| 51|\n",
"| false| trackerror| 69|\n",
"| NULL| trackerror| 33|\n",
"| false| trackdone|36102|\n",
"| NULL| trackdone|24077|\n",
"| false| remote| 121|\n",
"| NULL| remote| 859|\n",
"| false| logout| 491|\n",
"| NULL| logout| 745|\n",
"| true| fwdbtn|16571|\n",
"| NULL| fwdbtn|11445|\n",
"| true| endplay| 2943|\n",
"| NULL| endplay| 1447|\n",
"| true| backbtn| 650|\n",
"| NULL| backbtn| 402|\n",
"+-------+--------------------+-----+\n",
"only showing top 20 rows\n",
"+----------------------------+-----+\n",
"|reason_end |count|\n",
"+----------------------------+-----+\n",
"|trackdone |60179|\n",
"|fwdbtn |28016|\n",
"|NULL |4435 |\n",
"|endplay |4390 |\n",
"|unexpected-exit-while-paused|1675 |\n",
"|logout |1236 |\n",
"|backbtn |1052 |\n",
"|remote |980 |\n",
"|unknown |474 |\n",
"|trackerror |102 |\n",
"|unexpected-exit |87 |\n",
"+----------------------------+-----+\n",
"\n",
" Raw DF Rows: 105477\n",
" Transformed DF Rows: 102626\n",
" Row difference is 2851\n"
@@ -222,8 +256,25 @@
"\n",
" table_name = \"spotify_data\"\n",
" raw_df = read_json(spark, music_source_paths())\n",
" raw_df.show(1)\n",
" transformed_df = clean_track_df(raw_df)\n",
" load_data(transformed_df, table_name)\n",
" #load_data(transformed_df, table_name)\n",
" # transformed_df.show(5)\n",
" skip_reason_df = transformed_df.groupBy(\"skipped\", \"reason_end\").count().orderBy(F.desc(\"reason_end\")).show()\n",
" # reason_df = transformed_df.groupBy(\"reason_end\").orderBy(F.desc(\"count\")).show(5)\n",
" # skip_df = transformed_df.groupBy(\"skipped\").count().orderBy(F.desc(\"count\")).show(5)\n",
" skip_join_bool = transformed_df.select(\"reason_end\", \"was_skipped\").distinct().orderBy(\"reason_end\").show(truncate=False))\n",
" #join(transformed_df.F.col(\"reason_end\"))\n",
"\n",
" # missing_boolean = set(rename_map) - set(df.columns)\n",
"\n",
"\n",
"# Assuming 'df' is your DataFrame and 'timestamp_column' is the column with timestamps\n",
" # df = df.withColumn(\"time_of_day\", \n",
" # F.when((F.hour(\"timestamp_column\") >= 5) & (F.hour(\"timestamp_column\") < 12), \"morning\")\n",
" # .when((F.hour(\"timestamp_column\") >= 12) & (F.hour(\"timestamp_column\") < 18), \"afternoon\")\n",
" # .when((F.hour(\"timestamp_column\") >= 18) & (F.hour(\"timestamp_column\") < 22), \"evening\")\n",
" # .otherwise(\"night\"))\n",
"\n",
" print(f\" Raw DF Rows: {raw_df.count()}\")\n",
" print(f\" Transformed DF Rows: {transformed_df.count()}\")\n",
@@ -238,38 +289,26 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"id": "faa6a8cb",
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'music_df' is not defined",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m## Potential Test Help\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m print(metadata_df(music_df).rdd.getNumPartitions()) \u001b[38;5;66;03m# Partitions for Pyspark\u001b[39;00m\n\u001b[32m 3\u001b[39m print(len(metadata_df(music_df).columns)) \u001b[38;5;66;03m# # of columns in JSON\u001b[39;00m\n\u001b[32m 4\u001b[39m null_counts = null_counts_by_column(metadata_df(music_df)) \u001b[38;5;66;03m# Null counts by column\u001b[39;00m\n\u001b[32m 5\u001b[39m null_counts.show(vertical=\u001b[38;5;28;01mTrue\u001b[39;00m) \u001b[38;5;66;03m# Print Null counts by column_df\u001b[39;00m\n",
"\u001b[31mNameError\u001b[39m: name 'music_df' is not defined"
]
}
],
"outputs": [],
"source": [
"## Potential Test Help\n",
"print(metadata_df(music_df).rdd.getNumPartitions()) # Partitions for Pyspark\n",
"print(len(metadata_df(music_df).columns)) # # of columns in JSON\n",
"null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n",
"null_counts.show(vertical=True) # Print Null counts by column_df\n",
"total_rows = music_df.count\n",
"# ## Potential Test Help\n",
"# print(metadata_df(music_df).rdd.getNumPartitions()) # Partitions for Pyspark\n",
"# print(len(metadata_df(music_df).columns)) # # of columns in JSON\n",
"# null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n",
"# null_counts.show(vertical=True) # Print Null counts by column_df\n",
"# total_rows = music_df.count\n",
"\n",
"df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n",
" ## same can be done for podcasts\n",
"# df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n",
"# ## same can be done for podcasts\n",
"\n",
"df.show(5)\n",
"print(f\"Original Rows {df.count()}\")\n",
"# df.show(5)\n",
"# print(f\"Original Rows {df.count()}\")\n",
"\n",
"track_df.show(5)\n",
"print(f\"Track Rows {track_df.count()}\")\n",
"# track_df.show(5)\n",
"# print(f\"Track Rows {track_df.count()}\")\n",
"\n"
]
}