CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

transformed

Browse Source
This commit is contained in:
CC
2026-10-18 16:28:55 +01:00
parent aa58be0932
commit 73015e9230
1 changed files with 58 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
spotify.ipynb
View File

@@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 397, "execution_count": 459,
"id": "e9bf1d26", "id": "e9bf1d26",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -15,7 +15,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 398, "execution_count": 460,
"id": "11bb5dc2", "id": "11bb5dc2",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -54,22 +54,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 461,
"id": "a898326e", "id": "a898326e",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"ename": "NameError",
"evalue": "name 'Str' is not defined",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[399]\u001b[39m\u001b[32m, line 33\u001b[39m\n\u001b[32m 29\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;28;01min\u001b[39;00m columns:\n\u001b[32m 30\u001b[39m df = df.withColumn(column, F.to_timestamp(column))\n\u001b[32m 31\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m 32\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m33\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_minutes_played(df: DataFrame, src: Str, dest: Str, decimals: Int) -> DataFrame:\n\u001b[32m 34\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m (df.withColumn(dest, F.round(F.col(src) / \u001b[32m60000\u001b[39m, \u001b[32m2\u001b[39m)))\n\u001b[32m 35\u001b[39m \n\u001b[32m 36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_time_parts(df: DataFrame, ts_col=\u001b[33m\"ts\"\u001b[39m) -> DataFrame:\n",
"\u001b[31mNameError\u001b[39m: name 'Str' is not defined"
]
}
],
"source": [ "source": [
" ## Transform ##\n", " ## Transform ##\n",
"def null_counts_by_column(df):\n", "def null_counts_by_column(df):\n",
@@ -103,13 +91,21 @@
" df = df.withColumn(column, F.to_timestamp(column))\n", " df = df.withColumn(column, F.to_timestamp(column))\n",
" return df\n", " return df\n",
"\n", "\n",
"def parse_booleans(df: DataFrame, columns) -> DataFrame:\n",
" df = df.select([F.col(c).cast(\"string\") for c in columns])\n",
" return df\n",
"\n",
"def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n", "def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n",
" return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n", " return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n",
"\n", "\n",
"def add_time_parts(df: DataFrame, ts_col=\"ts\") -> DataFrame:\n", "def add_time_parts(df: DataFrame, ts_col) -> DataFrame:\n",
" return (df\n", " return (df\n",
" .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n", " .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
" .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n", " .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n",
" .withColumn('time', F.date_format('ts', 'HH:mm:ss'))\n",
" .withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n",
" .withColumn('offline_time', F.date_format('offline_timestamp', 'HH:mm:ss'))\n",
" .withColumn('offline_date', F.date_format('offline_timestamp', 'yyyy-MM-dd')) \n",
" )\n", " )\n",
"\n", "\n",
"def clean_track_df(df: DataFrame) -> DataFrame:\n", "def clean_track_df(df: DataFrame) -> DataFrame:\n",
@@ -117,14 +113,15 @@
" .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", " .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
" .transform(track_df)\n", " .transform(track_df)\n",
" .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n", " .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n",
" # .transform(parse_booleans, [\"offline\", \"shuffle\", \"skipped\"]) ##returns only the selected columns\n",
" .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n", " .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n",
" .transform(add_time_parts)\n", " .transform(add_time_parts, \"ts\")\n",
" )" " )"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 395, "execution_count": 462,
"id": "c609291c", "id": "c609291c",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -146,7 +143,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 400, "execution_count": 463,
"id": "4cc14bd0", "id": "4cc14bd0",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -160,23 +157,36 @@
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 1 row\n", "only showing top 1 row\n",
"+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", "+------------------+-------------+---------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+--------+----------+------------+------------+\n",
"| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username|\n", "| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username| time| date|offline_time|offline_date|\n",
"+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", "+------------------+-------------+---------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+--------+----------+------------+------------+\n",
"|Last Day Of Summer| Summer Walker| Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n", "|Last Day Of Summer|Summer Walker|Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|22:58:31|2023-04-06| 22:57:10| 2023-04-06|\n",
"|Last Day Of Summer| Summer Walker| Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n", "|Last Day Of Summer|Summer Walker|Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|12:39:14|2023-04-07| 12:39:00| 2023-04-07|\n",
"| Famouz| JHAYCO|No Me Conoce - Remix| 5.15| false|2023-04-07 12:39:15| ios| trackdone| clickrow| true| false|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n", "+------------------+-------------+---------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+--------+----------+------------+------------+\n",
"| 4K| El Alfa| 4K| 3.36| false|2023-04-07 12:45:51| ios| trackdone| trackdone| true| false|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n", "only showing top 2 rows\n",
"| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 0.3| false|2023-04-07 12:49:13| ios| fwdbtn| trackdone| true| true|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n", "root\n",
"+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n", " |-- album_name: string (nullable = true)\n",
"only showing top 5 rows\n", " |-- artist_name: string (nullable = true)\n",
"+--------------------+-----+\n", " |-- track_name: string (nullable = true)\n",
"| username|count|\n", " |-- ms_played: double (nullable = true)\n",
"+--------------------+-----+\n", " |-- offline: boolean (nullable = true)\n",
"| csm3-us|64556|\n", " |-- offline_timestamp: timestamp (nullable = true)\n",
"|fm30ajetkgrrkbvae...|38070|\n", " |-- platform: string (nullable = true)\n",
"+--------------------+-----+\n", " |-- reason_end: string (nullable = true)\n",
"\n" " |-- reason_start: string (nullable = true)\n",
" |-- shuffle: boolean (nullable = true)\n",
" |-- skipped: boolean (nullable = true)\n",
" |-- track_uri: string (nullable = true)\n",
" |-- ts: timestamp (nullable = true)\n",
" |-- username: string (nullable = true)\n",
" |-- time: string (nullable = true)\n",
" |-- date: string (nullable = true)\n",
" |-- offline_time: string (nullable = true)\n",
" |-- offline_date: string (nullable = true)\n",
"\n",
" Raw DF Rows: 105477\n",
" Transformed DF Rows: 102626\n",
" Row difference is 2851\n"
] ]
} }
], ],
@@ -186,18 +196,21 @@
"app_name = \"ml_media\"\n", "app_name = \"ml_media\"\n",
"spark = start_spark(app_name)\n", "spark = start_spark(app_name)\n",
"\n", "\n",
"df = read_json(spark, music_source_paths())\n", "raw_df = read_json(spark, music_source_paths())\n",
"df.show(1)\n", "raw_df.show(1)\n",
"\n", "\n",
"track_df = clean_track_df(df)\n", "transformed_df = clean_track_df(raw_df)\n",
"transformed_df.show(2)\n",
"transformed_df.printSchema()\n",
"\n", "\n",
"# rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", "print(f\" Raw DF Rows: {raw_df.count()}\")\n",
"# track_df = track_df(rd_df)\n", "print(f\" Transformed DF Rows: {transformed_df.count()}\")\n",
"print(f\" Row difference is {raw_df.count() - transformed_df.count()}\") \n",
"\n", "\n",
"# track_df = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\")).withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\")).withColumn(\"ms_played\", F.round(F.col(\"ms_played\") / 60000, 2)).withColumn('time', F.date_format('ts', 'HH:mm:ss')).withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n", "spark.stop()\n",
"track_df.show(5)\n",
" \n", " \n",
"track_df.groupBy(\"username\").count().show()" "\n",
"# transformed_df.groupBy(\"username\").count().show()"
] ]
}, },
{ {