CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

descriptions

Browse Source
This commit is contained in:
CC
2026-10-18 23:26:25 +01:00
parent 72b501a5ab
commit dbcbad6d64
4 changed files with 545 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
spotify_bronze_silver.ipynb
View File

@@ -59,13 +59,14 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": null,
"id": "a898326e",
"metadata": {},
"outputs": [],
"source": [
" ## Transform ##\n",
"def null_counts_by_column(df):\n",
" \"\"\"Count nulls in each column of the DataFrame.\"\"\"\n",
" null_counts = df.select([\n",
" F.count(F.when(F.col(c).isNull(), c)).alias(c)\n",
" for c in df.columns\n",
@@ -73,9 +74,11 @@
" return null_counts\n",
"\n",
"def metadata_df(df):\n",
" \"\"\"Rename verbose album/artist/track metadata columns to short names.\"\"\"\n",
" return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
"\n",
"def rename_columns(df, rename_map, drop_columns):\n",
" \"\"\"Drop specified columns and rename remaining ones per the rename map.\"\"\"\n",
" missing_rename = set(rename_map) - set(df.columns)\n",
" missing_dropped = set(drop_columns) - set(df.columns)\n",
"\n",
@@ -89,21 +92,26 @@
" return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n",
"\n",
"def track_df(df):\n",
" \"\"\"Filter to rows where track_name is not null.\"\"\"\n",
" return df.where(df[\"track_name\"].isNotNull())\n",
"\n",
"def parse_timestamps(df: DataFrame, columns) -> DataFrame:\n",
" \"\"\"Cast each column in the list to timestamp type.\"\"\"\n",
" for column in columns:\n",
" df = df.withColumn(column, F.to_timestamp(column))\n",
" return df\n",
"\n",
"def parse_booleans(df: DataFrame, columns) -> DataFrame:\n",
" \"\"\"Cast each column in the list to string type.\"\"\"\n",
" df = df.select([F.col(c).cast(\"string\") for c in columns])\n",
" return df\n",
"\n",
"def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n",
" \"\"\"Convert milliseconds played to minutes and round to 2 decimals.\"\"\"\n",
" return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n",
"\n",
"def add_time_parts(df: DataFrame, ts_col) -> DataFrame:\n",
" \"\"\"Add time/date/day-of-week/hour parts derived from the timestamp columns.\"\"\"\n",
" return (df\n",
" .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
" .withColumn(\"offline_timestamped\", F.to_timestamp(\"offline_timestamp\"))\n",
@@ -116,6 +124,7 @@
" )\n",
"\n",
"def clean_track_df(df: DataFrame) -> DataFrame:\n",
" \"\"\"Apply the full transform pipeline: rename, filter tracks, add minutes and time parts.\"\"\"\n",
" return(df\n",
" .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
" .transform(track_df)\n",