CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

transform func

Browse Source
This commit is contained in:
CC
2026-10-18 15:50:03 +01:00
parent a3b9c2897f
commit aa58be0932
1 changed files with 66 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
spotify.ipynb
View File

@@ -2,12 +2,12 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 110, "execution_count": 397,
"id": "e9bf1d26", "id": "e9bf1d26",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"from pyspark.sql import SparkSession\n", "from pyspark.sql import SparkSession, DataFrame\n",
"from pyspark.sql import functions as F\n", "from pyspark.sql import functions as F\n",
"from pathlib import Path\n", "from pathlib import Path\n",
"import glob" "import glob"
@@ -15,7 +15,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 275, "execution_count": 398,
"id": "11bb5dc2", "id": "11bb5dc2",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -54,10 +54,22 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 289, "execution_count": null,
"id": "a898326e", "id": "a898326e",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"ename": "NameError",
"evalue": "name 'Str' is not defined",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[399]\u001b[39m\u001b[32m, line 33\u001b[39m\n\u001b[32m 29\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;28;01min\u001b[39;00m columns:\n\u001b[32m 30\u001b[39m df = df.withColumn(column, F.to_timestamp(column))\n\u001b[32m 31\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m 32\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m33\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_minutes_played(df: DataFrame, src: Str, dest: Str, decimals: Int) -> DataFrame:\n\u001b[32m 34\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m (df.withColumn(dest, F.round(F.col(src) / \u001b[32m60000\u001b[39m, \u001b[32m2\u001b[39m)))\n\u001b[32m 35\u001b[39m \n\u001b[32m 36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_time_parts(df: DataFrame, ts_col=\u001b[33m\"ts\"\u001b[39m) -> DataFrame:\n",
"\u001b[31mNameError\u001b[39m: name 'Str' is not defined"
]
}
],
"source": [ "source": [
" ## Transform ##\n", " ## Transform ##\n",
"def null_counts_by_column(df):\n", "def null_counts_by_column(df):\n",
@@ -85,12 +97,34 @@
"\n", "\n",
"def track_df(df):\n", "def track_df(df):\n",
" return df.where(df[\"track_name\"].isNotNull())\n", " return df.where(df[\"track_name\"].isNotNull())\n",
"\n" "\n",
"def parse_timestamps(df: DataFrame, columns) -> DataFrame:\n",
" for column in columns:\n",
" df = df.withColumn(column, F.to_timestamp(column))\n",
" return df\n",
"\n",
"def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n",
" return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n",
"\n",
"def add_time_parts(df: DataFrame, ts_col=\"ts\") -> DataFrame:\n",
" return (df\n",
" .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
" .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n",
" )\n",
"\n",
"def clean_track_df(df: DataFrame) -> DataFrame:\n",
" return(df\n",
" .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
" .transform(track_df)\n",
" .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n",
" .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n",
" .transform(add_time_parts)\n",
" )"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 290, "execution_count": 395,
"id": "c609291c", "id": "c609291c",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -101,18 +135,18 @@
" \"master_metadata_album_artist_name\": \"artist_name\",\n", " \"master_metadata_album_artist_name\": \"artist_name\",\n",
" \"master_metadata_track_name\": \"track_name\",\n", " \"master_metadata_track_name\": \"track_name\",\n",
" \"spotify_episode_uri\" : \"episode_uri\",\n", " \"spotify_episode_uri\" : \"episode_uri\",\n",
" \"spotify_track_uri\" : \"spotify_track_uri\"\n", " \"spotify_track_uri\" : \"track_uri\"\n",
"}\n", "}\n",
"\n", "\n",
"TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n", "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n",
"\n", "\n",
"MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\"]\n", "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\", \"conn_country\", \"spotify_episode_uri\", \"episode_name\", \"episode_show_name\"]\n",
"\n" "\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 291, "execution_count": 400,
"id": "4cc14bd0", "id": "4cc14bd0",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -126,35 +160,22 @@
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 1 row\n", "only showing top 1 row\n",
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
"|conn_country|episode_name|episode_show_name| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", "| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username|\n",
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
"| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n", "|Last Day Of Summer| Summer Walker| Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n",
"| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n", "|Last Day Of Summer| Summer Walker| Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n",
"| GB| NULL| NULL| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n", "| Famouz| JHAYCO|No Me Conoce - Remix| 5.15| false|2023-04-07 12:39:15| ios| trackdone| clickrow| true| false|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n",
"| GB| NULL| NULL| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n", "| 4K| El Alfa| 4K| 3.36| false|2023-04-07 12:45:51| ios| trackdone| trackdone| true| false|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n",
"| GB| NULL| NULL| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n", "| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 0.3| false|2023-04-07 12:49:13| ios| fwdbtn| trackdone| true| true|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n",
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", "+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
"only showing top 5 rows\n", "only showing top 5 rows\n",
"root\n", "+--------------------+-----+\n",
" |-- conn_country: string (nullable = true)\n", "| username|count|\n",
" |-- episode_name: string (nullable = true)\n", "+--------------------+-----+\n",
" |-- episode_show_name: string (nullable = true)\n", "| csm3-us|64556|\n",
" |-- album_name: string (nullable = true)\n", "|fm30ajetkgrrkbvae...|38070|\n",
" |-- artist_name: string (nullable = true)\n", "+--------------------+-----+\n",
" |-- track_name: string (nullable = true)\n",
" |-- ms_played: long (nullable = true)\n",
" |-- offline: boolean (nullable = true)\n",
" |-- offline_timestamp: long (nullable = true)\n",
" |-- platform: string (nullable = true)\n",
" |-- reason_end: string (nullable = true)\n",
" |-- reason_start: string (nullable = true)\n",
" |-- shuffle: boolean (nullable = true)\n",
" |-- skipped: boolean (nullable = true)\n",
" |-- episode_uri: string (nullable = true)\n",
" |-- spotify_track_uri: string (nullable = true)\n",
" |-- ts: timestamp (nullable = true)\n",
" |-- username: string (nullable = true)\n",
"\n" "\n"
] ]
} }
@@ -168,13 +189,15 @@
"df = read_json(spark, music_source_paths())\n", "df = read_json(spark, music_source_paths())\n",
"df.show(1)\n", "df.show(1)\n",
"\n", "\n",
"rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", "track_df = clean_track_df(df)\n",
"track_df = track_df(rd_df)\n",
"\n", "\n",
"# rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
"# track_df = track_df(rd_df)\n",
"\n", "\n",
"timestamp = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\"))\n", "# track_df = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\")).withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\")).withColumn(\"ms_played\", F.round(F.col(\"ms_played\") / 60000, 2)).withColumn('time', F.date_format('ts', 'HH:mm:ss')).withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n",
"timestamp.show(5)\n", "track_df.show(5)\n",
"timestamp.printSchema()" "\n",
"track_df.groupBy(\"username\").count().show()"
] ]
}, },
{ {