transform func
This commit is contained in:
CC
109
spotify.ipynb
109
spotify.ipynb
@@ -2,12 +2,12 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 110,
|
||||
"execution_count": 397,
|
||||
"id": "e9bf1d26",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"from pyspark.sql import SparkSession, DataFrame\n",
|
||||
"from pyspark.sql import functions as F\n",
|
||||
"from pathlib import Path\n",
|
||||
"import glob"
|
||||
@@ -15,7 +15,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 275,
|
||||
"execution_count": 398,
|
||||
"id": "11bb5dc2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -54,10 +54,22 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 289,
|
||||
"execution_count": null,
|
||||
"id": "a898326e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "NameError",
|
||||
"evalue": "name 'Str' is not defined",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||||
"\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[399]\u001b[39m\u001b[32m, line 33\u001b[39m\n\u001b[32m 29\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;28;01min\u001b[39;00m columns:\n\u001b[32m 30\u001b[39m df = df.withColumn(column, F.to_timestamp(column))\n\u001b[32m 31\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m df\n\u001b[32m 32\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m33\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_minutes_played(df: DataFrame, src: Str, dest: Str, decimals: Int) -> DataFrame:\n\u001b[32m 34\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m (df.withColumn(dest, F.round(F.col(src) / \u001b[32m60000\u001b[39m, \u001b[32m2\u001b[39m)))\n\u001b[32m 35\u001b[39m \n\u001b[32m 36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m add_time_parts(df: DataFrame, ts_col=\u001b[33m\"ts\"\u001b[39m) -> DataFrame:\n",
|
||||
"\u001b[31mNameError\u001b[39m: name 'Str' is not defined"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
" ## Transform ##\n",
|
||||
"def null_counts_by_column(df):\n",
|
||||
@@ -85,12 +97,34 @@
|
||||
"\n",
|
||||
"def track_df(df):\n",
|
||||
" return df.where(df[\"track_name\"].isNotNull())\n",
|
||||
"\n"
|
||||
"\n",
|
||||
"def parse_timestamps(df: DataFrame, columns) -> DataFrame:\n",
|
||||
" for column in columns:\n",
|
||||
" df = df.withColumn(column, F.to_timestamp(column))\n",
|
||||
" return df\n",
|
||||
"\n",
|
||||
"def add_minutes_played(df: DataFrame, src, dest) -> DataFrame:\n",
|
||||
" return (df.withColumn(dest, F.round(F.col(src) / 60000, 2)))\n",
|
||||
"\n",
|
||||
"def add_time_parts(df: DataFrame, ts_col=\"ts\") -> DataFrame:\n",
|
||||
" return (df\n",
|
||||
" .withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
|
||||
" .withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\"))\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"def clean_track_df(df: DataFrame) -> DataFrame:\n",
|
||||
" return(df\n",
|
||||
" .transform(rename_columns, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
|
||||
" .transform(track_df)\n",
|
||||
" .transform(parse_timestamps, [\"ts\", \"offline_timestamp\"])\n",
|
||||
" .transform(add_minutes_played, \"ms_played\", \"ms_played\")\n",
|
||||
" .transform(add_time_parts)\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 290,
|
||||
"execution_count": 395,
|
||||
"id": "c609291c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -101,18 +135,18 @@
|
||||
" \"master_metadata_album_artist_name\": \"artist_name\",\n",
|
||||
" \"master_metadata_track_name\": \"track_name\",\n",
|
||||
" \"spotify_episode_uri\" : \"episode_uri\",\n",
|
||||
" \"spotify_track_uri\" : \"spotify_track_uri\"\n",
|
||||
" \"spotify_track_uri\" : \"track_uri\"\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n",
|
||||
"\n",
|
||||
"MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\"]\n",
|
||||
"MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\", \"conn_country\", \"spotify_episode_uri\", \"episode_name\", \"episode_show_name\"]\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 291,
|
||||
"execution_count": 400,
|
||||
"id": "4cc14bd0",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@@ -126,35 +160,22 @@
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
|
||||
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"only showing top 1 row\n",
|
||||
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n",
|
||||
"|conn_country|episode_name|episode_show_name| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n",
|
||||
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n",
|
||||
"| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n",
|
||||
"| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n",
|
||||
"| GB| NULL| NULL| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n",
|
||||
"| GB| NULL| NULL| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n",
|
||||
"| GB| NULL| NULL| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n",
|
||||
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n",
|
||||
"+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
|
||||
"| album_name| artist_name| track_name|ms_played|offline| offline_timestamp|platform| reason_end|reason_start|shuffle|skipped| track_uri| ts|username|\n",
|
||||
"+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
|
||||
"|Last Day Of Summer| Summer Walker| Girls Need Love| 1.34| false|2023-04-06 22:57:10| ios|unexpected-exit-w...| trackdone| true| false|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n",
|
||||
"|Last Day Of Summer| Summer Walker| Girls Need Love| 0.17| false|2023-04-07 12:39:00| ios| endplay| appload| true| true|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n",
|
||||
"| Famouz| JHAYCO|No Me Conoce - Remix| 5.15| false|2023-04-07 12:39:15| ios| trackdone| clickrow| true| false|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n",
|
||||
"| 4K| El Alfa| 4K| 3.36| false|2023-04-07 12:45:51| ios| trackdone| trackdone| true| false|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n",
|
||||
"| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 0.3| false|2023-04-07 12:49:13| ios| fwdbtn| trackdone| true| true|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n",
|
||||
"+------------------+--------------+--------------------+---------+-------+-------------------+--------+--------------------+------------+-------+-------+--------------------+-------------------+--------+\n",
|
||||
"only showing top 5 rows\n",
|
||||
"root\n",
|
||||
" |-- conn_country: string (nullable = true)\n",
|
||||
" |-- episode_name: string (nullable = true)\n",
|
||||
" |-- episode_show_name: string (nullable = true)\n",
|
||||
" |-- album_name: string (nullable = true)\n",
|
||||
" |-- artist_name: string (nullable = true)\n",
|
||||
" |-- track_name: string (nullable = true)\n",
|
||||
" |-- ms_played: long (nullable = true)\n",
|
||||
" |-- offline: boolean (nullable = true)\n",
|
||||
" |-- offline_timestamp: long (nullable = true)\n",
|
||||
" |-- platform: string (nullable = true)\n",
|
||||
" |-- reason_end: string (nullable = true)\n",
|
||||
" |-- reason_start: string (nullable = true)\n",
|
||||
" |-- shuffle: boolean (nullable = true)\n",
|
||||
" |-- skipped: boolean (nullable = true)\n",
|
||||
" |-- episode_uri: string (nullable = true)\n",
|
||||
" |-- spotify_track_uri: string (nullable = true)\n",
|
||||
" |-- ts: timestamp (nullable = true)\n",
|
||||
" |-- username: string (nullable = true)\n",
|
||||
"+--------------------+-----+\n",
|
||||
"| username|count|\n",
|
||||
"+--------------------+-----+\n",
|
||||
"| csm3-us|64556|\n",
|
||||
"|fm30ajetkgrrkbvae...|38070|\n",
|
||||
"+--------------------+-----+\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
@@ -168,13 +189,15 @@
|
||||
"df = read_json(spark, music_source_paths())\n",
|
||||
"df.show(1)\n",
|
||||
"\n",
|
||||
"rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
|
||||
"track_df = track_df(rd_df)\n",
|
||||
"track_df = clean_track_df(df)\n",
|
||||
"\n",
|
||||
"# rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
|
||||
"# track_df = track_df(rd_df)\n",
|
||||
"\n",
|
||||
"timestamp = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
|
||||
"timestamp.show(5)\n",
|
||||
"timestamp.printSchema()"
|
||||
"# track_df = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\")).withColumn(\"offline_timestamp\", F.to_timestamp(\"offline_timestamp\")).withColumn(\"ms_played\", F.round(F.col(\"ms_played\") / 60000, 2)).withColumn('time', F.date_format('ts', 'HH:mm:ss')).withColumn('date', F.date_format('ts', 'yyyy-MM-dd'))\n",
|
||||
"track_df.show(5)\n",
|
||||
"\n",
|
||||
"track_df.groupBy(\"username\").count().show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user