From a3b9c2897f2289a76ae091902a4baa899e8e242a Mon Sep 17 00:00:00 2001 From: CC Date: Sun, 18 Oct 2026 13:44:19 +0100 Subject: [PATCH] timestamp --- spotify.ipynb | 105 +++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 62 deletions(-) diff --git a/spotify.ipynb b/spotify.ipynb index 014d6a3..e584990 100644 --- a/spotify.ipynb +++ b/spotify.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 275, "id": "11bb5dc2", "metadata": {}, "outputs": [], @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 212, + "execution_count": 289, "id": "a898326e", "metadata": {}, "outputs": [], @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 213, + "execution_count": 290, "id": "c609291c", "metadata": {}, "outputs": [], @@ -106,12 +106,13 @@ "\n", "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n", "\n", - "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\"]" + "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\"]\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 214, + "execution_count": 291, "id": "4cc14bd0", "metadata": {}, "outputs": [ @@ -123,22 +124,38 @@ "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", - "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "only showing top 1 row\n", + "+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", + "|conn_country|episode_name|episode_show_name| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", + "+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", + "| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n", + "| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n", + "| GB| NULL| NULL| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n", + "| GB| NULL| NULL| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n", + "| GB| NULL| NULL| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n", + "+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n", "only showing top 5 rows\n", - "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", - "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", - "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", - "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", - "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", - "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", - "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", - "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", - "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", - "only showing top 5 rows\n" + "root\n", + " |-- conn_country: string (nullable = true)\n", + " |-- episode_name: string (nullable = true)\n", + " |-- episode_show_name: string (nullable = true)\n", + " |-- album_name: string (nullable = true)\n", + " |-- artist_name: string (nullable = true)\n", + " |-- track_name: string (nullable = true)\n", + " |-- ms_played: long (nullable = true)\n", + " |-- offline: boolean (nullable = true)\n", + " |-- offline_timestamp: long (nullable = true)\n", + " |-- platform: string (nullable = true)\n", + " |-- reason_end: string (nullable = true)\n", + " |-- reason_start: string (nullable = true)\n", + " |-- shuffle: boolean (nullable = true)\n", + " |-- skipped: boolean (nullable = true)\n", + " |-- episode_uri: string (nullable = true)\n", + " |-- spotify_track_uri: string (nullable = true)\n", + " |-- ts: timestamp (nullable = true)\n", + " |-- username: string (nullable = true)\n", + "\n" ] } ], @@ -149,11 +166,15 @@ "spark = start_spark(app_name)\n", "\n", "df = read_json(spark, music_source_paths())\n", - "df.show(5)\n", + "df.show(1)\n", "\n", - "track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS))\n", + "rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n", + "track_df = track_df(rd_df)\n", "\n", - "track_df.show(5)\n" + "\n", + "timestamp = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\"))\n", + "timestamp.show(5)\n", + "timestamp.printSchema()" ] }, { @@ -234,46 +255,6 @@ "print(f\"Track Rows {track_df.count()}\")\n", "\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a0e9641e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "7\n", - "21\n", - "-RECORD 0----------------------\n", - " conn_country | 0 \n", - " episode_name | 105093 \n", - " episode_show_name | 105093 \n", - " incognito_mode | 0 \n", - " ip_addr_decrypted | 4492 \n", - " album_name | 2851 \n", - " artist_name | 2851 \n", - " track_name | 2851 \n", - " ms_played | 0 \n", - " offline | 279 \n", - " offline_timestamp | 2 \n", - " platform | 0 \n", - " reason_end | 4490 \n", - " reason_start | 0 \n", - " shuffle | 0 \n", - " skipped | 44457 \n", - " spotify_episode_uri | 105093 \n", - " spotify_track_uri | 2851 \n", - " ts | 0 \n", - " user_agent_decrypted | 24700 \n", - " username | 0 \n", - "\n" - ] - } - ], - "source": [] } ], "metadata": {