CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

timestamp

Browse Source
This commit is contained in:
CC
2026-10-18 13:44:19 +01:00
parent 4cafebb07f
commit a3b9c2897f
1 changed files with 43 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
spotify.ipynb
View File

@@ -15,7 +15,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 111, "execution_count": 275,
"id": "11bb5dc2", "id": "11bb5dc2",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -54,7 +54,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 212, "execution_count": 289,
"id": "a898326e", "id": "a898326e",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -90,7 +90,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 213, "execution_count": 290,
"id": "c609291c", "id": "c609291c",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -106,12 +106,13 @@
"\n", "\n",
"TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n", "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n",
"\n", "\n",
"MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\"]" "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\", \"incognito_mode\"]\n",
"\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 214, "execution_count": 291,
"id": "4cc14bd0", "id": "4cc14bd0",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -123,22 +124,38 @@
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 1 row\n",
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n",
"|conn_country|episode_name|episode_show_name| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n",
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n",
"| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06 22:58:31| csm3-us|\n",
"| GB| NULL| NULL|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07 12:39:14| csm3-us|\n",
"| GB| NULL| NULL| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07 12:45:50| csm3-us|\n",
"| GB| NULL| NULL| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07 12:49:12| csm3-us|\n",
"| GB| NULL| NULL| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07 12:49:32| csm3-us|\n",
"+------------+------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+-------------------+--------+\n",
"only showing top 5 rows\n", "only showing top 5 rows\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "root\n",
"|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", " |-- conn_country: string (nullable = true)\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", " |-- episode_name: string (nullable = true)\n",
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", " |-- episode_show_name: string (nullable = true)\n",
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", " |-- album_name: string (nullable = true)\n",
"| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", " |-- artist_name: string (nullable = true)\n",
"| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", " |-- track_name: string (nullable = true)\n",
"| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", " |-- ms_played: long (nullable = true)\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", " |-- offline: boolean (nullable = true)\n",
"only showing top 5 rows\n" " |-- offline_timestamp: long (nullable = true)\n",
" |-- platform: string (nullable = true)\n",
" |-- reason_end: string (nullable = true)\n",
" |-- reason_start: string (nullable = true)\n",
" |-- shuffle: boolean (nullable = true)\n",
" |-- skipped: boolean (nullable = true)\n",
" |-- episode_uri: string (nullable = true)\n",
" |-- spotify_track_uri: string (nullable = true)\n",
" |-- ts: timestamp (nullable = true)\n",
" |-- username: string (nullable = true)\n",
"\n"
] ]
} }
], ],
@@ -149,11 +166,15 @@
"spark = start_spark(app_name)\n", "spark = start_spark(app_name)\n",
"\n", "\n",
"df = read_json(spark, music_source_paths())\n", "df = read_json(spark, music_source_paths())\n",
"df.show(5)\n", "df.show(1)\n",
"\n", "\n",
"track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS))\n", "rd_df = rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS)\n",
"track_df = track_df(rd_df)\n",
"\n", "\n",
"track_df.show(5)\n" "\n",
"timestamp = track_df.withColumn(\"ts\", F.to_timestamp(\"ts\"))\n",
"timestamp.show(5)\n",
"timestamp.printSchema()"
] ]
}, },
{ {
@@ -234,46 +255,6 @@
"print(f\"Track Rows {track_df.count()}\")\n", "print(f\"Track Rows {track_df.count()}\")\n",
"\n" "\n"
] ]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a0e9641e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"7\n",
"21\n",
"-RECORD 0----------------------\n",
" conn_country | 0 \n",
" episode_name | 105093 \n",
" episode_show_name | 105093 \n",
" incognito_mode | 0 \n",
" ip_addr_decrypted | 4492 \n",
" album_name | 2851 \n",
" artist_name | 2851 \n",
" track_name | 2851 \n",
" ms_played | 0 \n",
" offline | 279 \n",
" offline_timestamp | 2 \n",
" platform | 0 \n",
" reason_end | 4490 \n",
" reason_start | 0 \n",
" shuffle | 0 \n",
" skipped | 44457 \n",
" spotify_episode_uri | 105093 \n",
" spotify_track_uri | 2851 \n",
" ts | 0 \n",
" user_agent_decrypted | 24700 \n",
" username | 0 \n",
"\n"
]
}
],
"source": []
} }
], ],
"metadata": { "metadata": {