2026-10-17 23:51:14 +01:00
|
|
|
{
|
|
|
|
|
"cells": [
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
2026-10-18 11:48:40 +01:00
|
|
|
"execution_count": 110,
|
2026-10-17 23:51:14 +01:00
|
|
|
"id": "e9bf1d26",
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [],
|
|
|
|
|
"source": [
|
|
|
|
|
"from pyspark.sql import SparkSession\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"from pyspark.sql import functions as F\n",
|
|
|
|
|
"from pathlib import Path\n",
|
|
|
|
|
"import glob"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
"execution_count": 111,
|
|
|
|
|
"id": "11bb5dc2",
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [],
|
|
|
|
|
"source": [
|
|
|
|
|
" ## Extract ##\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"BASE = Path.cwd() / \"ingest\"\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"def start_spark(app_name):\n",
|
|
|
|
|
" spark = SparkSession.builder.appName(app_name).getOrCreate()\n",
|
|
|
|
|
" spark.sparkContext.setLogLevel(\"ERROR\")\n",
|
|
|
|
|
" return spark\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
|
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"def music_source_paths():\n",
|
|
|
|
|
" return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"def video_source_paths():\n",
|
|
|
|
|
" return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"def read_json(spark, paths):\n",
|
|
|
|
|
" if not paths:\n",
|
|
|
|
|
" raise FileNotFoundError(\"No JSON files matched the pattern\")\n",
|
|
|
|
|
" return (\n",
|
|
|
|
|
" spark.read\n",
|
|
|
|
|
" .format(\"json\")\n",
|
|
|
|
|
" .option(\"multiLine\", True)\n",
|
|
|
|
|
" .json(paths) # list of paths, one read\n",
|
|
|
|
|
" # .withColumn(\"source_file\", F.input_file_name())\n",
|
|
|
|
|
" # .withColumn(\n",
|
|
|
|
|
" # \"source_folder\",\n",
|
|
|
|
|
" # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n",
|
|
|
|
|
" # )\n",
|
|
|
|
|
" )"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
2026-10-18 11:48:40 +01:00
|
|
|
"execution_count": 186,
|
|
|
|
|
"id": "a898326e",
|
2026-10-17 23:51:14 +01:00
|
|
|
"metadata": {},
|
2026-10-18 11:48:40 +01:00
|
|
|
"outputs": [],
|
2026-10-17 23:51:14 +01:00
|
|
|
"source": [
|
2026-10-18 11:48:40 +01:00
|
|
|
" ## Transform ##\n",
|
|
|
|
|
"def null_counts_by_column(df):\n",
|
|
|
|
|
" null_counts = df.select([\n",
|
|
|
|
|
" F.count(F.when(F.col(c).isNull(), c)).alias(c)\n",
|
|
|
|
|
" for c in df.columns\n",
|
|
|
|
|
" ])\n",
|
|
|
|
|
" return null_counts\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"def metadata_df(df):\n",
|
|
|
|
|
" return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"def rename_columns(df, rename_map):\n",
|
|
|
|
|
" return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"def track_df(df):\n",
|
|
|
|
|
" return df.where(df[\"track_name\"].isNotNull())\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"def drop_pii_columns(df):\n",
|
|
|
|
|
" return df.drop(\"ip_addr_decrypted\", \"user_agent_decrypted\")\n"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
"execution_count": null,
|
2026-10-18 11:48:40 +01:00
|
|
|
"id": "c609291c",
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [],
|
|
|
|
|
"source": [
|
|
|
|
|
"## Config \n",
|
|
|
|
|
"SPOTIFY_RENAMES = {\n",
|
|
|
|
|
" \"master_metadata_album_album_name\": \"album_name\",\n",
|
|
|
|
|
" \"master_metadata_album_artist_name\": \"artist_name\",\n",
|
|
|
|
|
" \"master_metadata_track_name\": \"track_name\",\n",
|
|
|
|
|
"}\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
"execution_count": null,
|
|
|
|
|
"id": "4cc14bd0",
|
2026-10-17 23:51:14 +01:00
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [
|
|
|
|
|
{
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
"text": [
|
2026-10-18 11:48:40 +01:00
|
|
|
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"only showing top 5 rows\n",
|
|
|
|
|
"Original Rows 105477\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"only showing top 5 rows\n",
|
|
|
|
|
"Track Rows 102626\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|username|\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n",
|
|
|
|
|
"| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n",
|
|
|
|
|
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
|
|
|
|
|
"only showing top 5 rows\n",
|
|
|
|
|
"Dropped Rows 102626\n"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"source": [
|
2026-10-18 11:48:40 +01:00
|
|
|
" ## MAIN ## \n",
|
|
|
|
|
"\n",
|
|
|
|
|
"app_name = \"ml_media\"\n",
|
|
|
|
|
"spark = start_spark(app_name)\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"df = read_json(spark, music_source_paths())\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES))\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"drop_df = drop_pii_columns(track_df) \n"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
2026-10-18 11:48:40 +01:00
|
|
|
"execution_count": null,
|
|
|
|
|
"id": "faa6a8cb",
|
2026-10-17 23:51:14 +01:00
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [
|
|
|
|
|
{
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
"text": [
|
2026-10-18 11:48:40 +01:00
|
|
|
"7\n",
|
|
|
|
|
"21\n",
|
|
|
|
|
"-RECORD 0----------------------\n",
|
|
|
|
|
" conn_country | 0 \n",
|
|
|
|
|
" episode_name | 105093 \n",
|
|
|
|
|
" episode_show_name | 105093 \n",
|
|
|
|
|
" incognito_mode | 0 \n",
|
|
|
|
|
" ip_addr_decrypted | 4492 \n",
|
|
|
|
|
" album_name | 2851 \n",
|
|
|
|
|
" artist_name | 2851 \n",
|
|
|
|
|
" track_name | 2851 \n",
|
|
|
|
|
" ms_played | 0 \n",
|
|
|
|
|
" offline | 279 \n",
|
|
|
|
|
" offline_timestamp | 2 \n",
|
|
|
|
|
" platform | 0 \n",
|
|
|
|
|
" reason_end | 4490 \n",
|
|
|
|
|
" reason_start | 0 \n",
|
|
|
|
|
" shuffle | 0 \n",
|
|
|
|
|
" skipped | 44457 \n",
|
|
|
|
|
" spotify_episode_uri | 105093 \n",
|
|
|
|
|
" spotify_track_uri | 2851 \n",
|
|
|
|
|
" ts | 0 \n",
|
|
|
|
|
" user_agent_decrypted | 24700 \n",
|
|
|
|
|
" username | 0 \n",
|
|
|
|
|
"\n"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
2026-10-18 11:48:40 +01:00
|
|
|
"data": {
|
|
|
|
|
"text/plain": [
|
|
|
|
|
"2851"
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"execution_count": 155,
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"output_type": "execute_result"
|
2026-10-17 23:51:14 +01:00
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"source": [
|
2026-10-18 11:48:40 +01:00
|
|
|
"## Potential Test Help\n",
|
|
|
|
|
"print(metadata_df(music_df).rdd.getNumPartitions()) # Partitions for Pyspark\n",
|
|
|
|
|
"print(len(metadata_df(music_df).columns)) # # of columns in JSON\n",
|
|
|
|
|
"null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n",
|
|
|
|
|
"null_counts.show(vertical=True) # Print Null counts by column_df\n",
|
|
|
|
|
"total_rows = music_df.count\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n",
|
|
|
|
|
" ## same can be done for podcasts\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"df.show(5)\n",
|
|
|
|
|
"print(f\"Original Rows {df.count()}\")\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"track_df.show(5)\n",
|
|
|
|
|
"print(f\"Track Rows {track_df.count()}\")\n",
|
2026-10-17 23:51:14 +01:00
|
|
|
"\n",
|
2026-10-18 11:48:40 +01:00
|
|
|
"drop_df.show(5)\n",
|
|
|
|
|
"drop_df.count()\n"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"cell_type": "code",
|
2026-10-18 11:48:40 +01:00
|
|
|
"execution_count": null,
|
2026-10-17 23:51:14 +01:00
|
|
|
"id": "a0e9641e",
|
|
|
|
|
"metadata": {},
|
|
|
|
|
"outputs": [
|
|
|
|
|
{
|
2026-10-18 01:04:48 +01:00
|
|
|
"name": "stdout",
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
"text": [
|
2026-10-18 11:48:40 +01:00
|
|
|
"7\n",
|
|
|
|
|
"21\n",
|
2026-10-18 01:04:48 +01:00
|
|
|
"-RECORD 0----------------------\n",
|
|
|
|
|
" conn_country | 0 \n",
|
|
|
|
|
" episode_name | 105093 \n",
|
|
|
|
|
" episode_show_name | 105093 \n",
|
|
|
|
|
" incognito_mode | 0 \n",
|
|
|
|
|
" ip_addr_decrypted | 4492 \n",
|
|
|
|
|
" album_name | 2851 \n",
|
|
|
|
|
" artist_name | 2851 \n",
|
|
|
|
|
" track_name | 2851 \n",
|
|
|
|
|
" ms_played | 0 \n",
|
|
|
|
|
" offline | 279 \n",
|
|
|
|
|
" offline_timestamp | 2 \n",
|
|
|
|
|
" platform | 0 \n",
|
|
|
|
|
" reason_end | 4490 \n",
|
|
|
|
|
" reason_start | 0 \n",
|
|
|
|
|
" shuffle | 0 \n",
|
|
|
|
|
" skipped | 44457 \n",
|
|
|
|
|
" spotify_episode_uri | 105093 \n",
|
|
|
|
|
" spotify_track_uri | 2851 \n",
|
|
|
|
|
" ts | 0 \n",
|
|
|
|
|
" user_agent_decrypted | 24700 \n",
|
|
|
|
|
" username | 0 \n",
|
|
|
|
|
"\n"
|
2026-10-17 23:51:14 +01:00
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
],
|
2026-10-18 11:48:40 +01:00
|
|
|
"source": []
|
2026-10-17 23:51:14 +01:00
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"metadata": {
|
|
|
|
|
"kernelspec": {
|
|
|
|
|
"display_name": ".venv",
|
|
|
|
|
"language": "python",
|
|
|
|
|
"name": "python3"
|
|
|
|
|
},
|
|
|
|
|
"language_info": {
|
|
|
|
|
"codemirror_mode": {
|
|
|
|
|
"name": "ipython",
|
|
|
|
|
"version": 3
|
|
|
|
|
},
|
|
|
|
|
"file_extension": ".py",
|
|
|
|
|
"mimetype": "text/x-python",
|
|
|
|
|
"name": "python",
|
|
|
|
|
"nbconvert_exporter": "python",
|
|
|
|
|
"pygments_lexer": "ipython3",
|
|
|
|
|
"version": "3.13.7"
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"nbformat": 4,
|
|
|
|
|
"nbformat_minor": 5
|
|
|
|
|
}
|
