CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

module rename and drop

Browse Source
This commit is contained in:
CC
2026-10-18 12:12:02 +01:00
parent 25f6d45169
commit 4cafebb07f
1 changed files with 58 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
spotify.ipynb
View File

@@ -54,7 +54,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 189, "execution_count": 212,
"id": "a898326e", "id": "a898326e",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -70,22 +70,27 @@
"def metadata_df(df):\n", "def metadata_df(df):\n",
" return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n", " return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
"\n", "\n",
"def rename_columns(df, rename_map):\n", "def rename_columns(df, rename_map, drop_columns):\n",
" missing = set(rename_map) - set(df.columns)\n", " missing_rename = set(rename_map) - set(df.columns)\n",
" if missing:\n", " missing_dropped = set(drop_columns) - set(df.columns)\n",
" print(f\"Warning: columns not found: {missing}\")\n", "\n",
" if missing_rename:\n",
" print(f\"Warning: columns not found: {missing_rename}\")\n",
" if missing_dropped:\n",
" print(f\"Warning: columns not found: {missing_dropped}\")\n",
"\n",
" df = df.drop(*drop_columns)\n",
" \n",
" return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n", " return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n",
"\n", "\n",
"def track_df(df):\n", "def track_df(df):\n",
" return df.where(df[\"track_name\"].isNotNull())\n", " return df.where(df[\"track_name\"].isNotNull())\n",
"\n", "\n"
"def drop_pii_columns(df):\n",
" return df.drop(\"ip_addr_decrypted\", \"user_agent_decrypted\")\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 191, "execution_count": 213,
"id": "c609291c", "id": "c609291c",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -95,14 +100,18 @@
" \"master_metadata_album_album_name\": \"album_name\",\n", " \"master_metadata_album_album_name\": \"album_name\",\n",
" \"master_metadata_album_artist_name\": \"artist_name\",\n", " \"master_metadata_album_artist_name\": \"artist_name\",\n",
" \"master_metadata_track_name\": \"track_name\",\n", " \"master_metadata_track_name\": \"track_name\",\n",
" \"spotify_episode_uri\" : \"episode_uri\",\n",
" \"spotify_track_uri\" : \"spotify_track_uri\"\n",
"}\n", "}\n",
"\n", "\n",
"TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n" "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n",
"\n",
"MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\"]"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 214,
"id": "4cc14bd0", "id": "4cc14bd0",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -120,29 +129,16 @@
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 5 rows\n", "only showing top 5 rows\n",
"Original Rows 105477\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n",
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 5 rows\n",
"Track Rows 102626\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
"|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|username|\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n",
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n",
"| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n",
"| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n",
"| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
"only showing top 5 rows\n", "only showing top 5 rows\n"
"Dropped Rows 102626\n"
] ]
} }
], ],
@@ -153,15 +149,16 @@
"spark = start_spark(app_name)\n", "spark = start_spark(app_name)\n",
"\n", "\n",
"df = read_json(spark, music_source_paths())\n", "df = read_json(spark, music_source_paths())\n",
"df.show(5)\n",
"\n", "\n",
"track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES))\n", "track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS))\n",
"\n", "\n",
"drop_df = drop_pii_columns(track_df) \n" "track_df.show(5)\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 215,
"id": "faa6a8cb", "id": "faa6a8cb",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -193,18 +190,30 @@
" ts | 0 \n", " ts | 0 \n",
" user_agent_decrypted | 24700 \n", " user_agent_decrypted | 24700 \n",
" username | 0 \n", " username | 0 \n",
"\n" "\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
"only showing top 5 rows\n",
"Original Rows 105477\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
"|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n",
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n",
"| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n",
"| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n",
"| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n",
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n",
"only showing top 5 rows\n",
"Track Rows 102626\n"
] ]
},
{
"data": {
"text/plain": [
"2851"
]
},
"execution_count": 155,
"metadata": {},
"output_type": "execute_result"
} }
], ],
"source": [ "source": [
@@ -223,9 +232,7 @@
"\n", "\n",
"track_df.show(5)\n", "track_df.show(5)\n",
"print(f\"Track Rows {track_df.count()}\")\n", "print(f\"Track Rows {track_df.count()}\")\n",
"\n", "\n"
"drop_df.show(5)\n",
"drop_df.count()\n"
] ]
}, },
{ {