{ "cells": [ { "cell_type": "code", "execution_count": 110, "id": "e9bf1d26", "metadata": {}, "outputs": [], "source": [ "from pyspark.sql import SparkSession\n", "from pyspark.sql import functions as F\n", "from pathlib import Path\n", "import glob" ] }, { "cell_type": "code", "execution_count": 111, "id": "11bb5dc2", "metadata": {}, "outputs": [], "source": [ " ## Extract ##\n", "\n", "BASE = Path.cwd() / \"ingest\"\n", "\n", "def start_spark(app_name):\n", " spark = SparkSession.builder.appName(app_name).getOrCreate()\n", " spark.sparkContext.setLogLevel(\"ERROR\")\n", " return spark\n", "\n", "\n", "def music_source_paths():\n", " return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n", "\n", "def video_source_paths():\n", " return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n", "\n", "def read_json(spark, paths):\n", " if not paths:\n", " raise FileNotFoundError(\"No JSON files matched the pattern\")\n", " return (\n", " spark.read\n", " .format(\"json\")\n", " .option(\"multiLine\", True)\n", " .json(paths) # list of paths, one read\n", " # .withColumn(\"source_file\", F.input_file_name())\n", " # .withColumn(\n", " # \"source_folder\",\n", " # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n", " # )\n", " )" ] }, { "cell_type": "code", "execution_count": 212, "id": "a898326e", "metadata": {}, "outputs": [], "source": [ " ## Transform ##\n", "def null_counts_by_column(df):\n", " null_counts = df.select([\n", " F.count(F.when(F.col(c).isNull(), c)).alias(c)\n", " for c in df.columns\n", " ])\n", " return null_counts\n", "\n", "def metadata_df(df):\n", " return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n", "\n", "def rename_columns(df, rename_map, drop_columns):\n", " missing_rename = set(rename_map) - set(df.columns)\n", " missing_dropped = set(drop_columns) - set(df.columns)\n", "\n", " if missing_rename:\n", " print(f\"Warning: columns not found: {missing_rename}\")\n", " if missing_dropped:\n", " print(f\"Warning: columns not found: {missing_dropped}\")\n", "\n", " df = df.drop(*drop_columns)\n", " \n", " return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n", "\n", "def track_df(df):\n", " return df.where(df[\"track_name\"].isNotNull())\n", "\n" ] }, { "cell_type": "code", "execution_count": 213, "id": "c609291c", "metadata": {}, "outputs": [], "source": [ "## Config \n", "SPOTIFY_RENAMES = {\n", " \"master_metadata_album_album_name\": \"album_name\",\n", " \"master_metadata_album_artist_name\": \"artist_name\",\n", " \"master_metadata_track_name\": \"track_name\",\n", " \"spotify_episode_uri\" : \"episode_uri\",\n", " \"spotify_track_uri\" : \"spotify_track_uri\"\n", "}\n", "\n", "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n", "\n", "MUSIC_DROP_COLUMNS = [\"ip_addr_decrypted\", \"user_agent_decrypted\"]" ] }, { "cell_type": "code", "execution_count": 214, "id": "4cc14bd0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "only showing top 5 rows\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "only showing top 5 rows\n" ] } ], "source": [ " ## MAIN ## \n", "\n", "app_name = \"ml_media\"\n", "spark = start_spark(app_name)\n", "\n", "df = read_json(spark, music_source_paths())\n", "df.show(5)\n", "\n", "track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES, MUSIC_DROP_COLUMNS))\n", "\n", "track_df.show(5)\n" ] }, { "cell_type": "code", "execution_count": 215, "id": "faa6a8cb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "7\n", "21\n", "-RECORD 0----------------------\n", " conn_country | 0 \n", " episode_name | 105093 \n", " episode_show_name | 105093 \n", " incognito_mode | 0 \n", " ip_addr_decrypted | 4492 \n", " album_name | 2851 \n", " artist_name | 2851 \n", " track_name | 2851 \n", " ms_played | 0 \n", " offline | 279 \n", " offline_timestamp | 2 \n", " platform | 0 \n", " reason_end | 4490 \n", " reason_start | 0 \n", " shuffle | 0 \n", " skipped | 44457 \n", " spotify_episode_uri | 105093 \n", " spotify_track_uri | 2851 \n", " ts | 0 \n", " user_agent_decrypted | 24700 \n", " username | 0 \n", "\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", "only showing top 5 rows\n", "Original Rows 105477\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|episode_uri| spotify_track_uri| ts|username|\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-----------+--------------------+--------------------+--------+\n", "only showing top 5 rows\n", "Track Rows 102626\n" ] } ], "source": [ "## Potential Test Help\n", "print(metadata_df(music_df).rdd.getNumPartitions()) # Partitions for Pyspark\n", "print(len(metadata_df(music_df).columns)) # # of columns in JSON\n", "null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n", "null_counts.show(vertical=True) # Print Null counts by column_df\n", "total_rows = music_df.count\n", "\n", "df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n", " ## same can be done for podcasts\n", "\n", "df.show(5)\n", "print(f\"Original Rows {df.count()}\")\n", "\n", "track_df.show(5)\n", "print(f\"Track Rows {track_df.count()}\")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a0e9641e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "7\n", "21\n", "-RECORD 0----------------------\n", " conn_country | 0 \n", " episode_name | 105093 \n", " episode_show_name | 105093 \n", " incognito_mode | 0 \n", " ip_addr_decrypted | 4492 \n", " album_name | 2851 \n", " artist_name | 2851 \n", " track_name | 2851 \n", " ms_played | 0 \n", " offline | 279 \n", " offline_timestamp | 2 \n", " platform | 0 \n", " reason_end | 4490 \n", " reason_start | 0 \n", " shuffle | 0 \n", " skipped | 44457 \n", " spotify_episode_uri | 105093 \n", " spotify_track_uri | 2851 \n", " ts | 0 \n", " user_agent_decrypted | 24700 \n", " username | 0 \n", "\n" ] } ], "source": [] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.7" } }, "nbformat": 4, "nbformat_minor": 5 }