diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..64c2541 Binary files /dev/null and b/.DS_Store differ diff --git a/spotify.ipynb b/spotify.ipynb index 21aa187..5a68a53 100644 --- a/spotify.ipynb +++ b/spotify.ipynb @@ -2,495 +2,232 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 110, "id": "e9bf1d26", "metadata": {}, "outputs": [], "source": [ - "# Import functions\n", "from pyspark.sql import SparkSession\n", - "from pyspark.sql.functions import col, current_timestamp\n", - "\n", - "log_file = \n", - "spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n", - "spark.sparkContext.setLogLevel(\"error\")\n", - "\n", - "# Define variables used in code below\n", - "\n", - "def file_path():\n", - " file_path = pwd\n", - "\n", - "file_path = \"/databricks-datasets/structured-streaming/events\"\n", - "username = spark.sql(\"SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')\").first()[0]\n", - "table_name = f\"{username}_etl_quickstart\"\n", - "checkpoint_path = f\"/tmp/{username}/_checkpoint/etl_quickstart\"\n", - "\n", - "# Clear out data from previous demo execution\n", - "spark.sql(f\"DROP TABLE IF EXISTS {table_name}\")\n", - "dbutils.fs.rm(checkpoint_path, True)\n", - "\n", - "# Configure Auto Loader to ingest JSON data to a Delta table\n", - "(spark.readStream\n", - " .format(\"cloudFiles\")\n", - " .option(\"cloudFiles.format\", \"json\")\n", - " .option(\"cloudFiles.schemaLocation\", checkpoint_path)\n", - " .load(file_path)\n", - " .select(\"*\", col(\"_metadata.file_path\").alias(\"source_file\"), current_timestamp().alias(\"processing_time\"))\n", - " .writeStream\n", - " .option(\"checkpointLocation\", checkpoint_path)\n", - " .trigger(availableNow=True)\n", - " .toTable(table_name))" + "from pyspark.sql import functions as F\n", + "from pathlib import Path\n", + "import glob" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 111, "id": "11bb5dc2", "metadata": {}, + "outputs": [], + "source": [ + " ## Extract ##\n", + "\n", + "BASE = Path.cwd() / \"ingest\"\n", + "\n", + "def start_spark(app_name):\n", + " spark = SparkSession.builder.appName(app_name).getOrCreate()\n", + " spark.sparkContext.setLogLevel(\"ERROR\")\n", + " return spark\n", + "\n", + "\n", + "def music_source_paths():\n", + " return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n", + "\n", + "def video_source_paths():\n", + " return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n", + "\n", + "def read_json(spark, paths):\n", + " if not paths:\n", + " raise FileNotFoundError(\"No JSON files matched the pattern\")\n", + " return (\n", + " spark.read\n", + " .format(\"json\")\n", + " .option(\"multiLine\", True)\n", + " .json(paths) # list of paths, one read\n", + " # .withColumn(\"source_file\", F.input_file_name())\n", + " # .withColumn(\n", + " # \"source_folder\",\n", + " # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n", + " # )\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 186, + "id": "a898326e", + "metadata": {}, + "outputs": [], + "source": [ + " ## Transform ##\n", + "def null_counts_by_column(df):\n", + " null_counts = df.select([\n", + " F.count(F.when(F.col(c).isNull(), c)).alias(c)\n", + " for c in df.columns\n", + " ])\n", + " return null_counts\n", + "\n", + "def metadata_df(df):\n", + " return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n", + "\n", + "def rename_columns(df, rename_map):\n", + " return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n", + "\n", + "def track_df(df):\n", + " return df.where(df[\"track_name\"].isNotNull())\n", + "\n", + "def drop_pii_columns(df):\n", + " return df.drop(\"ip_addr_decrypted\", \"user_agent_decrypted\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c609291c", + "metadata": {}, + "outputs": [], + "source": [ + "## Config \n", + "SPOTIFY_RENAMES = {\n", + " \"master_metadata_album_album_name\": \"album_name\",\n", + " \"master_metadata_album_artist_name\": \"artist_name\",\n", + " \"master_metadata_track_name\": \"track_name\",\n", + "}\n", + "\n", + "TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cc14bd0", + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", + "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", + "+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "only showing top 5 rows\n", + "Original Rows 105477\n", + "+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n", + "+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n", + "| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n", + "+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n", + "only showing top 5 rows\n", + "Track Rows 102626\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", + "|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|username|\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", + "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n", + "| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n", + "| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n", + "| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n", + "| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n", + "+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n", + "only showing top 5 rows\n", + "Dropped Rows 102626\n" + ] + } + ], + "source": [ + " ## MAIN ## \n", + "\n", + "app_name = \"ml_media\"\n", + "spark = start_spark(app_name)\n", + "\n", + "df = read_json(spark, music_source_paths())\n", + "\n", + "track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES))\n", + "\n", + "drop_df = drop_pii_columns(track_df) \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faa6a8cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7\n", + "21\n", + "-RECORD 0----------------------\n", + " conn_country | 0 \n", + " episode_name | 105093 \n", + " episode_show_name | 105093 \n", + " incognito_mode | 0 \n", + " ip_addr_decrypted | 4492 \n", + " album_name | 2851 \n", + " artist_name | 2851 \n", + " track_name | 2851 \n", + " ms_played | 0 \n", + " offline | 279 \n", + " offline_timestamp | 2 \n", + " platform | 0 \n", + " reason_end | 4490 \n", + " reason_start | 0 \n", + " shuffle | 0 \n", + " skipped | 44457 \n", + " spotify_episode_uri | 105093 \n", + " spotify_track_uri | 2851 \n", + " ts | 0 \n", + " user_agent_decrypted | 24700 \n", + " username | 0 \n", + "\n" + ] + }, { "data": { "text/plain": [ - "'/Users/carlacastillo/Documents/Gitea/media_ml'" + "2851" ] }, - "execution_count": 3, + "execution_count": 155, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%pwd" + "## Potential Test Help\n", + "print(metadata_df(music_df).rdd.getNumPartitions()) # Partitions for Pyspark\n", + "print(len(metadata_df(music_df).columns)) # # of columns in JSON\n", + "null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n", + "null_counts.show(vertical=True) # Print Null counts by column_df\n", + "total_rows = music_df.count\n", + "\n", + "df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n", + " ## same can be done for podcasts\n", + "\n", + "df.show(5)\n", + "print(f\"Original Rows {df.count()}\")\n", + "\n", + "track_df.show(5)\n", + "print(f\"Track Rows {track_df.count()}\")\n", + "\n", + "drop_df.show(5)\n", + "drop_df.count()\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "a898326e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current working directory: /Users/carlacastillo/Documents/Gitea/media_ml\n" - ] - } - ], - "source": [ - "from pyspark.sql import SparkSession\n", - "\n", - "import os \n", - "cwd = os.getcwd() \n", - "print(\"Current working directory:\", cwd)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "0a0bbedc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Music\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Video_2023-2024.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2024_3.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Video_2018-2024.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json\n", - "Video\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-playback.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-episodes.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-30.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-1.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-26.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-51.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-47.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-10.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers-requests.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-lists.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-11.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-1.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-shows.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-46.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-50.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-shows.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-27.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-episodes.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-31.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-watchlist.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-movies.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-57.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-41.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-6.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-16.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-ratings.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-shows.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-36.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-20.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-profile.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-21.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-movies.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-37.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-60.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-collaborations.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-1.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-settings.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-stats.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-episodes.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-17.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-7.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-40.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-56.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-2.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-38.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-14.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-4.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-43.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-55.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-22.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-34.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-seasons.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-18.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-calendar.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-8.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-lists.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-59.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-58.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-9.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-favorites.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-19.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-35.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-23.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-54.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-42.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-5.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-15.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-following.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-39.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-shows.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-comments.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-collection_items.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-seasons.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-people.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-24.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-32.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-friends.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-49.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-seasons.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-28.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-movies.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-12.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-2.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-lists.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-45.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-53.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-52.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-movies.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-44.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-3.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-13.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-activities.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-last-activities.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-29.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-48.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-2.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-33.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-25.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json\n", - "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers.json\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(113)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 113\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 113\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(129)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 129\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 129\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(89)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 89\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 89\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(111)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 111\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 111\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(134)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 134\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 134\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(121)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 121\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 121\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(118)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 118\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 118\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(146)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 146\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 146\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(132)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 132\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 132\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(98)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 98\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 98\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(115)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 115\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 115\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(96)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 96\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 96\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(97)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 97\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 97\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(86)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 86\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 86\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(142)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 142\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 142\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(90)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 90\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 90\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(145)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 145\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 145\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(105)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 105\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 105\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(104)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 104\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 104\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(100)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 100\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 100\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(116)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 116\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 116\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 4\n", - "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 4\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 4\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4_piece0\n", - "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4_piece0 of size 40107 dropped from memory (free 454906095)\n", - "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", - "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.10.130:61896 in memory (size: 39.2 KiB, free: 434.4 MiB)\n", - "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0\n", - "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_4_piece0\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4\n", - "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4 of size 264472 dropped from memory (free 455170567)\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 2\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 4\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(95)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 95\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 95\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(106)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 106\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 106\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(103)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 103\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 103\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(141)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 141\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 141\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(137)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 137\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 137\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(136)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 136\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 136\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(101)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 101\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 101\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(99)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 99\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 99\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(140)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 140\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 140\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(102)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 102\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 102\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(93)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 93\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 93\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(83)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 83\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 83\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(119)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 119\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 119\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(128)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 128\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 128\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(138)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 138\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 138\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(87)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 87\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 87\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(130)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 130\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 130\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(108)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 108\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 108\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(110)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 110\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 110\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(109)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 109\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 109\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(117)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 117\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 117\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(147)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 147\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 147\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(125)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 125\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 125\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(114)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 114\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 114\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(85)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 85\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 85\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 5\n", - "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 5\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 5\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5_piece0\n", - "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5_piece0 of size 8326 dropped from memory (free 455178893)\n", - "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", - "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.10.130:61896 in memory (size: 8.1 KiB, free: 434.4 MiB)\n", - "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0\n", - "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_5_piece0\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5\n", - "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5 of size 18072 dropped from memory (free 455196965)\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 2\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 5\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(144)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 144\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 144\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(139)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 139\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 139\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(107)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 107\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 107\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(124)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 124\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 124\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 6\n", - "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 6\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 6\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6\n", - "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6 of size 264384 dropped from memory (free 455461349)\n", - "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6_piece0\n", - "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6_piece0 of size 40065 dropped from memory (free 455501414)\n", - "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", - "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.10.130:61896 in memory (size: 39.1 KiB, free: 434.4 MiB)\n", - "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0\n", - "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_6_piece0\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 2\n", - "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 6\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(126)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 126\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 126\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(122)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 122\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 122\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(91)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 91\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 91\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(112)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 112\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 112\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(133)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 133\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 133\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(120)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 120\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 120\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(127)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 127\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 127\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(92)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 92\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 92\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(143)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 143\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 143\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(148)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 148\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 148\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(131)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 131\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 131\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(135)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 135\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 135\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(94)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 94\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 94\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(123)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 123\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 123\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(88)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 88\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 88\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(84)\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 84\n", - "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 84\n" - ] - } - ], - "source": [ - "music_path = os.getcwd() + \"/ingest/**_sesh/*.json\"\n", - "video_path = os.getcwd() + \"/ingest/trakt**/*.json\"\n", - "\n", - "print(\"Music\")\n", - "for file in glob.iglob(music_path, recursive=False):\n", - "\n", - " print(file)\n", - "\n", - "print(\"Video\")\n", - "for file in glob.iglob(video_path, recursive=True):\n", - " print(file)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 96, "id": "a0e9641e", "metadata": {}, "outputs": [ @@ -498,29 +235,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "root\n", - " |-- conn_country: string (nullable = true)\n", - " |-- episode_name: string (nullable = true)\n", - " |-- episode_show_name: string (nullable = true)\n", - " |-- incognito_mode: boolean (nullable = true)\n", - " |-- ip_addr_decrypted: string (nullable = true)\n", - " |-- master_metadata_album_album_name: string (nullable = true)\n", - " |-- master_metadata_album_artist_name: string (nullable = true)\n", - " |-- master_metadata_track_name: string (nullable = true)\n", - " |-- ms_played: long (nullable = true)\n", - " |-- offline: boolean (nullable = true)\n", - " |-- offline_timestamp: long (nullable = true)\n", - " |-- platform: string (nullable = true)\n", - " |-- reason_end: string (nullable = true)\n", - " |-- reason_start: string (nullable = true)\n", - " |-- shuffle: boolean (nullable = true)\n", - " |-- skipped: boolean (nullable = true)\n", - " |-- spotify_episode_uri: string (nullable = true)\n", - " |-- spotify_track_uri: string (nullable = true)\n", - " |-- ts: string (nullable = true)\n", - " |-- user_agent_decrypted: string (nullable = true)\n", - " |-- username: string (nullable = true)\n", - "\n", + "7\n", + "21\n", "-RECORD 0----------------------\n", " conn_country | 0 \n", " episode_name | 105093 \n", @@ -547,69 +263,7 @@ ] } ], - "source": [ - "from pyspark.sql import SparkSession\n", - "from pyspark.sql import functions as F\n", - "from pathlib import Path\n", - "import glob\n", - "\n", - "BASE = Path.cwd() / \"ingest\"\n", - "\n", - "def music_source_paths():\n", - " return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n", - "\n", - "def video_source_paths():\n", - " return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n", - "\n", - "def read_json(spark, paths):\n", - " if not paths:\n", - " raise FileNotFoundError(\"No JSON files matched the pattern\")\n", - " return (\n", - " spark.read\n", - " .format(\"json\")\n", - " .option(\"multiLine\", True)\n", - " .json(paths) # list of paths, one read\n", - " # .withColumn(\"source_file\", F.input_file_name())\n", - " # .withColumn(\n", - " # \"source_folder\",\n", - " # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n", - " # )\n", - " )\n", - "\n", - "\n", - "def null_counts_by_column(df):\n", - " null_counts = df.select([\n", - " F.count(F.when(F.col(c).isNull(), c)).alias(c)\n", - " for c in df.columns\n", - " ])\n", - " return null_counts\n", - "\n", - "def metadata_df(df):\n", - " return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n", - "\n", - "## MAIN ## \n", - "\n", - "spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n", - "spark.sparkContext.setLogLevel(\"WARN\")\n", - "\n", - "music_df = read_json(spark, music_source_paths())\n", - "\n", - "music_df.printSchema()\n", - "# music_df.show(10)\n", - "\n", - "\n", - "null_counts = null_counts_by_column(metadata_df(music_df))\n", - "null_counts.show(vertical=True, truncate=False)\n", - "\n", - "\n", - "# total_rows = music_df.count\n", - "# print(music_df.count())\n", - "\n", - "\n", - "# print(group)\n", - "# display(group)\n", - "\n" - ] + "source": [] } ], "metadata": { diff --git a/spotify.py b/spotify.py index 83b927d..9b0b496 100644 --- a/spotify.py +++ b/spotify.py @@ -7,9 +7,6 @@ spark.sparkContext.setLogLevel("error") # Define variables used in code below -def file_path(): - file_path = pwd - file_path = "/databricks-datasets/structured-streaming/events" username = spark.sql("SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')").first()[0] table_name = f"{username}_etl_quickstart"