column renames and drops
This commit is contained in:
CC
750
spotify.ipynb
750
spotify.ipynb
@@ -2,495 +2,232 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 110,
|
||||
"id": "e9bf1d26",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import functions\n",
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"from pyspark.sql.functions import col, current_timestamp\n",
|
||||
"\n",
|
||||
"log_file = \n",
|
||||
"spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n",
|
||||
"spark.sparkContext.setLogLevel(\"error\")\n",
|
||||
"\n",
|
||||
"# Define variables used in code below\n",
|
||||
"\n",
|
||||
"def file_path():\n",
|
||||
" file_path = pwd\n",
|
||||
"\n",
|
||||
"file_path = \"/databricks-datasets/structured-streaming/events\"\n",
|
||||
"username = spark.sql(\"SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')\").first()[0]\n",
|
||||
"table_name = f\"{username}_etl_quickstart\"\n",
|
||||
"checkpoint_path = f\"/tmp/{username}/_checkpoint/etl_quickstart\"\n",
|
||||
"\n",
|
||||
"# Clear out data from previous demo execution\n",
|
||||
"spark.sql(f\"DROP TABLE IF EXISTS {table_name}\")\n",
|
||||
"dbutils.fs.rm(checkpoint_path, True)\n",
|
||||
"\n",
|
||||
"# Configure Auto Loader to ingest JSON data to a Delta table\n",
|
||||
"(spark.readStream\n",
|
||||
" .format(\"cloudFiles\")\n",
|
||||
" .option(\"cloudFiles.format\", \"json\")\n",
|
||||
" .option(\"cloudFiles.schemaLocation\", checkpoint_path)\n",
|
||||
" .load(file_path)\n",
|
||||
" .select(\"*\", col(\"_metadata.file_path\").alias(\"source_file\"), current_timestamp().alias(\"processing_time\"))\n",
|
||||
" .writeStream\n",
|
||||
" .option(\"checkpointLocation\", checkpoint_path)\n",
|
||||
" .trigger(availableNow=True)\n",
|
||||
" .toTable(table_name))"
|
||||
"from pyspark.sql import functions as F\n",
|
||||
"from pathlib import Path\n",
|
||||
"import glob"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 111,
|
||||
"id": "11bb5dc2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
" ## Extract ##\n",
|
||||
"\n",
|
||||
"BASE = Path.cwd() / \"ingest\"\n",
|
||||
"\n",
|
||||
"def start_spark(app_name):\n",
|
||||
" spark = SparkSession.builder.appName(app_name).getOrCreate()\n",
|
||||
" spark.sparkContext.setLogLevel(\"ERROR\")\n",
|
||||
" return spark\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def music_source_paths():\n",
|
||||
" return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n",
|
||||
"\n",
|
||||
"def video_source_paths():\n",
|
||||
" return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n",
|
||||
"\n",
|
||||
"def read_json(spark, paths):\n",
|
||||
" if not paths:\n",
|
||||
" raise FileNotFoundError(\"No JSON files matched the pattern\")\n",
|
||||
" return (\n",
|
||||
" spark.read\n",
|
||||
" .format(\"json\")\n",
|
||||
" .option(\"multiLine\", True)\n",
|
||||
" .json(paths) # list of paths, one read\n",
|
||||
" # .withColumn(\"source_file\", F.input_file_name())\n",
|
||||
" # .withColumn(\n",
|
||||
" # \"source_folder\",\n",
|
||||
" # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n",
|
||||
" # )\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 186,
|
||||
"id": "a898326e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
" ## Transform ##\n",
|
||||
"def null_counts_by_column(df):\n",
|
||||
" null_counts = df.select([\n",
|
||||
" F.count(F.when(F.col(c).isNull(), c)).alias(c)\n",
|
||||
" for c in df.columns\n",
|
||||
" ])\n",
|
||||
" return null_counts\n",
|
||||
"\n",
|
||||
"def metadata_df(df):\n",
|
||||
" return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
|
||||
"\n",
|
||||
"def rename_columns(df, rename_map):\n",
|
||||
" return df.toDF(*[rename_map.get(c, c) for c in df.columns])\n",
|
||||
"\n",
|
||||
"def track_df(df):\n",
|
||||
" return df.where(df[\"track_name\"].isNotNull())\n",
|
||||
"\n",
|
||||
"def drop_pii_columns(df):\n",
|
||||
" return df.drop(\"ip_addr_decrypted\", \"user_agent_decrypted\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c609291c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## Config \n",
|
||||
"SPOTIFY_RENAMES = {\n",
|
||||
" \"master_metadata_album_album_name\": \"album_name\",\n",
|
||||
" \"master_metadata_album_artist_name\": \"artist_name\",\n",
|
||||
" \"master_metadata_track_name\": \"track_name\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"TRAKT_RENAMES = {\"watched_at\": \"ts\", \"title\": \"video_title\"}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4cc14bd0",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted|master_metadata_album_album_name|master_metadata_album_artist_name|master_metadata_track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
|
||||
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO| No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2| Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
|
||||
"+------------+------------+-----------------+--------------+-----------------+--------------------------------+---------------------------------+--------------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"only showing top 5 rows\n",
|
||||
"Original Rows 105477\n",
|
||||
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"|conn_country|episode_name|episode_show_name|incognito_mode|ip_addr_decrypted| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|user_agent_decrypted|username|\n",
|
||||
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| unknown| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 31.94.22.158| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| unknown| csm3-us|\n",
|
||||
"+------------+------------+-----------------+--------------+-----------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------------------+--------+\n",
|
||||
"only showing top 5 rows\n",
|
||||
"Track Rows 102626\n",
|
||||
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
|
||||
"|conn_country|episode_name|episode_show_name|incognito_mode| album_name| artist_name| track_name|ms_played|offline|offline_timestamp|platform| reason_end|reason_start|shuffle|skipped|spotify_episode_uri| spotify_track_uri| ts|username|\n",
|
||||
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
|
||||
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 80505| false| 1680818230| ios|unexpected-exit-w...| trackdone| true| false| NULL|spotify:track:644...|2023-04-06T21:58:31Z| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false|Last Day Of Summer| Summer Walker| Girls Need Love| 10100| false| 1680867540| ios| endplay| appload| true| true| NULL|spotify:track:644...|2023-04-07T11:39:14Z| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| Famouz| JHAYCO|No Me Conoce - Remix| 309120| false| 1680867555| ios| trackdone| clickrow| true| false| NULL|spotify:track:2be...|2023-04-07T11:45:50Z| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| 4K| El Alfa| 4K| 201500| false| 1680867951| ios| trackdone| trackdone| true| false| NULL|spotify:track:4xI...|2023-04-07T11:49:12Z| csm3-us|\n",
|
||||
"| GB| NULL| NULL| false| Sauce Boyz 2|Eladio Carrion| No Te Deseo el Mal| 18282| false| 1680868153| ios| fwdbtn| trackdone| true| true| NULL|spotify:track:3HQ...|2023-04-07T11:49:32Z| csm3-us|\n",
|
||||
"+------------+------------+-----------------+--------------+------------------+--------------+--------------------+---------+-------+-----------------+--------+--------------------+------------+-------+-------+-------------------+--------------------+--------------------+--------+\n",
|
||||
"only showing top 5 rows\n",
|
||||
"Dropped Rows 102626\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
" ## MAIN ## \n",
|
||||
"\n",
|
||||
"app_name = \"ml_media\"\n",
|
||||
"spark = start_spark(app_name)\n",
|
||||
"\n",
|
||||
"df = read_json(spark, music_source_paths())\n",
|
||||
"\n",
|
||||
"track_df = track_df(rename_columns(music_df, SPOTIFY_RENAMES))\n",
|
||||
"\n",
|
||||
"drop_df = drop_pii_columns(track_df) \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "faa6a8cb",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"7\n",
|
||||
"21\n",
|
||||
"-RECORD 0----------------------\n",
|
||||
" conn_country | 0 \n",
|
||||
" episode_name | 105093 \n",
|
||||
" episode_show_name | 105093 \n",
|
||||
" incognito_mode | 0 \n",
|
||||
" ip_addr_decrypted | 4492 \n",
|
||||
" album_name | 2851 \n",
|
||||
" artist_name | 2851 \n",
|
||||
" track_name | 2851 \n",
|
||||
" ms_played | 0 \n",
|
||||
" offline | 279 \n",
|
||||
" offline_timestamp | 2 \n",
|
||||
" platform | 0 \n",
|
||||
" reason_end | 4490 \n",
|
||||
" reason_start | 0 \n",
|
||||
" shuffle | 0 \n",
|
||||
" skipped | 44457 \n",
|
||||
" spotify_episode_uri | 105093 \n",
|
||||
" spotify_track_uri | 2851 \n",
|
||||
" ts | 0 \n",
|
||||
" user_agent_decrypted | 24700 \n",
|
||||
" username | 0 \n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'/Users/carlacastillo/Documents/Gitea/media_ml'"
|
||||
"2851"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"execution_count": 155,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%pwd"
|
||||
"## Potential Test Help\n",
|
||||
"print(metadata_df(music_df).rdd.getNumPartitions()) # Partitions for Pyspark\n",
|
||||
"print(len(metadata_df(music_df).columns)) # # of columns in JSON\n",
|
||||
"null_counts = null_counts_by_column(metadata_df(music_df)) # Null counts by column\n",
|
||||
"null_counts.show(vertical=True) # Print Null counts by column_df\n",
|
||||
"total_rows = music_df.count\n",
|
||||
"\n",
|
||||
"df.count()-track_df.count() ## assert the difference in rows for tracks is equal to the nulls count\n",
|
||||
" ## same can be done for podcasts\n",
|
||||
"\n",
|
||||
"df.show(5)\n",
|
||||
"print(f\"Original Rows {df.count()}\")\n",
|
||||
"\n",
|
||||
"track_df.show(5)\n",
|
||||
"print(f\"Track Rows {track_df.count()}\")\n",
|
||||
"\n",
|
||||
"drop_df.show(5)\n",
|
||||
"drop_df.count()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a898326e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Current working directory: /Users/carlacastillo/Documents/Gitea/media_ml\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"\n",
|
||||
"import os \n",
|
||||
"cwd = os.getcwd() \n",
|
||||
"print(\"Current working directory:\", cwd)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "0a0bbedc",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Music\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Video_2023-2024.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2024_3.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Video_2018-2024.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json\n",
|
||||
"Video\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-playback.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-episodes.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-30.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-26.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-51.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-47.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-10.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers-requests.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-lists.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-11.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-46.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-50.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-27.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-episodes.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-31.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-watchlist.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-57.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-41.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-6.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-16.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-ratings.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-36.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-20.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-profile.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-21.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-37.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-60.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-collaborations.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-settings.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-stats.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-episodes.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-17.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-7.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-40.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-56.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-38.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-14.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-4.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-43.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-55.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-22.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-34.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-seasons.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-18.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-calendar.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-8.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-lists.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-59.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-58.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-9.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-favorites.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-19.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-35.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-23.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-54.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-42.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-5.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-15.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-following.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-39.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-comments.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-collection_items.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-seasons.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-people.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-24.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-32.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-friends.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-49.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-seasons.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-28.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-12.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-lists.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-45.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-53.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-52.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-44.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-3.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-13.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-activities.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-last-activities.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-29.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-48.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-33.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-25.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers.json\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(113)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 113\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 113\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(129)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 129\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 129\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(89)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 89\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 89\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(111)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 111\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 111\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(134)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 134\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 134\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(121)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 121\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 121\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(118)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 118\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 118\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(146)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 146\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 146\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(132)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 132\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 132\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(98)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 98\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 98\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(115)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 115\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 115\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(96)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 96\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 96\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(97)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 97\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 97\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(86)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 86\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 86\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(142)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 142\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 142\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(90)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 90\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 90\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(145)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 145\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 145\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(105)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 105\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 105\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(104)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 104\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 104\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(100)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 100\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 100\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(116)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 116\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 116\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4_piece0 of size 40107 dropped from memory (free 454906095)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.10.130:61896 in memory (size: 39.2 KiB, free: 434.4 MiB)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_4_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4 of size 264472 dropped from memory (free 455170567)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 2\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(95)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 95\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 95\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(106)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 106\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 106\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(103)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 103\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 103\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(141)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 141\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 141\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(137)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 137\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 137\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(136)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 136\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 136\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(101)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 101\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 101\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(99)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 99\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 99\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(140)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 140\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 140\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(102)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 102\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 102\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(93)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 93\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 93\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(83)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 83\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 83\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(119)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 119\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 119\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(128)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 128\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 128\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(138)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 138\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 138\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(87)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 87\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 87\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(130)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 130\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 130\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(108)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 108\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 108\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(110)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 110\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 110\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(109)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 109\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 109\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(117)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 117\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 117\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(147)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 147\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 147\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(125)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 125\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 125\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(114)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 114\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 114\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(85)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 85\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 85\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5_piece0 of size 8326 dropped from memory (free 455178893)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.10.130:61896 in memory (size: 8.1 KiB, free: 434.4 MiB)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_5_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5 of size 18072 dropped from memory (free 455196965)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 2\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(144)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 144\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 144\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(139)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 139\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 139\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(107)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 107\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 107\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(124)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 124\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 124\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6 of size 264384 dropped from memory (free 455461349)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6_piece0 of size 40065 dropped from memory (free 455501414)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.10.130:61896 in memory (size: 39.1 KiB, free: 434.4 MiB)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_6_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 2\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(126)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 126\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 126\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(122)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 122\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 122\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(91)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 91\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 91\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(112)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 112\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 112\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(133)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 133\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 133\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(120)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 120\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 120\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(127)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 127\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 127\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(92)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 92\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 92\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(143)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 143\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 143\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(148)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 148\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 148\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(131)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 131\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 131\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(135)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 135\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 135\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(94)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 94\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 94\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(123)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 123\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 123\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(88)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 88\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 88\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(84)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 84\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 84\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"music_path = os.getcwd() + \"/ingest/**_sesh/*.json\"\n",
|
||||
"video_path = os.getcwd() + \"/ingest/trakt**/*.json\"\n",
|
||||
"\n",
|
||||
"print(\"Music\")\n",
|
||||
"for file in glob.iglob(music_path, recursive=False):\n",
|
||||
"\n",
|
||||
" print(file)\n",
|
||||
"\n",
|
||||
"print(\"Video\")\n",
|
||||
"for file in glob.iglob(video_path, recursive=True):\n",
|
||||
" print(file)\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 96,
|
||||
"id": "a0e9641e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@@ -498,29 +235,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"root\n",
|
||||
" |-- conn_country: string (nullable = true)\n",
|
||||
" |-- episode_name: string (nullable = true)\n",
|
||||
" |-- episode_show_name: string (nullable = true)\n",
|
||||
" |-- incognito_mode: boolean (nullable = true)\n",
|
||||
" |-- ip_addr_decrypted: string (nullable = true)\n",
|
||||
" |-- master_metadata_album_album_name: string (nullable = true)\n",
|
||||
" |-- master_metadata_album_artist_name: string (nullable = true)\n",
|
||||
" |-- master_metadata_track_name: string (nullable = true)\n",
|
||||
" |-- ms_played: long (nullable = true)\n",
|
||||
" |-- offline: boolean (nullable = true)\n",
|
||||
" |-- offline_timestamp: long (nullable = true)\n",
|
||||
" |-- platform: string (nullable = true)\n",
|
||||
" |-- reason_end: string (nullable = true)\n",
|
||||
" |-- reason_start: string (nullable = true)\n",
|
||||
" |-- shuffle: boolean (nullable = true)\n",
|
||||
" |-- skipped: boolean (nullable = true)\n",
|
||||
" |-- spotify_episode_uri: string (nullable = true)\n",
|
||||
" |-- spotify_track_uri: string (nullable = true)\n",
|
||||
" |-- ts: string (nullable = true)\n",
|
||||
" |-- user_agent_decrypted: string (nullable = true)\n",
|
||||
" |-- username: string (nullable = true)\n",
|
||||
"\n",
|
||||
"7\n",
|
||||
"21\n",
|
||||
"-RECORD 0----------------------\n",
|
||||
" conn_country | 0 \n",
|
||||
" episode_name | 105093 \n",
|
||||
@@ -547,69 +263,7 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"from pyspark.sql import functions as F\n",
|
||||
"from pathlib import Path\n",
|
||||
"import glob\n",
|
||||
"\n",
|
||||
"BASE = Path.cwd() / \"ingest\"\n",
|
||||
"\n",
|
||||
"def music_source_paths():\n",
|
||||
" return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n",
|
||||
"\n",
|
||||
"def video_source_paths():\n",
|
||||
" return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n",
|
||||
"\n",
|
||||
"def read_json(spark, paths):\n",
|
||||
" if not paths:\n",
|
||||
" raise FileNotFoundError(\"No JSON files matched the pattern\")\n",
|
||||
" return (\n",
|
||||
" spark.read\n",
|
||||
" .format(\"json\")\n",
|
||||
" .option(\"multiLine\", True)\n",
|
||||
" .json(paths) # list of paths, one read\n",
|
||||
" # .withColumn(\"source_file\", F.input_file_name())\n",
|
||||
" # .withColumn(\n",
|
||||
" # \"source_folder\",\n",
|
||||
" # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n",
|
||||
" # )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def null_counts_by_column(df):\n",
|
||||
" null_counts = df.select([\n",
|
||||
" F.count(F.when(F.col(c).isNull(), c)).alias(c)\n",
|
||||
" for c in df.columns\n",
|
||||
" ])\n",
|
||||
" return null_counts\n",
|
||||
"\n",
|
||||
"def metadata_df(df):\n",
|
||||
" return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
|
||||
"\n",
|
||||
"## MAIN ## \n",
|
||||
"\n",
|
||||
"spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n",
|
||||
"spark.sparkContext.setLogLevel(\"WARN\")\n",
|
||||
"\n",
|
||||
"music_df = read_json(spark, music_source_paths())\n",
|
||||
"\n",
|
||||
"music_df.printSchema()\n",
|
||||
"# music_df.show(10)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"null_counts = null_counts_by_column(metadata_df(music_df))\n",
|
||||
"null_counts.show(vertical=True, truncate=False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# total_rows = music_df.count\n",
|
||||
"# print(music_df.count())\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# print(group)\n",
|
||||
"# display(group)\n",
|
||||
"\n"
|
||||
]
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
Reference in New Issue
Block a user