637 lines
42 KiB
Plaintext
637 lines
42 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e9bf1d26",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Import functions\n",
|
|
"from pyspark.sql import SparkSession\n",
|
|
"from pyspark.sql.functions import col, current_timestamp\n",
|
|
"\n",
|
|
"log_file = \n",
|
|
"spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n",
|
|
"spark.sparkContext.setLogLevel(\"error\")\n",
|
|
"\n",
|
|
"# Define variables used in code below\n",
|
|
"\n",
|
|
"def file_path():\n",
|
|
" file_path = pwd\n",
|
|
"\n",
|
|
"file_path = \"/databricks-datasets/structured-streaming/events\"\n",
|
|
"username = spark.sql(\"SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')\").first()[0]\n",
|
|
"table_name = f\"{username}_etl_quickstart\"\n",
|
|
"checkpoint_path = f\"/tmp/{username}/_checkpoint/etl_quickstart\"\n",
|
|
"\n",
|
|
"# Clear out data from previous demo execution\n",
|
|
"spark.sql(f\"DROP TABLE IF EXISTS {table_name}\")\n",
|
|
"dbutils.fs.rm(checkpoint_path, True)\n",
|
|
"\n",
|
|
"# Configure Auto Loader to ingest JSON data to a Delta table\n",
|
|
"(spark.readStream\n",
|
|
" .format(\"cloudFiles\")\n",
|
|
" .option(\"cloudFiles.format\", \"json\")\n",
|
|
" .option(\"cloudFiles.schemaLocation\", checkpoint_path)\n",
|
|
" .load(file_path)\n",
|
|
" .select(\"*\", col(\"_metadata.file_path\").alias(\"source_file\"), current_timestamp().alias(\"processing_time\"))\n",
|
|
" .writeStream\n",
|
|
" .option(\"checkpointLocation\", checkpoint_path)\n",
|
|
" .trigger(availableNow=True)\n",
|
|
" .toTable(table_name))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "11bb5dc2",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'/Users/carlacastillo/Documents/Gitea/media_ml'"
|
|
]
|
|
},
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"%pwd"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a898326e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Current working directory: /Users/carlacastillo/Documents/Gitea/media_ml\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from pyspark.sql import SparkSession\n",
|
|
"\n",
|
|
"import os \n",
|
|
"cwd = os.getcwd() \n",
|
|
"print(\"Current working directory:\", cwd)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"id": "0a0bbedc",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Music\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Video_2023-2024.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2024_3.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Video_2018-2024.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json\n",
|
|
"Video\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-playback.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-episodes.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-30.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-1.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-26.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-51.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-47.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-10.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers-requests.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-lists.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-11.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-1.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-shows.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-46.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-50.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-shows.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-27.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-episodes.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-31.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-watchlist.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-movies.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-57.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-41.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-6.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-16.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-ratings.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-shows.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-36.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-20.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-profile.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-21.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-movies.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-37.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-60.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-collaborations.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-1.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-settings.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-stats.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-episodes.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-17.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-7.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-40.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-56.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-2.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-38.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-14.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-4.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-43.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-55.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-22.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-34.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-seasons.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-18.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-calendar.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-8.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-lists.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-59.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-58.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-9.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-favorites.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-19.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-35.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-23.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-54.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-42.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-5.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-15.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-following.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-39.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-shows.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-comments.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-collection_items.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-seasons.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-people.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-24.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-32.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-friends.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-49.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-seasons.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-28.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-movies.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-12.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-2.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-lists.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-45.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-53.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-52.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-movies.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-44.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-3.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-13.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-activities.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-last-activities.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-29.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-48.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-2.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-33.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-25.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json\n",
|
|
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers.json\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(113)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 113\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 113\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(129)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 129\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 129\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(89)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 89\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 89\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(111)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 111\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 111\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(134)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 134\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 134\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(121)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 121\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 121\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(118)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 118\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 118\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(146)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 146\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 146\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(132)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 132\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 132\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(98)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 98\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 98\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(115)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 115\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 115\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(96)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 96\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 96\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(97)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 97\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 97\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(86)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 86\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 86\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(142)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 142\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 142\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(90)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 90\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 90\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(145)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 145\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 145\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(105)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 105\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 105\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(104)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 104\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 104\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(100)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 100\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 100\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(116)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 116\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 116\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 4\n",
|
|
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 4\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 4\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4_piece0 of size 40107 dropped from memory (free 454906095)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.10.130:61896 in memory (size: 39.2 KiB, free: 434.4 MiB)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_4_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4\n",
|
|
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4 of size 264472 dropped from memory (free 455170567)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 2\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 4\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(95)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 95\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 95\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(106)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 106\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 106\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(103)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 103\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 103\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(141)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 141\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 141\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(137)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 137\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 137\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(136)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 136\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 136\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(101)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 101\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 101\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(99)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 99\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 99\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(140)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 140\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 140\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(102)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 102\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 102\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(93)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 93\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 93\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(83)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 83\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 83\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(119)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 119\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 119\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(128)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 128\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 128\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(138)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 138\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 138\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(87)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 87\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 87\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(130)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 130\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 130\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(108)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 108\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 108\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(110)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 110\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 110\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(109)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 109\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 109\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(117)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 117\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 117\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(147)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 147\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 147\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(125)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 125\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 125\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(114)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 114\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 114\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(85)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 85\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 85\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 5\n",
|
|
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 5\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 5\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5_piece0 of size 8326 dropped from memory (free 455178893)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.10.130:61896 in memory (size: 8.1 KiB, free: 434.4 MiB)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_5_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5\n",
|
|
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5 of size 18072 dropped from memory (free 455196965)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 2\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 5\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(144)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 144\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 144\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(139)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 139\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 139\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(107)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 107\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 107\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(124)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 124\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 124\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 6\n",
|
|
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 6\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 6\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6\n",
|
|
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6 of size 264384 dropped from memory (free 455461349)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6_piece0 of size 40065 dropped from memory (free 455501414)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.10.130:61896 in memory (size: 39.1 KiB, free: 434.4 MiB)\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_6_piece0\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 2\n",
|
|
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 6\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(126)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 126\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 126\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(122)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 122\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 122\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(91)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 91\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 91\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(112)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 112\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 112\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(133)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 133\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 133\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(120)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 120\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 120\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(127)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 127\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 127\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(92)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 92\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 92\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(143)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 143\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 143\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(148)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 148\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 148\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(131)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 131\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 131\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(135)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 135\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 135\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(94)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 94\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 94\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(123)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 123\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 123\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(88)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 88\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 88\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(84)\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 84\n",
|
|
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 84\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"music_path = os.getcwd() + \"/ingest/**_sesh/*.json\"\n",
|
|
"video_path = os.getcwd() + \"/ingest/trakt**/*.json\"\n",
|
|
"\n",
|
|
"print(\"Music\")\n",
|
|
"for file in glob.iglob(music_path, recursive=False):\n",
|
|
"\n",
|
|
" print(file)\n",
|
|
"\n",
|
|
"print(\"Video\")\n",
|
|
"for file in glob.iglob(video_path, recursive=True):\n",
|
|
" print(file)\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 96,
|
|
"id": "a0e9641e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"root\n",
|
|
" |-- conn_country: string (nullable = true)\n",
|
|
" |-- episode_name: string (nullable = true)\n",
|
|
" |-- episode_show_name: string (nullable = true)\n",
|
|
" |-- incognito_mode: boolean (nullable = true)\n",
|
|
" |-- ip_addr_decrypted: string (nullable = true)\n",
|
|
" |-- master_metadata_album_album_name: string (nullable = true)\n",
|
|
" |-- master_metadata_album_artist_name: string (nullable = true)\n",
|
|
" |-- master_metadata_track_name: string (nullable = true)\n",
|
|
" |-- ms_played: long (nullable = true)\n",
|
|
" |-- offline: boolean (nullable = true)\n",
|
|
" |-- offline_timestamp: long (nullable = true)\n",
|
|
" |-- platform: string (nullable = true)\n",
|
|
" |-- reason_end: string (nullable = true)\n",
|
|
" |-- reason_start: string (nullable = true)\n",
|
|
" |-- shuffle: boolean (nullable = true)\n",
|
|
" |-- skipped: boolean (nullable = true)\n",
|
|
" |-- spotify_episode_uri: string (nullable = true)\n",
|
|
" |-- spotify_track_uri: string (nullable = true)\n",
|
|
" |-- ts: string (nullable = true)\n",
|
|
" |-- user_agent_decrypted: string (nullable = true)\n",
|
|
" |-- username: string (nullable = true)\n",
|
|
"\n",
|
|
"-RECORD 0----------------------\n",
|
|
" conn_country | 0 \n",
|
|
" episode_name | 105093 \n",
|
|
" episode_show_name | 105093 \n",
|
|
" incognito_mode | 0 \n",
|
|
" ip_addr_decrypted | 4492 \n",
|
|
" album_name | 2851 \n",
|
|
" artist_name | 2851 \n",
|
|
" track_name | 2851 \n",
|
|
" ms_played | 0 \n",
|
|
" offline | 279 \n",
|
|
" offline_timestamp | 2 \n",
|
|
" platform | 0 \n",
|
|
" reason_end | 4490 \n",
|
|
" reason_start | 0 \n",
|
|
" shuffle | 0 \n",
|
|
" skipped | 44457 \n",
|
|
" spotify_episode_uri | 105093 \n",
|
|
" spotify_track_uri | 2851 \n",
|
|
" ts | 0 \n",
|
|
" user_agent_decrypted | 24700 \n",
|
|
" username | 0 \n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from pyspark.sql import SparkSession\n",
|
|
"from pyspark.sql import functions as F\n",
|
|
"from pathlib import Path\n",
|
|
"import glob\n",
|
|
"\n",
|
|
"BASE = Path.cwd() / \"ingest\"\n",
|
|
"\n",
|
|
"def music_source_paths():\n",
|
|
" return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n",
|
|
"\n",
|
|
"def video_source_paths():\n",
|
|
" return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n",
|
|
"\n",
|
|
"def read_json(spark, paths):\n",
|
|
" if not paths:\n",
|
|
" raise FileNotFoundError(\"No JSON files matched the pattern\")\n",
|
|
" return (\n",
|
|
" spark.read\n",
|
|
" .format(\"json\")\n",
|
|
" .option(\"multiLine\", True)\n",
|
|
" .json(paths) # list of paths, one read\n",
|
|
" # .withColumn(\"source_file\", F.input_file_name())\n",
|
|
" # .withColumn(\n",
|
|
" # \"source_folder\",\n",
|
|
" # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n",
|
|
" # )\n",
|
|
" )\n",
|
|
"\n",
|
|
"\n",
|
|
"def null_counts_by_column(df):\n",
|
|
" null_counts = df.select([\n",
|
|
" F.count(F.when(F.col(c).isNull(), c)).alias(c)\n",
|
|
" for c in df.columns\n",
|
|
" ])\n",
|
|
" return null_counts\n",
|
|
"\n",
|
|
"def metadata_df(df):\n",
|
|
" return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n",
|
|
"\n",
|
|
"## MAIN ## \n",
|
|
"\n",
|
|
"spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n",
|
|
"spark.sparkContext.setLogLevel(\"WARN\")\n",
|
|
"\n",
|
|
"music_df = read_json(spark, music_source_paths())\n",
|
|
"\n",
|
|
"music_df.printSchema()\n",
|
|
"# music_df.show(10)\n",
|
|
"\n",
|
|
"\n",
|
|
"null_counts = null_counts_by_column(metadata_df(music_df))\n",
|
|
"null_counts.show(vertical=True, truncate=False)\n",
|
|
"\n",
|
|
"\n",
|
|
"# total_rows = music_df.count\n",
|
|
"# print(music_df.count())\n",
|
|
"\n",
|
|
"\n",
|
|
"# print(group)\n",
|
|
"# display(group)\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.13.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|