ml
This commit is contained in:
CC
560
spotify.ipynb
Normal file
560
spotify.ipynb
Normal file
@@ -0,0 +1,560 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e9bf1d26",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import functions\n",
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"from pyspark.sql.functions import col, current_timestamp\n",
|
||||
"\n",
|
||||
"log_file = \n",
|
||||
"spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n",
|
||||
"spark.sparkContext.setLogLevel(\"error\")\n",
|
||||
"\n",
|
||||
"# Define variables used in code below\n",
|
||||
"\n",
|
||||
"def file_path():\n",
|
||||
" file_path = pwd\n",
|
||||
"\n",
|
||||
"file_path = \"/databricks-datasets/structured-streaming/events\"\n",
|
||||
"username = spark.sql(\"SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')\").first()[0]\n",
|
||||
"table_name = f\"{username}_etl_quickstart\"\n",
|
||||
"checkpoint_path = f\"/tmp/{username}/_checkpoint/etl_quickstart\"\n",
|
||||
"\n",
|
||||
"# Clear out data from previous demo execution\n",
|
||||
"spark.sql(f\"DROP TABLE IF EXISTS {table_name}\")\n",
|
||||
"dbutils.fs.rm(checkpoint_path, True)\n",
|
||||
"\n",
|
||||
"# Configure Auto Loader to ingest JSON data to a Delta table\n",
|
||||
"(spark.readStream\n",
|
||||
" .format(\"cloudFiles\")\n",
|
||||
" .option(\"cloudFiles.format\", \"json\")\n",
|
||||
" .option(\"cloudFiles.schemaLocation\", checkpoint_path)\n",
|
||||
" .load(file_path)\n",
|
||||
" .select(\"*\", col(\"_metadata.file_path\").alias(\"source_file\"), current_timestamp().alias(\"processing_time\"))\n",
|
||||
" .writeStream\n",
|
||||
" .option(\"checkpointLocation\", checkpoint_path)\n",
|
||||
" .trigger(availableNow=True)\n",
|
||||
" .toTable(table_name))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "11bb5dc2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'/Users/carlacastillo/Documents/Gitea/media_ml'"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%pwd"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a898326e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Current working directory: /Users/carlacastillo/Documents/Gitea/media_ml\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"\n",
|
||||
"import os \n",
|
||||
"cwd = os.getcwd() \n",
|
||||
"print(\"Current working directory:\", cwd)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "0a0bbedc",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Music\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Video_2023-2024.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2024_3.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Video_2018-2024.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json\n",
|
||||
"Video\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-playback.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-episodes.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-30.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-26.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-51.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-47.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-10.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers-requests.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-lists.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-11.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-46.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-50.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-27.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-episodes.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-31.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-watchlist.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-57.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-41.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-6.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-16.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-ratings.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-36.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-20.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-profile.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-21.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-37.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-60.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-collaborations.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-1.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-settings.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-stats.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-episodes.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-17.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-7.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-40.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-56.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-38.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-14.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-4.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-43.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-55.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-22.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-34.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-seasons.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-18.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-calendar.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-8.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-lists.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-59.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-58.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-9.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-favorites.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-19.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-35.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-23.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-54.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-42.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-5.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-15.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-following.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-39.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-shows.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-comments.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-collection_items.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-seasons.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-people.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-24.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-32.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-friends.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-49.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-seasons.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-28.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-12.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-lists.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-45.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-53.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-52.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-movies.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-44.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-3.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-13.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-activities.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-last-activities.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-29.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-48.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-2.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-33.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-25.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json\n",
|
||||
"/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers.json\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(113)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 113\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 113\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(129)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 129\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 129\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(89)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 89\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 89\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(111)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 111\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 111\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(134)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 134\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 134\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(121)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 121\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 121\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(118)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 118\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 118\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(146)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 146\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 146\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(132)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 132\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 132\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(98)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 98\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 98\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(115)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 115\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 115\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(96)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 96\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 96\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(97)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 97\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 97\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(86)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 86\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 86\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(142)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 142\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 142\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(90)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 90\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 90\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(145)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 145\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 145\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(105)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 105\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 105\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(104)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 104\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 104\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(100)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 100\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 100\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(116)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 116\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 116\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4_piece0 of size 40107 dropped from memory (free 454906095)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.10.130:61896 in memory (size: 39.2 KiB, free: 434.4 MiB)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_4_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4 of size 264472 dropped from memory (free 455170567)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 2\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 4\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(95)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 95\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 95\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(106)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 106\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 106\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(103)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 103\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 103\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(141)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 141\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 141\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(137)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 137\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 137\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(136)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 136\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 136\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(101)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 101\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 101\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(99)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 99\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 99\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(140)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 140\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 140\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(102)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 102\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 102\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(93)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 93\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 93\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(83)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 83\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 83\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(119)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 119\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 119\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(128)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 128\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 128\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(138)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 138\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 138\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(87)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 87\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 87\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(130)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 130\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 130\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(108)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 108\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 108\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(110)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 110\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 110\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(109)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 109\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 109\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(117)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 117\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 117\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(147)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 147\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 147\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(125)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 125\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 125\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(114)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 114\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 114\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(85)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 85\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 85\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5_piece0 of size 8326 dropped from memory (free 455178893)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.10.130:61896 in memory (size: 8.1 KiB, free: 434.4 MiB)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_5_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5 of size 18072 dropped from memory (free 455196965)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 2\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 5\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(144)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 144\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 144\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(139)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 139\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 139\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(107)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 107\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 107\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(124)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 124\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 124\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6 of size 264384 dropped from memory (free 455461349)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6_piece0 of size 40065 dropped from memory (free 455501414)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.10.130:61896 in memory (size: 39.1 KiB, free: 434.4 MiB)\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_6_piece0\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 2\n",
|
||||
"26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 6\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(126)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 126\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 126\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(122)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 122\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 122\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(91)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 91\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 91\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(112)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 112\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 112\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(133)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 133\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 133\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(120)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 120\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 120\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(127)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 127\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 127\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(92)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 92\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 92\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(143)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 143\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 143\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(148)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 148\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 148\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(131)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 131\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 131\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(135)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 135\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 135\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(94)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 94\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 94\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(123)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 123\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 123\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(88)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 88\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 88\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(84)\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 84\n",
|
||||
"26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 84\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"music_path = os.getcwd() + \"/ingest/**_sesh/*.json\"\n",
|
||||
"video_path = os.getcwd() + \"/ingest/trakt**/*.json\"\n",
|
||||
"\n",
|
||||
"print(\"Music\")\n",
|
||||
"for file in glob.iglob(music_path, recursive=False):\n",
|
||||
"\n",
|
||||
" print(file)\n",
|
||||
"\n",
|
||||
"print(\"Video\")\n",
|
||||
"for file in glob.iglob(video_path, recursive=True):\n",
|
||||
" print(file)\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a0e9641e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "SyntaxError",
|
||||
"evalue": "expected ':' (3030084487.py, line 14)",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
" \u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[29]\u001b[39m\u001b[32m, line 14\u001b[39m\n\u001b[31m \u001b[39m\u001b[31mif music_file.endswith(\".json\")\u001b[39m\n ^\n\u001b[31mSyntaxError\u001b[39m\u001b[31m:\u001b[39m expected ':'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"import os \n",
|
||||
"import glob \n",
|
||||
"\n",
|
||||
"def music_source_paths():\n",
|
||||
" music_path = os.getcwd() + \"/ingest/*_sesh/*.json\"\n",
|
||||
" music_files = []\n",
|
||||
" for file in glob.iglob(music_path, recursive=True):\n",
|
||||
" music_files.append(file)\n",
|
||||
" return(music_files)\n",
|
||||
"\n",
|
||||
"# def music_df(spark, music_files):\n",
|
||||
"# for music_file in music_files:\n",
|
||||
"# if music_file.endswith(\".json\")\n",
|
||||
"\n",
|
||||
"def video_ingest_path(file_path):\n",
|
||||
" video_path = os.getcwd() + \"/ingest/trakt_*\"\n",
|
||||
" video_files = []\n",
|
||||
" for file in glob.iglob(video_path, recursive=True):\n",
|
||||
" return(file)\n",
|
||||
"\n",
|
||||
"spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n",
|
||||
"spark.sparkContext.setLogLevel(\"INFO\")\n",
|
||||
"\n",
|
||||
"music_json = music_source_paths()\n",
|
||||
"print(music_json)\n",
|
||||
"music_df = spark.read.json(*music_json)\n",
|
||||
"\n",
|
||||
"music_df.show(5)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.13.7"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user