diff --git a/.gitignore b/.gitignore index cac109b..4aa5034 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,4 @@ ipython_config.py # Remove previous ipynb_checkpoints # git rm -r .ipynb_checkpoints/ +spark-warehouse \ No newline at end of file diff --git a/cc_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf b/ingest/cc_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf similarity index 100% rename from cc_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf rename to ingest/cc_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf diff --git a/cc_sesh/Streaming_History_Audio_2021-2022_0.json b/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json similarity index 100% rename from cc_sesh/Streaming_History_Audio_2021-2022_0.json rename to ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json diff --git a/cc_sesh/Streaming_History_Audio_2022-2023_1.json b/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json similarity index 100% rename from cc_sesh/Streaming_History_Audio_2022-2023_1.json rename to ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json diff --git a/cc_sesh/Streaming_History_Audio_2023-2024_2.json b/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json similarity index 100% rename from cc_sesh/Streaming_History_Audio_2023-2024_2.json rename to ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json diff --git a/cc_sesh/Streaming_History_Video_2023-2024.json b/ingest/cc_sesh/Streaming_History_Video_2023-2024.json similarity index 100% rename from cc_sesh/Streaming_History_Video_2023-2024.json rename to ingest/cc_sesh/Streaming_History_Video_2023-2024.json diff --git a/cs_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf b/ingest/cs_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf similarity index 100% rename from cs_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf rename to ingest/cs_sesh/ReadMeFirst_ExtendedStreamingHistory.pdf diff --git a/cs_sesh/Streaming_History_Audio_2015-2022_0.json b/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json similarity index 100% rename from cs_sesh/Streaming_History_Audio_2015-2022_0.json rename to ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json diff --git a/cs_sesh/Streaming_History_Audio_2022-2023_1.json b/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json similarity index 100% rename from cs_sesh/Streaming_History_Audio_2022-2023_1.json rename to ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json diff --git a/cs_sesh/Streaming_History_Audio_2023-2024_2.json b/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json similarity index 100% rename from cs_sesh/Streaming_History_Audio_2023-2024_2.json rename to ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json diff --git a/cs_sesh/Streaming_History_Audio_2024_3.json b/ingest/cs_sesh/Streaming_History_Audio_2024_3.json similarity index 100% rename from cs_sesh/Streaming_History_Audio_2024_3.json rename to ingest/cs_sesh/Streaming_History_Audio_2024_3.json diff --git a/cs_sesh/Streaming_History_Video_2018-2024.json b/ingest/cs_sesh/Streaming_History_Video_2018-2024.json similarity index 100% rename from cs_sesh/Streaming_History_Video_2018-2024.json rename to ingest/cs_sesh/Streaming_History_Video_2018-2024.json diff --git a/trakt-export-acclaimedcat245/collection-episodes-1.json b/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json similarity index 100% rename from trakt-export-acclaimedcat245/collection-episodes-1.json rename to ingest/trakt-export-acclaimedcat245/collection-episodes-1.json diff --git a/trakt-export-acclaimedcat245/collection-episodes-2.json b/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json similarity index 100% rename from trakt-export-acclaimedcat245/collection-episodes-2.json rename to ingest/trakt-export-acclaimedcat245/collection-episodes-2.json diff --git a/trakt-export-acclaimedcat245/collection-episodes-3.json b/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json similarity index 100% rename from trakt-export-acclaimedcat245/collection-episodes-3.json rename to ingest/trakt-export-acclaimedcat245/collection-episodes-3.json diff --git a/trakt-export-acclaimedcat245/collection-episodes-4.json b/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json similarity index 100% rename from trakt-export-acclaimedcat245/collection-episodes-4.json rename to ingest/trakt-export-acclaimedcat245/collection-episodes-4.json diff --git a/trakt-export-acclaimedcat245/collection-movies.json b/ingest/trakt-export-acclaimedcat245/collection-movies.json similarity index 100% rename from trakt-export-acclaimedcat245/collection-movies.json rename to ingest/trakt-export-acclaimedcat245/collection-movies.json diff --git a/trakt-export-acclaimedcat245/collection-shows.json b/ingest/trakt-export-acclaimedcat245/collection-shows.json similarity index 100% rename from trakt-export-acclaimedcat245/collection-shows.json rename to ingest/trakt-export-acclaimedcat245/collection-shows.json diff --git a/trakt-export-acclaimedcat245/comments-episodes.json b/ingest/trakt-export-acclaimedcat245/comments-episodes.json similarity index 100% rename from trakt-export-acclaimedcat245/comments-episodes.json rename to ingest/trakt-export-acclaimedcat245/comments-episodes.json diff --git a/trakt-export-acclaimedcat245/comments-lists.json b/ingest/trakt-export-acclaimedcat245/comments-lists.json similarity index 100% rename from trakt-export-acclaimedcat245/comments-lists.json rename to ingest/trakt-export-acclaimedcat245/comments-lists.json diff --git a/trakt-export-acclaimedcat245/comments-movies.json b/ingest/trakt-export-acclaimedcat245/comments-movies.json similarity index 100% rename from trakt-export-acclaimedcat245/comments-movies.json rename to ingest/trakt-export-acclaimedcat245/comments-movies.json diff --git a/trakt-export-acclaimedcat245/comments-seasons.json b/ingest/trakt-export-acclaimedcat245/comments-seasons.json similarity index 100% rename from trakt-export-acclaimedcat245/comments-seasons.json rename to ingest/trakt-export-acclaimedcat245/comments-seasons.json diff --git a/trakt-export-acclaimedcat245/comments-shows.json b/ingest/trakt-export-acclaimedcat245/comments-shows.json similarity index 100% rename from trakt-export-acclaimedcat245/comments-shows.json rename to ingest/trakt-export-acclaimedcat245/comments-shows.json diff --git a/trakt-export-acclaimedcat245/hidden-calendar.json b/ingest/trakt-export-acclaimedcat245/hidden-calendar.json similarity index 100% rename from trakt-export-acclaimedcat245/hidden-calendar.json rename to ingest/trakt-export-acclaimedcat245/hidden-calendar.json diff --git a/trakt-export-acclaimedcat245/hidden-progress-collected.json b/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json similarity index 100% rename from trakt-export-acclaimedcat245/hidden-progress-collected.json rename to ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json diff --git a/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json b/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json similarity index 100% rename from trakt-export-acclaimedcat245/hidden-progress-watched-reset.json rename to ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json diff --git a/trakt-export-acclaimedcat245/hidden-progress-watched.json b/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json similarity index 100% rename from trakt-export-acclaimedcat245/hidden-progress-watched.json rename to ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json diff --git a/trakt-export-acclaimedcat245/hidden-recommendations.json b/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json similarity index 100% rename from trakt-export-acclaimedcat245/hidden-recommendations.json rename to ingest/trakt-export-acclaimedcat245/hidden-recommendations.json diff --git a/trakt-export-acclaimedcat245/likes-comments.json b/ingest/trakt-export-acclaimedcat245/likes-comments.json similarity index 100% rename from trakt-export-acclaimedcat245/likes-comments.json rename to ingest/trakt-export-acclaimedcat245/likes-comments.json diff --git a/trakt-export-acclaimedcat245/likes-lists.json b/ingest/trakt-export-acclaimedcat245/likes-lists.json similarity index 100% rename from trakt-export-acclaimedcat245/likes-lists.json rename to ingest/trakt-export-acclaimedcat245/likes-lists.json diff --git a/trakt-export-acclaimedcat245/lists-collaborations.json b/ingest/trakt-export-acclaimedcat245/lists-collaborations.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-collaborations.json rename to ingest/trakt-export-acclaimedcat245/lists-collaborations.json diff --git a/trakt-export-acclaimedcat245/lists-favorites.json b/ingest/trakt-export-acclaimedcat245/lists-favorites.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-favorites.json rename to ingest/trakt-export-acclaimedcat245/lists-favorites.json diff --git a/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json b/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json rename to ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json diff --git a/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json b/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json rename to ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json diff --git a/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json b/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json rename to ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json diff --git a/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json b/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json rename to ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json diff --git a/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json b/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json rename to ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json diff --git a/trakt-export-acclaimedcat245/lists-lists.json b/ingest/trakt-export-acclaimedcat245/lists-lists.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-lists.json rename to ingest/trakt-export-acclaimedcat245/lists-lists.json diff --git a/trakt-export-acclaimedcat245/lists-watchlist.json b/ingest/trakt-export-acclaimedcat245/lists-watchlist.json similarity index 100% rename from trakt-export-acclaimedcat245/lists-watchlist.json rename to ingest/trakt-export-acclaimedcat245/lists-watchlist.json diff --git a/trakt-export-acclaimedcat245/network-followers-requests.json b/ingest/trakt-export-acclaimedcat245/network-followers-requests.json similarity index 100% rename from trakt-export-acclaimedcat245/network-followers-requests.json rename to ingest/trakt-export-acclaimedcat245/network-followers-requests.json diff --git a/trakt-export-acclaimedcat245/network-followers.json b/ingest/trakt-export-acclaimedcat245/network-followers.json similarity index 100% rename from trakt-export-acclaimedcat245/network-followers.json rename to ingest/trakt-export-acclaimedcat245/network-followers.json diff --git a/trakt-export-acclaimedcat245/network-following.json b/ingest/trakt-export-acclaimedcat245/network-following.json similarity index 100% rename from trakt-export-acclaimedcat245/network-following.json rename to ingest/trakt-export-acclaimedcat245/network-following.json diff --git a/trakt-export-acclaimedcat245/network-friends.json b/ingest/trakt-export-acclaimedcat245/network-friends.json similarity index 100% rename from trakt-export-acclaimedcat245/network-friends.json rename to ingest/trakt-export-acclaimedcat245/network-friends.json diff --git a/trakt-export-acclaimedcat245/notes-activities.json b/ingest/trakt-export-acclaimedcat245/notes-activities.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-activities.json rename to ingest/trakt-export-acclaimedcat245/notes-activities.json diff --git a/trakt-export-acclaimedcat245/notes-collection_items.json b/ingest/trakt-export-acclaimedcat245/notes-collection_items.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-collection_items.json rename to ingest/trakt-export-acclaimedcat245/notes-collection_items.json diff --git a/trakt-export-acclaimedcat245/notes-episodes.json b/ingest/trakt-export-acclaimedcat245/notes-episodes.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-episodes.json rename to ingest/trakt-export-acclaimedcat245/notes-episodes.json diff --git a/trakt-export-acclaimedcat245/notes-movies.json b/ingest/trakt-export-acclaimedcat245/notes-movies.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-movies.json rename to ingest/trakt-export-acclaimedcat245/notes-movies.json diff --git a/trakt-export-acclaimedcat245/notes-people.json b/ingest/trakt-export-acclaimedcat245/notes-people.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-people.json rename to ingest/trakt-export-acclaimedcat245/notes-people.json diff --git a/trakt-export-acclaimedcat245/notes-ratings.json b/ingest/trakt-export-acclaimedcat245/notes-ratings.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-ratings.json rename to ingest/trakt-export-acclaimedcat245/notes-ratings.json diff --git a/trakt-export-acclaimedcat245/notes-seasons.json b/ingest/trakt-export-acclaimedcat245/notes-seasons.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-seasons.json rename to ingest/trakt-export-acclaimedcat245/notes-seasons.json diff --git a/trakt-export-acclaimedcat245/notes-shows.json b/ingest/trakt-export-acclaimedcat245/notes-shows.json similarity index 100% rename from trakt-export-acclaimedcat245/notes-shows.json rename to ingest/trakt-export-acclaimedcat245/notes-shows.json diff --git a/trakt-export-acclaimedcat245/ratings-episodes.json b/ingest/trakt-export-acclaimedcat245/ratings-episodes.json similarity index 100% rename from trakt-export-acclaimedcat245/ratings-episodes.json rename to ingest/trakt-export-acclaimedcat245/ratings-episodes.json diff --git a/trakt-export-acclaimedcat245/ratings-movies.json b/ingest/trakt-export-acclaimedcat245/ratings-movies.json similarity index 100% rename from trakt-export-acclaimedcat245/ratings-movies.json rename to ingest/trakt-export-acclaimedcat245/ratings-movies.json diff --git a/trakt-export-acclaimedcat245/ratings-seasons.json b/ingest/trakt-export-acclaimedcat245/ratings-seasons.json similarity index 100% rename from trakt-export-acclaimedcat245/ratings-seasons.json rename to ingest/trakt-export-acclaimedcat245/ratings-seasons.json diff --git a/trakt-export-acclaimedcat245/ratings-shows.json b/ingest/trakt-export-acclaimedcat245/ratings-shows.json similarity index 100% rename from trakt-export-acclaimedcat245/ratings-shows.json rename to ingest/trakt-export-acclaimedcat245/ratings-shows.json diff --git a/trakt-export-acclaimedcat245/user-last-activities.json b/ingest/trakt-export-acclaimedcat245/user-last-activities.json similarity index 100% rename from trakt-export-acclaimedcat245/user-last-activities.json rename to ingest/trakt-export-acclaimedcat245/user-last-activities.json diff --git a/trakt-export-acclaimedcat245/user-profile.json b/ingest/trakt-export-acclaimedcat245/user-profile.json similarity index 100% rename from trakt-export-acclaimedcat245/user-profile.json rename to ingest/trakt-export-acclaimedcat245/user-profile.json diff --git a/trakt-export-acclaimedcat245/user-settings.json b/ingest/trakt-export-acclaimedcat245/user-settings.json similarity index 100% rename from trakt-export-acclaimedcat245/user-settings.json rename to ingest/trakt-export-acclaimedcat245/user-settings.json diff --git a/trakt-export-acclaimedcat245/user-stats.json b/ingest/trakt-export-acclaimedcat245/user-stats.json similarity index 100% rename from trakt-export-acclaimedcat245/user-stats.json rename to ingest/trakt-export-acclaimedcat245/user-stats.json diff --git a/trakt-export-acclaimedcat245/watched-history-1.json b/ingest/trakt-export-acclaimedcat245/watched-history-1.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-1.json rename to ingest/trakt-export-acclaimedcat245/watched-history-1.json diff --git a/trakt-export-acclaimedcat245/watched-history-10.json b/ingest/trakt-export-acclaimedcat245/watched-history-10.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-10.json rename to ingest/trakt-export-acclaimedcat245/watched-history-10.json diff --git a/trakt-export-acclaimedcat245/watched-history-11.json b/ingest/trakt-export-acclaimedcat245/watched-history-11.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-11.json rename to ingest/trakt-export-acclaimedcat245/watched-history-11.json diff --git a/trakt-export-acclaimedcat245/watched-history-12.json b/ingest/trakt-export-acclaimedcat245/watched-history-12.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-12.json rename to ingest/trakt-export-acclaimedcat245/watched-history-12.json diff --git a/trakt-export-acclaimedcat245/watched-history-13.json b/ingest/trakt-export-acclaimedcat245/watched-history-13.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-13.json rename to ingest/trakt-export-acclaimedcat245/watched-history-13.json diff --git a/trakt-export-acclaimedcat245/watched-history-14.json b/ingest/trakt-export-acclaimedcat245/watched-history-14.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-14.json rename to ingest/trakt-export-acclaimedcat245/watched-history-14.json diff --git a/trakt-export-acclaimedcat245/watched-history-15.json b/ingest/trakt-export-acclaimedcat245/watched-history-15.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-15.json rename to ingest/trakt-export-acclaimedcat245/watched-history-15.json diff --git a/trakt-export-acclaimedcat245/watched-history-16.json b/ingest/trakt-export-acclaimedcat245/watched-history-16.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-16.json rename to ingest/trakt-export-acclaimedcat245/watched-history-16.json diff --git a/trakt-export-acclaimedcat245/watched-history-17.json b/ingest/trakt-export-acclaimedcat245/watched-history-17.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-17.json rename to ingest/trakt-export-acclaimedcat245/watched-history-17.json diff --git a/trakt-export-acclaimedcat245/watched-history-18.json b/ingest/trakt-export-acclaimedcat245/watched-history-18.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-18.json rename to ingest/trakt-export-acclaimedcat245/watched-history-18.json diff --git a/trakt-export-acclaimedcat245/watched-history-19.json b/ingest/trakt-export-acclaimedcat245/watched-history-19.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-19.json rename to ingest/trakt-export-acclaimedcat245/watched-history-19.json diff --git a/trakt-export-acclaimedcat245/watched-history-2.json b/ingest/trakt-export-acclaimedcat245/watched-history-2.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-2.json rename to ingest/trakt-export-acclaimedcat245/watched-history-2.json diff --git a/trakt-export-acclaimedcat245/watched-history-20.json b/ingest/trakt-export-acclaimedcat245/watched-history-20.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-20.json rename to ingest/trakt-export-acclaimedcat245/watched-history-20.json diff --git a/trakt-export-acclaimedcat245/watched-history-21.json b/ingest/trakt-export-acclaimedcat245/watched-history-21.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-21.json rename to ingest/trakt-export-acclaimedcat245/watched-history-21.json diff --git a/trakt-export-acclaimedcat245/watched-history-22.json b/ingest/trakt-export-acclaimedcat245/watched-history-22.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-22.json rename to ingest/trakt-export-acclaimedcat245/watched-history-22.json diff --git a/trakt-export-acclaimedcat245/watched-history-23.json b/ingest/trakt-export-acclaimedcat245/watched-history-23.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-23.json rename to ingest/trakt-export-acclaimedcat245/watched-history-23.json diff --git a/trakt-export-acclaimedcat245/watched-history-24.json b/ingest/trakt-export-acclaimedcat245/watched-history-24.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-24.json rename to ingest/trakt-export-acclaimedcat245/watched-history-24.json diff --git a/trakt-export-acclaimedcat245/watched-history-25.json b/ingest/trakt-export-acclaimedcat245/watched-history-25.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-25.json rename to ingest/trakt-export-acclaimedcat245/watched-history-25.json diff --git a/trakt-export-acclaimedcat245/watched-history-26.json b/ingest/trakt-export-acclaimedcat245/watched-history-26.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-26.json rename to ingest/trakt-export-acclaimedcat245/watched-history-26.json diff --git a/trakt-export-acclaimedcat245/watched-history-27.json b/ingest/trakt-export-acclaimedcat245/watched-history-27.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-27.json rename to ingest/trakt-export-acclaimedcat245/watched-history-27.json diff --git a/trakt-export-acclaimedcat245/watched-history-28.json b/ingest/trakt-export-acclaimedcat245/watched-history-28.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-28.json rename to ingest/trakt-export-acclaimedcat245/watched-history-28.json diff --git a/trakt-export-acclaimedcat245/watched-history-29.json b/ingest/trakt-export-acclaimedcat245/watched-history-29.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-29.json rename to ingest/trakt-export-acclaimedcat245/watched-history-29.json diff --git a/trakt-export-acclaimedcat245/watched-history-3.json b/ingest/trakt-export-acclaimedcat245/watched-history-3.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-3.json rename to ingest/trakt-export-acclaimedcat245/watched-history-3.json diff --git a/trakt-export-acclaimedcat245/watched-history-30.json b/ingest/trakt-export-acclaimedcat245/watched-history-30.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-30.json rename to ingest/trakt-export-acclaimedcat245/watched-history-30.json diff --git a/trakt-export-acclaimedcat245/watched-history-31.json b/ingest/trakt-export-acclaimedcat245/watched-history-31.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-31.json rename to ingest/trakt-export-acclaimedcat245/watched-history-31.json diff --git a/trakt-export-acclaimedcat245/watched-history-32.json b/ingest/trakt-export-acclaimedcat245/watched-history-32.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-32.json rename to ingest/trakt-export-acclaimedcat245/watched-history-32.json diff --git a/trakt-export-acclaimedcat245/watched-history-33.json b/ingest/trakt-export-acclaimedcat245/watched-history-33.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-33.json rename to ingest/trakt-export-acclaimedcat245/watched-history-33.json diff --git a/trakt-export-acclaimedcat245/watched-history-34.json b/ingest/trakt-export-acclaimedcat245/watched-history-34.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-34.json rename to ingest/trakt-export-acclaimedcat245/watched-history-34.json diff --git a/trakt-export-acclaimedcat245/watched-history-35.json b/ingest/trakt-export-acclaimedcat245/watched-history-35.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-35.json rename to ingest/trakt-export-acclaimedcat245/watched-history-35.json diff --git a/trakt-export-acclaimedcat245/watched-history-36.json b/ingest/trakt-export-acclaimedcat245/watched-history-36.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-36.json rename to ingest/trakt-export-acclaimedcat245/watched-history-36.json diff --git a/trakt-export-acclaimedcat245/watched-history-37.json b/ingest/trakt-export-acclaimedcat245/watched-history-37.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-37.json rename to ingest/trakt-export-acclaimedcat245/watched-history-37.json diff --git a/trakt-export-acclaimedcat245/watched-history-38.json b/ingest/trakt-export-acclaimedcat245/watched-history-38.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-38.json rename to ingest/trakt-export-acclaimedcat245/watched-history-38.json diff --git a/trakt-export-acclaimedcat245/watched-history-39.json b/ingest/trakt-export-acclaimedcat245/watched-history-39.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-39.json rename to ingest/trakt-export-acclaimedcat245/watched-history-39.json diff --git a/trakt-export-acclaimedcat245/watched-history-4.json b/ingest/trakt-export-acclaimedcat245/watched-history-4.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-4.json rename to ingest/trakt-export-acclaimedcat245/watched-history-4.json diff --git a/trakt-export-acclaimedcat245/watched-history-40.json b/ingest/trakt-export-acclaimedcat245/watched-history-40.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-40.json rename to ingest/trakt-export-acclaimedcat245/watched-history-40.json diff --git a/trakt-export-acclaimedcat245/watched-history-41.json b/ingest/trakt-export-acclaimedcat245/watched-history-41.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-41.json rename to ingest/trakt-export-acclaimedcat245/watched-history-41.json diff --git a/trakt-export-acclaimedcat245/watched-history-42.json b/ingest/trakt-export-acclaimedcat245/watched-history-42.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-42.json rename to ingest/trakt-export-acclaimedcat245/watched-history-42.json diff --git a/trakt-export-acclaimedcat245/watched-history-43.json b/ingest/trakt-export-acclaimedcat245/watched-history-43.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-43.json rename to ingest/trakt-export-acclaimedcat245/watched-history-43.json diff --git a/trakt-export-acclaimedcat245/watched-history-44.json b/ingest/trakt-export-acclaimedcat245/watched-history-44.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-44.json rename to ingest/trakt-export-acclaimedcat245/watched-history-44.json diff --git a/trakt-export-acclaimedcat245/watched-history-45.json b/ingest/trakt-export-acclaimedcat245/watched-history-45.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-45.json rename to ingest/trakt-export-acclaimedcat245/watched-history-45.json diff --git a/trakt-export-acclaimedcat245/watched-history-46.json b/ingest/trakt-export-acclaimedcat245/watched-history-46.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-46.json rename to ingest/trakt-export-acclaimedcat245/watched-history-46.json diff --git a/trakt-export-acclaimedcat245/watched-history-47.json b/ingest/trakt-export-acclaimedcat245/watched-history-47.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-47.json rename to ingest/trakt-export-acclaimedcat245/watched-history-47.json diff --git a/trakt-export-acclaimedcat245/watched-history-48.json b/ingest/trakt-export-acclaimedcat245/watched-history-48.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-48.json rename to ingest/trakt-export-acclaimedcat245/watched-history-48.json diff --git a/trakt-export-acclaimedcat245/watched-history-49.json b/ingest/trakt-export-acclaimedcat245/watched-history-49.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-49.json rename to ingest/trakt-export-acclaimedcat245/watched-history-49.json diff --git a/trakt-export-acclaimedcat245/watched-history-5.json b/ingest/trakt-export-acclaimedcat245/watched-history-5.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-5.json rename to ingest/trakt-export-acclaimedcat245/watched-history-5.json diff --git a/trakt-export-acclaimedcat245/watched-history-50.json b/ingest/trakt-export-acclaimedcat245/watched-history-50.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-50.json rename to ingest/trakt-export-acclaimedcat245/watched-history-50.json diff --git a/trakt-export-acclaimedcat245/watched-history-51.json b/ingest/trakt-export-acclaimedcat245/watched-history-51.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-51.json rename to ingest/trakt-export-acclaimedcat245/watched-history-51.json diff --git a/trakt-export-acclaimedcat245/watched-history-52.json b/ingest/trakt-export-acclaimedcat245/watched-history-52.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-52.json rename to ingest/trakt-export-acclaimedcat245/watched-history-52.json diff --git a/trakt-export-acclaimedcat245/watched-history-53.json b/ingest/trakt-export-acclaimedcat245/watched-history-53.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-53.json rename to ingest/trakt-export-acclaimedcat245/watched-history-53.json diff --git a/trakt-export-acclaimedcat245/watched-history-54.json b/ingest/trakt-export-acclaimedcat245/watched-history-54.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-54.json rename to ingest/trakt-export-acclaimedcat245/watched-history-54.json diff --git a/trakt-export-acclaimedcat245/watched-history-55.json b/ingest/trakt-export-acclaimedcat245/watched-history-55.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-55.json rename to ingest/trakt-export-acclaimedcat245/watched-history-55.json diff --git a/trakt-export-acclaimedcat245/watched-history-56.json b/ingest/trakt-export-acclaimedcat245/watched-history-56.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-56.json rename to ingest/trakt-export-acclaimedcat245/watched-history-56.json diff --git a/trakt-export-acclaimedcat245/watched-history-57.json b/ingest/trakt-export-acclaimedcat245/watched-history-57.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-57.json rename to ingest/trakt-export-acclaimedcat245/watched-history-57.json diff --git a/trakt-export-acclaimedcat245/watched-history-58.json b/ingest/trakt-export-acclaimedcat245/watched-history-58.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-58.json rename to ingest/trakt-export-acclaimedcat245/watched-history-58.json diff --git a/trakt-export-acclaimedcat245/watched-history-59.json b/ingest/trakt-export-acclaimedcat245/watched-history-59.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-59.json rename to ingest/trakt-export-acclaimedcat245/watched-history-59.json diff --git a/trakt-export-acclaimedcat245/watched-history-6.json b/ingest/trakt-export-acclaimedcat245/watched-history-6.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-6.json rename to ingest/trakt-export-acclaimedcat245/watched-history-6.json diff --git a/trakt-export-acclaimedcat245/watched-history-60.json b/ingest/trakt-export-acclaimedcat245/watched-history-60.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-60.json rename to ingest/trakt-export-acclaimedcat245/watched-history-60.json diff --git a/trakt-export-acclaimedcat245/watched-history-7.json b/ingest/trakt-export-acclaimedcat245/watched-history-7.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-7.json rename to ingest/trakt-export-acclaimedcat245/watched-history-7.json diff --git a/trakt-export-acclaimedcat245/watched-history-8.json b/ingest/trakt-export-acclaimedcat245/watched-history-8.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-8.json rename to ingest/trakt-export-acclaimedcat245/watched-history-8.json diff --git a/trakt-export-acclaimedcat245/watched-history-9.json b/ingest/trakt-export-acclaimedcat245/watched-history-9.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-history-9.json rename to ingest/trakt-export-acclaimedcat245/watched-history-9.json diff --git a/trakt-export-acclaimedcat245/watched-movies-1.json b/ingest/trakt-export-acclaimedcat245/watched-movies-1.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-movies-1.json rename to ingest/trakt-export-acclaimedcat245/watched-movies-1.json diff --git a/trakt-export-acclaimedcat245/watched-movies-2.json b/ingest/trakt-export-acclaimedcat245/watched-movies-2.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-movies-2.json rename to ingest/trakt-export-acclaimedcat245/watched-movies-2.json diff --git a/trakt-export-acclaimedcat245/watched-playback.json b/ingest/trakt-export-acclaimedcat245/watched-playback.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-playback.json rename to ingest/trakt-export-acclaimedcat245/watched-playback.json diff --git a/trakt-export-acclaimedcat245/watched-shows-1.json b/ingest/trakt-export-acclaimedcat245/watched-shows-1.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-shows-1.json rename to ingest/trakt-export-acclaimedcat245/watched-shows-1.json diff --git a/trakt-export-acclaimedcat245/watched-shows-2.json b/ingest/trakt-export-acclaimedcat245/watched-shows-2.json similarity index 100% rename from trakt-export-acclaimedcat245/watched-shows-2.json rename to ingest/trakt-export-acclaimedcat245/watched-shows-2.json diff --git a/spotify.ipynb b/spotify.ipynb new file mode 100644 index 0000000..4f5c3e3 --- /dev/null +++ b/spotify.ipynb @@ -0,0 +1,560 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e9bf1d26", + "metadata": {}, + "outputs": [], + "source": [ + "# Import functions\n", + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import col, current_timestamp\n", + "\n", + "log_file = \n", + "spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n", + "spark.sparkContext.setLogLevel(\"error\")\n", + "\n", + "# Define variables used in code below\n", + "\n", + "def file_path():\n", + " file_path = pwd\n", + "\n", + "file_path = \"/databricks-datasets/structured-streaming/events\"\n", + "username = spark.sql(\"SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')\").first()[0]\n", + "table_name = f\"{username}_etl_quickstart\"\n", + "checkpoint_path = f\"/tmp/{username}/_checkpoint/etl_quickstart\"\n", + "\n", + "# Clear out data from previous demo execution\n", + "spark.sql(f\"DROP TABLE IF EXISTS {table_name}\")\n", + "dbutils.fs.rm(checkpoint_path, True)\n", + "\n", + "# Configure Auto Loader to ingest JSON data to a Delta table\n", + "(spark.readStream\n", + " .format(\"cloudFiles\")\n", + " .option(\"cloudFiles.format\", \"json\")\n", + " .option(\"cloudFiles.schemaLocation\", checkpoint_path)\n", + " .load(file_path)\n", + " .select(\"*\", col(\"_metadata.file_path\").alias(\"source_file\"), current_timestamp().alias(\"processing_time\"))\n", + " .writeStream\n", + " .option(\"checkpointLocation\", checkpoint_path)\n", + " .trigger(availableNow=True)\n", + " .toTable(table_name))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "11bb5dc2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/Users/carlacastillo/Documents/Gitea/media_ml'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a898326e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current working directory: /Users/carlacastillo/Documents/Gitea/media_ml\n" + ] + } + ], + "source": [ + "from pyspark.sql import SparkSession\n", + "\n", + "import os \n", + "cwd = os.getcwd() \n", + "print(\"Current working directory:\", cwd)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "0a0bbedc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Music\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Video_2023-2024.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2024_3.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Video_2018-2024.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json\n", + "Video\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-playback.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-episodes.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-30.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-1.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-26.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-51.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-47.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-10.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers-requests.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-lists.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-11.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-1.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-shows.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-46.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-50.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-shows.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-27.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-episodes.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-31.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-watchlist.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-movies.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-57.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-41.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-6.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-16.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-ratings.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-shows.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-36.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-20.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-profile.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-21.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-movies.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-37.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-60.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-collaborations.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-1.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-settings.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-stats.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-episodes.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-17.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-7.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-40.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-56.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-2.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-38.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-14.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-4.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-43.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-55.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-22.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-34.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-seasons.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-18.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-calendar.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-8.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-lists.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-59.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-58.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-9.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-favorites.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-19.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-35.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-23.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-54.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-42.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-5.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-15.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-following.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-39.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-shows.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-comments.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-collection_items.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-seasons.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-people.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-24.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-32.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-friends.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-49.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-seasons.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-28.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-movies.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-12.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-2.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-lists.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-45.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-53.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-52.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-movies.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-44.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-3.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-13.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-activities.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-last-activities.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-29.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-48.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-2.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-33.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-25.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json\n", + "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers.json\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(113)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 113\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 113\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(129)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 129\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 129\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(89)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 89\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 89\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(111)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 111\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 111\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(134)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 134\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 134\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(121)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 121\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 121\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(118)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 118\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 118\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(146)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 146\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 146\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(132)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 132\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 132\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(98)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 98\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 98\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(115)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 115\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 115\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(96)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 96\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 96\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(97)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 97\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 97\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(86)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 86\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 86\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(142)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 142\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 142\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(90)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 90\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 90\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(145)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 145\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 145\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(105)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 105\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 105\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(104)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 104\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 104\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(100)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 100\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 100\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(116)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 116\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 116\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 4\n", + "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 4\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 4\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4_piece0\n", + "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4_piece0 of size 40107 dropped from memory (free 454906095)\n", + "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", + "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.10.130:61896 in memory (size: 39.2 KiB, free: 434.4 MiB)\n", + "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0\n", + "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_4_piece0\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4\n", + "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4 of size 264472 dropped from memory (free 455170567)\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 2\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 4\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(95)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 95\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 95\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(106)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 106\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 106\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(103)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 103\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 103\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(141)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 141\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 141\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(137)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 137\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 137\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(136)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 136\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 136\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(101)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 101\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 101\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(99)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 99\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 99\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(140)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 140\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 140\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(102)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 102\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 102\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(93)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 93\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 93\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(83)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 83\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 83\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(119)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 119\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 119\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(128)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 128\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 128\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(138)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 138\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 138\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(87)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 87\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 87\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(130)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 130\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 130\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(108)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 108\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 108\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(110)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 110\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 110\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(109)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 109\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 109\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(117)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 117\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 117\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(147)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 147\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 147\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(125)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 125\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 125\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(114)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 114\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 114\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(85)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 85\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 85\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 5\n", + "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 5\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 5\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5_piece0\n", + "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5_piece0 of size 8326 dropped from memory (free 455178893)\n", + "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", + "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.10.130:61896 in memory (size: 8.1 KiB, free: 434.4 MiB)\n", + "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0\n", + "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_5_piece0\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5\n", + "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5 of size 18072 dropped from memory (free 455196965)\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 2\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 5\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(144)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 144\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 144\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(139)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 139\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 139\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(107)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 107\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 107\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(124)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 124\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 124\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 6\n", + "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 6\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 6\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6\n", + "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6 of size 264384 dropped from memory (free 455461349)\n", + "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6_piece0\n", + "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6_piece0 of size 40065 dropped from memory (free 455501414)\n", + "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", + "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.10.130:61896 in memory (size: 39.1 KiB, free: 434.4 MiB)\n", + "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0\n", + "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_6_piece0\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 2\n", + "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 6\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(126)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 126\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 126\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(122)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 122\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 122\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(91)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 91\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 91\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(112)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 112\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 112\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(133)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 133\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 133\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(120)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 120\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 120\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(127)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 127\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 127\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(92)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 92\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 92\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(143)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 143\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 143\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(148)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 148\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 148\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(131)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 131\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 131\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(135)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 135\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 135\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(94)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 94\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 94\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(123)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 123\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 123\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(88)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 88\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 88\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(84)\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 84\n", + "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 84\n" + ] + } + ], + "source": [ + "music_path = os.getcwd() + \"/ingest/**_sesh/*.json\"\n", + "video_path = os.getcwd() + \"/ingest/trakt**/*.json\"\n", + "\n", + "print(\"Music\")\n", + "for file in glob.iglob(music_path, recursive=False):\n", + "\n", + " print(file)\n", + "\n", + "print(\"Video\")\n", + "for file in glob.iglob(video_path, recursive=True):\n", + " print(file)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0e9641e", + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "expected ':' (3030084487.py, line 14)", + "output_type": "error", + "traceback": [ + " \u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[29]\u001b[39m\u001b[32m, line 14\u001b[39m\n\u001b[31m \u001b[39m\u001b[31mif music_file.endswith(\".json\")\u001b[39m\n ^\n\u001b[31mSyntaxError\u001b[39m\u001b[31m:\u001b[39m expected ':'\n" + ] + } + ], + "source": [ + "from pyspark.sql import SparkSession\n", + "import os \n", + "import glob \n", + "\n", + "def music_source_paths():\n", + " music_path = os.getcwd() + \"/ingest/*_sesh/*.json\"\n", + " music_files = []\n", + " for file in glob.iglob(music_path, recursive=True):\n", + " music_files.append(file)\n", + " return(music_files)\n", + "\n", + "# def music_df(spark, music_files):\n", + "# for music_file in music_files:\n", + "# if music_file.endswith(\".json\")\n", + "\n", + "def video_ingest_path(file_path):\n", + " video_path = os.getcwd() + \"/ingest/trakt_*\"\n", + " video_files = []\n", + " for file in glob.iglob(video_path, recursive=True):\n", + " return(file)\n", + "\n", + "spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n", + "spark.sparkContext.setLogLevel(\"INFO\")\n", + "\n", + "music_json = music_source_paths()\n", + "print(music_json)\n", + "music_df = spark.read.json(*music_json)\n", + "\n", + "music_df.show(5)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}