{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "e9bf1d26", "metadata": {}, "outputs": [], "source": [ "# Import functions\n", "from pyspark.sql import SparkSession\n", "from pyspark.sql.functions import col, current_timestamp\n", "\n", "log_file = \n", "spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n", "spark.sparkContext.setLogLevel(\"error\")\n", "\n", "# Define variables used in code below\n", "\n", "def file_path():\n", " file_path = pwd\n", "\n", "file_path = \"/databricks-datasets/structured-streaming/events\"\n", "username = spark.sql(\"SELECT regexp_replace(session_user(), '[^a-zA-Z0-9]', '_')\").first()[0]\n", "table_name = f\"{username}_etl_quickstart\"\n", "checkpoint_path = f\"/tmp/{username}/_checkpoint/etl_quickstart\"\n", "\n", "# Clear out data from previous demo execution\n", "spark.sql(f\"DROP TABLE IF EXISTS {table_name}\")\n", "dbutils.fs.rm(checkpoint_path, True)\n", "\n", "# Configure Auto Loader to ingest JSON data to a Delta table\n", "(spark.readStream\n", " .format(\"cloudFiles\")\n", " .option(\"cloudFiles.format\", \"json\")\n", " .option(\"cloudFiles.schemaLocation\", checkpoint_path)\n", " .load(file_path)\n", " .select(\"*\", col(\"_metadata.file_path\").alias(\"source_file\"), current_timestamp().alias(\"processing_time\"))\n", " .writeStream\n", " .option(\"checkpointLocation\", checkpoint_path)\n", " .trigger(availableNow=True)\n", " .toTable(table_name))" ] }, { "cell_type": "code", "execution_count": 3, "id": "11bb5dc2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/Users/carlacastillo/Documents/Gitea/media_ml'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%pwd" ] }, { "cell_type": "code", "execution_count": null, "id": "a898326e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Current working directory: /Users/carlacastillo/Documents/Gitea/media_ml\n" ] } ], "source": [ "from pyspark.sql import SparkSession\n", "\n", "import os \n", "cwd = os.getcwd() \n", "print(\"Current working directory:\", cwd)\n" ] }, { "cell_type": "code", "execution_count": 16, "id": "0a0bbedc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Music\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2022-2023_1.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Video_2023-2024.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2021-2022_0.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cc_sesh/Streaming_History_Audio_2023-2024_2.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2024_3.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Video_2018-2024.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2022-2023_1.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2023-2024_2.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/cs_sesh/Streaming_History_Audio_2015-2022_0.json\n", "Video\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-playback.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-episodes.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-30.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-1.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-26.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25877187-tv-recommendations-couchmoney-tv.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-51.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-47.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-10.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched-reset.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers-requests.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714752-movies-recommendations-couchmoney-tv.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-recommendations.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-lists.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-11.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-1.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-shows.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-46.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-50.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-shows.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-27.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-episodes.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-31.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-watchlist.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-movies.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-57.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-41.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-6.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-16.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-ratings.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-shows.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-36.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-20.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-1.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-profile.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-21.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-movies.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-37.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-60.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-collaborations.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26876568-tna-shows.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-1.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-settings.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-stats.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-episodes.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-17.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-7.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-40.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-56.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-26139626-tv-recommendations-eng-only-couchmoney-tv.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-movies-2.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-38.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-14.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-4.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-43.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-55.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-3.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-22.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-34.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-seasons.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-18.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-calendar.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-collected.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-8.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-lists.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-59.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-58.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-9.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-favorites.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-19.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-35.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-23.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-2.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-54.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-42.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-5.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-15.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-following.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-39.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/comments-shows.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-comments.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-collection_items.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-seasons.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-people.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-24.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-32.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-friends.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-49.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-seasons.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/hidden-progress-watched.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-28.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-movies.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-12.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-2.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/likes-lists.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-45.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-53.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-52.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/lists-list-25714753-movies-based-on-recently-watched-couchmoney-tv.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/ratings-movies.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-44.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-3.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-13.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/notes-activities.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/user-last-activities.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-29.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-48.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-shows-2.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-33.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/watched-history-25.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/collection-episodes-4.json\n", "/Users/carlacastillo/Documents/Gitea/media_ml/ingest/trakt-export-acclaimedcat245/network-followers.json\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(113)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 113\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 113\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(129)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 129\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 129\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(89)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 89\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 89\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(111)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 111\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 111\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(134)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 134\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 134\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(121)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 121\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 121\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(118)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 118\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 118\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(146)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 146\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 146\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(132)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 132\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 132\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(98)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 98\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 98\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(115)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 115\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 115\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(96)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 96\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 96\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(97)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 97\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 97\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(86)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 86\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 86\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(142)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 142\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 142\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(90)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 90\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 90\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(145)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 145\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 145\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(105)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 105\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 105\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(104)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 104\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 104\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(100)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 100\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 100\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(116)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 116\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 116\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(4)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 4\n", "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 4\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 4\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 4\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4_piece0\n", "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4_piece0 of size 40107 dropped from memory (free 454906095)\n", "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_4_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.10.130:61896 in memory (size: 39.2 KiB, free: 434.4 MiB)\n", "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_4_piece0\n", "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_4_piece0\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_4\n", "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_4 of size 264472 dropped from memory (free 455170567)\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 4, response is 2\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 4\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(95)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 95\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 95\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(106)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 106\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 106\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(103)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 103\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 103\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(141)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 141\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 141\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(137)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 137\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 137\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(136)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 136\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 136\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(101)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 101\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 101\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(99)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 99\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 99\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(140)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 140\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 140\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(102)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 102\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 102\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(93)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 93\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 93\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(83)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 83\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 83\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(119)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 119\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 119\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(128)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 128\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 128\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(138)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 138\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 138\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(87)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 87\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 87\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(130)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 130\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 130\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(108)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 108\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 108\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(110)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 110\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 110\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(109)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 109\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 109\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(117)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 117\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 117\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(147)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 147\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 147\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(125)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 125\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 125\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(114)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 114\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 114\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(85)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 85\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 85\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(5)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 5\n", "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 5\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 5\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 5\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5_piece0\n", "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5_piece0 of size 8326 dropped from memory (free 455178893)\n", "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_5_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.10.130:61896 in memory (size: 8.1 KiB, free: 434.4 MiB)\n", "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_5_piece0\n", "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_5_piece0\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_5\n", "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_5 of size 18072 dropped from memory (free 455196965)\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 5, response is 2\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 5\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(144)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 144\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 144\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(139)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 139\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 139\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(107)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 107\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 107\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(124)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 124\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 124\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanBroadcast(6)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning broadcast 6\n", "26/10/17 23:19:36 DEBUG TorrentBroadcast: Unpersisting TorrentBroadcast 6\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: removing broadcast 6\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing broadcast 6\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6\n", "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6 of size 264384 dropped from memory (free 455461349)\n", "26/10/17 23:19:36 DEBUG BlockManager: Removing block broadcast_6_piece0\n", "26/10/17 23:19:36 DEBUG MemoryStore: Block broadcast_6_piece0 of size 40065 dropped from memory (free 455501414)\n", "26/10/17 23:19:36 DEBUG BlockManagerMasterEndpoint: Updating block info on master broadcast_6_piece0 for BlockManagerId(driver, 192.168.10.130, 61896, None)\n", "26/10/17 23:19:36 DEBUG BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.10.130:61896 in memory (size: 39.1 KiB, free: 434.4 MiB)\n", "26/10/17 23:19:36 DEBUG BlockManagerMaster: Updated info of block broadcast_6_piece0\n", "26/10/17 23:19:36 DEBUG BlockManager: Told master about block broadcast_6_piece0\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Done removing broadcast 6, response is 2\n", "26/10/17 23:19:36 DEBUG BlockManagerStorageEndpoint: Sent response: 2 to 192.168.10.130:61895\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned broadcast 6\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(126)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 126\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 126\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(122)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 122\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 122\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(91)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 91\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 91\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(112)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 112\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 112\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(133)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 133\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 133\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(120)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 120\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 120\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(127)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 127\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 127\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(92)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 92\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 92\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(143)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 143\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 143\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(148)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 148\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 148\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(131)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 131\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 131\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(135)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 135\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 135\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(94)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 94\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 94\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(123)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 123\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 123\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(88)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 88\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 88\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Got cleaning task CleanAccum(84)\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaning accumulator 84\n", "26/10/17 23:19:36 DEBUG ContextCleaner: Cleaned accumulator 84\n" ] } ], "source": [ "music_path = os.getcwd() + \"/ingest/**_sesh/*.json\"\n", "video_path = os.getcwd() + \"/ingest/trakt**/*.json\"\n", "\n", "print(\"Music\")\n", "for file in glob.iglob(music_path, recursive=False):\n", "\n", " print(file)\n", "\n", "print(\"Video\")\n", "for file in glob.iglob(video_path, recursive=True):\n", " print(file)\n", "\n" ] }, { "cell_type": "code", "execution_count": 96, "id": "a0e9641e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "root\n", " |-- conn_country: string (nullable = true)\n", " |-- episode_name: string (nullable = true)\n", " |-- episode_show_name: string (nullable = true)\n", " |-- incognito_mode: boolean (nullable = true)\n", " |-- ip_addr_decrypted: string (nullable = true)\n", " |-- master_metadata_album_album_name: string (nullable = true)\n", " |-- master_metadata_album_artist_name: string (nullable = true)\n", " |-- master_metadata_track_name: string (nullable = true)\n", " |-- ms_played: long (nullable = true)\n", " |-- offline: boolean (nullable = true)\n", " |-- offline_timestamp: long (nullable = true)\n", " |-- platform: string (nullable = true)\n", " |-- reason_end: string (nullable = true)\n", " |-- reason_start: string (nullable = true)\n", " |-- shuffle: boolean (nullable = true)\n", " |-- skipped: boolean (nullable = true)\n", " |-- spotify_episode_uri: string (nullable = true)\n", " |-- spotify_track_uri: string (nullable = true)\n", " |-- ts: string (nullable = true)\n", " |-- user_agent_decrypted: string (nullable = true)\n", " |-- username: string (nullable = true)\n", "\n", "-RECORD 0----------------------\n", " conn_country | 0 \n", " episode_name | 105093 \n", " episode_show_name | 105093 \n", " incognito_mode | 0 \n", " ip_addr_decrypted | 4492 \n", " album_name | 2851 \n", " artist_name | 2851 \n", " track_name | 2851 \n", " ms_played | 0 \n", " offline | 279 \n", " offline_timestamp | 2 \n", " platform | 0 \n", " reason_end | 4490 \n", " reason_start | 0 \n", " shuffle | 0 \n", " skipped | 44457 \n", " spotify_episode_uri | 105093 \n", " spotify_track_uri | 2851 \n", " ts | 0 \n", " user_agent_decrypted | 24700 \n", " username | 0 \n", "\n" ] } ], "source": [ "from pyspark.sql import SparkSession\n", "from pyspark.sql import functions as F\n", "from pathlib import Path\n", "import glob\n", "\n", "BASE = Path.cwd() / \"ingest\"\n", "\n", "def music_source_paths():\n", " return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n", "\n", "def video_source_paths():\n", " return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n", "\n", "def read_json(spark, paths):\n", " if not paths:\n", " raise FileNotFoundError(\"No JSON files matched the pattern\")\n", " return (\n", " spark.read\n", " .format(\"json\")\n", " .option(\"multiLine\", True)\n", " .json(paths) # list of paths, one read\n", " # .withColumn(\"source_file\", F.input_file_name())\n", " # .withColumn(\n", " # \"source_folder\",\n", " # F.regexp_extract(\"source_file\", r\"/ingest/([^/]+)/\", 1),\n", " # )\n", " )\n", "\n", "\n", "def null_counts_by_column(df):\n", " null_counts = df.select([\n", " F.count(F.when(F.col(c).isNull(), c)).alias(c)\n", " for c in df.columns\n", " ])\n", " return null_counts\n", "\n", "def metadata_df(df):\n", " return df.withColumnRenamed(\"master_metadata_album_album_name\", \"album_name\").withColumnRenamed(\"master_metadata_album_artist_name\", \"artist_name\").withColumnRenamed(\"master_metadata_track_name\", \"track_name\")\n", "\n", "## MAIN ## \n", "\n", "spark = SparkSession.builder.appName(\"media_ml\").getOrCreate()\n", "spark.sparkContext.setLogLevel(\"WARN\")\n", "\n", "music_df = read_json(spark, music_source_paths())\n", "\n", "music_df.printSchema()\n", "# music_df.show(10)\n", "\n", "\n", "null_counts = null_counts_by_column(metadata_df(music_df))\n", "null_counts.show(vertical=True, truncate=False)\n", "\n", "\n", "# total_rows = music_df.count\n", "# print(music_df.count())\n", "\n", "\n", "# print(group)\n", "# display(group)\n", "\n" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.7" } }, "nbformat": 4, "nbformat_minor": 5 }