CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

descriptions

Browse Source
This commit is contained in:
CC
2026-10-18 23:25:48 +01:00
parent 8fc40a9015
commit 72b501a5ab
1 changed files with 34 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
spotify_bronze_silver.ipynb
View File

@@ -17,7 +17,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": null,
"id": "11bb5dc2", "id": "11bb5dc2",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -27,17 +27,21 @@
"BASE = Path.cwd() / \"ingest\"\n", "BASE = Path.cwd() / \"ingest\"\n",
"\n", "\n",
"def start_spark(app_name):\n", "def start_spark(app_name):\n",
" \"\"\"Initialize and return a SparkSession with the PostgreSQL JDBC driver.\"\"\"\n",
" jar_path = str(Path.cwd() / \"postgresql.jar\")\n", " jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
" spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n", " spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
" return spark\n", " return spark\n",
"\n", "\n",
"def music_source_paths():\n", "def music_source_paths():\n",
" \"\"\"Return sorted list of Spotify listening history JSON file paths.\"\"\"\n",
" return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n", " return sorted(glob.glob(str(BASE / \"*_sesh\" / \"*.json\")))\n",
"\n", "\n",
"def video_source_paths():\n", "def video_source_paths():\n",
" \"\"\"Return sorted list of Trakt video watching history JSON file paths.\"\"\"\n",
" return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n", " return sorted(glob.glob(str(BASE / \"trakt_*\" / \"*.json\")))\n",
"\n", "\n",
"def read_json(spark, paths):\n", "def read_json(spark, paths):\n",
" \"\"\"Read a list of JSON files into a multi-line Spark DataFrame.\"\"\"\n",
" if not paths:\n", " if not paths:\n",
" raise FileNotFoundError(\"No JSON files matched the pattern\")\n", " raise FileNotFoundError(\"No JSON files matched the pattern\")\n",
" return (\n", " return (\n",
@@ -192,7 +196,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 19,
"id": "4cc14bd0", "id": "4cc14bd0",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -200,7 +204,32 @@
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"[Stage 2:> (0 + 6) / 7]\r" " \r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"+----------------------------+--------------+\n",
"|reason_end |skipped_values|\n",
"+----------------------------+--------------+\n",
"|NULL |[] |\n",
"|backbtn |[true] |\n",
"|endplay |[true] |\n",
"|fwdbtn |[true] |\n",
"|logout |[false] |\n",
"|remote |[false] |\n",
"|trackdone |[false] |\n",
"|trackerror |[false] |\n",
"|unexpected-exit |[false] |\n",
"|unexpected-exit-while-paused|[false] |\n",
"|unknown |[false] |\n",
"+----------------------------+--------------+\n",
"\n",
" Raw DF Rows: 105477\n",
" Transformed DF Rows: 102626\n",
" Row difference is 2851\n"
] ]
} }
], ],
@@ -256,7 +285,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 20,
"id": "35878e16", "id": "35878e16",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -285,7 +314,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 21,
"id": "faa6a8cb", "id": "faa6a8cb",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],