From f03814ab934a1a88fc474edc3b9c84985139c6cd Mon Sep 17 00:00:00 2001 From: CC Date: Sun, 18 Oct 2026 18:38:51 +0100 Subject: [PATCH] silver load from jdbc --- spotify.ipynb => spotify_bronze_silver.ipynb | 4 +- spotify_silver_gold.ipynb | 147 +++++++++++++++++++ 2 files changed, 149 insertions(+), 2 deletions(-) rename spotify.ipynb => spotify_bronze_silver.ipynb (99%) create mode 100644 spotify_silver_gold.ipynb diff --git a/spotify.ipynb b/spotify_bronze_silver.ipynb similarity index 99% rename from spotify.ipynb rename to spotify_bronze_silver.ipynb index bd5e54d..ef26d5c 100644 --- a/spotify.ipynb +++ b/spotify_bronze_silver.ipynb @@ -185,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "4cc14bd0", "metadata": {}, "outputs": [ @@ -228,7 +228,7 @@ " print(f\" Raw DF Rows: {raw_df.count()}\")\n", " print(f\" Transformed DF Rows: {transformed_df.count()}\")\n", " print(f\" Row difference is {raw_df.count() - transformed_df.count()}\") \n", - " spark.stop\n", + " spark.stop()\n", "if __name__ == \"__main__\":\n", " main()\n", "\n", diff --git a/spotify_silver_gold.ipynb b/spotify_silver_gold.ipynb new file mode 100644 index 0000000..dc0696a --- /dev/null +++ b/spotify_silver_gold.ipynb @@ -0,0 +1,147 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 13, + "id": "09a05cc1", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession, DataFrame\n", + "from pyspark.sql import functions as F\n", + "from pathlib import Path\n", + "import glob\n", + "import os\n", + "from dotenv import load_dotenv, find_dotenv" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "2c1aeb30", + "metadata": {}, + "outputs": [], + "source": [ + " ## Extract (Load in) ##\n", + "def start_spark(app_name):\n", + " jar_path = str(Path.cwd() / \"postgresql.jar\")\n", + " spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n", + " spark.sparkContext.setLogLevel(\"ERROR\")\n", + " return spark\n", + "\n", + "def get_jdbc_config():\n", + " \"\"\"Load env vars and return a Postgres URL\"\"\"\n", + " load_dotenv(find_dotenv())\n", + " \n", + " username=os.getenv(\"DATABASE_USERNAME\")\n", + " password=os.getenv(\"DATABASE_PASSWORD\")\n", + " host=os.getenv(\"DATABASE_IP\")\n", + " port=int(os.getenv(\"DATABASE_PORT\", 5432))\n", + " database=os.getenv(\"DATABASE_NAME\")\n", + " \n", + " missing = [k for k, v in {\n", + " \"DATABASE_USERNAME\": username,\n", + " \"DATABASE_PASSWORD\": password,\n", + " \"DATABASE_IP\": host,\n", + " \"DATABASE_NAME\": database,}.items() if not v]\n", + " if missing:\n", + " raise ValueError(f\"Missing env vars: {missing}\")\n", + "\n", + " url = f\"jdbc:postgresql://{host}:{port}/{database}\"\n", + " properties = {\n", + " \"user\": username,\n", + " \"password\": password,\n", + " \"driver\": \"org.postgresql.Driver\"\n", + " }\n", + " return url, properties\n", + "\n", + "def read_jdbc(spark, table_name):\n", + " url, properties = get_jdbc_config()\n", + " return spark.read.jdbc(url=url, table=table_name, properties=properties)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "9631ba07", + "metadata": {}, + "outputs": [], + "source": [ + " ## Transform ##" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7673c72", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 0:> (0 + 1) / 1]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n", + "|album_name| artist_name| track_name|ms_played|offline|offline_timestamp| platform|reason_end|reason_start|shuffle|skipped| track_uri| ts| username| offline_timestamped| time| date|offline_time|offline_date|\n", + "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n", + "|Elton John| Elton John| Your Song| 0.42| false| 1665740300104|iOS 16.0 (iPhone1...| endplay| clickrow| false| NULL|spotify:track:38z...|2022-10-14 10:38:46|fm30ajetkgrrkbvae...|+54755-03-18 14:5...|10:38:46|2022-10-14| 14:55:04|+54755-03-18|\n", + "| Lights 10|Ellie Goulding| Your Song| 3.11| false| 1665740325724|iOS 16.0 (iPhone1...| endplay| clickrow| true| NULL|spotify:track:6XK...|2022-10-14 10:42:32|fm30ajetkgrrkbvae...|+54755-03-18 22:0...|10:42:32|2022-10-14| 22:02:04|+54755-03-18|\n", + "| 19| Adele|Make You Feel My ...| 3.53| false| 1665740551741|iOS 16.0 (iPhone1...| trackdone| clickrow| false| NULL|spotify:track:273...|2022-10-14 10:46:27|fm30ajetkgrrkbvae...|+54755-03-21 12:4...|10:46:27|2022-10-14| 12:49:01|+54755-03-21|\n", + "| May It Be| Enya| May It Be| 0.07| false| 1665740786443|iOS 16.0 (iPhone1...| backbtn| trackdone| true| NULL|spotify:track:1wI...|2022-10-14 10:46:32|fm30ajetkgrrkbvae...|+54755-03-24 06:0...|10:46:32|2022-10-14| 06:00:43|+54755-03-24|\n", + "| 19| Adele|Make You Feel My ...| 0.13| false| 1665740791987|iOS 16.0 (iPhone1...| fwdbtn| backbtn| true| NULL|spotify:track:273...|2022-10-14 10:46:40|fm30ajetkgrrkbvae...|+54755-03-24 07:3...|10:46:40|2022-10-14| 07:33:07|+54755-03-24|\n", + "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n", + "only showing top 5 rows\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + " ## MAIN ## \n", + "def main():\n", + " app_name = \"ml_media_silver\"\n", + " spark = start_spark(app_name)\n", + "\n", + " table_name = \"spotify_data\"\n", + " silver_df = read_jdbc(spark, table_name)\n", + " silver_df.show(1)\n", + "\n", + " spark.stop()\n", + "if __name__ == \"__main__\":\n", + " main()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}