silver load from jdbc

2026-10-18 18:38:51 +01:00
parent dc6b08eeeb
commit f03814ab93
2 changed files with 149 additions and 2 deletions
--- a/spotify_silver_gold.ipynb
+++ b/spotify_silver_gold.ipynb
@@ -0,0 +1,147 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "09a05cc1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pyspark.sql import SparkSession, DataFrame\n",
+    "from pyspark.sql import functions as F\n",
+    "from pathlib import Path\n",
+    "import glob\n",
+    "import os\n",
+    "from dotenv import load_dotenv, find_dotenv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "2c1aeb30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "                            ## Extract (Load in) ##\n",
+    "def start_spark(app_name):\n",
+    "    jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
+    "    spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
+    "    spark.sparkContext.setLogLevel(\"ERROR\")\n",
+    "    return spark\n",
+    "\n",
+    "def get_jdbc_config():\n",
+    "    \"\"\"Load env vars and return a Postgres URL\"\"\"\n",
+    "    load_dotenv(find_dotenv())\n",
+    "    \n",
+    "    username=os.getenv(\"DATABASE_USERNAME\")\n",
+    "    password=os.getenv(\"DATABASE_PASSWORD\")\n",
+    "    host=os.getenv(\"DATABASE_IP\")\n",
+    "    port=int(os.getenv(\"DATABASE_PORT\", 5432))\n",
+    "    database=os.getenv(\"DATABASE_NAME\")\n",
+    "    \n",
+    "    missing = [k for k, v in {\n",
+    "        \"DATABASE_USERNAME\": username,\n",
+    "        \"DATABASE_PASSWORD\": password,\n",
+    "        \"DATABASE_IP\": host,\n",
+    "        \"DATABASE_NAME\": database,}.items() if not v]\n",
+    "    if missing:\n",
+    "        raise ValueError(f\"Missing env vars: {missing}\")\n",
+    "\n",
+    "    url = f\"jdbc:postgresql://{host}:{port}/{database}\"\n",
+    "    properties = {\n",
+    "        \"user\": username,\n",
+    "        \"password\": password,\n",
+    "        \"driver\": \"org.postgresql.Driver\"\n",
+    "    }\n",
+    "    return url, properties\n",
+    "\n",
+    "def read_jdbc(spark, table_name):\n",
+    "    url, properties = get_jdbc_config()\n",
+    "    return spark.read.jdbc(url=url, table=table_name, properties=properties)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "9631ba07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "                                ## Transform ##"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f7673c72",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[Stage 0:>                                                          (0 + 1) / 1]\r"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n",
+      "|album_name|   artist_name|          track_name|ms_played|offline|offline_timestamp|            platform|reason_end|reason_start|shuffle|skipped|           track_uri|                 ts|            username| offline_timestamped|    time|      date|offline_time|offline_date|\n",
+      "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n",
+      "|Elton John|    Elton John|           Your Song|     0.42|  false|    1665740300104|iOS 16.0 (iPhone1...|   endplay|    clickrow|  false|   NULL|spotify:track:38z...|2022-10-14 10:38:46|fm30ajetkgrrkbvae...|+54755-03-18 14:5...|10:38:46|2022-10-14|    14:55:04|+54755-03-18|\n",
+      "| Lights 10|Ellie Goulding|           Your Song|     3.11|  false|    1665740325724|iOS 16.0 (iPhone1...|   endplay|    clickrow|   true|   NULL|spotify:track:6XK...|2022-10-14 10:42:32|fm30ajetkgrrkbvae...|+54755-03-18 22:0...|10:42:32|2022-10-14|    22:02:04|+54755-03-18|\n",
+      "|        19|         Adele|Make You Feel My ...|     3.53|  false|    1665740551741|iOS 16.0 (iPhone1...| trackdone|    clickrow|  false|   NULL|spotify:track:273...|2022-10-14 10:46:27|fm30ajetkgrrkbvae...|+54755-03-21 12:4...|10:46:27|2022-10-14|    12:49:01|+54755-03-21|\n",
+      "| May It Be|          Enya|           May It Be|     0.07|  false|    1665740786443|iOS 16.0 (iPhone1...|   backbtn|   trackdone|   true|   NULL|spotify:track:1wI...|2022-10-14 10:46:32|fm30ajetkgrrkbvae...|+54755-03-24 06:0...|10:46:32|2022-10-14|    06:00:43|+54755-03-24|\n",
+      "|        19|         Adele|Make You Feel My ...|     0.13|  false|    1665740791987|iOS 16.0 (iPhone1...|    fwdbtn|     backbtn|   true|   NULL|spotify:track:273...|2022-10-14 10:46:40|fm30ajetkgrrkbvae...|+54755-03-24 07:3...|10:46:40|2022-10-14|    07:33:07|+54755-03-24|\n",
+      "+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n",
+      "only showing top 5 rows\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                                                \r"
+     ]
+    }
+   ],
+   "source": [
+    "                                    ## MAIN ## \n",
+    "def main():\n",
+    "    app_name = \"ml_media_silver\"\n",
+    "    spark = start_spark(app_name)\n",
+    "\n",
+    "    table_name = \"spotify_data\"\n",
+    "    silver_df = read_jdbc(spark, table_name)\n",
+    "    silver_df.show(1)\n",
+    "\n",
+    "    spark.stop()\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}