CC/media_ml
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

silver load from jdbc

Browse Source
This commit is contained in:
CC
2026-10-18 18:38:51 +01:00
parent dc6b08eeeb
commit f03814ab93
2 changed files with 149 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
spotify.ipynb → spotify_bronze_silver.ipynb
View File

@@ -185,7 +185,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"id": "4cc14bd0",
"metadata": {},
"outputs": [
@@ -228,7 +228,7 @@
" print(f\" Raw DF Rows: {raw_df.count()}\")\n",
" print(f\" Transformed DF Rows: {transformed_df.count()}\")\n",
" print(f\" Row difference is {raw_df.count() - transformed_df.count()}\") \n",
" spark.stop\n",
" spark.stop()\n",
"if __name__ == \"__main__\":\n",
" main()\n",
"\n",

147
spotify_silver_gold.ipynb Normal file
View File

@@ -0,0 +1,147 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 13,
"id": "09a05cc1",
"metadata": {},
"outputs": [],
"source": [
"from pyspark.sql import SparkSession, DataFrame\n",
"from pyspark.sql import functions as F\n",
"from pathlib import Path\n",
"import glob\n",
"import os\n",
"from dotenv import load_dotenv, find_dotenv"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "2c1aeb30",
"metadata": {},
"outputs": [],
"source": [
" ## Extract (Load in) ##\n",
"def start_spark(app_name):\n",
" jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
" spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
" spark.sparkContext.setLogLevel(\"ERROR\")\n",
" return spark\n",
"\n",
"def get_jdbc_config():\n",
" \"\"\"Load env vars and return a Postgres URL\"\"\"\n",
" load_dotenv(find_dotenv())\n",
" \n",
" username=os.getenv(\"DATABASE_USERNAME\")\n",
" password=os.getenv(\"DATABASE_PASSWORD\")\n",
" host=os.getenv(\"DATABASE_IP\")\n",
" port=int(os.getenv(\"DATABASE_PORT\", 5432))\n",
" database=os.getenv(\"DATABASE_NAME\")\n",
" \n",
" missing = [k for k, v in {\n",
" \"DATABASE_USERNAME\": username,\n",
" \"DATABASE_PASSWORD\": password,\n",
" \"DATABASE_IP\": host,\n",
" \"DATABASE_NAME\": database,}.items() if not v]\n",
" if missing:\n",
" raise ValueError(f\"Missing env vars: {missing}\")\n",
"\n",
" url = f\"jdbc:postgresql://{host}:{port}/{database}\"\n",
" properties = {\n",
" \"user\": username,\n",
" \"password\": password,\n",
" \"driver\": \"org.postgresql.Driver\"\n",
" }\n",
" return url, properties\n",
"\n",
"def read_jdbc(spark, table_name):\n",
" url, properties = get_jdbc_config()\n",
" return spark.read.jdbc(url=url, table=table_name, properties=properties)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "9631ba07",
"metadata": {},
"outputs": [],
"source": [
" ## Transform ##"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f7673c72",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[Stage 0:> (0 + 1) / 1]\r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n",
"|album_name| artist_name| track_name|ms_played|offline|offline_timestamp| platform|reason_end|reason_start|shuffle|skipped| track_uri| ts| username| offline_timestamped| time| date|offline_time|offline_date|\n",
"+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n",
"|Elton John| Elton John| Your Song| 0.42| false| 1665740300104|iOS 16.0 (iPhone1...| endplay| clickrow| false| NULL|spotify:track:38z...|2022-10-14 10:38:46|fm30ajetkgrrkbvae...|+54755-03-18 14:5...|10:38:46|2022-10-14| 14:55:04|+54755-03-18|\n",
"| Lights 10|Ellie Goulding| Your Song| 3.11| false| 1665740325724|iOS 16.0 (iPhone1...| endplay| clickrow| true| NULL|spotify:track:6XK...|2022-10-14 10:42:32|fm30ajetkgrrkbvae...|+54755-03-18 22:0...|10:42:32|2022-10-14| 22:02:04|+54755-03-18|\n",
"| 19| Adele|Make You Feel My ...| 3.53| false| 1665740551741|iOS 16.0 (iPhone1...| trackdone| clickrow| false| NULL|spotify:track:273...|2022-10-14 10:46:27|fm30ajetkgrrkbvae...|+54755-03-21 12:4...|10:46:27|2022-10-14| 12:49:01|+54755-03-21|\n",
"| May It Be| Enya| May It Be| 0.07| false| 1665740786443|iOS 16.0 (iPhone1...| backbtn| trackdone| true| NULL|spotify:track:1wI...|2022-10-14 10:46:32|fm30ajetkgrrkbvae...|+54755-03-24 06:0...|10:46:32|2022-10-14| 06:00:43|+54755-03-24|\n",
"| 19| Adele|Make You Feel My ...| 0.13| false| 1665740791987|iOS 16.0 (iPhone1...| fwdbtn| backbtn| true| NULL|spotify:track:273...|2022-10-14 10:46:40|fm30ajetkgrrkbvae...|+54755-03-24 07:3...|10:46:40|2022-10-14| 07:33:07|+54755-03-24|\n",
"+----------+--------------+--------------------+---------+-------+-----------------+--------------------+----------+------------+-------+-------+--------------------+-------------------+--------------------+--------------------+--------+----------+------------+------------+\n",
"only showing top 5 rows\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" \r"
]
}
],
"source": [
" ## MAIN ## \n",
"def main():\n",
" app_name = \"ml_media_silver\"\n",
" spark = start_spark(app_name)\n",
"\n",
" table_name = \"spotify_data\"\n",
" silver_df = read_jdbc(spark, table_name)\n",
" silver_df.show(1)\n",
"\n",
" spark.stop()\n",
"if __name__ == \"__main__\":\n",
" main()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}