191 lines
6.0 KiB
Plaintext
191 lines
6.0 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "09a05cc1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pyspark.sql import SparkSession, DataFrame\n",
|
|
"from pyspark.sql import functions as F\n",
|
|
"from pathlib import Path\n",
|
|
"import glob\n",
|
|
"import os\n",
|
|
"from dotenv import load_dotenv, find_dotenv"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2c1aeb30",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
" ## Extract (Load in) ##\n",
|
|
"def start_spark(app_name):\n",
|
|
" jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
|
|
" spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
|
|
" spark.log.level(\"ERROR\")\n",
|
|
" return spark\n",
|
|
"\n",
|
|
"def get_jdbc_config():\n",
|
|
" \"\"\"Load env vars and return a Postgres URL\"\"\"\n",
|
|
" load_dotenv(find_dotenv())\n",
|
|
" \n",
|
|
" username=os.getenv(\"DATABASE_USERNAME\")\n",
|
|
" password=os.getenv(\"DATABASE_PASSWORD\")\n",
|
|
" host=os.getenv(\"DATABASE_IP\")\n",
|
|
" port=int(os.getenv(\"DATABASE_PORT\", 5432))\n",
|
|
" database=os.getenv(\"DATABASE_NAME\")\n",
|
|
" \n",
|
|
" missing = [k for k, v in {\n",
|
|
" \"DATABASE_USERNAME\": username,\n",
|
|
" \"DATABASE_PASSWORD\": password,\n",
|
|
" \"DATABASE_IP\": host,\n",
|
|
" \"DATABASE_NAME\": database,}.items() if not v]\n",
|
|
" if missing:\n",
|
|
" raise ValueError(f\"Missing env vars: {missing}\")\n",
|
|
"\n",
|
|
" url = f\"jdbc:postgresql://{host}:{port}/{database}\"\n",
|
|
" properties = {\n",
|
|
" \"user\": username,\n",
|
|
" \"password\": password,\n",
|
|
" \"driver\": \"org.postgresql.Driver\"\n",
|
|
" }\n",
|
|
" return url, properties\n",
|
|
"\n",
|
|
"def read_jdbc(spark, table_name):\n",
|
|
" url, properties = get_jdbc_config()\n",
|
|
" return spark.read.jdbc(url=url, table=table_name, properties=properties)\n",
|
|
"\n",
|
|
"def databricks_read(df, table_name):\n",
|
|
" \"\"\"Read the data from catalog.\"\"\"\n",
|
|
" catalog = \"workspace\"\n",
|
|
" schema = \"default\"\n",
|
|
" path = f\"{catalog}.{schema}.{table_name}\"\n",
|
|
" return df.read.table(path)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"id": "9631ba07",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
" ## Transform ##"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f7673c72",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[Stage 0:> (0 + 1) / 1]\r"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"+--------------------+--------------------+-----+\n",
|
|
"| track_name| username|count|\n",
|
|
"+--------------------+--------------------+-----+\n",
|
|
"| Mujeriego| csm3-us| 93|\n",
|
|
"| Angelito| csm3-us| 16|\n",
|
|
"|Navidad en Mi Barrio| csm3-us| 1|\n",
|
|
"| Como Perro Y Gato| csm3-us| 1|\n",
|
|
"| The Taste of Ink|fm30ajetkgrrkbvae...| 2|\n",
|
|
"| Hasta el Amanecer| csm3-us| 4|\n",
|
|
"| Tu Foto| csm3-us| 1|\n",
|
|
"|Love Is Not Over ...| csm3-us| 3|\n",
|
|
"|Like I'm Gonna Lo...|fm30ajetkgrrkbvae...| 2|\n",
|
|
"| Lo Siento BB:/| csm3-us| 5|\n",
|
|
"| Superbeast| csm3-us| 1|\n",
|
|
"| Amor de Hospital| csm3-us| 11|\n",
|
|
"| All Right Now|fm30ajetkgrrkbvae...| 2|\n",
|
|
"|Dancing On The Ce...|fm30ajetkgrrkbvae...| 4|\n",
|
|
"|Prologue / The Da...|fm30ajetkgrrkbvae...| 4|\n",
|
|
"|That Way (feat. J...| csm3-us| 11|\n",
|
|
"| Blame| csm3-us| 29|\n",
|
|
"| Speak Now|fm30ajetkgrrkbvae...| 2|\n",
|
|
"|A Little Respect ...|fm30ajetkgrrkbvae...| 3|\n",
|
|
"| Rockstar|fm30ajetkgrrkbvae...| 4|\n",
|
|
"+--------------------+--------------------+-----+\n",
|
|
"only showing top 20 rows\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" \r"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
" ## MAIN ## \n",
|
|
"def main():\n",
|
|
" app_name = \"ml_media_silver\"\n",
|
|
" spark = start_spark(app_name)\n",
|
|
"\n",
|
|
" bronze_table_name = \"bronze_spotify_data\"\n",
|
|
" silver_table_name = \"silver_spotify_data\"\n",
|
|
" silver_df = read_jdbc(spark, bronze_table_name)\n",
|
|
"\n",
|
|
" silver_df.groupBy(\"track_name\", \"username\").count().show()\n",
|
|
"\n",
|
|
" #spark.stop()\n",
|
|
"if __name__ == \"__main__\":\n",
|
|
" main()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c9a695ae",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"## Databricks ##\n",
|
|
" ## MAIN ## \n",
|
|
"\n",
|
|
"# app_name = \"ml_media_silver\"\n",
|
|
"# bronze_table_name = \"bronze_spotify_data\"\n",
|
|
"# silver_table_name = \"silver_spotify_data\"\n",
|
|
"# silver_df = databricks_read(spark, bronze_table_name)\n",
|
|
"# silver_df.show(1)\n",
|
|
"\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.13.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|