media_ml/spotify_silver_gold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "09a05cc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession, DataFrame\n",
    "from pyspark.sql import functions as F\n",
    "from pathlib import Path\n",
    "import glob\n",
    "import os\n",
    "from dotenv import load_dotenv, find_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c1aeb30",
   "metadata": {},
   "outputs": [],
   "source": [
    "                            ## Extract (Load in) ##\n",
    "def start_spark(app_name):\n",
    "    \"\"\"Initialize and return a SparkSession with the PostgreSQL JDBC driver.\"\"\"\n",
    "    jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
    "    spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
    "    return spark\n",
    "\n",
    "def get_jdbc_config():\n",
    "    \"\"\"Load env vars and return a Postgres URL\"\"\"\n",
    "    load_dotenv(find_dotenv())\n",
    "    \n",
    "    username=os.getenv(\"DATABASE_USERNAME\")\n",
    "    password=os.getenv(\"DATABASE_PASSWORD\")\n",
    "    host=os.getenv(\"DATABASE_IP\")\n",
    "    port=int(os.getenv(\"DATABASE_PORT\", 5432))\n",
    "    database=os.getenv(\"DATABASE_NAME\")\n",
    "    \n",
    "    missing = [k for k, v in {\n",
    "        \"DATABASE_USERNAME\": username,\n",
    "        \"DATABASE_PASSWORD\": password,\n",
    "        \"DATABASE_IP\": host,\n",
    "        \"DATABASE_NAME\": database,}.items() if not v]\n",
    "    if missing:\n",
    "        raise ValueError(f\"Missing env vars: {missing}\")\n",
    "\n",
    "    url = f\"jdbc:postgresql://{host}:{port}/{database}\"\n",
    "    properties = {\n",
    "        \"user\": username,\n",
    "        \"password\": password,\n",
    "        \"driver\": \"org.postgresql.Driver\"\n",
    "    }\n",
    "    return url, properties\n",
    "\n",
    "def read_jdbc(spark, table_name):\n",
    "    \"\"\"Read a table from Postgres via JDBC into a Spark DataFrame.\"\"\"\n",
    "    url, properties = get_jdbc_config()\n",
    "    return spark.read.jdbc(url=url, table=table_name, properties=properties)\n",
    "\n",
    "def databricks_read(df, table_name):\n",
    "    \"\"\"Read the data from catalog.\"\"\"\n",
    "    catalog = \"workspace\"\n",
    "    schema = \"default\"\n",
    "    path = f\"{catalog}.{schema}.{table_name}\"\n",
    "    return df.read.table(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9631ba07",
   "metadata": {},
   "outputs": [],
   "source": [
    "                                ## Transform ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7673c72",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+-----+\n",
      "|          track_name|            username|count|\n",
      "+--------------------+--------------------+-----+\n",
      "|           Mujeriego|             csm3-us|   93|\n",
      "|            Angelito|             csm3-us|   16|\n",
      "|Navidad en Mi Barrio|             csm3-us|    1|\n",
      "|   Como Perro Y Gato|             csm3-us|    1|\n",
      "|    The Taste of Ink|fm30ajetkgrrkbvae...|    2|\n",
      "|   Hasta el Amanecer|             csm3-us|    4|\n",
      "|             Tu Foto|             csm3-us|    1|\n",
      "|Love Is Not Over ...|             csm3-us|    3|\n",
      "|Like I'm Gonna Lo...|fm30ajetkgrrkbvae...|    2|\n",
      "|      Lo Siento BB:/|             csm3-us|    5|\n",
      "|          Superbeast|             csm3-us|    1|\n",
      "|    Amor de Hospital|             csm3-us|   11|\n",
      "|       All Right Now|fm30ajetkgrrkbvae...|    2|\n",
      "|Dancing On The Ce...|fm30ajetkgrrkbvae...|    4|\n",
      "|Prologue / The Da...|fm30ajetkgrrkbvae...|    4|\n",
      "|That Way (feat. J...|             csm3-us|   11|\n",
      "|               Blame|             csm3-us|   29|\n",
      "|           Speak Now|fm30ajetkgrrkbvae...|    2|\n",
      "|A Little Respect ...|fm30ajetkgrrkbvae...|    3|\n",
      "|            Rockstar|fm30ajetkgrrkbvae...|    4|\n",
      "+--------------------+--------------------+-----+\n",
      "only showing top 20 rows\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "                                    ## MAIN ## \n",
    "def main():\n",
    "    app_name = \"ml_media_silver\"\n",
    "    spark = start_spark(app_name)\n",
    "\n",
    "    bronze_table_name = \"bronze_spotify_data\"\n",
    "    silver_table_name = \"silver_spotify_data\"\n",
    "    silver_df = read_jdbc(spark, bronze_table_name)\n",
    "\n",
    "    silver_df.groupBy(\"track_name\", \"username\").count().show()\n",
    "\n",
    "    #spark.stop()\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9a695ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Databricks ##\n",
    "                                    ## MAIN ## \n",
    "\n",
    "# app_name = \"ml_media_silver\"\n",
    "# bronze_table_name = \"bronze_spotify_data\"\n",
    "# silver_table_name = \"silver_spotify_data\"\n",
    "# silver_df = databricks_read(spark, bronze_table_name)\n",
    "# silver_df.show(1)\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}