databricks

spotify_silver_gold.ipynb

@@ -17,7 +17,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "2c1aeb30",
    "metadata": {},
    "outputs": [],
@@ -26,7 +26,7 @@
     "def start_spark(app_name):\n",
     "    jar_path = str(Path.cwd() / \"postgresql.jar\")\n",
     "    spark = SparkSession.builder.appName(app_name).config(\"spark.jars\", jar_path).config(\"spark.driver.extraClassPath\", jar_path).getOrCreate()\n",
-    "    spark.sparkContext.setLogLevel(\"ERROR\")\n",
+    "    spark.log.level(\"ERROR\")\n",
     "    return spark\n",
     "\n",
     "def get_jdbc_config():\n",
@@ -57,7 +57,14 @@
     "\n",
     "def read_jdbc(spark, table_name):\n",
     "    url, properties = get_jdbc_config()\n",
-    "    return spark.read.jdbc(url=url, table=table_name, properties=properties)"
+    "    return spark.read.jdbc(url=url, table=table_name, properties=properties)\n",
+    "\n",
+    "def databricks_read(df, table_name):\n",
+    "    \"\"\"Read the data from catalog.\"\"\"\n",
+    "    catalog = \"workspace\"\n",
+    "    schema = \"default\"\n",
+    "    path = f\"{catalog}.{schema}.{table_name}\"\n",
+    "    return df.read.table(path)"
    ]
   },
   {
@@ -72,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "id": "f7673c72",
    "metadata": {},
    "outputs": [
@@ -128,14 +135,34 @@
     "    app_name = \"ml_media_silver\"\n",
     "    spark = start_spark(app_name)\n",
     "\n",
-    "    table_name = \"spotify_data\"\n",
-    "    silver_df = read_jdbc(spark, table_name)\n",
+    "    bronze_table_name = \"bronze_spotify_data\"\n",
+    "    silver_table_name = \"silver_spotify_data\"\n",
+    "    silver_df = read_jdbc(spark, bronze_table_name)\n",
     "\n",
     "    silver_df.groupBy(\"track_name\", \"username\").count().show()\n",
     "\n",
     "    #spark.stop()\n",
     "if __name__ == \"__main__\":\n",
-    "    main()"
+    "    main()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c9a695ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Databricks ##\n",
+    "                                    ## MAIN ## \n",
+    "\n",
+    "# app_name = \"ml_media_silver\"\n",
+    "# bronze_table_name = \"bronze_spotify_data\"\n",
+    "# silver_table_name = \"silver_spotify_data\"\n",
+    "# silver_df = databricks_read(spark, bronze_table_name)\n",
+    "# silver_df.show(1)\n",
+    "\n",
+    "\n"
    ]
   }
  ],