LeetCode Pyspark Solution

Author: bitoollearner

Solved/2142. The Number of Passengers in Each Bus I (Medium)-(Solved).ipynb

@@ -4,7 +4,10 @@
    "cell_type": "markdown",
    "metadata": {
     "application/vnd.databricks.v1+cell": {
-     "cellMetadata": {},
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
      "inputWidgets": {},
      "nuid": "a23f6d56-6d8d-44bc-b796-cedda9e17c21",
      "showTitle": false,
@@ -21,7 +24,10 @@
    "execution_count": 0,
    "metadata": {
     "application/vnd.databricks.v1+cell": {
-     "cellMetadata": {},
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
      "inputWidgets": {},
      "nuid": "0a69d78b-43d7-4f9e-a05d-026985c2539f",
      "showTitle": false,
@@ -40,7 +46,10 @@
    "cell_type": "markdown",
    "metadata": {
     "application/vnd.databricks.v1+cell": {
-     "cellMetadata": {},
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
      "inputWidgets": {},
      "nuid": "2a54b88b-06bd-43e4-978a-c0c5d33ca5bb",
      "showTitle": false,
@@ -122,15 +131,27 @@
    "execution_count": 0,
    "metadata": {
     "application/vnd.databricks.v1+cell": {
-     "cellMetadata": {},
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
      "inputWidgets": {},
      "nuid": "c260f510-542b-4067-a6f4-ffed86995056",
      "showTitle": false,
      "tableResultSettingsMap": {},
      "title": ""
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "output_type": "stream",
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+------+------------+\n|bus_id|arrival_time|\n+------+------------+\n|     1|           2|\n|     2|           4|\n|     3|           7|\n+------+------------+\n\n+------------+------------+\n|passenger_id|arrival_time|\n+------------+------------+\n|          11|           1|\n|          12|           5|\n|          13|           6|\n|          14|           7|\n+------------+------------+\n\n"
+     ]
+    }
+   ],
    "source": [
     "buses_data_2142 = [\n",
     "    (1, 2),\n",
@@ -153,15 +174,144 @@
     "passengers_df_2142 = spark.createDataFrame(passengers_data_2142, passengers_columns_2142)\n",
     "passengers_df_2142.show()"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "afaca1d0-79e0-4ffc-9c77-79c6fc7e38f4",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "joined_df_2142 = passengers_df_2142.alias(\"p\")\\\n",
+    "                        .join( buses_df_2142.alias(\"b\"), col(\"p.arrival_time\") <= col(\"b.arrival_time\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "25a51bb8-bf85-4f6b-b942-84c2d4dbc22d",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "windowSpec = Window.partitionBy(\"p.passenger_id\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "9dfccb64-9516-4036-afce-8b228a9a05e6",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "passenger_bus_df_2142 = joined_df_2142\\\n",
+    "                            .withColumn( \"min_bus_time\", min(\"b.arrival_time\").over(windowSpec))\\\n",
+    "                                .where(col(\"b.arrival_time\") == col(\"min_bus_time\")) \\\n",
+    "                                    .select(\"p.passenger_id\", \"b.bus_id\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "45a272fb-7400-4f5a-acc5-43e18e8745fc",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "bus_counts_df_2142 = passenger_bus_df_2142\\\n",
+    "                            .groupBy(\"bus_id\")\\\n",
+    "                                .agg(count(\"passenger_id\").alias(\"passengers_cnt\")\n",
+    "                                     )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "860f8f74-09f1-4067-8df0-45d9fcf6c3b5",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "output_type": "stream",
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+------+--------------+\n|bus_id|passengers_cnt|\n+------+--------------+\n|     1|             1|\n|     2|             0|\n|     3|             3|\n+------+--------------+\n\n"
+     ]
+    }
+   ],
+   "source": [
+    "buses_df_2142\\\n",
+    "    .join(bus_counts_df_2142, \"bus_id\", \"left\").fillna(0).orderBy(\"bus_id\")\\\n",
+    "        .select('bus_id','passengers_cnt').show()"
+   ]
   }
  ],
  "metadata": {
   "application/vnd.databricks.v1+notebook": {
-   "computePreferences": null,
+   "computePreferences": {
+    "hardware": {
+     "accelerator": null,
+     "gpuPoolId": null,
+     "memory": null
+    }
+   },
    "dashboards": [],
    "environmentMetadata": {
     "base_environment": "",
-    "environment_version": "1"
+    "environment_version": "2"
    },
    "inputWidgetPreferences": null,
    "language": "python",