|
4 | 4 | "cell_type": "markdown",
|
5 | 5 | "metadata": {
|
6 | 6 | "application/vnd.databricks.v1+cell": {
|
7 |
| - "cellMetadata": {}, |
| 7 | + "cellMetadata": { |
| 8 | + "byteLimit": 2048000, |
| 9 | + "rowLimit": 10000 |
| 10 | + }, |
8 | 11 | "inputWidgets": {},
|
9 | 12 | "nuid": "f6ac9266-658c-4507-8a37-ecc81334b06d",
|
10 | 13 | "showTitle": false,
|
|
21 | 24 | "execution_count": 0,
|
22 | 25 | "metadata": {
|
23 | 26 | "application/vnd.databricks.v1+cell": {
|
24 |
| - "cellMetadata": {}, |
| 27 | + "cellMetadata": { |
| 28 | + "byteLimit": 2048000, |
| 29 | + "rowLimit": 10000 |
| 30 | + }, |
25 | 31 | "inputWidgets": {},
|
26 | 32 | "nuid": "d6194993-9312-4b46-8365-d6a73d8cb0b0",
|
27 | 33 | "showTitle": false,
|
|
40 | 46 | "cell_type": "markdown",
|
41 | 47 | "metadata": {
|
42 | 48 | "application/vnd.databricks.v1+cell": {
|
43 |
| - "cellMetadata": {}, |
| 49 | + "cellMetadata": { |
| 50 | + "byteLimit": 2048000, |
| 51 | + "rowLimit": 10000 |
| 52 | + }, |
44 | 53 | "inputWidgets": {},
|
45 | 54 | "nuid": "23f6a680-8f01-4976-83fa-88103e49c6b8",
|
46 | 55 | "showTitle": false,
|
|
139 | 148 | "execution_count": 0,
|
140 | 149 | "metadata": {
|
141 | 150 | "application/vnd.databricks.v1+cell": {
|
142 |
| - "cellMetadata": {}, |
| 151 | + "cellMetadata": { |
| 152 | + "byteLimit": 2048000, |
| 153 | + "rowLimit": 10000 |
| 154 | + }, |
143 | 155 | "inputWidgets": {},
|
144 | 156 | "nuid": "a762f84d-f622-4403-b695-4066e5bdbaf0",
|
145 | 157 | "showTitle": false,
|
146 | 158 | "tableResultSettingsMap": {},
|
147 | 159 | "title": ""
|
148 | 160 | }
|
149 | 161 | },
|
150 |
| - "outputs": [], |
| 162 | + "outputs": [ |
| 163 | + { |
| 164 | + "output_type": "stream", |
| 165 | + "name": "stdout", |
| 166 | + "output_type": "stream", |
| 167 | + "text": [ |
| 168 | + "+----------+------------+\n|student_id|student_name|\n+----------+------------+\n| 1| Alice|\n| 2| Bob|\n+----------+------------+\n\n+----------+------------+\n|student_id|student_name|\n+----------+------------+\n| 3| Tom|\n+----------+------------+\n\n+----------+------------+\n|student_id|student_name|\n+----------+------------+\n| 3| Tom|\n| 2| Jerry|\n| 10| Alice|\n+----------+------------+\n\n" |
| 169 | + ] |
| 170 | + } |
| 171 | + ], |
151 | 172 | "source": [
|
152 | 173 | "schoolA_data_1623 = [(1, \"Alice\"), (2, \"Bob\")]\n",
|
153 | 174 | "\n",
|
|
167 | 188 | "schoolC_df_1623 = spark.createDataFrame(schoolC_data_1623, schoolC_columns_1623)\n",
|
168 | 189 | "schoolC_df_1623.show()"
|
169 | 190 | ]
|
| 191 | + }, |
| 192 | + { |
| 193 | + "cell_type": "code", |
| 194 | + "execution_count": 0, |
| 195 | + "metadata": { |
| 196 | + "application/vnd.databricks.v1+cell": { |
| 197 | + "cellMetadata": { |
| 198 | + "byteLimit": 2048000, |
| 199 | + "rowLimit": 10000 |
| 200 | + }, |
| 201 | + "inputWidgets": {}, |
| 202 | + "nuid": "0cf73bc0-f144-422f-aaa6-f6192d24cdcb", |
| 203 | + "showTitle": false, |
| 204 | + "tableResultSettingsMap": {}, |
| 205 | + "title": "" |
| 206 | + } |
| 207 | + }, |
| 208 | + "outputs": [], |
| 209 | + "source": [ |
| 210 | + "A = schoolA_df_1623.alias(\"A\")\n", |
| 211 | + "B = schoolB_df_1623.alias(\"B\")\n", |
| 212 | + "C = schoolC_df_1623.alias(\"C\")" |
| 213 | + ] |
| 214 | + }, |
| 215 | + { |
| 216 | + "cell_type": "code", |
| 217 | + "execution_count": 0, |
| 218 | + "metadata": { |
| 219 | + "application/vnd.databricks.v1+cell": { |
| 220 | + "cellMetadata": { |
| 221 | + "byteLimit": 2048000, |
| 222 | + "rowLimit": 10000 |
| 223 | + }, |
| 224 | + "inputWidgets": {}, |
| 225 | + "nuid": "fd5c5e79-9609-4fab-8009-5adbe0731079", |
| 226 | + "showTitle": false, |
| 227 | + "tableResultSettingsMap": {}, |
| 228 | + "title": "" |
| 229 | + } |
| 230 | + }, |
| 231 | + "outputs": [], |
| 232 | + "source": [ |
| 233 | + "triplets_df_1623 = A.crossJoin(B).crossJoin(C) \\\n", |
| 234 | + " .select(\n", |
| 235 | + " col(\"A.student_id\").alias(\"id_A\"),\n", |
| 236 | + " col(\"A.student_name\").alias(\"member_A\"),\n", |
| 237 | + " col(\"B.student_id\").alias(\"id_B\"),\n", |
| 238 | + " col(\"B.student_name\").alias(\"member_B\"),\n", |
| 239 | + " col(\"C.student_id\").alias(\"id_C\"),\n", |
| 240 | + " col(\"C.student_name\").alias(\"member_C\")\n", |
| 241 | + " )" |
| 242 | + ] |
| 243 | + }, |
| 244 | + { |
| 245 | + "cell_type": "code", |
| 246 | + "execution_count": 0, |
| 247 | + "metadata": { |
| 248 | + "application/vnd.databricks.v1+cell": { |
| 249 | + "cellMetadata": { |
| 250 | + "byteLimit": 2048000, |
| 251 | + "rowLimit": 10000 |
| 252 | + }, |
| 253 | + "inputWidgets": {}, |
| 254 | + "nuid": "1de95fdd-d2f4-4436-beaa-793b60cfacbc", |
| 255 | + "showTitle": false, |
| 256 | + "tableResultSettingsMap": {}, |
| 257 | + "title": "" |
| 258 | + } |
| 259 | + }, |
| 260 | + "outputs": [ |
| 261 | + { |
| 262 | + "output_type": "stream", |
| 263 | + "name": "stdout", |
| 264 | + "output_type": "stream", |
| 265 | + "text": [ |
| 266 | + "+--------+--------+--------+\n|member_A|member_B|member_C|\n+--------+--------+--------+\n| Alice| Tom| Jerry|\n| Bob| Tom| Alice|\n+--------+--------+--------+\n\n" |
| 267 | + ] |
| 268 | + } |
| 269 | + ], |
| 270 | + "source": [ |
| 271 | + "triplets_df_1623\\\n", |
| 272 | + " .filter(\n", |
| 273 | + " (col(\"id_A\") != col(\"id_B\")) &\n", |
| 274 | + " (col(\"id_A\") != col(\"id_C\")) &\n", |
| 275 | + " (col(\"id_B\") != col(\"id_C\")) &\n", |
| 276 | + " (col(\"member_A\") != col(\"member_B\")) &\n", |
| 277 | + " (col(\"member_A\") != col(\"member_C\")) &\n", |
| 278 | + " (col(\"member_B\") != col(\"member_C\"))\n", |
| 279 | + " )\\\n", |
| 280 | + " .select(\"member_A\", \"member_B\", \"member_C\").show()" |
| 281 | + ] |
170 | 282 | }
|
171 | 283 | ],
|
172 | 284 | "metadata": {
|
173 | 285 | "application/vnd.databricks.v1+notebook": {
|
174 |
| - "computePreferences": null, |
| 286 | + "computePreferences": { |
| 287 | + "hardware": { |
| 288 | + "accelerator": null, |
| 289 | + "gpuPoolId": null, |
| 290 | + "memory": null |
| 291 | + } |
| 292 | + }, |
175 | 293 | "dashboards": [],
|
176 | 294 | "environmentMetadata": {
|
177 | 295 | "base_environment": "",
|
178 |
| - "environment_version": "1" |
| 296 | + "environment_version": "2" |
179 | 297 | },
|
180 | 298 | "inputWidgetPreferences": null,
|
181 | 299 | "language": "python",
|
|
0 commit comments