|
4 | 4 | "cell_type": "markdown",
|
5 | 5 | "metadata": {
|
6 | 6 | "application/vnd.databricks.v1+cell": {
|
7 |
| - "cellMetadata": {}, |
| 7 | + "cellMetadata": { |
| 8 | + "byteLimit": 2048000, |
| 9 | + "rowLimit": 10000 |
| 10 | + }, |
8 | 11 | "inputWidgets": {},
|
9 | 12 | "nuid": "83d38679-f742-44bb-952b-b541e178ddc8",
|
10 | 13 | "showTitle": false,
|
|
21 | 24 | "execution_count": 0,
|
22 | 25 | "metadata": {
|
23 | 26 | "application/vnd.databricks.v1+cell": {
|
24 |
| - "cellMetadata": {}, |
| 27 | + "cellMetadata": { |
| 28 | + "byteLimit": 2048000, |
| 29 | + "rowLimit": 10000 |
| 30 | + }, |
25 | 31 | "inputWidgets": {},
|
26 | 32 | "nuid": "1daf4387-60f2-4c66-b119-d9ce2cd54aa2",
|
27 | 33 | "showTitle": false,
|
|
40 | 46 | "cell_type": "markdown",
|
41 | 47 | "metadata": {
|
42 | 48 | "application/vnd.databricks.v1+cell": {
|
43 |
| - "cellMetadata": {}, |
| 49 | + "cellMetadata": { |
| 50 | + "byteLimit": 2048000, |
| 51 | + "rowLimit": 10000 |
| 52 | + }, |
44 | 53 | "inputWidgets": {},
|
45 | 54 | "nuid": "83ab9b20-fd4f-4392-ae0c-ff0580c3c7c1",
|
46 | 55 | "showTitle": false,
|
|
86 | 95 | "**Subscriptions table:**\n",
|
87 | 96 | "| account_id | start_date | end_date |\n",
|
88 | 97 | "|------------|------------|------------|\n",
|
89 |
| - "| 9 | 2020-02-18 | 2021-10-30 |\n", |
| 98 | + "| 9 | 2021-02-18 | 2021-10-30 |\n", |
90 | 99 | "| 3 | 2021-09-21 | 2021-11-13 |\n",
|
91 | 100 | "| 11 | 2020-02-28 | 2020-08-18 |\n",
|
92 | 101 | "| 13 | 2021-04-20 | 2021-09-22 |\n",
|
93 |
| - "| 4 | 2020-10-26 | 2021-05-08 |\n", |
| 102 | + "| 4 | 2021-01-26 | 2021-05-08 |\n", |
94 | 103 | "| 5 | 2020-09-11 | 2021-01-17 |\n",
|
95 | 104 | "\n",
|
96 | 105 | "**Streams table:**\n",
|
|
117 | 126 | "execution_count": 0,
|
118 | 127 | "metadata": {
|
119 | 128 | "application/vnd.databricks.v1+cell": {
|
120 |
| - "cellMetadata": {}, |
| 129 | + "cellMetadata": { |
| 130 | + "byteLimit": 2048000, |
| 131 | + "rowLimit": 10000 |
| 132 | + }, |
121 | 133 | "inputWidgets": {},
|
122 | 134 | "nuid": "e39af07f-85df-4e21-8a5f-c65604a811f3",
|
123 | 135 | "showTitle": false,
|
124 | 136 | "tableResultSettingsMap": {},
|
125 | 137 | "title": ""
|
126 | 138 | }
|
127 | 139 | },
|
128 |
| - "outputs": [], |
| 140 | + "outputs": [ |
| 141 | + { |
| 142 | + "output_type": "stream", |
| 143 | + "name": "stdout", |
| 144 | + "output_type": "stream", |
| 145 | + "text": [ |
| 146 | + "+----------+----------+----------+\n|account_id|start_date| end_date|\n+----------+----------+----------+\n| 9|2021-02-18|2021-10-30|\n| 3|2021-09-21|2021-11-13|\n| 11|2020-02-28|2020-08-18|\n| 13|2021-04-20|2021-09-22|\n| 4|2021-01-26|2021-05-08|\n| 5|2020-09-11|2021-01-17|\n+----------+----------+----------+\n\n+----------+----------+-----------+\n|session_id|account_id|stream_date|\n+----------+----------+-----------+\n| 14| 9| 2020-05-16|\n| 16| 3| 2021-10-27|\n| 18| 11| 2020-04-29|\n| 17| 13| 2021-08-08|\n| 19| 4| 2020-12-31|\n| 13| 5| 2021-01-05|\n+----------+----------+-----------+\n\n" |
| 147 | + ] |
| 148 | + } |
| 149 | + ], |
129 | 150 | "source": [
|
130 | 151 | "subscriptions_data_2020 = [\n",
|
131 |
| - " (9, \"2020-02-18\", \"2021-10-30\"),\n", |
| 152 | + " (9, \"2021-02-18\", \"2021-10-30\"),\n", |
132 | 153 | " (3, \"2021-09-21\", \"2021-11-13\"),\n",
|
133 | 154 | " (11, \"2020-02-28\", \"2020-08-18\"),\n",
|
134 | 155 | " (13, \"2021-04-20\", \"2021-09-22\"),\n",
|
135 |
| - " (4, \"2020-10-26\", \"2021-05-08\"),\n", |
136 |
| - " (5, \"2020-09-11\", \"2021-01-17\"),\n", |
| 156 | + " (4, \"2021-01-26\", \"2021-05-08\"),\n", |
| 157 | + " (5, \"2020-09-11\", \"2021-01-17\")\n", |
137 | 158 | "]\n",
|
138 | 159 | "\n",
|
139 | 160 | "subscriptions_columns_2020 = [\"account_id\", \"start_date\", \"end_date\"]\n",
|
|
146 | 167 | " (18, 11, \"2020-04-29\"),\n",
|
147 | 168 | " (17, 13, \"2021-08-08\"),\n",
|
148 | 169 | " (19, 4, \"2020-12-31\"),\n",
|
149 |
| - " (13, 5, \"2021-01-05\"),\n", |
| 170 | + " (13, 5, \"2021-01-05\")\n", |
150 | 171 | "]\n",
|
151 | 172 | "\n",
|
152 | 173 | "streams_columns_2020 = [\"session_id\", \"account_id\", \"stream_date\"]\n",
|
153 | 174 | "streams_df_2020 = spark.createDataFrame(streams_data_2020, streams_columns_2020)\n",
|
154 | 175 | "streams_df_2020.show()"
|
155 | 176 | ]
|
| 177 | + }, |
| 178 | + { |
| 179 | + "cell_type": "code", |
| 180 | + "execution_count": 0, |
| 181 | + "metadata": { |
| 182 | + "application/vnd.databricks.v1+cell": { |
| 183 | + "cellMetadata": { |
| 184 | + "byteLimit": 2048000, |
| 185 | + "rowLimit": 10000 |
| 186 | + }, |
| 187 | + "inputWidgets": {}, |
| 188 | + "nuid": "0901c1c2-69b2-4da0-b035-e904e9da7390", |
| 189 | + "showTitle": false, |
| 190 | + "tableResultSettingsMap": {}, |
| 191 | + "title": "" |
| 192 | + } |
| 193 | + }, |
| 194 | + "outputs": [], |
| 195 | + "source": [ |
| 196 | + "subscriptions_df_2020 = subscriptions_df_2020\\\n", |
| 197 | + " .withColumn(\"start_date\", col(\"start_date\").cast(\"date\")) \\\n", |
| 198 | + " .withColumn(\"end_date\", col(\"end_date\").cast(\"date\"))" |
| 199 | + ] |
| 200 | + }, |
| 201 | + { |
| 202 | + "cell_type": "code", |
| 203 | + "execution_count": 0, |
| 204 | + "metadata": { |
| 205 | + "application/vnd.databricks.v1+cell": { |
| 206 | + "cellMetadata": { |
| 207 | + "byteLimit": 2048000, |
| 208 | + "rowLimit": 10000 |
| 209 | + }, |
| 210 | + "inputWidgets": {}, |
| 211 | + "nuid": "903bc630-1067-4606-a2b3-05b63ed59c29", |
| 212 | + "showTitle": false, |
| 213 | + "tableResultSettingsMap": {}, |
| 214 | + "title": "" |
| 215 | + } |
| 216 | + }, |
| 217 | + "outputs": [], |
| 218 | + "source": [ |
| 219 | + "streams_df_2020 = streams_df_2020\\\n", |
| 220 | + " .withColumn(\"stream_date\", col(\"stream_date\").cast(\"date\"))" |
| 221 | + ] |
| 222 | + }, |
| 223 | + { |
| 224 | + "cell_type": "code", |
| 225 | + "execution_count": 0, |
| 226 | + "metadata": { |
| 227 | + "application/vnd.databricks.v1+cell": { |
| 228 | + "cellMetadata": { |
| 229 | + "byteLimit": 2048000, |
| 230 | + "rowLimit": 10000 |
| 231 | + }, |
| 232 | + "inputWidgets": {}, |
| 233 | + "nuid": "4b4707d2-78f1-464e-8d32-84556314f070", |
| 234 | + "showTitle": false, |
| 235 | + "tableResultSettingsMap": {}, |
| 236 | + "title": "" |
| 237 | + } |
| 238 | + }, |
| 239 | + "outputs": [], |
| 240 | + "source": [ |
| 241 | + "subs_2021 = subscriptions_df_2020.filter(year(\"start_date\") == 2021)" |
| 242 | + ] |
| 243 | + }, |
| 244 | + { |
| 245 | + "cell_type": "code", |
| 246 | + "execution_count": 0, |
| 247 | + "metadata": { |
| 248 | + "application/vnd.databricks.v1+cell": { |
| 249 | + "cellMetadata": { |
| 250 | + "byteLimit": 2048000, |
| 251 | + "rowLimit": 10000 |
| 252 | + }, |
| 253 | + "inputWidgets": {}, |
| 254 | + "nuid": "3cc33f83-171a-4bc0-8f34-b5ede8d4f5ef", |
| 255 | + "showTitle": false, |
| 256 | + "tableResultSettingsMap": {}, |
| 257 | + "title": "" |
| 258 | + } |
| 259 | + }, |
| 260 | + "outputs": [], |
| 261 | + "source": [ |
| 262 | + "streams_2021 = streams_df_2020.filter(year(\"stream_date\") == 2021)" |
| 263 | + ] |
| 264 | + }, |
| 265 | + { |
| 266 | + "cell_type": "code", |
| 267 | + "execution_count": 0, |
| 268 | + "metadata": { |
| 269 | + "application/vnd.databricks.v1+cell": { |
| 270 | + "cellMetadata": { |
| 271 | + "byteLimit": 2048000, |
| 272 | + "rowLimit": 10000 |
| 273 | + }, |
| 274 | + "inputWidgets": {}, |
| 275 | + "nuid": "528f1f88-1726-41d2-a401-51727e965d43", |
| 276 | + "showTitle": false, |
| 277 | + "tableResultSettingsMap": {}, |
| 278 | + "title": "" |
| 279 | + } |
| 280 | + }, |
| 281 | + "outputs": [ |
| 282 | + { |
| 283 | + "output_type": "stream", |
| 284 | + "name": "stdout", |
| 285 | + "output_type": "stream", |
| 286 | + "text": [ |
| 287 | + "+--------------+\n|accounts_count|\n+--------------+\n| 2|\n+--------------+\n\n" |
| 288 | + ] |
| 289 | + } |
| 290 | + ], |
| 291 | + "source": [ |
| 292 | + "subs_2021.join(streams_2021, \"account_id\", \"left\")\\\n", |
| 293 | + " .filter(streams_2021[\"session_id\"].isNull())\\\n", |
| 294 | + " .select(countDistinct(\"account_id\").alias(\"accounts_count\")).show()" |
| 295 | + ] |
156 | 296 | }
|
157 | 297 | ],
|
158 | 298 | "metadata": {
|
159 | 299 | "application/vnd.databricks.v1+notebook": {
|
160 |
| - "computePreferences": null, |
| 300 | + "computePreferences": { |
| 301 | + "hardware": { |
| 302 | + "accelerator": null, |
| 303 | + "gpuPoolId": null, |
| 304 | + "memory": null |
| 305 | + } |
| 306 | + }, |
161 | 307 | "dashboards": [],
|
162 | 308 | "environmentMetadata": {
|
163 | 309 | "base_environment": "",
|
164 |
| - "environment_version": "1" |
| 310 | + "environment_version": "2" |
165 | 311 | },
|
166 | 312 | "inputWidgetPreferences": null,
|
167 | 313 | "language": "python",
|
|
0 commit comments