{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d91d9c4-e0d9-4747-aac3-2ad59f3f80b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wmfdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "50446eed-3a67-4146-92cc-e7e782530b13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPARK_HOME: /usr/lib/spark3\n",
      "Using Hadoop client lib jars at 3.2.0, provided by Spark.\n",
      "PYSPARK_PYTHON=/opt/conda-analytics/bin/python3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "23/07/13 19:44:54 WARN SparkConf: Note that spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in mesos/standalone/kubernetes and LOCAL_DIRS in YARN).\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12000. Attempting port 12001.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12001. Attempting port 12002.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12002. Attempting port 12003.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12003. Attempting port 12004.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12004. Attempting port 12005.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12005. Attempting port 12006.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12006. Attempting port 12007.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12007. Attempting port 12008.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12008. Attempting port 12009.\n",
      "23/07/13 19:44:54 WARN Utils: Service 'sparkDriver' could not bind on port 12009. Attempting port 12010.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4041. Attempting port 4042.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4042. Attempting port 4043.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4043. Attempting port 4044.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4044. Attempting port 4045.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4045. Attempting port 4046.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4046. Attempting port 4047.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4047. Attempting port 4048.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4048. Attempting port 4049.\n",
      "23/07/13 19:44:55 WARN Utils: Service 'SparkUI' could not bind on port 4049. Attempting port 4050.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13000. Attempting port 13001.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13001. Attempting port 13002.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13002. Attempting port 13003.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13003. Attempting port 13004.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13004. Attempting port 13005.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13005. Attempting port 13006.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13006. Attempting port 13007.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13007. Attempting port 13008.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13008. Attempting port 13009.\n",
      "23/07/13 19:45:02 WARN Utils: Service 'org.apache.spark.network.netty.NettyBlockTransferService' could not bind on port 13009. Attempting port 13010.\n",
      "23/07/13 19:45:03 WARN YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!\n"
     ]
    }
   ],
   "source": [
    "session = wmfdata.spark.create_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "df4575c7-ace3-485f-8332-6abbe56d0408",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"\"\"SELECT * from event.mediawiki_visual_editor_feature_use\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "532739d5-1674-4169-93d9-e0046fc00e34",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = session.sql(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e979c5b6-92f3-4dd3-b386-d7bf5e1e9bae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import functions as fun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "de26e70d-e6b3-4c80-99f4-37c7342fdeca",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu = df.select(fun.monotonically_increasing_id().alias(\"_id\"), \"*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "76b5e57c-dd8e-42d7-a4f6-6d4bd3f90fff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 3:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------------------------------------------------------------+-----------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+-------------+-------------------------------------+----------+----+-----+---+----+\n",
      "|_id|_schema|agent                                |custom_data                                                                                                                                                                |dt                      |http                                                                                                                                                                 |mediawiki                                     |meta                                                                                                                                                                 |name                         |page                                                              |performer                                                                    |user_agent_map                                                                                                                                      |is_wmf_domain|normalized_host                      |datacenter|year|month|day|hour|\n",
      "+---+-------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------------------------------------------------------------+-----------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+-------------+-------------------------------------+----------+----+-----+---+----+\n",
      "|0  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, mergeCells}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}         |2023-06-30T21:45:52.035Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:46:22.001Z, 5b73b36e-4d92-4862-b69a-fde718fb999f, 7b917860-6f58-4b5d-96ce-43e1d1d44bb6, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|1  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}               |2023-06-30T21:28:33.722Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:29:02.348Z, 9fb3a82a-0c6d-4909-82b3-40e6a3d5c258, a81c4854-4c91-4bc0-9e32-95f627aad979, mediawiki.visual_editor_feature_use, null}|vefu.window-open-from-command|{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|2  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, textStyle/bold}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}     |2023-06-30T21:36:13.637Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:36:33.393Z, 94cafaad-3b78-494e-904d-46ef151f8eef, 7b144317-92d5-404b-b4ce-d5a084d78175, mediawiki.visual_editor_feature_use, null}|vefu.toggle-selection        |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|3  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |2023-06-30T21:29:07.922Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:29:24.509Z, 4e8c70e8-ffbc-4163-b627-baf9f2893973, 993c52c2-0ec1-4b07-a15a-58db11e6b21a, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|4  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}               |2023-06-30T21:27:55.269Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:28:17.707Z, baf7033c-db94-4ccb-8cdc-745cd8ac380d, e01331ef-e495-46ff-8e34-8b0ef8f43535, mediawiki.visual_editor_feature_use, null}|vefu.panel-switch            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|5  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |2023-06-30T21:29:11.835Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:29:24.514Z, d695d25e-8e9b-4c06-8482-226e7344a6dd, 685bd4aa-d07e-49ed-9abb-90a550e33b10, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|6  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, citoid}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}             |2023-06-30T21:41:40.686Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:41:53.080Z, 4ee07017-ca31-4f83-80cb-ceecfe5f1198, 360b872c-4fda-4cb5-ba49-a6d15a8bcd8b, mediawiki.visual_editor_feature_use, null}|vefu.panel-switch            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|7  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |2023-06-30T21:28:52.267Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:29:02.352Z, ef297053-0924-4228-a95a-d25d5fe5063d, e85b353b-0b01-4b6b-bd15-72cb544fc40e, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|8  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, format}, editing_session_id -> {string, 4ae29ea8492333893949}, editor_interface -> {string, visualeditor}, integration -> {string, page}}             |2023-06-30T21:32:44.989Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0}, null, null}                                 |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:33:08.631Z, 7a46b104-2bf6-4a24-9fbd-2d95fd5825eb, a0904ab4-daac-472f-9010-1d63e20ecf12, mediawiki.visual_editor_feature_use, null}|vefu.mwHeading-1             |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Firefox, device_family -> Mac, wmf_app_version -> -}|true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|9  |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |2023-06-30T21:29:02.534Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:29:24.505Z, 81ef9aeb-6d25-4c6b-93bf-8beffb903bc3, 6f15c8d6-fd29-4479-b5c0-03858c68b16f, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|10 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, table}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}              |2023-06-30T21:46:01.814Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:46:22.029Z, 6d6684a4-8138-4913-ba50-74f8cccce10c, f1e4e546-569f-4906-9679-e76acb740ecf, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|11 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, textStyle/underline}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}|2023-06-30T21:40:33.394Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:40:39.049Z, 36e0e12b-4ea6-44a3-a288-0ad008248e90, 5a9984ef-ccbf-4039-8053-9649c40d40bf, mediawiki.visual_editor_feature_use, null}|vefu.toggle-selection        |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|12 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, reference}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}          |2023-06-30T21:38:22.859Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:38:27.701Z, afdc8984-8f55-45b5-a72e-8fc89db0dc58, 8c920340-26ef-466c-92b3-ed11a09d84b3, mediawiki.visual_editor_feature_use, null}|vefu.window-open-from-command|{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|13 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, table}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}              |2023-06-30T21:45:59.708Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:46:22.026Z, b45fffe0-8bfe-4dbb-a345-f1d10c8e4679, 13674b93-4937-421a-bf92-82494c1388f4, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|14 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, specialCharacter}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}   |2023-06-30T21:43:48.723Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:44:14.828Z, cfb5e1d1-8139-4ec1-8584-718f5156c284, e60adc74-d98f-4636-a229-ba6740a069db, mediawiki.visual_editor_feature_use, null}|vefu.dialog-abort            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|15 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, language}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}           |2023-06-30T21:36:55.277Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:37:25.345Z, cf45d6d5-dfe2-46f6-bd51-94d7ecc5cf19, f9e81dc4-1a1d-4590-9e67-dea95fd7560b, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|16 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, citoid}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}             |2023-06-30T21:41:51.618Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:41:53.076Z, bd6c8416-d14b-4bf2-946e-f1f19f490ffa, e481a6f3-9ac3-4e03-9271-7bed8952a059, mediawiki.visual_editor_feature_use, null}|vefu.dialog-reuse-choose     |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|17 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, sumCells}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}           |2023-06-30T21:46:04.524Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:46:22.031Z, 607fd547-b414-42a6-a767-d23b98b667a8, 4d6bec6b-48b0-43e4-86e1-cb394ee78aa0, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|18 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |2023-06-30T21:29:00.646Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:29:02.353Z, 7cc2e883-638e-4562-969f-3c92884dbe5c, d88346e4-dcc7-4b7d-aa4a-03c16ef7665f, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "|19 |null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, language}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}           |2023-06-30T21:40:24.401Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:40:39.045Z, 84855bb9-a109-4001-8280-af955c870720, 465a4f23-ad28-4ee1-ac0d-73021dca2e22, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -} |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |\n",
      "+---+-------+-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------------------------------------------------------------+-----------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+-------------+-------------------------------------+----------+----+-----+---+----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_vefu.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "49a526c1-3154-4949-af94-cfa814a13c9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_vefu.write.saveAsTable(\"cjming.mp_vefu_copy_with_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "04a4d9a5-b547-4841-96ff-5ec289da5f7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu_cd = df.select(fun.monotonically_increasing_id().alias(\"_id\"), \"custom_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "026fd098-f5b1-46c6-bf52-14ad1818b4ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|_id|custom_data                                                                                                                                                                |\n",
      "+---+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|0  |{feature -> {string, mergeCells}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}         |\n",
      "|1  |{feature -> {string, link}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}               |\n",
      "|2  |{feature -> {string, textStyle/bold}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}     |\n",
      "|3  |{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |\n",
      "|4  |{feature -> {string, link}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}               |\n",
      "|5  |{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |\n",
      "|6  |{feature -> {string, citoid}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}             |\n",
      "|7  |{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |\n",
      "|8  |{feature -> {string, format}, editing_session_id -> {string, 4ae29ea8492333893949}, editor_interface -> {string, visualeditor}, integration -> {string, page}}             |\n",
      "|9  |{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |\n",
      "|10 |{feature -> {string, table}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}              |\n",
      "|11 |{feature -> {string, textStyle/underline}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}|\n",
      "|12 |{feature -> {string, reference}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}          |\n",
      "|13 |{feature -> {string, table}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}              |\n",
      "|14 |{feature -> {string, specialCharacter}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}   |\n",
      "|15 |{feature -> {string, language}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}           |\n",
      "|16 |{feature -> {string, citoid}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}             |\n",
      "|17 |{feature -> {string, sumCells}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}           |\n",
      "|18 |{feature -> {string, link/internal}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |\n",
      "|19 |{feature -> {string, language}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}           |\n",
      "+---+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_vefu_cd.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9eec156b-544f-4380-90e1-df1792ddffa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu_cd1 = df_vefu.select(\"_id\", fun.explode(\"custom_data\").alias(\"custom_data_name\", \"cd_value\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "de069a9f-14df-4b83-b355-7976409633f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+------------------+------------------------------+\n",
      "|_id|custom_data_name  |cd_value                      |\n",
      "+---+------------------+------------------------------+\n",
      "|0  |feature           |{string, mergeCells}          |\n",
      "|0  |editing_session_id|{string, b55da8bc535cc0cb9fef}|\n",
      "|0  |editor_interface  |{string, visualeditor}        |\n",
      "|0  |integration       |{string, page}                |\n",
      "|1  |feature           |{string, link}                |\n",
      "|1  |editing_session_id|{string, b55da8bc535cc0cb9fef}|\n",
      "|1  |editor_interface  |{string, visualeditor}        |\n",
      "|1  |integration       |{string, page}                |\n",
      "|2  |feature           |{string, textStyle/bold}      |\n",
      "|2  |editing_session_id|{string, b55da8bc535cc0cb9fef}|\n",
      "|2  |editor_interface  |{string, visualeditor}        |\n",
      "|2  |integration       |{string, page}                |\n",
      "|3  |feature           |{string, link/internal}       |\n",
      "|3  |editing_session_id|{string, b55da8bc535cc0cb9fef}|\n",
      "|3  |editor_interface  |{string, visualeditor}        |\n",
      "|3  |integration       |{string, page}                |\n",
      "|4  |feature           |{string, link}                |\n",
      "|4  |editing_session_id|{string, b55da8bc535cc0cb9fef}|\n",
      "|4  |editor_interface  |{string, visualeditor}        |\n",
      "|4  |integration       |{string, page}                |\n",
      "+---+------------------+------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_vefu_cd1.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "eebcad61-b40d-4a93-98b7-b03a9bf5d47d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu_cd2 = df_vefu_cd1.select(\"_id\", \"custom_data_name\", \"cd_value.*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f07ab1bb-4330-4454-a315-80d2537f7bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu_custom_data = df_vefu_cd2.withColumnRenamed(\"data_type\", \"custom_data_type\").withColumnRenamed(\"value\", \"custom_data_value\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dd33f248-c588-4983-bace-a8da0d9dbfbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 6:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+------------------+----------------+--------------------+\n",
      "|_id|custom_data_name  |custom_data_type|custom_data_value   |\n",
      "+---+------------------+----------------+--------------------+\n",
      "|0  |feature           |string          |mergeCells          |\n",
      "|0  |editing_session_id|string          |b55da8bc535cc0cb9fef|\n",
      "|0  |editor_interface  |string          |visualeditor        |\n",
      "|0  |integration       |string          |page                |\n",
      "|1  |feature           |string          |link                |\n",
      "|1  |editing_session_id|string          |b55da8bc535cc0cb9fef|\n",
      "|1  |editor_interface  |string          |visualeditor        |\n",
      "|1  |integration       |string          |page                |\n",
      "|2  |feature           |string          |textStyle/bold      |\n",
      "|2  |editing_session_id|string          |b55da8bc535cc0cb9fef|\n",
      "|2  |editor_interface  |string          |visualeditor        |\n",
      "|2  |integration       |string          |page                |\n",
      "|3  |feature           |string          |link/internal       |\n",
      "|3  |editing_session_id|string          |b55da8bc535cc0cb9fef|\n",
      "|3  |editor_interface  |string          |visualeditor        |\n",
      "|3  |integration       |string          |page                |\n",
      "|4  |feature           |string          |link                |\n",
      "|4  |editing_session_id|string          |b55da8bc535cc0cb9fef|\n",
      "|4  |editor_interface  |string          |visualeditor        |\n",
      "|4  |integration       |string          |page                |\n",
      "+---+------------------+----------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_vefu_custom_data.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "171dbc34-7305-4ab2-9156-1520354df1d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_vefu_custom_data.write.saveAsTable(\"cjming.mp_vefu_custom_data_vertical\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "253a9545-4ae8-4082-ad66-403a341041d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu_cd_flat = df_vefu_custom_data.groupBy(\"_id\").agg(fun.first(\"custom_data_name\").alias(\"custom_data_name1\"),\n",
    "                                 fun.first(\"custom_data_type\").alias(\"custom_data_type1\"),\n",
    "                                 fun.first(\"custom_data_value\").alias(\"custom_data_value1\"),\n",
    "                                 fun.last(\"custom_data_name\").alias(\"custom_data_name2\"),\n",
    "                                 fun.last(\"custom_data_type\").alias(\"custom_data_type2\"),\n",
    "                                 fun.last(\"custom_data_value\").alias(\"custom_data_value2\")\n",
    "                                                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e23bb5a7-4682-4113-95bb-79d9bb3a3c0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-----------------+-----------------+------------------+-----------------+-----------------+------------------+\n",
      "|_id        |custom_data_name1|custom_data_type1|custom_data_value1|custom_data_name2|custom_data_type2|custom_data_value2|\n",
      "+-----------+-----------------+-----------------+------------------+-----------------+-----------------+------------------+\n",
      "|93         |feature          |string           |mergeCells        |integration      |string           |page              |\n",
      "|171        |feature          |string           |table             |integration      |string           |page              |\n",
      "|232        |feature          |string           |mwGallery         |integration      |string           |page              |\n",
      "|271        |feature          |string           |format            |integration      |string           |page              |\n",
      "|405        |feature          |string           |transclusion      |integration      |string           |page              |\n",
      "|849        |feature          |string           |preview           |integration      |string           |page              |\n",
      "|1238       |feature          |string           |media             |integration      |string           |page              |\n",
      "|1365       |feature          |string           |preview           |integration      |string           |page              |\n",
      "|1785       |feature          |string           |media             |integration      |string           |page              |\n",
      "|1898       |feature          |string           |mwSave            |integration      |string           |page              |\n",
      "|1990       |feature          |string           |mergeCells        |integration      |string           |page              |\n",
      "|2154       |feature          |string           |transclusion      |integration      |string           |page              |\n",
      "|8589934633 |feature          |string           |citoid            |integration      |string           |page              |\n",
      "|8589934863 |feature          |string           |link              |integration      |string           |page              |\n",
      "|8589934969 |feature          |string           |clipboard         |integration      |string           |page              |\n",
      "|17179869353|feature          |string           |transclusion      |integration      |string           |page              |\n",
      "|17179869371|feature          |string           |editor-switch     |integration      |string           |discussiontools   |\n",
      "|372        |feature          |string           |format            |integration      |string           |page              |\n",
      "|707        |feature          |string           |preview           |integration      |string           |page              |\n",
      "|722        |feature          |string           |preview           |integration      |string           |page              |\n",
      "+-----------+-----------------+-----------------+------------------+-----------------+-----------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_vefu_cd_flat.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "2b5a51cb-7988-4d83-8fe8-8bb90e7d966a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vefu_core_custom_data_2 = df_vefu.join(df_vefu_cd_flat, df_vefu._id == df_vefu_cd_flat._id, \"inner\").drop(df_vefu_cd_flat._id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5dc31712-45ce-48ba-a15f-0b8d9d8c4e41",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------------------------------------------------------------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+-------------+-------------------------------------+----------+----+-----+---+----+-----------+-----------------+-----------------+------------------+-----------------+-----------------+------------------+\n",
      "|_schema|agent                                |custom_data                                                                                                                                                                     |dt                      |http                                                                                                                                                                 |mediawiki                                     |meta                                                                                                                                                                 |name                         |page                                                              |performer                                                                      |user_agent_map                                                                                                                                          |is_wmf_domain|normalized_host                      |datacenter|year|month|day|hour|_id        |custom_data_name1|custom_data_type1|custom_data_value1|custom_data_name2|custom_data_type2|custom_data_value2|\n",
      "+-------+-------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------------------------------------------------------------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+-------------+-------------------------------------+----------+----+-----+---+----+-----------+-----------------+-----------------+------------------+-----------------+-----------------+------------------+\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, mergeCells}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}              |2023-06-30T21:45:48.811Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:45:50.933Z, a04d119b-c02d-4eab-899b-d73f1aaaf3aa, 18ee7500-abe4-49cf-b0c4-ee4bdd182f37, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |93         |feature          |string           |mergeCells        |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, table}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                   |2023-06-30T21:46:05.638Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:46:22.110Z, 43c6018d-a668-41e4-80df-5621aca3d916, 9c98c453-978b-4e19-985d-557e9194fbe9, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |171        |feature          |string           |table             |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, mwGallery}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}               |2023-06-30T21:48:37.612Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:49:07.307Z, 5425b248-0f90-4613-b052-e008de8e751a, c43ecd18-a09c-4968-89df-784b4f82a70d, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |232        |feature          |string           |mwGallery         |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, format}, editing_session_id -> {string, 4ae29ea8492333893949}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                  |2023-06-30T21:32:42.474Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0}, null, null}                                 |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:33:08.633Z, 31f950bc-58c4-43aa-9a14-0c01e15c1a4f, 2c4d8183-8cb3-4de7-924a-e952073bcd33, mediawiki.visual_editor_feature_use, null}|vefu.mwHeading-1             |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Firefox, device_family -> Mac, wmf_app_version -> -}    |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |271        |feature          |string           |format            |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, transclusion}, editing_session_id -> {string, 4c6e6171ccd00b8c33c8}, editor_interface -> {string, visualeditor}, integration -> {string, page}}            |2023-05-06T14:45:53.191Z|{null, null, null, {user-agent -> Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0}, null, null}                                     |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-05-06T14:45:58.257Z, 900a90d6-9b3e-442c-85af-8fe6d7ae340c, 520f408b-f636-4217-a603-92283c76139c, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, null, null, null, 0, null, null, null, null, null, null, null, null}    |{os_family -> Windows, os_major -> 10, os_minor -> -, browser_major -> 112, browser_family -> Firefox, device_family -> Other, wmf_app_version -> -}    |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|5    |6  |14  |405        |feature          |string           |transclusion      |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, preview}, editing_session_id -> {string, 87e61321af1feddc886067a718ece408}, editor_interface -> {string, wikitext}, integration -> {string, page}}         |2023-06-22T15:03:29.255Z|{null, null, null, {user-agent -> Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}       |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-22T15:03:45.202Z, bde3403e-5864-4680-9620-c5417e6f07cb, ddc54eb2-2480-4e57-8110-46e13afdad1b, mediawiki.visual_editor_feature_use, null}|vefu.preview-realtime-loaded |{null, null, null, null, null, null, null, null, null, null, null}|{null, 21, null, null, 58457, null, null, null, null, null, null, null, null}  |{os_family -> Chrome OS, os_major -> 14541, os_minor -> 0, browser_major -> 114, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}|true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |22 |15  |849        |feature          |string           |preview           |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, media}, editing_session_id -> {string, 1673dacff11d1d6815ec}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                   |2023-06-23T19:23:44.718Z|{null, null, null, {user-agent -> Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36}, null, null}      |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-23T19:23:56.342Z, 362e0423-2dd2-4d0a-9b74-69184704309b, fa95f683-2e02-470a-8188-02b2c0ffaaa0, mediawiki.visual_editor_feature_use, null}|vefu.dialog-insert           |{null, null, null, null, null, null, null, null, null, null, null}|{null, null, null, null, 0, null, null, null, null, null, null, null, null}    |{os_family -> Windows, os_major -> 10, os_minor -> -, browser_major -> 117, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|codfw     |2023|6    |23 |19  |1238       |feature          |string           |media             |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, preview}, editing_session_id -> {string, e0485c2f7341551d33af9c9dc9d67bec}, editor_interface -> {string, wikitext}, integration -> {string, page}}         |2023-06-22T14:24:18.457Z|{null, null, null, {user-agent -> Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}       |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-22T14:24:49.662Z, 22275b8f-e244-4a41-bcec-fa4868c2ac4b, ab641b1f-81f7-4a1d-9ec5-ed403d594f04, mediawiki.visual_editor_feature_use, null}|vefu.preview-realtime-loaded |{null, null, null, null, null, null, null, null, null, null, null}|{null, 20, null, null, 58457, null, null, null, null, null, null, null, null}  |{os_family -> Chrome OS, os_major -> 14541, os_minor -> 0, browser_major -> 114, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}|true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |22 |14  |1365       |feature          |string           |preview           |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, media}, editing_session_id -> {string, 7d00c93e27bfc0d40c3c}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                   |2023-06-19T14:50:41.002Z|{null, null, null, {user-agent -> Mozilla/5.0 (X11; Linux ppc64le; rv:102.0) Gecko/20100101 Firefox/102.0}, null, null}                                              |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-19T14:51:10.500Z, 0602544a-7ce9-4337-9d02-dbd3e9da2aa4, 21811523-8b94-432a-a06d-6a5746b62987, mediawiki.visual_editor_feature_use, null}|vefu.search-change-query     |{null, null, null, null, null, null, null, null, null, null, null}|{null, 423, null, null, 44719, null, null, null, null, null, null, null, null} |{os_family -> Linux, os_major -> -, os_minor -> -, browser_major -> 102, browser_family -> Firefox, device_family -> Other, wmf_app_version -> -}       |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |19 |14  |1785       |feature          |string           |media             |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, mwSave}, editing_session_id -> {string, q4ptsd08hl4d0r4r6do1pkgv6mg3q2vh}, editor_interface -> {string, visualeditor}, integration -> {string, page}}      |2023-05-15T23:49:56.994Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0}, null, null}                                 |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-05-15T23:50:13.554Z, 2c94b2f7-b48b-4b18-9d34-f6635b8b2b6a, 0eea3924-2869-4d94-97d4-1e1b6c47e094, mediawiki.visual_editor_feature_use, null}|vefu.dialog-save             |{null, null, null, null, null, null, null, null, null, null, null}|{null, 1055, null, null, 27425, null, null, null, null, null, null, null, null}|{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 113, browser_family -> Firefox, device_family -> Mac, wmf_app_version -> -}    |true         |{wikipedia, test, [], org, wikipedia}|codfw     |2023|5    |15 |23  |1898       |feature          |string           |mwSave            |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, mergeCells}, editing_session_id -> {string, 330b8af277bd732497a87636343ca6eb}, editor_interface -> {string, visualeditor}, integration -> {string, page}}  |2023-06-16T07:18:21.107Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-16T07:18:39.696Z, fab26947-5d81-428c-864a-4050a612f704, f8477cf3-817c-4669-9484-393a4fdd1297, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 28, null, null, 19605, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |16 |7   |1990       |feature          |string           |mergeCells        |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, transclusion}, editing_session_id -> {string, 4c6e6171ccd00b8c33c8}, editor_interface -> {string, visualeditor}, integration -> {string, page}}            |2023-05-06T15:59:54.247Z|{null, null, null, {user-agent -> Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0}, null, null}                                     |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-05-06T16:00:11.843Z, 95f1b8ff-859f-456f-8127-3e0174e7274c, 68e5db5f-7f7d-40bb-b5bf-6de9a48f61ac, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, null, null, null, 0, null, null, null, null, null, null, null, null}    |{os_family -> Windows, os_major -> 10, os_minor -> -, browser_major -> 112, browser_family -> Firefox, device_family -> Other, wmf_app_version -> -}    |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|5    |6  |16  |2154       |feature          |string           |transclusion      |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, citoid}, editing_session_id -> {string, d933eb7a72ab8439bad3}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                  |2023-04-14T22:30:34.431Z|{null, null, null, {user-agent -> Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36}, null, null}      |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-04-14T22:30:57.202Z, 88c6a959-133a-4b9b-86ad-ad31dfd31d14, 8efe6a8a-80d1-4273-9d2f-472c331ac910, mediawiki.visual_editor_feature_use, null}|vefu.window-open-from-command|{null, null, null, null, null, null, null, null, null, null, null}|{null, 2267, null, null, 49714, null, null, null, null, null, null, null, null}|{os_family -> Windows, os_major -> 10, os_minor -> -, browser_major -> 111, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|codfw     |2023|4    |14 |22  |8589934633 |feature          |string           |citoid            |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, link}, editing_session_id -> {string, 0ddb10e5d6ff472afed2}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                    |2023-06-10T08:34:14.207Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-10T08:34:17.515Z, 92d67ab1-5cff-49d3-9950-62db209af88d, f07aba3e-876f-412a-85f1-433f1388efda, mediawiki.visual_editor_feature_use, null}|vefu.dialog-abort            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 0, null, null, 58389, null, null, null, null, null, null, null, null}   |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |10 |8   |8589934863 |feature          |string           |link              |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, clipboard}, editing_session_id -> {string, ddb7fe10db308281e9a2}, editor_interface -> {string, wikitext-2017}, integration -> {string, page}}              |2023-06-05T17:04:16.312Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0}, null, null}                                 |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-05T17:04:30.809Z, f2142eae-7a07-4c1c-b4c3-82755208a069, 5e10b658-7264-41a5-b7ee-518f93d64c03, mediawiki.visual_editor_feature_use, null}|vefu.paste                   |{null, null, null, null, null, null, null, null, null, null, null}|{null, 27, null, null, 50439, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 113, browser_family -> Firefox, device_family -> Mac, wmf_app_version -> -}    |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |5  |17  |8589934969 |feature          |string           |clipboard         |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, transclusion}, editing_session_id -> {string, 0f5a694c978676ac4c1a}, editor_interface -> {string, visualeditor}, integration -> {string, page}}            |2023-05-04T14:41:25.473Z|{null, null, null, {user-agent -> Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36}, null, null}                |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-05-04T14:41:52.147Z, 997314d8-9442-4bc0-ace4-259e1e686bdc, 85db52cd-601a-4de4-9495-2a0ef07d11e0, mediawiki.visual_editor_feature_use, null}|vefu.context-show            |{null, null, null, null, null, null, null, null, null, null, null}|{null, 1, null, null, 40567, null, null, null, null, null, null, null, null}   |{os_family -> Linux, os_major -> -, os_minor -> -, browser_major -> 112, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}        |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|5    |4  |14  |17179869353|feature          |string           |transclusion      |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, editor-switch}, editing_session_id -> {string, 8fff3bca5953ef0c279e}, editor_interface -> {string, visualeditor}, integration -> {string, discussiontools}}|2023-06-30T10:23:01.235Z|{null, null, null, {user-agent -> Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}             |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T10:23:16.943Z, 37e953fa-78df-4739-af91-639817c59377, 77967d25-1e8f-439c-aa66-4642d99471e9, mediawiki.visual_editor_feature_use, null}|vefu.source-nwe-desktop      |{null, null, null, null, null, null, null, null, null, null, null}|{null, null, null, null, 0, null, null, null, null, null, null, null, null}    |{os_family -> Android, os_major -> 10, os_minor -> -, browser_major -> 114, browser_family -> Chrome, device_family -> K, wmf_app_version -> -}         |true         |{wikipedia, test, [], org, wikipedia}|codfw     |2023|6    |30 |10  |17179869371|feature          |string           |editor-switch     |integration      |string           |discussiontools   |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, format}, editing_session_id -> {string, b55da8bc535cc0cb9fef}, editor_interface -> {string, visualeditor}, integration -> {string, page}}                  |2023-06-30T21:33:06.232Z|{null, null, null, {user-agent -> Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}|{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-30T21:33:26.549Z, e39f71e8-4d4b-4939-8bba-0c43c4730341, 8597600c-f03a-444f-aa67-5cf4a7e42512, mediawiki.visual_editor_feature_use, null}|vefu.mwPreformatted          |{null, null, null, null, null, null, null, null, null, null, null}|{null, 29, null, null, 50439, null, null, null, null, null, null, null, null}  |{os_family -> Mac OS X, os_major -> 10, os_minor -> 15, browser_major -> 114, browser_family -> Chrome, device_family -> Mac, wmf_app_version -> -}     |true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |30 |21  |372        |feature          |string           |format            |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, preview}, editing_session_id -> {string, 197154a87e85c5c9b8411bcf4c176083}, editor_interface -> {string, wikitext}, integration -> {string, page}}         |2023-06-18T04:19:51.527Z|{null, null, null, {user-agent -> Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}       |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-18T04:20:12.173Z, 859c53b5-c7fb-46c9-991a-6323d17f2310, f9d4f6ec-4518-4c33-9a3a-cee0c2601d31, mediawiki.visual_editor_feature_use, null}|vefu.preview-realtime-inuse  |{null, null, null, null, null, null, null, null, null, null, null}|{null, 2, null, null, 58457, null, null, null, null, null, null, null, null}   |{os_family -> Chrome OS, os_major -> 14541, os_minor -> 0, browser_major -> 114, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}|true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |18 |4   |707        |feature          |string           |preview           |integration      |string           |page              |\n",
      "|null   |{null, mediawiki_js, desktop_browser}|{feature -> {string, preview}, editing_session_id -> {string, 7841434794a892528b4d907fc7ee7375}, editor_interface -> {string, wikitext}, integration -> {string, page}}         |2023-06-18T04:06:06.207Z|{null, null, null, {user-agent -> Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36}, null, null}       |{testwiki, null, null, null, null, null, null}|{test.wikipedia.org, 2023-06-18T04:06:36.346Z, 7360046a-8ae8-4bfb-84d0-9779a226805f, a6eec0b4-a311-477b-a670-548e4d157594, mediawiki.visual_editor_feature_use, null}|vefu.preview-realtime-inuse  |{null, null, null, null, null, null, null, null, null, null, null}|{null, 1, null, null, 58457, null, null, null, null, null, null, null, null}   |{os_family -> Chrome OS, os_major -> 14541, os_minor -> 0, browser_major -> 114, browser_family -> Chrome, device_family -> Other, wmf_app_version -> -}|true         |{wikipedia, test, [], org, wikipedia}|eqiad     |2023|6    |18 |4   |722        |feature          |string           |preview           |integration      |string           |page              |\n",
      "+-------+-------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------------------------------------------------------------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+-------------+-------------------------------------+----------+----+-----+---+----+-----------+-----------------+-----------------+------------------+-----------------+-----------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_vefu_core_custom_data_2.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bc37b5cc-713d-42e0-8847-3269a6a1ecb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_vefu_core_custom_data_2.write.saveAsTable(\"cjming.mp_vefu_core_custom_data_2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02755165-64b5-4c7a-9365-70d69d378d20",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}