WMDE Autumn Banner Campaign 2019: Interim Report

Feedback should be send to goran.milovanovic_ext@wikimedia.de.

The campaign is run from 2019/11/01 to 2019/11/09.

CURRENT UPDATE: Complete dataset as of 2019/11/09.

0. Data Acquisiton

NOTE: the Data Acquisition code chunk is not fully reproducible from this Report. The data are collected by running the script ABC_2019_PRODUCTION.R from the stat1004 WMF statistics server, collecting the data as .tsv and .csv files, aggregating on the server with ABC_2019_Aggregation.R, copying manually and processing locally.

0.1 Daily Update

# !diagnostics off
### --- script: ABC_2019_PRODUCTION.R
### --- Data Acquisition for the Autumn Banner Campaign 2019
### --- run from stat1004
### --- /home/goransm/Analytics/NewEditors/Campaigns/2019AuBC/

### --- libraries
library(tidyverse)
library(data.table)
library(lubridate)

### --- dir structure
campaignPath <- '/home/goransm/Analytics/NewEditors/Campaigns/2019AuBC/'
dataDir <- paste0(campaignPath, "_data/")
analyticsDir <- paste0(campaignPath, "_analytics/")

### --- determine cetDay
cetDay <- Sys.time()
cetDay
attr(cetDay, "tzone") <- "Europe/Berlin"
# - one day behind for crontab
# - (i.e. waiting for wmf.webrequest to complete is data acquisition)
cetDay <- ymd(
  strsplit(as.character(cetDay), 
           split = " ", 
           fixed = T)[[1]][1]
) - 1

### ----------------------------------------------------------
### --- Banner Impressions
### ----------------------------------------------------------

# - function: wmde_collect_banner_impressions
wmde_collect_banner_impressions <- function(uri_host, 
                                            uri_path, 
                                            uri_query, 
                                            cetDay,
                                            queryFile,
                                            fileName,
                                            dataDir) {
  
  # - NOTE:
  # - expected format for cetDay is: YYYY-MM-DD
  
  # - to dataDir
  setwd(dataDir)
  
  # - libraries
  library(stringr)
  
  # - WHERE condition: create datetime_condition
  cet_condition <- seq(
    from = as.POSIXct(paste0(cetDay," 0:00"), tz = "Europe/Berlin"),
    to = as.POSIXct(paste0(cetDay," 23:00"), tz = "Europe/Berlin"),
    by = "hour"
  ) 
  attr(cet_condition, "tzone") <- "UTC"
  cet_condition <- as.character(cet_condition)
  cet_condition <- unlist(str_extract_all(cet_condition, "^([[:digit:]]|\\s|-)*"))
  cet_years <- sapply(
    strsplit(cet_condition, split = " ", fixed = T), function(x) {
      strsplit(x, split = "-")[[1]][1]
    })
  cet_months <- sapply(
    strsplit(cet_condition, split = " ", fixed = T), function(x) {
      strsplit(x, split = "-")[[1]][2]
    })
  cet_months <- gsub("^0", "", cet_months)
  cet_days <- sapply(
    strsplit(cet_condition, split = " ", fixed = T), function(x) {
      strsplit(x, split = "-")[[1]][3]
    })
  cet_days <- gsub("^0", "", cet_days)
  cet_hours <- sapply(strsplit(cet_condition, split = " ", fixed = T), 
                      function(x) {
                        x[2]
                      })
  cet_hours <- gsub("^0", "", cet_hours)
  datetimeCondition <- paste0(
    "year = ", cet_years, " AND ",
    "month = ", cet_months, " AND ",
    "day = ", cet_days, " AND ", 
    "hour = ", cet_hours
  )
  datetimeCondition <- paste("(", 
                             datetimeCondition, 
                             ")",
                             collapse = " OR ", 
                             sep = "")
  
  # - WHERE condition: create uri_path_condition
  if (length(uri_path) > 1) {
    uri_path_condition <- paste0("(",
                                 paste(
                                   paste0("uri_path = '", uri_path, "'"),
                                   collapse = " OR ", sep = " "),
                                 ")"
    )
  } else {
    uri_path_condition = paste0("uri_path = '", uri_path, "'")
  }
  
  # - WHERE condition: create uri_host_condition
  if (length(uri_host) > 1) {
    uri_host_condition <- paste0("(",
                                 paste(
                                   paste0("uri_host = '", uri_host, "'"),
                                   collapse = " OR ", sep = " "),
                                 ")"
    )
  } else {
    uri_host_condition = paste0("uri_host = '", uri_host, "'")
  }
  
  # - WHERE condition: create uri_query_condition
  if (length(uri_query) > 1) {
    uri_query_condition <- paste0("(",
                                  paste(
                                    paste0("uri_query LIKE '%", uri_query, "%'"),
                                    collapse = " OR ", sep = " "),
                                  ")"
    )
  } else {
    uri_query_condition = paste0("uri_query LIKE '%", uri_query, "%'")
  }
  
  # - compose HiveQL query
  hiveQuery <- paste0(
    "USE wmf;
    SELECT uri_query FROM webrequest
    WHERE (",
    uri_host_condition, " AND ",
    uri_path_condition, " AND ",
    uri_query_condition, " AND ",
    "(", datetimeCondition, ")",
    ");"
    )
  
  # - write hql
  write(hiveQuery, queryFile)
  # - execute hql script:
  hiveArgs <- '/usr/local/bin/beeline -f'
  hiveInput <- paste0(queryFile, ' > ', fileName)
  # - command:
  hiveCommand <- paste(hiveArgs, hiveInput)
  return(
    system(command = hiveCommand, wait = TRUE)
  )
}

# - set params to wmde_collect_banner_impressions
# - for the Autumn Banner Campaign 2019
uri_host <- c('de.wikipedia.org', 'de.m.wikipedia.org')
uri_path  <- '/beacon/impression'
uri_query <- c('WMDE_neweditors_autumn_2019')
queryFile <- 'abc2019_BannerImpressions.hql'
fileName <- paste0(dataDir, "bannerImpressions_", cetDay, ".tsv")

# - collect Banner Impression data
wmde_collect_banner_impressions(uri_host,
                                uri_path,
                                uri_query,
                                cetDay,
                                queryFile,
                                fileName,
                                dataDir)

# - function: wmde_process_banner_impressions
wmde_process_banner_impressions <- function(fileName,
                                            dataDir, 
                                            cetDay, 
                                            campaignName) {
  
  # - to dataDir
  setwd(dataDir)
  
  # - libraries
  library(stringr)
  library(dplyr)
  
  # - load
  bannerData <- read.delim(fileName, 
                           stringsAsFactors = F)
  colnames(bannerData) <- 'uri_query'
  
  # - clean
  wStart <- which(bannerData$uri_query == "uri_query")
  bannerData <- bannerData[(wStart + 1):(dim(bannerData)[1] - 2), ]
  
  # - split
  bannerData <- strsplit(bannerData, split = "&", fixed = T)
  # - extract relevant fields
  # - banner:
  banner <- sapply(bannerData, function(x) {
    x[which(grepl("^banner=", x))]
  })
  banner <- gsub("^banner=", "", banner)
  # - recordImpressionSampleRate:
  recordImpressionSampleRate <- sapply(bannerData, function(x) {
    x[which(grepl("^recordImpressionSampleRate=", x))]
  })
  recordImpressionSampleRate <- as.numeric(
    gsub("^recordImpressionSampleRate=", "", recordImpressionSampleRate)
  )
  # - result:
  result <- sapply(bannerData, function(x) {
    x[which(grepl("^result=", x))]
  })
  result <- gsub("^result=", "", result)
  
  # - compose table:
  bannerObservations <- data.frame(banner = banner, 
                                   recordImpressionSampleRate = recordImpressionSampleRate, 
                                   result = result, 
                                   stringsAsFactors = F)
  
  # - filter for result=show
  bannerObservations <- dplyr::filter(bannerObservations,
                                      result == "show")
  
  # - correction for recordImpressionSampleRate
  bannerObservations$recordImpressionSampleRate <- 
    1/bannerObservations$recordImpressionSampleRate
  
  # - aggregate:
  bannerObservations <- bannerObservations %>% 
    dplyr::select(banner, recordImpressionSampleRate) %>% 
    dplyr::group_by(banner) %>% 
    dplyr::summarise(impressions = sum(recordImpressionSampleRate))
  
  # - add cetDay, me
  bannerObservations$date <- cetDay
  bannerObservations$campaign <- campaignName
  
  # - store:
  write.csv(bannerObservations, 
            paste0("bannerImpressionsAggregated_",
                   cetDay,
                   ".csv"
            )
  )
  
}

# - wrangle Banner Impression data
campaignName <- "2019_AuBC"
wmde_process_banner_impressions(fileName = fileName, 
                                dataDir = dataDir, 
                                cetDay = cetDay,
                                campaignName = campaignName)

### ----------------------------------------------------------
### --- Pageviews
### ----------------------------------------------------------

# - function: wmde_collect_pageviews
wmde_collect_pageviews <- function(uri_host,
                                   uri_path,
                                   cetDay,
                                   queryFile,
                                   fileName,
                                   dataDir) {
  
  # - NOTE:
  # - expected format for cetDay is: YYYY-MM-DD
  
  # - to dataDir
  setwd(dataDir)
  
  # - libraries
  library(stringr)
  
  # - WHERE condition: create datetime_condition
  cet_condition <- seq(
    from = as.POSIXct(paste0(cetDay," 0:00"), tz = "Europe/Berlin"),
    to = as.POSIXct(paste0(cetDay," 23:00"), tz = "Europe/Berlin"),
    by = "hour"
  ) 
  attr(cet_condition, "tzone") <- "UTC"
  cet_condition <- as.character(cet_condition)
  cet_condition <- unlist(str_extract_all(cet_condition, "^([[:digit:]]|\\s|-)*"))
  cet_years <- sapply(
    strsplit(cet_condition, split = " ", fixed = T), function(x) {
      strsplit(x, split = "-")[[1]][1]
    })
  cet_months <- sapply(
    strsplit(cet_condition, split = " ", fixed = T), function(x) {
      strsplit(x, split = "-")[[1]][2]
    })
  cet_months <- gsub("^0", "", cet_months)
  cet_days <- sapply(
    strsplit(cet_condition, split = " ", fixed = T), function(x) {
      strsplit(x, split = "-")[[1]][3]
    })
  cet_days <- gsub("^0", "", cet_days)
  cet_hours <- sapply(strsplit(cet_condition, split = " ", fixed = T), 
                      function(x) {
                        x[2]
                      })
  cet_hours <- gsub("^0", "", cet_hours)
  datetimeCondition <- paste0(
    "year = ", cet_years, " AND ",
    "month = ", cet_months, " AND ",
    "day = ", cet_days, " AND ", 
    "hour = ", cet_hours
  )
  datetimeCondition <- paste("(", 
                             datetimeCondition, 
                             ")",
                             collapse = " OR ", 
                             sep = "")
  
  # - WHERE condition: create uri_path_condition
  if (length(uri_path) > 1) {
    uri_path_condition <- paste0("(",
                                 paste(
                                   paste0("uri_path = '", uri_path, "'"),
                                   collapse = " OR ", sep = " "),
                                 ")"
    )
  } else {
    uri_path_condition = paste0("uri_path = '", uri_path, "'")
  }
  
  # - WHERE condition: create uri_host_condition
  if (length(uri_host) > 1) {
    uri_host_condition <- paste0("(",
                                 paste(
                                   paste0("uri_host = '", uri_host, "'"),
                                   collapse = " OR ", sep = " "),
                                 ")"
    )
  } else {
    uri_host_condition = paste0("uri_host = '", uri_host, "'")
  }
  
  # - compose HiveQL query
  hiveQuery <- paste0(
    "USE wmf;
    SELECT uri_path, uri_query, referer FROM webrequest
    WHERE (",
    uri_host_condition, " AND ",
    uri_path_condition, " AND ",
    "(", datetimeCondition, ")",
    ");"
    )
  
  # - write hql
  write(hiveQuery, queryFile)
  # - execute hql script:
  hiveArgs <- '/usr/local/bin/beeline -f'
  hiveInput <- paste0(queryFile, ' > ', fileName)
  # - command:
  hiveCommand <- paste(hiveArgs, hiveInput)
  return(
    system(command = hiveCommand, wait = TRUE)
  )
}

# - set params to wmde_collect_pageviews
# - for the Autumn Banner Campaign 2019
uri_host <- c('de.wikipedia.org', 'de.m.wikipedia.org')
uri_path  <- c(
  '/wiki/Wikipedia:Wikipedia_vor_Ort',
  '/wiki/Wikipedia:Kontor_Hamburg/Aktionstag_2019',
  '/wiki/Wikipedia:WikiWedding/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:WikiB%C3%A4r/Aktionstag_Wikipedia_2019',
  '/wiki/Wikipedia:Ruhrgebiet/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:K%C3%B6ln/Aktionstag_Wikipedia_2019',
  '/wiki/Wikipedia:Hannover/Aktionstag_Wikipedia_2019',
  '/wiki/Wikipedia:Frankfurt/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:Augsburg/Aktionstag_Wikipedia_2019',
  '/wiki/Wikipedia:WikiMUC/2019-11-10_Wikipedia_vor_Ort',
  '/wiki/Wikipedia:Freiburg_im_Breisgau/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:L%C3%B6rrach/Aktionstag_Wikipedia_2019',
  '/wiki/Wikipedia:Bodensee/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:Ober%C3%B6sterreich/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:Wien/Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:Nieder%C3%B6sterreich/Aktionstag_Wikipedia_vor_Ort_2019',
  '/wiki/Wikipedia:Wikimedia_Deutschland/LerneWikipedia')

# - set params for wmde_collect_pageviews
queryFile <- 'abc2019_Pageviews.hql'
fileName <- paste0("pageviews_", cetDay, ".tsv")

# - collect Pageviews data
wmde_collect_pageviews(uri_host,
                       uri_path,
                       cetDay,
                       queryFile,
                       fileName,
                       dataDir)

### --- Wrangle Pageviews
# - function: wmde_process_pageviews
wmde_process_pageviews <- function(fileName,
                                   dataDir, 
                                   uri_query_filter, 
                                   cetDay = cetDay,
                                   campaignName = campaignName) {
  
  # - to dataDir
  setwd(dataDir)
  
  # - libraries
  library(stringr)
  library(dplyr)
  library(tidyr)
  library(data.table)
  
  # - load
  pageviewsData <- readLines(fileName)
  wStart <- which(grepl("^uri_path", pageviewsData))
  pageviewsData <- pageviewsData[(wStart + 2):(length(pageviewsData) - 2)]
  pageviewsData <- data.frame(dat = pageviewsData, 
                              stringsAsFactors = F)
  pageviewsData <- separate(pageviewsData,
                            dat,
                            into = c('uri_path', 'uri_query', 'referer'),
                            sep = "\t")
  # - apply uri_query_filter
  # - NOTE: Autumn 2019, looking in both: uri_query, referer
  w_uri_query <- which(grepl(uri_query_filter, pageviewsData$uri_query))
  w_uri_query_referer <- which(grepl(uri_query_filter, pageviewsData$referer))
  w_uri_query <- unique(c(w_uri_query, w_uri_query_referer))
  pageviewsData <- pageviewsData[w_uri_query, ]
  w_uri_query_referer <- which(grepl(uri_query_filter, pageviewsData$referer))
  w_uri_query_referer_delete <- setdiff(1:dim(pageviewsData)[1], w_uri_query_referer)
  pageviewsData$referer[w_uri_query_referer_delete] <- ''
  # - when there is no uri_query, use the query from the referer field if present there
  pageviewsData$referer <- str_extract(pageviewsData$referer, "\\?campaign=.*$")
  pageviewsData$referer[is.na(pageviewsData$referer)] <- ""
  pageviewsData$referer <- gsub("?campaign=", "", pageviewsData$referer, fixed = T)
  pageviewsData$uri_query <- gsub("?campaign=", "", pageviewsData$uri_query, fixed = T)
  pageviewsData$uri_query[pageviewsData$uri_query == ""] <- 
    pageviewsData$referer[pageviewsData$uri_query == ""]
  pageviewsData <- dplyr::filter(pageviewsData, 
                                 uri_query != "")
  pageviewsData$referer <- NULL
  # - clean up a bit:
  pageviewsData$uri_query <- gsub("/.*$", "", pageviewsData$uri_query)
  
  # - aggregate:
  pageviewsData <- pageviewsData %>% 
    dplyr::group_by(uri_query, uri_path) %>% 
    dplyr::summarise(pageviews = n())
  colnames(pageviewsData) <- c('Tag', 'Page', 'Pageviews')
  
  # - add cetDay, campaignName
  pageviewsData$date <- cetDay
  pageviewsData$campaign <- campaignName
  
  # - store:
  write.csv(pageviewsData, 
            paste0("pageviewsAggregated_",
                   strsplit(
                     strsplit(fileName, split = "_", fixed = T)[[1]][2],
                     split = ".", 
                     fixed = T)[[1]][1],
                   ".csv"
            )
  )
  
}

# - set params to wmde_process_pageviews
# - for the Autumn Banner Campaign 2019
uri_query_filter <- 'WMDE_neweditors_autumn_2019'

# - wrangle pageviews
wmde_process_pageviews(fileName = fileName,
                       dataDir = dataDir,
                       uri_query_filter = uri_query_filter, 
                       cetDay = cetDay,
                       campaignName = campaignName) 

### ----------------------------------------------------------
### --- User Registrations
### --- HiveQL query: event.ServerSideAccountCreation table
### ----------------------------------------------------------
hiveQL <- 
  "SELECT year, month, day, hour,  event.campaign, event.userId, event.userName 
    FROM event.serversideaccountcreation 
    WHERE 
      year = 2019 
      AND month >= 11 
      AND day >= 1 
      AND (event.campaign LIKE '%WMDE_neweditors_autumn_2019%');"
# - write hql
write(hiveQL, paste0(dataDir, 'user_registrations.hql'))
### --- output filename
filename <- paste0(dataDir, 'user_registrations.tsv')
### --- execute hql script:
hiveArgs <- '/usr/local/bin/beeline --silent --incremental=true --verbose=false -f'
hiveInput <- paste(paste0(dataDir, 'user_registrations.hql'),
                   " > ",
                   filename,
                   sep = "")
# - command:
hiveCommand <- paste(hiveArgs, hiveInput)
system(command = hiveCommand, wait = TRUE)

0.2 Data Aggregation

NOTE: Not run from this report; the data were already pre-processed and aggregated by the following R script before being submitted to analytical procedures:

# !diagnostics off
### --- script: ABC_2019_Aggregation.R
### --- Data Aggregation for the Autumn Banner Campaign 2019
### --- run from stat1004
### --- /home/goransm/Analytics/NewEditors/Campaigns/2019AuBC/

### --- dir structure
campaignPath <- '/home/goransm/Analytics/NewEditors/Campaigns/2019AuBC/'
dataDir <- paste0(campaignPath, "_data/")
analyticsDir <- paste0(campaignPath, "_analytics/")

### --- Report Banner Impression Data

# - function: wmde_report_banner_impressions
wmde_report_banner_impressions <- function(dataDir) {
  
  # - Setup
  library(data.table)
  library(dplyr)
  
  # - list files:
  lF <- list.files(dataDir)
  
  # - filter aggregated banner impression data
  lF <- lF[grepl("bannerImpressionsAggregated_", lF, fixed = T)]
  
  # - load files and merge
  bannerData <- vector(mode = "list", length = length(lF))
  for (i in 1:length(lF)) {
    if (grepl("csv$|tsv$", lF[i])) {
      bannerData[[i]] <- fread(paste0(dataDir, lF[i]))
    } else {
      bannerData[[i]] <- NULL
    }
  }
  bannerData <- rbindlist(bannerData)
  bannerData$V1 <- NULL
  
  # - aggregates
  perBannerTotals <- bannerData %>% 
    select(banner, impressions) %>% 
    group_by(banner) %>% 
    summarise(totalImpressions = sum(impressions))
  perDayTotals <- bannerData %>% 
    select(date, impressions) %>% 
    group_by(date) %>% 
    summarise(totalImpressions = sum(impressions))
  
  # - output
  return(
    list(bannerImpressionsReport = bannerData, 
         perBannerTotals = perBannerTotals, 
         perDayTotals = perDayTotals)
  )
  
}

# - Report banner impressions
bannerImpressionsData <- wmde_report_banner_impressions(dataDir)
bannerImpressionsFile <- bannerImpressionsData$bannerImpressionsReport
write.csv(bannerImpressionsFile, paste0(analyticsDir, "bannerImpressionsFile.csv"))
bannerTotals <- bannerImpressionsData$perBannerTotals
write.csv(bannerTotals, paste0(analyticsDir, "bannerTotals.csv"))
bannerDayTotals <- bannerImpressionsData$perDayTotals
write.csv(bannerDayTotals, paste0(analyticsDir, "bannerDayTotals.csv"))


### --- Report Pageviews Data

# - function: wmde_report_pageviews
wmde_report_pageviews <- function(dataDir) {
  
  # - Setup
  library(data.table)
  library(dplyr)
  
  # - list files:
  lF <- list.files(dataDir)
  
  # - filter aggregated banner impression data
  lF <- lF[grepl("pageviewsAggregated_", lF, fixed = T)]
  
  # - load files and merge
  pageviewsData <- vector(mode = "list", length = length(lF))
  for (i in 1:length(lF)) {
    if (grepl("csv$|tsv$", lF[i])) {
      pageviewsData[[i]] <- fread(paste0(dataDir, lF[i]))
    } else {
      pageviewsData[[i]] <- NULL
    }
  }
  pageviewsData <- rbindlist(pageviewsData)
  pageviewsData$V1 <- NULL
  
  # - aggregates
  perDayTotals <- pageviewsData %>% 
    select(date, Pageviews) %>% 
    group_by(date) %>% 
    summarise(totalPageviews = sum(Pageviews))
  perTagTotals <- pageviewsData %>% 
    select(Tag, Pageviews) %>% 
    group_by(Tag) %>% 
    summarise(totalPageviews = sum(Pageviews))
  perPageTotals <- pageviewsData %>% 
    select(Page, Pageviews) %>% 
    group_by(Page) %>% 
    summarise(totalPageviews = sum(Pageviews))
  perPageDayTotals <- pageviewsData %>% 
    select(Page, date, Pageviews) %>%
    group_by(Page, date) %>% 
    summarise(totalPageviews = sum(Pageviews))
  perTagDayTotals <- pageviewsData %>% 
    select(Tag, date, Pageviews) %>%
    group_by(Tag, date) %>% 
    summarise(totalPageviews = sum(Pageviews))
  perTagPageTotals <- pageviewsData %>% 
    select(Tag, Page, Pageviews) %>%
    group_by(Tag, Page) %>% 
    summarise(totalPageviews = sum(Pageviews))
  
  # - output
  return(
    list(pageviewsDataReport = pageviewsData, 
         perDayTotals = perDayTotals, 
         perTagTotals = perTagTotals, 
         perPageTotals = perPageTotals, 
         perPageDayTotals = perPageDayTotals, 
         perTagDayTotals = perTagDayTotals,
         perTagPageTotals = perTagPageTotals)
  )
  
}

# - Report pageviews:
pageviewsData <- wmde_report_pageviews(dataDir)
pageviewsReportFile <- pageviewsData$pageviewsDataReport
write.csv(pageviewsReportFile, paste0(analyticsDir, "pageviewsReportFile.csv"))
pageviews_perDayTotals <- pageviewsData$perDayTotals
write.csv(pageviews_perDayTotals, paste0(analyticsDir, "pageviews_perDayTotals.csv"))
pageviews_perTagTotals <- pageviewsData$perTagTotals
write.csv(pageviews_perTagTotals, paste0(analyticsDir, "pageviews_perTagTotals.csv"))
pageviews_perPageTotals <- pageviewsData$perPageTotals
write.csv(pageviews_perPageTotals, paste0(analyticsDir, "pageviews_perPageTotals.csv"))
pageviews_perPageDayTotals <- pageviewsData$perPageDayTotals
write.csv(pageviews_perPageDayTotals, paste0(analyticsDir, "pageviews_perPageDayTotals.csv"))
pageviews_perTagDayTotals <- pageviewsData$perTagDayTotals
write.csv(pageviews_perTagDayTotals, paste0(analyticsDir, "pageviews_perTagDayTotals.csv"))
pageviews_perTagPageTotals <- pageviewsData$perTagPageTotals
write.csv(pageviews_perTagPageTotals, paste0(analyticsDir, "perTagPageTotals.csv"))

### --- Report User Registrations

# - function: wmde_report_registrations_hive
wmde_report_registrations_hive <- function(dataDir, campaign) {
  
  # - Setup
  library(data.table)
  library(dplyr)
  
  # - list files:
  lF <- list.files(dataDir)
  
  # - filter aggregated user registration data
  lF <- lF[grepl("user_registrations.tsv", lF, fixed = T)]
  
  # - load file and wrangle columns
  registrationData <- fread(paste0(dataDir, lF), header = T)
  colnames(registrationData)[5] <- 'event_campaign'
  colnames(registrationData)[6] <- 'event_userId'
  registrationData$campaign <- campaign
  registrationData$day <- ifelse(nchar(registrationData$day) == 1,
                                 paste0("0", registrationData$day), 
                                 registrationData$day)
  registrationData$date <- paste(registrationData$year, 
                                 registrationData$month,
                                 registrationData$day,
                                 sep = "-")

  # - aggregates
  perDayTotals <- registrationData %>% 
    select(date) %>% 
    group_by(date) %>% 
    summarise(totalRegistrations = n())
  perTagTotals <- registrationData %>% 
    select(event_campaign) %>% 
    group_by(event_campaign) %>% 
    summarise(totalRegistrations = n())
  perTagDayTotals <- registrationData %>% 
    select(event_campaign, date) %>% 
    group_by(event_campaign, date) %>% 
    summarise(totalRegistrations = n())
  
  # - full registration dataset: non-aggregated registrations
  # - filter data:
  fullRegData <- registrationData %>% 
    select(event_userId, 
           event_campaign, 
           date, 
           campaign)
  
  # - output
  return(
    list(registrationsDataReport = registrationData, 
         perDayTotals = perDayTotals, 
         perTagTotals = perTagTotals, 
         perTagDayTotals = perTagDayTotals,
         fullRegistrationDataset = fullRegData)
  )
  
}

# - Report upon user registrations
campaign <- '2019_AuBC'
userRegData <- wmde_report_registrations_hive(dataDir, campaign)
userRegistrationsReportFile <- userRegData$registrationsDataReport
write.csv(userRegistrationsReportFile, paste0(analyticsDir, "userRegistrationsReportFile.csv"))
userRegistrations_perDayTotals <- userRegData$perDayTotals
write.csv(userRegistrations_perDayTotals, paste0(analyticsDir, "userRegistrations_perDayTotals.csv"))
userRegistrations_perTagTotals <- userRegData$perTagTotals
write.csv(userRegistrations_perTagTotals, paste0(analyticsDir, "userRegistrations_perTagTotals.csv"))
userRegistrations_perTagDayTotals <- userRegData$perTagDayTotals
write.csv(userRegistrations_perTagDayTotals, paste0(analyticsDir, "userRegistrations_perTagDayTotals.csv"))
fullUserRegistrations <- userRegData$fullRegistrationDataset
write.csv(fullUserRegistrations, paste0(analyticsDir, "fullRegistrationDataset.csv"))

1. Campaign Banners and Pages

This section presents all data and statistics on the campaign banners and pages.

1.1 Banner Impressions

1.1.1 Banner Impressions Overview

Chart 1.1.1 Daily Banner Impressions

dataSet <- read.csv(
  '_analytics/bannerImpressionsFile.csv',
                    header = T,
                    row.names = 1,
                    check.names = F,
                    stringsAsFactors = F)
  
# - Visualize w. {ggplot2}
ggplot(dataSet, aes(x = date,
                    y = impressions,
                    group = banner,
                    color = banner,
                    fill = banner,
                    label = impressions,
                    )) + 
  geom_path(size = .5) + 
  geom_point(size = 1.5) +
  scale_y_continuous(labels = comma) +
  ggtitle('Autumn Banner Campaign 2019: Banner Impressions') +
  theme_minimal() + 
  geom_text_repel(size = 3.5, show.legend = FALSE) + 
  scale_y_continuous(labels = comma) +
  theme(axis.text.x = element_text(angle = 90, size = 8)) +
  theme(plot.title = element_text(size = 10)) +
  theme(legend.title = element_blank()) + 
  theme(legend.position = "top")

1.1.1 Banner Impressions Overview: Table

Table 1.1.1. Daily Banner Impressions

### --- Full Dataset (Table Report)
datatable(dataSet)

1.1.2 Total Banner Impressions

Chart 1.1.2. Total Banner Impressions

dataSet <- read.csv(
  '_analytics/bannerTotals.csv',
                    header = T,
                    row.names = 1,
                    check.names = F,
                    stringsAsFactors = F)
ggplot(dataSet, aes(x = banner, 
                    y = totalImpressions, 
                    color = banner, 
                    fill = banner, 
                    label = totalImpressions)) + 
  geom_bar(width = .5, stat = "identity") + 
  scale_y_continuous(labels = comma) +
  ggtitle('Autumn Banner Campaign 2019: Banner Impressions') +
  theme_minimal() + 
  geom_label_repel(size = 3.5, color = "white", show.legend = FALSE) + 
  scale_y_continuous(labels = comma) +
  theme(axis.text.x = element_text(angle = 90, size = 8)) +
  theme(plot.title = element_text(size = 10)) +
  theme(legend.title = element_blank()) + 
  theme(legend.position = "right") + 
  theme(axis.text.x = element_blank())

1.1.3 Banner Impressions per Day

Chart 1.1.3. Banner Impressions per Day

dataSet <- read.csv(
  '_analytics/bannerDayTotals.csv',
                    header = T,
                    row.names = 1,
                    check.names = F,
                    stringsAsFactors = F)
ggplot(dataSet, aes(x = date, 
                    y = totalImpressions, 
                    label = totalImpressions)) +
  geom_path(size = .25, group = 1, color = "darkblue") +
  geom_point(size = 1.5, color = "darkblue") + 
  geom_point(size = 1, color = "white") + 
  scale_y_continuous(labels = comma) +
  ggtitle('Autumn Banner Campaign 2019: Banner Impressions') +
  theme_minimal() + 
  geom_label_repel(size = 3.5, show.legend = FALSE) + 
  scale_y_continuous(labels = comma) +
  theme(axis.text.x = element_text(angle = 90, size = 8)) +
  theme(plot.title = element_text(size = 10)) +
  theme(legend.title = element_blank()) + 
  theme(legend.position = "right") + 
  theme(axis.text.x = element_text(angle = 90))

1.2 Pageviews

1.2.1 Pageviews Overview

Chart 1.2.1. Pageviews Overview. Log scaling of the pageviews is necessary; the numbers reported in the data point labels are exact.

1.2.1 Pageviews Overview: Table

Table 1.2.1. Pageviews Overview

### --- Full Dataset (Table Report)
datatable(dataSet %>% arrange(desc(totalPageviews)))

1.2.2 Pageviews Overview: totals per Page

Chart 1.2.2. Pageviews Overview: totals per Page

1.2.3 Pageviews Overview: totals per day

Chart 1.2.3. Pageviews Overview: totals per day

1.2.3 Pageviews Overview: totals per Tag/Page

Chart 1.2.3. Pageviews Overview: totals per Tag/Page

1.2.4 Pageviews per Tag, daily totals

Chart 1.2.4. Pageviews per Tag, daily totals. Log scaling of the pageviews is necessary; the numbers reported in the data point labels are exact.

dataSet <- read.csv(
  '_analytics/pageviews_perTagDayTotals.csv',
                    header = T,
                    row.names = 1,
                    check.names = F,
                    stringsAsFactors = F)
campaignTags <- c('WMDE_neweditors_autumn_2019_nl_lp1',
                  'WMDE_neweditors_autumn_2019_nl_lp2',
                  'WMDE_neweditors_autumn_2019_flyer',
                  'WMDE_neweditors_autumn_2019_bnr')
dataSet <- filter(dataSet, 
                  grepl(campaignTags[1], dataSet$Tag)| 
                    grepl(campaignTags[2], dataSet$Tag)| 
                    grepl(campaignTags[3], dataSet$Tag)| 
                    grepl(campaignTags[4], dataSet$Tag))
dataSet$Tag[grepl(campaignTags[1], dataSet$Tag, fixed = T)] <- "nl_lp1"
dataSet$Tag[grepl(campaignTags[2], dataSet$Tag, fixed = T)] <- "nl_lp2"
dataSet$Tag[grepl(campaignTags[3], dataSet$Tag, fixed = T)] <- "flyer"
dataSet$Tag[grepl(campaignTags[4], dataSet$Tag, fixed = T)] <- "bnr"
dataSet <- dataSet %>% 
  group_by(Tag, date) %>% 
  summarise(totalPageviews = sum(totalPageviews))
ggplot(dataSet, aes(x = date,
                    y = log10(totalPageviews),
                    group = Tag,
                    color = Tag,
                    fill = Tag,
                    label = totalPageviews,
                    )) + 
  geom_path(size = .25) + 
  geom_point(size = 1.5) +
  geom_point(size = 1, color = "white") +
  scale_y_continuous(labels = comma) +
  ggtitle('Autumn Banner Campaign 2019: Pageviews per Tag, daily totals') +
  theme_minimal() + 
  geom_text_repel(size = 3.5, show.legend = FALSE) + 
  scale_y_continuous(labels = comma) +
  theme(axis.text.x = element_text(angle = 90, size = 8)) +
  theme(plot.title = element_text(size = 10)) +
  theme(legend.title = element_blank()) + 
  theme(legend.position = "top")

2. User Registrations

All data on user registrations are presented in this section.

2.1 Registrations per tag and day

Chart 2.1. Registrations per tag and day. Please note: points with no data labels signify 0 user registrations.