popular_edge_ngrams = ( spark.read.table('discovery.cirrus_index') .where(F.col('wiki') == 'enwiki') .where(F.col('snapshot') == '20250302') .where(F.col('index_name').startswith('enwiki_content')) .select( 'title', F.split(F.col('title'), ' ').alias('title_words') ) .select( 'title', F.array( F.col('title_words')[0], F.concat_ws(' ', F.col('title_words')[0], F.col('title_words')[1]), F.concat_ws(' ', F.col('title_words')[0], F.col('title_words')[1], F.col('title_words')[2]), ).alias('edge_ngrams') ) .select(F.explode('edge_ngrams').alias('edge_ngram')) .groupBy(F.col('edge_ngram')) .agg(F.sum(F.lit(1)).alias('count')) .orderBy(F.col('count').desc()) .limit(50) ).toPandas() | | edge_ngram | count | |---:|:-------------|--------:| | 0 | List | 123691 | | 1 | List of | 123654 | | 2 | The | 119575 | | 3 | John | 52991 | | 4 | William | 30714 | | 5 | James | 21894 | | 6 | Robert | 20720 | | 7 | David | 20348 | | 8 | George | 20274 | | 9 | Thomas | 17692 | | 10 | Charles | 16912 | | 11 | National | 15608 | | 12 | Michael | 14067 | | 13 | Richard | 13498 | | 14 | Peter | 13224 | | 15 | Paul | 12181 | | 16 | New | 11701 | | 17 | Henry | 11671 | | 18 | A | 11581 | | 19 | Battle | 11573 | | 20 | Battle of | 10991 | | 21 | USS | 10288 | | 22 | Joseph | 10189 | | 23 | Edward | 9038 | | 24 | 2022 | 8827 | | 25 | St. | 8767 | | 26 | 2024 | 8604 | | 27 | 2023 | 8481 | | 28 | La | 8347 | | 29 | United | 8282 | | 30 | 2019 | 8196 | | 31 | Frank | 8124 | | 32 | 2018 | 8086 | | 33 | St | 7988 | | 34 | 2016 | 7628 | | 35 | Mark | 7559 | | 36 | Daniel | 7517 | | 37 | 2017 | 7454 | | 38 | HMS | 7416 | | 39 | 2014 | 7337 | | 40 | South | 7323 | | 41 | 2015 | 7271 | | 42 | Mount | 7261 | | 43 | San | 7226 | | 44 | 2021 | 7048 | | 45 | Jack | 7044 | | 46 | Tom | 6847 | | 47 | Alexander | 6666 | | 48 | North | 6652 | | 49 | 2013 | 6570 |