.DELETE_ON_ERROR:

all: models

models: \
	models/enwiki.draft_quality.gradient_boosting.model \
	models/enwiki.draft_quality_with_sentiment.gradient_boosting.model

#datasets/enwiki.draft_quality.50k_stratified.json: \
#	       datasets/enwiki.draft_quality.201508-201608.tsv.bz2
#       ( \
#	 bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | \
#	 tsv2json str int str int str | \
#	 grep '"draft_quality": "OK"' | shuf -n 26257; \
#	 bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | \
#	 tsv2json str int str int str | \
#	 grep -v '"draft_quality": "OK"' \
#       ) > \
#       datasets/enwiki.draft_quality.50k_stratified.json

datasets/enwiki.draft_quality.201508-201608.json.bz2: \
		datasets/enwiki.draft_quality.201508-201608.tsv.bz2
	bzcat $< | \
	tsv2json str int str int str | bzip2 -c > $@

datasets/enwiki.draft_quality.201508-201608.with_text.json.bz2: \
		datasets/enwiki.draft_quality.201508-201608.json.bz2
	bzcat $< | \
	revscoring fetch_text --host https://en.wikipedia.org \
	  --verbose | bzip2 -c > $@

datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2: \
		datasets/enwiki.draft_quality.201508-201608.with_text.json.bz2
	bzcat $< | \
	wikiclass extract_from_text \
	  draftquality.feature_lists.enwiki.draft_quality \
	  draftquality.feature_lists.enwiki.draft_quality_with_sentiment \
	  --extractors 12 \
	  --verbose | bzip2 -c > $@

tuning_reports/enwiki.draft_quality.md: \
		datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
	bzcat $< | \
	revscoring tune \
	  config/classifiers.params.yaml \
	  draftquality.feature_lists.enwiki.draft_quality \
	  draft_quality \
	  --cv-timeout=60 \
	  --scoring=accuracy \
	  --debug > $@

datasets/enwiki.draft_quality.201508-201608.10k.with_cache.json: \
		datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
	bzcat $< | shuf -n 10000 > $@

datasets/enwiki.draft_quality.201508-201608.test.10k.with_cache.json: \
		datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
	bzcat $< | shuf -n 10000 > $@

datasets/enwiki.draft_quality.201508-201608.shuffled.with_cache.json.bz2: \
		datasets/enwiki.draft_quality.201508-201608.with_cache.json.bz2
	bzcat $< | shuf | bzip2 -c > $@

datasets/enwiki.draft_quality.201508-201608.train.700k.with_cache.json.bz2: \
		datasets/enwiki.draft_quality.201508-201608.shuffled.with_cache.json.bz2
	bzcat $< | tail -n +100000 | head -n 700000 | bzip2 -c > $@

datasets/enwiki.draft_quality.201508-201608.test.100k.with_cache.json.bz2: \
		datasets/enwiki.draft_quality.201508-201608.shuffled.with_cache.json.bz2
	bzcat $< | head -n 100000 | bzip2 -c > $@

models/enwiki.draft_quality_with_sentiment.gradient_boosting.model: \
		datasets/enwiki.draft_quality.201508-201608.train.700k.with_cache.json.bz2
	bzcat $< | \
	revscoring train_model \
	  revscoring.scorer_models.GradientBoosting \
	  draftquality.feature_lists.enwiki.draft_quality_with_sentiment \
	  draft_quality \
	  -p 'learning_rate=0.01' \
	  -p 'max_features="log2"' \
	  -p 'max_depth=7' \
	  -p 'n_estimators=700' \
	  --version 0.0.1 > $@

models/enwiki.draft_quality.gradient_boosting.model: \
		datasets/enwiki.draft_quality.201508-201608.train.700k.with_cache.json.bz2
	bzcat $< | \
	revscoring train_model \
	  revscoring.scorer_models.GradientBoosting \
	  draftquality.feature_lists.enwiki.draft_quality \
	  draft_quality \
	  -p 'learning_rate=0.01' \
	  -p 'max_features="log2"' \
	  -p 'max_depth=7' \
	  -p 'n_estimators=700' \
	  --version 0.0.1 > $@

tests: \
	test-wo-sentiment \
	test-sentiment

# NOTE test-inclusive model is left for manual pickup as "test"
test-wo-sentiment: \
		datasets/enwiki.draft_quality.201508-201608.test.100k.with_cache.json.bz2 \
		models/enwiki.draft_quality.gradient_boosting.model
	bzcat $< | \
	  revscoring test_model \
	  models/enwiki.draft_quality.gradient_boosting.model \
	  draft_quality \
	  --observations="<stdin>" \
	  -s 'table' -s 'accuracy' -s 'roc' -s 'f1' \
	  -s 'filter_rate_at_recall(min_recall=0.75)' \
	  -s 'filter_rate_at_recall(min_recall=0.9)' > $@

# NOTE test-inclusive model is left for manual pickup as "test"
test-sentiment: \
		datasets/enwiki.draft_quality.201508-201608.test.100k.with_cache.json.bz2 \
		models/enwiki.draft_quality_with_sentiment.gradient_boosting.model
	bzcat $< | \
	  revscoring test_model \
	  models/enwiki.draft_quality_with_sentiment.gradient_boosting.model \
	  --observations="<stdin>" \
	  draft_quality \
	  -s 'table' -s 'accuracy' -s 'roc' -s 'f1' \
	  -s 'filter_rate_at_recall(min_recall=0.75)' \
	  -s 'filter_rate_at_recall(min_recall=0.9)' > $@



############### Big dataset ###################################################

host=analytics-store.eqiad.wmnet
user=research

mysql_args = -h $(host) -u $(user)
mysqlc = mysql $(mysql_args)
mysqlc_import = mysqlimport $(mysql_args) --local

datasets/enwiki.draft_quality.201508-201608.tsv.bz2: \
		datasets/enwiki.draft_quality.201508.tsv.bz2 \
		datasets/enwiki.draft_quality.201509.tsv.bz2 \
		datasets/enwiki.draft_quality.201510.tsv.bz2 \
		datasets/enwiki.draft_quality.201511.tsv.bz2 \
		datasets/enwiki.draft_quality.201512.tsv.bz2 \
		datasets/enwiki.draft_quality.201601.tsv.bz2 \
		datasets/enwiki.draft_quality.201602.tsv.bz2 \
		datasets/enwiki.draft_quality.201603.tsv.bz2 \
		datasets/enwiki.draft_quality.201604.tsv.bz2 \
		datasets/enwiki.draft_quality.201605.tsv.bz2 \
		datasets/enwiki.draft_quality.201606.tsv.bz2 \
		datasets/enwiki.draft_quality.201607.tsv.bz2
	( \
	  bzcat datasets/enwiki.draft_quality.201508.tsv.bz2; \
	  bzcat datasets/enwiki.draft_quality.201509.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201510.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201511.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201512.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201601.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201602.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201603.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201604.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201605.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201606.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201607.tsv.bz2 | tail -n+2 \
	) | bzip2 -c > $@

datasets/enwiki.draft_quality.201508.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201508", @end="201509";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201509.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201509", @end="201510";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201510.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201510", @end="201511";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201511.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201511", @end="201512";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201512.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201512", @end="201601";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201601.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201601", @end="201602";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201602.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201602", @end="201603";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201603.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201603", @end="201604";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201604.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201604", @end="201605";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201605.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201605", @end="201606";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201606.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201606", @end="201607";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201607.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201607", @end="201608";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.75_not_OK_sample.tsv: \
		datasets/enwiki.draft_quality.201508-201608.tsv.bz2
	( \
	  bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | head -n 1; \
	  bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | grep -P 'spam$$' | shuf -n 25; \
	  bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | grep -P 'attack$$' | shuf -n 25; \
	  bzcat datasets/enwiki.draft_quality.201508-201608.tsv.bz2 | grep -P 'vandalism$$' | shuf -n 25 \
	) > $@

datasets/enwiki.draft_quality.201608-201701.tsv.bz2: \
		datasets/enwiki.draft_quality.201608.tsv.bz2 \
		datasets/enwiki.draft_quality.201609.tsv.bz2 \
		datasets/enwiki.draft_quality.201610.tsv.bz2 \
		datasets/enwiki.draft_quality.201611.tsv.bz2 \
		datasets/enwiki.draft_quality.201612.tsv.bz2
	( \
	  bzcat datasets/enwiki.draft_quality.201608.tsv.bz2; \
	  bzcat datasets/enwiki.draft_quality.201609.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201610.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201611.tsv.bz2 | tail -n+2; \
	  bzcat datasets/enwiki.draft_quality.201612.tsv.bz2 | tail -n+2 \
	) | bzip2 -c > $@

datasets/enwiki.draft_quality.201608.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201608", @end="201609";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201609.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201609", @end="201610";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201610.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201610", @end="201611";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201611.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201611", @end="201612";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201612.tsv.bz2: \
		sql/draft_quality.variables.sql
	echo 'SET @start="201612", @end="201701";' | \
	cat - $< | \
	$(mysqlc) enwiki | bzip2 -c > $@

datasets/enwiki.draft_quality.201608-201701.json: \
		datasets/enwiki.draft_quality.201608-201701.tsv.bz2
	bzcat $< | \
	tsv2json str int str int str > $@

datasets/enwiki.draft_quality.201608-201701.with_text.json.bz2: \
		datasets/enwiki.draft_quality.201608-201701.json
	cat $< | \
	revscoring fetch_text --host https://en.wikipedia.org \
	  --verbose | bzip2 -c > $@

datasets/enwiki.draft_quality.201608-201701.with_cache.json.bz2: \
		datasets/enwiki.draft_quality.201608-201701.with_text.json.bz2
	bzcat $< | \
	wikiclass extract_from_text \
	  draftquality.feature_lists.enwiki.draft_quality \
	  --verbose | bzip2 -c > $@