diff --git a/.gitignore b/.gitignore index f31d514..f023cee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,18 @@ .tox/ .*.sw? /_build /oozie/glent/coordinator-*.xml .mypy_cache .cache /airflow/airflow.db /airflow/airflow.cfg /airflow/unittests.cfg /airflow/logs __pycache__ # Created by scap scripts venv venv.zip .idea/ target/ +airflow/coverage.xml +coverage.xml diff --git a/airflow/tox.ini b/airflow/tox.ini index 555aceb..69493df 100644 --- a/airflow/tox.ini +++ b/airflow/tox.ini @@ -1,54 +1,55 @@ # Tox configuration [tox] minversion = 1.6 skipsdist = True # List the environment that will be run by default envlist = flake8,mypy,pytest [testenv] basepython = python3.7 # Default configuration. py26 and py27 will end up using this setenv = VIRTUAL_ENV={envdir} py_paths = dags plugins tests mypy_paths = dags plugins/wmf_airflow # Settings specific to the flake8 environment [testenv:flake8] basepython = python3.7 # The command to run: commands = flake8 {posargs:{[testenv]py_paths}} # We only need flake8 when linting, we do not care about the project dependencies deps = flake8 [flake8] ignore = W503 max-line-length = 100 [testenv:mypy] deps = mypy==0.720 ignore_errors = True commands = mypy {posargs:{[testenv]mypy_paths}} [testenv:pytest] basepython = python3.7 commands_pre = /bin/rm -rf {envtmpdir}/airflow /bin/mkdir -p {envtmpdir}/airflow /bin/ln -s /{toxinidir}/dags /{toxinidir}/plugins {envtmpdir}/airflow/ airflow initdb /usr/bin/env /bin/grep -E '^(plugins|dags)_folder' {envtmpdir}/airflow/airflow.cfg -commands = pytest {posargs:--pyargs .} +commands = pytest --cov=dags --cov-report xml {posargs:--pyargs .} deps = apache-airflow==1.10.6 Werkzeug==0.16.0 pytest + pytest-cov pytest-mock hmsclient pydruid==0.5.7 skein findspark passenv = XDG_CACHE_HOME REBUILD_FIXTURES SPARK_HOME setenv = AIRFLOW_HOME = {envtmpdir}/airflow whitelist_externals = /bin/ln,/bin/rm,/bin/mkdir,/usr/bin/env,/bin/grep diff --git a/sonar-project.properties b/sonar-project.properties index 2e7e6a3..7b7b297 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,8 +1,9 @@ sonar.host.url=https://sonarcloud.io sonar.organization=wmftest sonar.projectKey=discovery-analytics sonar.projectName=discovery-analytics sonar.exclusions=data/**/* sonar.sources=airflow/dags,spark sonar.analysis.gerritProjectName=wikimedia/discovery/analytics sonar.analysis.allowCommentOnMaster=1 +sonar.python.coverage.reportPaths=coverage.xml,airflow/coverage.xml diff --git a/tox.ini b/tox.ini index bd10123..6120336 100644 --- a/tox.ini +++ b/tox.ini @@ -1,50 +1,51 @@ # Tox configuration [tox] minversion = 1.6 skipsdist = True # List the environment that will be run by default envlist = flake8 airflow mypy pytest [testenv] basepython = python3.7 # Default configuration. py26 and py27 will end up using this setenv = VIRTUAL_ENV={envdir} # Settings specific to the flake8 environment [testenv:flake8] # The command to run: commands = flake8 {posargs:oozie/ spark/} # We only need flake8 when linting, we do not care about the project dependencies deps = flake8 [flake8] ignore = W503 [testenv:airflow] changedir = {toxinidir}/airflow commands = tox {posargs} whitelist_externals = tox passenv = XDG_CACHE_HOME REBUILD_FIXTURES SPARK_HOME [testenv:mypy] deps = mypy==0.720 pyspark-stubs==2.4.0post5 commands = mypy {posargs:spark/} [testenv:pytest] deps = findspark pytest + pytest-cov pytest-mock requests dnspython mwapi oresapi setenv = PYTHONPATH=spark/ passenv = XDG_CACHE_HOME SPARK_HOME JAVA_HOME -commands = pytest {posargs:spark/test} +commands = pytest --cov=spark --cov-report xml {posargs:spark/test}