From d1555d77fcab5ea2d383ac4bb9ecb85f73e56124 Mon Sep 17 00:00:00 2001 From: Nicola Tarocco Date: Sun, 18 Feb 2024 20:18:09 +0100 Subject: [PATCH 1/2] upgrade dependencies --- requirements.txt | 198 ++++++++++++++++++++++++++--------------------- 1 file changed, 111 insertions(+), 87 deletions(-) diff --git a/requirements.txt b/requirements.txt index aad6a78a..452741d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,109 +3,133 @@ # pip list --format freeze | grep -vi caimira | grep -v pip | grep -v setuptools >> requirements.txt .[app] -anyio==3.6.2 -appnope==0.1.3 -argon2-cffi==21.3.0 +anyio==4.2.0 +appnope==0.1.4 +argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 -asttokens==2.2.1 -attrs==22.2.0 -Babel==2.11.0 -backcall==0.2.0 -beautifulsoup4==4.11.2 -bleach==6.0.0 -certifi==2022.12.7 -cffi==1.15.1 -charset-normalizer==3.0.1 -cloudpickle==2.2.1 -comm==0.1.2 -contourpy==1.0.7 -cycler==0.11.0 -debugpy==1.6.6 +arrow==1.3.0 +asttokens==2.4.1 +async-lru==2.0.4 +attrs==23.2.0 +Babel==2.14.0 +beautifulsoup4==4.12.3 +bleach==6.1.0 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +cloudpickle==3.0.0 +comm==0.2.1 +contourpy==1.2.0 +cycler==0.12.1 +debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -entrypoints==0.4 -executing==1.2.0 -fastjsonschema==2.16.2 -fonttools==4.38.0 +exceptiongroup==1.2.0 +executing==2.0.1 +fastjsonschema==2.19.1 +fonttools==4.49.0 +fqdn==1.5.1 +h11==0.14.0 h3==3.7.6 -idna==3.4 -importlib-metadata==6.0.0 -importlib-resources==5.12.0 -ipykernel==6.22.0 +httpcore==1.0.3 +httpx==0.26.0 +idna==3.6 +importlib-metadata==7.0.1 +importlib-resources==6.1.1 +ipykernel==6.29.2 ipympl==0.9.3 -ipython==8.10.0 +ipython==8.18.1 ipython-genutils==0.2.0 -ipywidgets==7.7.3 -jedi==0.18.2 -Jinja2==3.0.3 -joblib==1.2.0 -json5==0.9.11 -jsonschema==4.17.3 -jupyter-client==6.1.12 -jupyter-core==5.3.0 -jupyter-server==1.23.6 -jupyterlab-pygments==0.2.2 -jupyterlab-server==2.20.0 -jupyterlab-widgets==1.1.3 -kiwisolver==1.4.4 -loky==3.3.0 -MarkupSafe==2.1.2 -matplotlib==3.7.0 +ipywidgets==7.8.1 +isoduration==20.11.0 +jedi==0.19.1 +Jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonpointer==2.4 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +jupyter-events==0.9.0 +jupyter-lsp==2.2.2 +jupyter_server==2.12.5 +jupyter_server_terminals==0.5.2 +jupyterlab==4.1.1 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.25.3 +jupyterlab-widgets==1.1.7 +kiwisolver==1.4.5 +loky==3.4.1 +MarkupSafe==2.1.5 +matplotlib==3.8.3 matplotlib-inline==0.1.6 memoization==0.4.0 -mistune==0.8.4 -nbclassic==0.5.3 -nbclient==0.5.13 -nbconvert==6.5.4 -nbformat==5.8.0 -nest-asyncio==1.5.6 -notebook==6.5.3 -notebook-shim==0.2.2 -numpy==1.24.2 -packaging==23.0 -pandas==1.5.3 -pandocfilters==1.5.0 +mistune==3.0.2 +nbclient==0.7.4 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +notebook==7.1.0 +notebook_shim==0.2.4 +numpy==1.26.4 +overrides==7.7.0 +packaging==23.2 +pandas==2.2.0 +pandocfilters==1.5.1 parso==0.8.3 -pexpect==4.8.0 -pickleshare==0.7.5 -Pillow==9.4.0 -platformdirs==3.0.0 -prometheus-client==0.16.0 -prompt-toolkit==3.0.37 -psutil==5.9.4 +pexpect==4.9.0 +pillow==10.2.0 +platformdirs==4.2.0 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +psutil==5.9.8 ptyprocess==0.7.0 pure-eval==0.2.2 py==1.11.0 pycparser==2.21 -Pygments==2.14.0 -pyparsing==3.0.9 -pyrsistent==0.19.3 +Pygments==2.17.2 +pyinstrument==4.6.2 +PyJWT==2.8.0 +pyparsing==3.1.1 python-dateutil==2.8.2 -pytz==2022.7.1 -pyzmq==25.0.0 -requests==2.28.2 +python-json-logger==2.0.7 +pytz==2024.1 +PyYAML==6.0.1 +pyzmq==25.1.2 +referencing==0.33.0 +requests==2.31.0 retry==0.9.2 -ruptures==1.1.8 -scikit-learn==1.2.1 -scipy==1.10.1 -Send2Trash==1.8.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rpds-py==0.18.0 +ruptures==1.1.9 +scikit-learn==1.4.1.post1 +scipy==1.12.0 +Send2Trash==1.8.2 six==1.16.0 sniffio==1.3.0 -soupsieve==2.4 -stack-data==0.6.2 -terminado==0.17.1 -threadpoolctl==3.1.0 -timezonefinder==6.1.9 +soupsieve==2.5 +stack-data==0.6.3 +terminado==0.18.0 +threadpoolctl==3.3.0 +timezonefinder==6.4.1 tinycss2==1.2.1 -tornado==6.2 -traitlets==5.9.0 -types-retry==0.9.9.2 -urllib3==1.26.14 -voila==0.2.16 -wcwidth==0.2.6 +tomli==2.0.1 +tornado==6.4 +traitlets==5.14.1 +types-python-dateutil==2.8.19.20240106 +types-retry==0.9.9.4 +typing_extensions==4.9.0 +tzdata==2024.1 +uri-template==1.3.0 +urllib3==2.2.0 +voila==0.5.5 +wcwidth==0.2.13 +webcolors==1.13 webencodings==0.5.1 -websocket-client==1.5.1 -websockets==10.4 -wheel==0.36.2 -widgetsnbextension==3.6.2 -zipp==3.14.0 +websocket-client==1.7.0 +websockets==12.0 +wheel==0.41.3 +widgetsnbextension==3.6.6 +zipp==3.17.0 From ddfe0b660ee7b6f901ad4bdc8adae5244a8b8b88 Mon Sep 17 00:00:00 2001 From: Nicola Tarocco Date: Mon, 26 Feb 2024 11:28:24 +0100 Subject: [PATCH 2/2] tests: make report generation time limit configurable --- .gitlab-ci.yml | 3 +-- README.md | 4 +--- app-config/openshift/deploymentconfig.yaml | 4 ++-- caimira/apps/calculator/__init__.py | 4 ++-- caimira/profiler.py | 4 ++++ caimira/tests/apps/calculator/test_report_generator.py | 6 ++++-- caimira/tests/apps/calculator/test_webapp.py | 9 +++------ 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc846dc9..13cf58bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,6 @@ include: variables: project_name: caimira - PY_VERSION: "3.9" # ################################################################################################### @@ -110,7 +109,7 @@ check_openshift_config_prod: - /kaniko/executor --context ${CI_PROJECT_DIR}/${DOCKER_CONTEXT_DIRECTORY} --dockerfile ${CI_PROJECT_DIR}/${DOCKERFILE_DIRECTORY}/Dockerfile --destination ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${IMAGE_TAG} # Print the full registry path of the pushed image - echo "Image pushed successfully to ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${IMAGE_TAG}" - + auth-service-image_builder: extends: diff --git a/README.md b/README.md index 94c68656..a2684aed 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,7 @@ pytest ./caimira ### Running the profiler -The profiler is enabled in one of the following cases: -- the calculator app is running in `debug` mode -- the environment variable `CAIMIRA_PROFILER_ENABLED` is set to 1 +The profiler is enabled when the environment variable `CAIMIRA_PROFILER_ENABLED` is set to 1. When visiting http://localhost:8080/profiler, you can start a new session and choose between [PyInstrument](https://github.com/joerick/pyinstrument) or [cProfile](https://docs.python.org/3/library/profile.html#module-cProfile). The app includes two different profilers, mainly because they can give different information. diff --git a/app-config/openshift/deploymentconfig.yaml b/app-config/openshift/deploymentconfig.yaml index 01023cd3..ac58a1ea 100644 --- a/app-config/openshift/deploymentconfig.yaml +++ b/app-config/openshift/deploymentconfig.yaml @@ -284,7 +284,7 @@ key: ARVE_API_KEY name: arve-api - name: DATA_SERVICE_ENABLED - value: 'False' + value: 'True' image: '${PROJECT_NAME}/calculator-app' ports: - containerPort: 8080 @@ -363,7 +363,7 @@ - name: CAIMIRA_CALCULATOR_PREFIX value: /calculator-open - name: DATA_SERVICE_ENABLED - value: 'False' + value: 'True' image: '${PROJECT_NAME}/calculator-app' ports: - containerPort: 8080 diff --git a/caimira/apps/calculator/__init__.py b/caimira/apps/calculator/__init__.py index d7b53f7d..8898f272 100644 --- a/caimira/apps/calculator/__init__.py +++ b/caimira/apps/calculator/__init__.py @@ -516,8 +516,8 @@ def make_app( 'filename': 'userguide.html.j2'}), ] - profiler = os.environ.get('CAIMIRA_PROFILER_ENABLED', 0) - if debug or profiler: + profiler_enabled = os.environ.get('CAIMIRA_PROFILER_ENABLED', 0) + if profiler_enabled: urls += [ (get_root_url(CaimiraProfiler.ROOT_URL), ProfilerPage), (get_root_url(r'{root_url}/(.*)'.format(root_url=CaimiraProfiler.ROOT_URL)), ProfilerReport), diff --git a/caimira/profiler.py b/caimira/profiler.py index 6c39212e..8f3c26a0 100644 --- a/caimira/profiler.py +++ b/caimira/profiler.py @@ -195,6 +195,10 @@ class CaimiraProfiler: @property def is_active(self): """Return True if a session is active.""" + enabled = os.environ.get("CAIMIRA_PROFILER_ENABLED", 0) + if not enabled: + return False + if not os.path.exists(self._cache_filepath): return False diff --git a/caimira/tests/apps/calculator/test_report_generator.py b/caimira/tests/apps/calculator/test_report_generator.py index 18190777..22355c5d 100644 --- a/caimira/tests/apps/calculator/test_report_generator.py +++ b/caimira/tests/apps/calculator/test_report_generator.py @@ -1,8 +1,8 @@ import concurrent.futures from functools import partial +import os import time -import numpy.testing import numpy as np import pytest @@ -16,7 +16,7 @@ def test_generate_report(baseline_form) -> None: # generate a report for it. Because this is what happens in the caimira # calculator, we confirm that the generation happens within a reasonable # time threshold. - time_limit: float = 20.0 # seconds + time_limit: float = float(os.environ.get("CAIMIRA_TESTS_CALCULATOR_TIMEOUT", 10.)) start = time.perf_counter() @@ -25,6 +25,8 @@ def test_generate_report(baseline_form) -> None: concurrent.futures.ThreadPoolExecutor, 1, )) end = time.perf_counter() + total = end-start + print(f"Time limit: {time_limit} | Time taken: {end} - {start} = {total} < {time_limit}") assert report != "" assert end - start < time_limit diff --git a/caimira/tests/apps/calculator/test_webapp.py b/caimira/tests/apps/calculator/test_webapp.py index 32d22352..dc6ea769 100644 --- a/caimira/tests/apps/calculator/test_webapp.py +++ b/caimira/tests/apps/calculator/test_webapp.py @@ -1,13 +1,13 @@ +import os from pathlib import Path import pytest import tornado.testing -from retry import retry import caimira.apps.calculator from caimira.apps.calculator.report_generator import generate_permalink -_TIMEOUT = 20. +_TIMEOUT = float(os.environ.get("CAIMIRA_TESTS_CALCULATOR_TIMEOUT", 10.)) @pytest.fixture @@ -34,7 +34,6 @@ async def test_404(http_server_client): assert resp.code == 404 -@retry(tries=10) class TestBasicApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): return caimira.apps.calculator.make_app() @@ -63,7 +62,6 @@ class TestBasicApp(tornado.testing.AsyncHTTPTestCase): assert 'expected number of new cases is' in response.body.decode() -@retry(tries=10) class TestCernApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): cern_theme = Path(caimira.apps.calculator.__file__).parent.parent / 'themes' / 'cern' @@ -76,7 +74,6 @@ class TestCernApp(tornado.testing.AsyncHTTPTestCase): assert 'expected number of new cases is' in response.body.decode() -retry(tries=10) class TestOpenApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): return caimira.apps.calculator.make_app(calculator_prefix="/mycalc") @@ -149,7 +146,7 @@ class TestCERNGenericPage(tornado.testing.AsyncHTTPTestCase): ] return tornado.web.Application(pages, **app.settings) - + @tornado.testing.gen_test(timeout=_TIMEOUT) def test_user_guide(self): response = yield self.http_client.fetch(self.get_url('/calculator/user-guide'))