From 6cc612107552a17c2b402d8d66da35fba70020cb Mon Sep 17 00:00:00 2001 From: Luis Aleixo Date: Fri, 16 Jun 2023 16:02:47 +0200 Subject: [PATCH 1/5] developed server side API connection to Service Data (WP3) --- app-config/calculator-app/app.sh | 3 ++ caimira/apps/calculator/__init__.py | 55 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/app-config/calculator-app/app.sh b/app-config/calculator-app/app.sh index 6a808f8c..15c94234 100755 --- a/app-config/calculator-app/app.sh +++ b/app-config/calculator-app/app.sh @@ -23,6 +23,9 @@ if [[ "$APP_NAME" == "calculator-app" ]]; then export "EXTRA_PAGES"="$EXTRA_PAGES" + export "DATA_SERVICE_CLIENT_EMAIL"="$DATA_SERVICE_CLIENT_EMAIL" + export "DATA_SERVICE_CLIENT_PASSWORD"="$DATA_SERVICE_CLIENT_PASSWORD" + echo "Starting the caimira webservice with: python -m caimira.apps.calculator ${args[@]}" python -m caimira.apps.calculator "${args[@]}" elif [[ "$APP_NAME" == "caimira-voila" ]]; then diff --git a/caimira/apps/calculator/__init__.py b/caimira/apps/calculator/__init__.py index b7e11c4c..49bd3b0c 100644 --- a/caimira/apps/calculator/__init__.py +++ b/caimira/apps/calculator/__init__.py @@ -44,6 +44,9 @@ LOG = logging.getLogger(__name__) class BaseRequestHandler(RequestHandler): + # Host URL for the CAiMIRA Data Service API + host: str = 'https://caimira-data-api.app.cern.ch' + async def prepare(self): """Called at the beginning of a request before `get`/`post`/etc.""" @@ -60,6 +63,47 @@ class BaseRequestHandler(RequestHandler): else: self.current_user = AnonymousUser() + async def data_service_connection(self): + client_email = self.settings["data_service_client_email"] + client_password = self.settings['data_service_client_password'] + + if (client_email == None or client_password == None): + # If the credentials are not defined, we skip the API Integration + return self.send_error(401) + + http_client = AsyncHTTPClient() + headers = {'Content-type': 'application/json'} + json_body = { "email": f"{client_email}", "password": f"{client_password}"} + + try: + response = await http_client.fetch(HTTPRequest( + url=self.host + '/login', + method='POST', + headers=headers, + body=json.dumps(json_body), + ), + raise_error=True) + except Exception as e: + print("Something went wrong: %s" % e) + + return json.loads(response.body)['access_token'] + + async def data_service_fetcher(self, access_token: str): + http_client = AsyncHTTPClient() + headers = {'Authorization': f'Bearer {access_token}'} + + try: + response = await http_client.fetch(HTTPRequest( + url=self.host + '/data', + method='GET', + headers=headers, + ), + raise_error=True) + except Exception as e: + print("Something went wrong: %s" % e) + + return json.loads(response.body) + def write_error(self, status_code: int, **kwargs) -> None: template = self.settings["template_environment"].get_template( "error.html.j2") @@ -105,6 +149,13 @@ class ConcentrationModel(BaseRequestHandler): pprint(requested_model_config) start = datetime.datetime.now() + # Data Service API Integration + try: + access_token = await self.data_service_connection() + service_data = await self.data_service_fetcher(access_token) + except Exception as e: + print("Something went wrong with the data service: %s" % e) + try: form = model_generator.FormData.from_dict(requested_model_config) except Exception as err: @@ -435,6 +486,10 @@ def make_app( arve_client_secret=os.environ.get('ARVE_CLIENT_SECRET', None), arve_api_key=os.environ.get('ARVE_API_KEY', None), + # Data Service API Credentials + data_service_client_email = os.environ.get('DATA_SERVICE_CLIENT_EMAIL', ''), + data_service_client_password = os.environ.get('DATA_SERVICE_CLIENT_PASSWORD', ''), + # Process parallelism controls. There is a balance between serving a single report # requests quickly or serving multiple requests concurrently. # The defaults are: handle one report at a time, and allow parallelism From 9c33ac1df093196cc68615b9472f50029913c1c8 Mon Sep 17 00:00:00 2001 From: Luis Aleixo Date: Tue, 11 Jul 2023 13:35:26 +0100 Subject: [PATCH 2/5] updated server-side logic --- caimira/apps/calculator/__init__.py | 68 +++++++---------------------- 1 file changed, 15 insertions(+), 53 deletions(-) diff --git a/caimira/apps/calculator/__init__.py b/caimira/apps/calculator/__init__.py index 49bd3b0c..f57449f5 100644 --- a/caimira/apps/calculator/__init__.py +++ b/caimira/apps/calculator/__init__.py @@ -19,7 +19,6 @@ import typing import uuid import zlib - import jinja2 import loky from tornado.web import Application, RequestHandler, StaticFileHandler @@ -29,6 +28,7 @@ import tornado.log from . import markdown_tools from . import model_generator from .report_generator import ReportGenerator, calculate_report_data +from .data_service import DataService from .user import AuthenticatedUser, AnonymousUser # The calculator version is based on a combination of the model version and the @@ -41,12 +41,10 @@ from .user import AuthenticatedUser, AnonymousUser __version__ = "4.11" LOG = logging.getLogger(__name__) - + class BaseRequestHandler(RequestHandler): - # Host URL for the CAiMIRA Data Service API - host: str = 'https://caimira-data-api.app.cern.ch' - + async def prepare(self): """Called at the beginning of a request before `get`/`post`/etc.""" @@ -63,47 +61,6 @@ class BaseRequestHandler(RequestHandler): else: self.current_user = AnonymousUser() - async def data_service_connection(self): - client_email = self.settings["data_service_client_email"] - client_password = self.settings['data_service_client_password'] - - if (client_email == None or client_password == None): - # If the credentials are not defined, we skip the API Integration - return self.send_error(401) - - http_client = AsyncHTTPClient() - headers = {'Content-type': 'application/json'} - json_body = { "email": f"{client_email}", "password": f"{client_password}"} - - try: - response = await http_client.fetch(HTTPRequest( - url=self.host + '/login', - method='POST', - headers=headers, - body=json.dumps(json_body), - ), - raise_error=True) - except Exception as e: - print("Something went wrong: %s" % e) - - return json.loads(response.body)['access_token'] - - async def data_service_fetcher(self, access_token: str): - http_client = AsyncHTTPClient() - headers = {'Authorization': f'Bearer {access_token}'} - - try: - response = await http_client.fetch(HTTPRequest( - url=self.host + '/data', - method='GET', - headers=headers, - ), - raise_error=True) - except Exception as e: - print("Something went wrong: %s" % e) - - return json.loads(response.body) - def write_error(self, status_code: int, **kwargs) -> None: template = self.settings["template_environment"].get_template( "error.html.j2") @@ -148,14 +105,15 @@ class ConcentrationModel(BaseRequestHandler): from pprint import pprint pprint(requested_model_config) start = datetime.datetime.now() - + # Data Service API Integration try: - access_token = await self.data_service_connection() - service_data = await self.data_service_fetcher(access_token) + data_service: DataService = self.settings["data_service"] + access_token = await data_service.login() + service_data = await data_service.fetch(access_token) except Exception as e: print("Something went wrong with the data service: %s" % e) - + try: form = model_generator.FormData.from_dict(requested_model_config) except Exception as err: @@ -468,6 +426,11 @@ def make_app( ) template_environment.globals['get_url']=get_root_url template_environment.globals['get_calculator_url']=get_root_calculator_url + + data_service_credentials = { + 'data_service_client_email': os.environ.get('DATA_SERVICE_CLIENT_EMAIL', None), + 'data_service_client_password': os.environ.get('DATA_SERVICE_CLIENT_PASSWORD', None), + } if debug: tornado.log.enable_pretty_logging() @@ -486,9 +449,8 @@ def make_app( arve_client_secret=os.environ.get('ARVE_CLIENT_SECRET', None), arve_api_key=os.environ.get('ARVE_API_KEY', None), - # Data Service API Credentials - data_service_client_email = os.environ.get('DATA_SERVICE_CLIENT_EMAIL', ''), - data_service_client_password = os.environ.get('DATA_SERVICE_CLIENT_PASSWORD', ''), + # Data Service Integration + data_service = DataService(data_service_credentials), # Process parallelism controls. There is a balance between serving a single report # requests quickly or serving multiple requests concurrently. From 38587c43efa4668fbb1d85ce8ecd4c110d09942c Mon Sep 17 00:00:00 2001 From: Luis Aleixo Date: Tue, 11 Jul 2023 13:36:29 +0100 Subject: [PATCH 3/5] added the data service as an external component (class) --- caimira/apps/calculator/data_service.py | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 caimira/apps/calculator/data_service.py diff --git a/caimira/apps/calculator/data_service.py b/caimira/apps/calculator/data_service.py new file mode 100644 index 00000000..4ac27fa1 --- /dev/null +++ b/caimira/apps/calculator/data_service.py @@ -0,0 +1,65 @@ +import dataclasses +import json +import logging + +from tornado.web import RequestHandler +from tornado.httpclient import AsyncHTTPClient, HTTPRequest + +LOG = logging.getLogger(__name__) + + +@dataclasses.dataclass +class DataService(RequestHandler): + ''' + Responsible for establishing a connection to a + database through a REST API by handling authentication + and fetching data. It utilizes the Tornado web framework + for asynchronous HTTP requests. + ''' + # Credentials used for authentication + credentials: dict + + # Host URL for the CAiMIRA Data Service API + host: str = 'https://caimira-data-api.app.cern.ch' + + async def login(self): + client_email = self.credentials["data_service_client_email"] + client_password = self.credentials['data_service_client_password'] + + if (client_email == None or client_password == None): + # If the credentials are not defined, an error is thrown. + return self.send_error(500) + + http_client = AsyncHTTPClient() + headers = {'Content-type': 'application/json'} + json_body = { "email": f"{client_email}", "password": f"{client_password}"} + + try: + response = await http_client.fetch(HTTPRequest( + url=self.host + '/login', + method='POST', + headers=headers, + body=json.dumps(json_body), + ), + raise_error=True) + except Exception as err: + LOG.error("Something went wrong: %s" % err) + self.send_error(500) + + return json.loads(response.body)['access_token'] + + async def fetch(self, access_token: str): + http_client = AsyncHTTPClient() + headers = {'Authorization': f'Bearer {access_token}'} + + try: + response = await http_client.fetch(HTTPRequest( + url=self.host + '/data', + method='GET', + headers=headers, + ), + raise_error=True) + except Exception as e: + print("Something went wrong: %s" % e) + + return json.loads(response.body) \ No newline at end of file From f9fb2a12fbe856ee4b96a91e0da5809dfe7d82b1 Mon Sep 17 00:00:00 2001 From: Luis Aleixo Date: Tue, 11 Jul 2023 13:36:36 +0100 Subject: [PATCH 4/5] added a test file --- caimira/tests/test_data_service.py | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 caimira/tests/test_data_service.py diff --git a/caimira/tests/test_data_service.py b/caimira/tests/test_data_service.py new file mode 100644 index 00000000..be2cb106 --- /dev/null +++ b/caimira/tests/test_data_service.py @@ -0,0 +1,87 @@ +from dataclasses import dataclass + +import unittest +from unittest.mock import patch, MagicMock +from tornado.httpclient import HTTPError + +from caimira.apps.calculator.data_service import DataService + +@dataclass +class MockResponse: + body: str + +class DataServiceTests(unittest.TestCase): + def setUp(self): + # Set up any necessary test data or configurations + self.credentials = { + "data_service_client_email": "test@example.com", + "data_service_client_password": "password123" + } + self.data_service = DataService(self.credentials) + + @patch('caimira.apps.calculator.data_service.AsyncHTTPClient') + async def test_login_successful(self, mock_http_client): + # Mock successful login response + mock_response = MockResponse('{"access_token": "dummy_token"}') + mock_fetch = MagicMock(return_value=mock_response) + mock_http_client.return_value.fetch = mock_fetch + + # Call the login method + access_token = await self.data_service.login() + + # Assert that the access token is returned correctly + self.assertEqual(access_token, "dummy_token") + + # Verify that the fetch method was called with the expected arguments + mock_fetch.assert_called_once_with( + url='https://caimira-data-api.app.cern.ch/login', + method='POST', + headers={'Content-type': 'application/json'}, + body='{"email": "test@example.com", "password": "password123"}' + ) + + @patch('caimira.apps.calculator.data_service.AsyncHTTPClient') + async def test_login_error(self, mock_http_client): + # Mock login error response + mock_fetch = MagicMock(side_effect=HTTPError(500)) + mock_http_client.return_value.fetch = mock_fetch + + # Call the login method + access_token = await self.data_service.login() + + # Assert that the login method returns None in case of an error + self.assertIsNone(access_token) + + @patch('caimira.apps.calculator.data_service.AsyncHTTPClient') + async def test_fetch_successful(self, mock_http_client): + # Mock successful fetch response + mock_response = MockResponse('{"data": "dummy_data"}') + mock_fetch = MagicMock(return_value=mock_response) + mock_http_client.return_value.fetch = mock_fetch + + # Call the fetch method with a mock access token + access_token = "dummy_token" + data = await self.data_service.fetch(access_token) + + # Assert that the data is returned correctly + self.assertEqual(data, {"data": "dummy_data"}) + + # Verify that the fetch method was called with the expected arguments + mock_fetch.assert_called_once_with( + url='https://caimira-data-api.app.cern.ch/data', + method='GET', + headers={'Authorization': 'Bearer dummy_token'} + ) + + @patch('caimira.apps.calculator.data_service.AsyncHTTPClient') + async def test_fetch_error(self, mock_http_client): + # Mock fetch error response + mock_fetch = MagicMock(side_effect=HTTPError(404)) + mock_http_client.return_value.fetch = mock_fetch + + # Call the fetch method with a mock access token + access_token = "dummy_token" + data = await self.data_service.fetch(access_token) + + # Assert that the fetch method returns None in case of an error + self.assertIsNone(data) From c9a64a2d925053199ce7257362f9686b2e585398 Mon Sep 17 00:00:00 2001 From: Luis Aleixo Date: Mon, 17 Jul 2023 17:24:23 +0200 Subject: [PATCH 5/5] handled exceptions --- caimira/apps/calculator/__init__.py | 12 ++++--- caimira/apps/calculator/data_service.py | 43 +++++++++++-------------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/caimira/apps/calculator/__init__.py b/caimira/apps/calculator/__init__.py index f57449f5..b3954128 100644 --- a/caimira/apps/calculator/__init__.py +++ b/caimira/apps/calculator/__init__.py @@ -38,7 +38,7 @@ from .user import AuthenticatedUser, AnonymousUser # calculator version. If the calculator needs to make breaking changes (e.g. change # form attributes) then it can also increase its MAJOR version without needing to # increase the overall CAiMIRA version (found at ``caimira.__version__``). -__version__ = "4.11" +__version__ = "4.12" LOG = logging.getLogger(__name__) @@ -107,13 +107,15 @@ class ConcentrationModel(BaseRequestHandler): start = datetime.datetime.now() # Data Service API Integration + data_service: DataService = self.settings["data_service"] try: - data_service: DataService = self.settings["data_service"] access_token = await data_service.login() service_data = await data_service.fetch(access_token) - except Exception as e: - print("Something went wrong with the data service: %s" % e) - + except Exception as err: + error_message = f"Something went wrong with the data service: {str(err)}" + LOG.error(error_message, exc_info=True) + self.send_error(500, reason=error_message) + try: form = model_generator.FormData.from_dict(requested_model_config) except Exception as err: diff --git a/caimira/apps/calculator/data_service.py b/caimira/apps/calculator/data_service.py index 4ac27fa1..2d480780 100644 --- a/caimira/apps/calculator/data_service.py +++ b/caimira/apps/calculator/data_service.py @@ -2,14 +2,13 @@ import dataclasses import json import logging -from tornado.web import RequestHandler from tornado.httpclient import AsyncHTTPClient, HTTPRequest LOG = logging.getLogger(__name__) @dataclasses.dataclass -class DataService(RequestHandler): +class DataService(): ''' Responsible for establishing a connection to a database through a REST API by handling authentication @@ -27,24 +26,20 @@ class DataService(RequestHandler): client_password = self.credentials['data_service_client_password'] if (client_email == None or client_password == None): - # If the credentials are not defined, an error is thrown. - return self.send_error(500) + # If the credentials are not defined, an exception is raised. + raise Exception("DataService credentials not set") http_client = AsyncHTTPClient() headers = {'Content-type': 'application/json'} json_body = { "email": f"{client_email}", "password": f"{client_password}"} - try: - response = await http_client.fetch(HTTPRequest( - url=self.host + '/login', - method='POST', - headers=headers, - body=json.dumps(json_body), - ), - raise_error=True) - except Exception as err: - LOG.error("Something went wrong: %s" % err) - self.send_error(500) + response = await http_client.fetch(HTTPRequest( + url=self.host + '/login', + method='POST', + headers=headers, + body=json.dumps(json_body), + ), + raise_error=True) return json.loads(response.body)['access_token'] @@ -52,14 +47,12 @@ class DataService(RequestHandler): http_client = AsyncHTTPClient() headers = {'Authorization': f'Bearer {access_token}'} - try: - response = await http_client.fetch(HTTPRequest( - url=self.host + '/data', - method='GET', - headers=headers, - ), - raise_error=True) - except Exception as e: - print("Something went wrong: %s" % e) + response = await http_client.fetch(HTTPRequest( + url=self.host + '/data', + method='GET', + headers=headers, + ), + raise_error=True) - return json.loads(response.body) \ No newline at end of file + return json.loads(response.body) + \ No newline at end of file