From 2dde48d10e7db898deb46706287f689a23520cd7 Mon Sep 17 00:00:00 2001 From: Patricia Morimoto Date: Wed, 13 Jun 2018 14:42:04 -0300 Subject: [PATCH 1/6] =?UTF-8?q?Altera=C3=A7=C3=A3o=20do=20m=C3=B3dulo=20ca?= =?UTF-8?q?talogmanager.persistence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Criação de `insert_file` nos DBManagers - Criação de `ChangesService.register` com o novo conteúdo do registro de mudança - Criação de `DatabaseService.add_file` para persistir arquivos usando o DBManager e registrando a mudança com `ChangesService.register` - Ajustes nos fixtures para executar os testes sem duplicação - Testes unitários e de integração do módulo persistence para o novo registro de artigos --- persistence/databases.py | 20 ++++++++++++ persistence/services.py | 27 +++++++++++++++- persistence/tests/conftest.py | 44 +++++++++++++++++++------ persistence/tests/test_changes.py | 33 +++++++++++++++---- persistence/tests/test_databases.py | 47 +++++++++++++++++++++++++++ persistence/tests/test_services.py | 50 ++++++++++++++--------------- requirements.txt | 1 + setup.py | 8 ++++- 8 files changed, 187 insertions(+), 43 deletions(-) diff --git a/persistence/databases.py b/persistence/databases.py index 01c5519..fe8a157 100644 --- a/persistence/databases.py +++ b/persistence/databases.py @@ -93,6 +93,10 @@ def get_attachment_properties(self, id, file_id): doc = self.read(id) return doc.get(self._attachments_properties_key, {}).get(file_id) + @abc.abstractmethod + def insert_file(self, file_id, content) -> None: + return NotImplemented + class InMemoryDBManager(BaseDBManager): @@ -233,6 +237,11 @@ def list_attachments(self, id): doc = self.read(id) return list(doc.get(self._attachments_key, {}).keys()) + def insert_file(self, file_id, content): + file = self.database.get(id) + if not file: + self.database.update({file_id: content}) + class CouchDBManager(BaseDBManager): @@ -387,6 +396,17 @@ def list_attachments(self, id): doc = self.read(id) return list(doc.get(self._attachments_key, {}).keys()) + def insert_file(self, file_id, content): + file = self.database.get(file_id) + if not file: + self.create(id=file_id, document={}) + doc = self.database.get(file_id) + self.database.put_attachment( + doc=doc, + content=content, + filename=file_id + ) + def sort_results(results, sort): scores = [list() for i in results] diff --git a/persistence/services.py b/persistence/services.py index 03399f8..6210458 100644 --- a/persistence/services.py +++ b/persistence/services.py @@ -1,6 +1,5 @@ from datetime import datetime from enum import Enum -from uuid import uuid4 from .databases import QueryOperator @@ -53,6 +52,20 @@ def register_change(self, ) return change_record['record_id'] + def register(self, record_id, change_type): + sequencial = str(self.seqnum_generator.new()) + change_record = { + 'change_id': sequencial, + 'document_id': record_id, + 'type': change_type.value, + 'created_date': str(datetime.utcnow().timestamp()), + } + self.changes_db_manager.create( + sequencial, + change_record + ) + return sequencial + class DatabaseService: """ @@ -222,6 +235,18 @@ def get_attachment_properties(self, document_id, file_id): """ return self.db_manager.get_attachment_properties(document_id, file_id) + def add_file(self, file_id, content): + """ + Persiste conteúdo de arquivo na base de dados identificado com ID + informado. + + Params: + file_id: ID do arquivo + content: conteúdo do arquivo + """ + self.db_manager.insert_file(file_id=file_id, content=content) + self.changes_service.register(file_id, ChangeType.CREATE) + def list_changes(self, last_sequence, limit): """ Busca registros de mudança a partir do sequencial informado e retorna diff --git a/persistence/tests/conftest.py b/persistence/tests/conftest.py index df1e4bb..8be6c32 100644 --- a/persistence/tests/conftest.py +++ b/persistence/tests/conftest.py @@ -76,24 +76,37 @@ def fin(): return s -@pytest.fixture(params=[ - CouchDBManager, - InMemoryDBManager -]) -def database_service(request, article_db_settings, change_db_settings, - seqnumber_generator): +@pytest.fixture( + params=[ + CouchDBManager, + InMemoryDBManager + ] +) +def database_service(request, + article_db_settings, + change_db_settings, + seqnum_db_settings): DBManager = request.param + s = SeqNumGenerator( + DBManager(**seqnum_db_settings), + 'CHANGE' + ) db_service = DatabaseService( DBManager(**article_db_settings), ChangesService( DBManager(**change_db_settings), - seqnumber_generator + s ) ) def fin(): - db_service.db_manager.drop_database() - db_service.changes_service.changes_db_manager.drop_database() + try: + db_service.db_manager.drop_database() + db_service.changes_service.changes_db_manager.drop_database() + s.db_manager.drop_database() + except Exception: + pass + request.addfinalizer(fin) return db_service @@ -112,11 +125,22 @@ def inmemory_db_setup(request, persistence_config, change_db_settings): ) def fin(): - inmemory_db_service.changes_service.changes_db_manager.drop_database() + try: + inmemory_db_service.db_manager.drop_database() + inmemory_db_service.changes_service.changes_db_manager.\ + drop_database() + inmemory_db_service.changes_service.db_manager.drop_database() + except Exception: + pass request.addfinalizer(fin) return inmemory_db_service +@pytest.fixture(params=[CouchDBManager, InMemoryDBManager]) +def db_manager_test(request, article_db_settings): + return request.param(**article_db_settings) + + @pytest.fixture def test_changes_records(request): changes_list = [] diff --git a/persistence/tests/test_changes.py b/persistence/tests/test_changes.py index 17b4732..3ea94c9 100644 --- a/persistence/tests/test_changes.py +++ b/persistence/tests/test_changes.py @@ -13,7 +13,7 @@ def get_article_record(content={'Test': 'ChangeRecord'}): created_date=datetime.utcnow()) -def test_register_create_change(database_service): +def test_register_change_create(database_service): article_record = get_article_record() change_id = database_service.changes_service.register_change( article_record, @@ -21,7 +21,8 @@ def test_register_create_change(database_service): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['document_id'] == article_record['document_id'] assert check_change['document_type'] == article_record['document_type'] @@ -29,6 +30,22 @@ def test_register_create_change(database_service): assert check_change['created_date'] is not None +def test_register_create(database_service): + document_id = 'ID-1234' + change_id = database_service.changes_service.register( + document_id, + ChangeType.CREATE + ) + assert change_id is not None + check_change = dict( + database_service.changes_service.changes_db_manager.database[change_id] + ) + assert check_change is not None + assert check_change['document_id'] == document_id + assert check_change['type'] == ChangeType.CREATE.value + assert check_change['created_date'] is not None + + def test_register_update_change(database_service): article_record = get_article_record({'Test': 'ChangeRecord2'}) change_id = database_service.changes_service.register_change( @@ -37,7 +54,8 @@ def test_register_update_change(database_service): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['document_id'] == article_record['document_id'] assert check_change['document_type'] == article_record['document_type'] @@ -53,7 +71,8 @@ def test_register_delete_change(database_service): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['document_id'] == article_record['document_id'] assert check_change['document_type'] == article_record['document_type'] @@ -84,7 +103,8 @@ def test_add_attachment_create_change(database_service, xml_test): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['attachment_id'] == attachment_id assert check_change['document_id'] == article_record['document_id'] @@ -116,7 +136,8 @@ def test_update_attachment_create_change(database_service, xml_test): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['attachment_id'] == attachment_id assert check_change['document_id'] == article_record['document_id'] diff --git a/persistence/tests/test_databases.py b/persistence/tests/test_databases.py index 6bab4f5..36f354e 100644 --- a/persistence/tests/test_databases.py +++ b/persistence/tests/test_databases.py @@ -489,3 +489,50 @@ def compare_documents(document, expected): assert document[k] == expected[k] else: assert document == expected + + +def test_add_file_insert_file_into_database_ok( + db_manager_test, + xml_test +): + db_manager_test.insert_file( + file_id='href_file1', + content=xml_test.encode('utf-8') + ) + file = db_manager_test.database.get('href_file1') + assert file is not None + + +def test_add_file_call_dbmanager_insert_file(database_service, xml_test): + with patch.object(database_service.db_manager, 'insert_file') \ + as mocked_insert_file: + database_service.add_file( + file_id='href_file1', + content=xml_test.encode('utf-8'), + ) + mocked_insert_file.assert_called_once_with( + file_id='href_file1', + content=xml_test.encode('utf-8') + ) + + +def test_add_file_call_changeservice_register(inmemory_db_setup, xml_test): + with patch.object(inmemory_db_setup.changes_service, 'register') \ + as mocked_change_register: + file_id = '/rawfile/href_file1' + inmemory_db_setup.add_file( + file_id=file_id, + content=xml_test.encode('utf-8'), + ) + mocked_change_register.assert_called_once_with( + file_id, ChangeType.CREATE + ) + + +def test_add_file_dbmanager_insert_into_database(inmemory_db_setup, xml_test): + inmemory_db_setup.add_file( + file_id='href_file1', + content=xml_test.encode('utf-8'), + ) + file = inmemory_db_setup.db_manager.database.get('href_file1') + assert file is not None diff --git a/persistence/tests/test_services.py b/persistence/tests/test_services.py index cdd2901..355f753 100644 --- a/persistence/tests/test_services.py +++ b/persistence/tests/test_services.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock +from unittest.mock import patch from persistence.databases import QueryOperator from persistence.services import ChangeType, SortOrder @@ -9,31 +9,31 @@ def test_list_changes_calls_db_manager_find(inmemory_db_setup, xml_test): _changes_db_manager = inmemory_db_setup.changes_service.changes_db_manager - _changes_db_manager.find = Mock() - _changes_db_manager.find.return_value = [] - last_sequence = '123456' - limit = 10 - expected_fields = [ - 'change_id', - 'document_id', - 'document_type', - 'type', - 'created_date' - ] - filter = { - 'change_id': [ - (QueryOperator.GREATER_THAN, last_sequence) + with patch.object(_changes_db_manager, 'find'): + _changes_db_manager.find.return_value = [] + last_sequence = '123456' + limit = 10 + expected_fields = [ + 'change_id', + 'document_id', + 'document_type', + 'type', + 'created_date' ] - } - sort = [{'change_id': SortOrder.ASC.value}] - inmemory_db_setup.list_changes(last_sequence=last_sequence, - limit=limit) - _changes_db_manager.find.assert_called_once_with( - fields=expected_fields, - limit=limit, - filter=filter, - sort=sort - ) + filter = { + 'change_id': [ + (QueryOperator.GREATER_THAN, last_sequence) + ] + } + sort = [{'change_id': SortOrder.ASC.value}] + inmemory_db_setup.list_changes(last_sequence=last_sequence, + limit=limit) + _changes_db_manager.find.assert_called_once_with( + fields=expected_fields, + limit=limit, + filter=filter, + sort=sort + ) def test_list_changes_returns_db_manager_find_all(inmemory_db_setup, diff --git a/requirements.txt b/requirements.txt index 34bea5d..81795cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ cornice==3.4.0 cornice-swagger==0.6.0 CouchDB==1.2 +freezegun==0.3.10 gunicorn==19.7.1 lxml==4.2.1 Mako==1.0.7 diff --git a/setup.py b/setup.py index 75ffca6..71cb7a9 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,13 @@ 'cornice>=3.4.0', ] -test_requires = ['webtest', 'pytest', 'pytest-cov', 'pytest-lazy-fixture'] +test_requires = [ + 'webtest', + 'pytest', + 'pytest-cov', + 'pytest-lazy-fixture', + 'freezegun' +] setup_requires = ['pytest-runner'] setup( From 61ffb0f82c211901a6a1e35faa40a1ef9d286163 Mon Sep 17 00:00:00 2001 From: Patricia Morimoto Date: Wed, 13 Jun 2018 15:06:50 -0300 Subject: [PATCH 2/6] Adding Prometheus metric to add_file --- persistence/services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/persistence/services.py b/persistence/services.py index 12e77ee..d8a87cc 100644 --- a/persistence/services.py +++ b/persistence/services.py @@ -267,6 +267,7 @@ def get_attachment_properties(self, document_id, file_id): """ return self.db_manager.get_attachment_properties(document_id, file_id) + @REQUEST_TIME_ATT_UPD.time() def add_file(self, file_id, content): """ Persiste conteúdo de arquivo na base de dados identificado com ID From 5d3bf665e789f3fbc80975ec0a701242845e9ae6 Mon Sep 17 00:00:00 2001 From: Patricia Morimoto Date: Wed, 13 Jun 2018 15:57:05 -0300 Subject: [PATCH 3/6] DBManager.insert_file returns absolut file URL --- persistence/databases.py | 16 +++++++----- persistence/tests/conftest.py | 38 +++++++++++++++-------------- persistence/tests/test_databases.py | 8 ++++-- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/persistence/databases.py b/persistence/databases.py index fe8a157..cc1d642 100644 --- a/persistence/databases.py +++ b/persistence/databases.py @@ -102,6 +102,7 @@ class InMemoryDBManager(BaseDBManager): def __init__(self, **kwargs): self._database_name = kwargs['database_name'] + self._database_url = kwargs['database_uri'] self._attachments_key = 'attachments' self._attachments_properties_key = 'attachments_properties' self._database = {} @@ -238,19 +239,21 @@ def list_attachments(self, id): return list(doc.get(self._attachments_key, {}).keys()) def insert_file(self, file_id, content): + id = '/'.join([self._database_url, file_id]) file = self.database.get(id) if not file: - self.database.update({file_id: content}) + self.database.update({id: content}) class CouchDBManager(BaseDBManager): def __init__(self, **kwargs): self._database_name = kwargs['database_name'] + self._database_url = kwargs['database_uri'] self._attachments_key = '_attachments' self._attachments_properties_key = 'attachments_properties' self._database = None - self._db_server = couchdb.Server(kwargs['database_uri']) + self._db_server = couchdb.Server(self._database_url) self._db_server.resource.credentials = ( kwargs['database_username'], kwargs['database_password'] @@ -397,14 +400,15 @@ def list_attachments(self, id): return list(doc.get(self._attachments_key, {}).keys()) def insert_file(self, file_id, content): - file = self.database.get(file_id) + id = '/'.join([self._database_url, file_id]) + file = self.database.get(id) if not file: - self.create(id=file_id, document={}) - doc = self.database.get(file_id) + self.create(id=id, document={}) + doc = self.database.get(id) self.database.put_attachment( doc=doc, content=content, - filename=file_id + filename=id ) diff --git a/persistence/tests/conftest.py b/persistence/tests/conftest.py index 8be6c32..e236fdb 100644 --- a/persistence/tests/conftest.py +++ b/persistence/tests/conftest.py @@ -18,17 +18,6 @@ from persistence.seqnum_generator import SeqNumGenerator -@pytest.yield_fixture -def persistence_config(request): - yield testing.setUp() - testing.tearDown() - - -@pytest.fixture -def fake_change_list(): - return ['Test1', 'Test2', 'Test3', 'Test4', 'Test5', 'Test6'] - - @pytest.fixture def article_db_settings(): return { @@ -39,6 +28,11 @@ def article_db_settings(): } +@pytest.fixture +def article_dbinmemory_settings(): + return {'database_uri': '/rawfile', 'database_name': 'articles'} + + @pytest.fixture def change_db_settings(): return { @@ -112,13 +106,15 @@ def fin(): @pytest.fixture -def inmemory_db_setup(request, persistence_config, change_db_settings): +def inmemory_db_setup(request): + db_host = '/rawfile' inmemory_db_service = DatabaseService( - InMemoryDBManager(database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='articles'), ChangesService( - InMemoryDBManager(database_name='changes'), + InMemoryDBManager(database_uri=db_host, database_name='changes'), SeqNumGenerator( - InMemoryDBManager(database_name='seqnum'), + InMemoryDBManager(database_uri=db_host, + database_name='seqnum'), 'CHANGE' ) ) @@ -136,9 +132,15 @@ def fin(): return inmemory_db_service -@pytest.fixture(params=[CouchDBManager, InMemoryDBManager]) -def db_manager_test(request, article_db_settings): - return request.param(**article_db_settings) +@pytest.fixture( + params=[ + (CouchDBManager, article_db_settings), + (InMemoryDBManager, article_dbinmemory_settings) + ] +) +def db_manager_test(request): + db_settings = request.param[1]() + return request.param[0](**db_settings) @pytest.fixture diff --git a/persistence/tests/test_databases.py b/persistence/tests/test_databases.py index 36f354e..c2e5b6d 100644 --- a/persistence/tests/test_databases.py +++ b/persistence/tests/test_databases.py @@ -499,7 +499,9 @@ def test_add_file_insert_file_into_database_ok( file_id='href_file1', content=xml_test.encode('utf-8') ) - file = db_manager_test.database.get('href_file1') + file = db_manager_test.database.get( + '/'.join([db_manager_test._database_url, 'href_file1']) + ) assert file is not None @@ -534,5 +536,7 @@ def test_add_file_dbmanager_insert_into_database(inmemory_db_setup, xml_test): file_id='href_file1', content=xml_test.encode('utf-8'), ) - file = inmemory_db_setup.db_manager.database.get('href_file1') + file = inmemory_db_setup.db_manager.database.get( + '/'.join([inmemory_db_setup.db_manager._database_url, 'href_file1']) + ) assert file is not None From 3db3ad2d5f98c683597c4174a174fd18732f9df6 Mon Sep 17 00:00:00 2001 From: Patricia Morimoto Date: Thu, 14 Jun 2018 16:57:27 -0300 Subject: [PATCH 4/6] =?UTF-8?q?Ajuste=20do=20sequencial=20na=20inser=C3=A7?= =?UTF-8?q?=C3=A3o=20do=20novo=20registro=20de=20mudan=C3=A7a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- persistence/services.py | 4 ++-- persistence/tests/conftest.py | 1 - persistence/tests/test_changes.py | 35 ++++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/persistence/services.py b/persistence/services.py index d8a87cc..aceddc3 100644 --- a/persistence/services.py +++ b/persistence/services.py @@ -79,7 +79,7 @@ def register_change(self, return change_record['record_id'] def register(self, record_id, change_type): - sequencial = str(self.seqnum_generator.new()) + sequencial = self.seqnum_generator.new() change_record = { 'change_id': sequencial, 'document_id': record_id, @@ -87,7 +87,7 @@ def register(self, record_id, change_type): 'created_date': str(datetime.utcnow().timestamp()), } self.changes_db_manager.create( - sequencial, + str(sequencial), change_record ) return sequencial diff --git a/persistence/tests/conftest.py b/persistence/tests/conftest.py index e236fdb..be68304 100644 --- a/persistence/tests/conftest.py +++ b/persistence/tests/conftest.py @@ -1,7 +1,6 @@ from datetime import datetime from random import randint -from pyramid import testing import pytest from persistence.databases import ( diff --git a/persistence/tests/test_changes.py b/persistence/tests/test_changes.py index 3ea94c9..58e7917 100644 --- a/persistence/tests/test_changes.py +++ b/persistence/tests/test_changes.py @@ -1,7 +1,8 @@ from datetime import datetime +from random import randint from uuid import uuid4 -from persistence.services import ChangeType +from persistence.services import ChangeType, SortOrder from persistence.models import get_record, RecordType @@ -30,7 +31,7 @@ def test_register_change_create(database_service): assert check_change['created_date'] is not None -def test_register_create(database_service): +def test_register_change(database_service): document_id = 'ID-1234' change_id = database_service.changes_service.register( document_id, @@ -38,7 +39,9 @@ def test_register_create(database_service): ) assert change_id is not None check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id] + database_service.changes_service.changes_db_manager.database[ + str(change_id) + ] ) assert check_change is not None assert check_change['document_id'] == document_id @@ -46,6 +49,32 @@ def test_register_create(database_service): assert check_change['created_date'] is not None +def test_register_change_must_keep_sequential_order(database_service): + changes_service = database_service.changes_service + change_type_list = list(ChangeType) + + for counter in range(1, 11): + changes_service.register( + 'ID-{}'.format(counter), + change_type_list[randint(0, len(change_type_list)-1)] + ) + sort = [{'change_id': SortOrder.ASC.value}] + changes_list = changes_service.changes_db_manager.find(fields=[], + filter={}, + sort=sort) + + assert changes_list is not None + assert len(changes_list) == 10 + assert all([ + isinstance(change_list['change_id'], int) + for change_list in changes_list + ]) + assert all([ + changes_list[i]['change_id'] < changes_list[i + 1]['change_id'] + for i in range(len(changes_list) - 1) + ]) + + def test_register_update_change(database_service): article_record = get_article_record({'Test': 'ChangeRecord2'}) change_id = database_service.changes_service.register_change( From 2cd813b727b9eb94c543cbf5f253b5535dab82ac Mon Sep 17 00:00:00 2001 From: Patricia Morimoto Date: Mon, 18 Jun 2018 14:46:24 -0300 Subject: [PATCH 5/6] =?UTF-8?q?Altera=C3=A7=C3=A3o=20da=20camada=20de=20pe?= =?UTF-8?q?rsist=C3=AAncia=20e=20ajustes=20do=20WEB=20App?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ajustes em `managers` para as mudanças em `persistence`. - Alteração do ArticleDocument para prever o registro de versões diferentes do artigo. - Ajuste do ArticleManager para as alterações do ArticleDocument, o gerenciamento de um DBManager para os arquivos brutos, e a implementação do método `add_document`. Ajuste do método post na view de artigo. - Criação de teste funcional para a funcionalidade de registro de novo artigo. --- api/tests/conftest.py | 44 +++++- api/tests/test_article.py | 13 +- api/views/article.py | 16 +- managers/__init__.py | 40 ++++- managers/article_manager.py | 17 ++- managers/models/article_model.py | 63 ++++++-- managers/tests/conftest.py | 39 +++-- managers/tests/test_article_manager.py | 199 ++++++++++++++++++------- managers/tests/test_article_model.py | 90 +++++------ managers/tests/test_managers.py | 99 ++++++++++-- persistence/tests/test_changes.py | 1 + tests/test_functional.py | 40 +++++ 12 files changed, 497 insertions(+), 164 deletions(-) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 688eaec..8773bbd 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -4,17 +4,57 @@ import pytest from pyramid import testing +from managers.article_manager import ArticleManager +from persistence.databases import InMemoryDBManager +from persistence.services import ChangesService +from persistence.seqnum_generator import SeqNumGenerator + @pytest.fixture def dummy_request(): request = testing.DummyRequest() request.db_settings = { - 'host': 'http://localhost', - 'port': '12345' + 'db_host': 'http://localhost', + 'db_port': '12345' } return request +@pytest.fixture +def inmemory_article_manager(request): + db_host = 'http://inmemory' + articles_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='articles') + files_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='files') + changes_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='changes') + changes_seq_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='changes_seqnum') + + def fin(): + try: + articles_dbmanager.drop_database() + files_dbmanager.drop_database() + changes_dbmanager.drop_database() + changes_seq_dbmanager.drop_database() + except Exception: + pass + + request.addfinalizer(fin) + return ArticleManager( + articles_dbmanager, + files_dbmanager, + ChangesService( + changes_dbmanager, + SeqNumGenerator( + changes_seq_dbmanager, + 'CHANGE' + ) + ) + ) + + @pytest.fixture def test_xml_file(): return """ diff --git a/api/tests/test_article.py b/api/tests/test_article.py index 46dcaa4..91171a4 100644 --- a/api/tests/test_article.py +++ b/api/tests/test_article.py @@ -356,16 +356,14 @@ def test_http_get_asset_file_succeeded(mocked_get_asset_file, assert response.content_type == expected[0] -@patch.object(managers, 'create_file') @patch.object(managers, 'post_article') def test_post_article_invalid_xml(mocked_post_article, - mocked_create_file, dummy_request, test_xml_file): xml_file = MockCGIFieldStorage("test_xml_file.xml", BytesIO(test_xml_file.encode('utf-8'))) error_msg = 'Invalid XML Content' - mocked_create_file.side_effect = \ + mocked_post_article.side_effect = \ managers.exceptions.ManagerFileError( message=error_msg ) @@ -402,10 +400,13 @@ def test_post_article_internal_error(mocked_post_article, assert excinfo.value.message == error_msg -@patch.object(managers, 'post_article') -def test_post_article_returns_article_version_url(mocked_post_article, +@patch.object(managers, '_get_article_manager') +def test_post_article_returns_article_version_url(mocked__get_article_manager, + inmemory_article_manager, dummy_request, test_xml_file): + mocked__get_article_manager.return_value = inmemory_article_manager + inmemory_db_settings = '/rawfile' xml_file = MockCGIFieldStorage("test_xml_file.xml", BytesIO(test_xml_file.encode('utf-8'))) dummy_request.POST = { @@ -415,7 +416,7 @@ def test_post_article_returns_article_version_url(mocked_post_article, article_api = ArticleAPI(dummy_request) response = article_api.collection_post() - mocked_post_article.assert_called_once() assert response.status_code == 201 assert response.json is not None assert response.json.get('url').endswith(xml_file.filename) + assert response.json.get('url').startswith(inmemory_db_settings) diff --git a/api/views/article.py b/api/views/article.py index 4ef0066..0f07f2e 100644 --- a/api/views/article.py +++ b/api/views/article.py @@ -45,18 +45,22 @@ def _get_file_property(self, file_field): raise HTTPBadRequest(detail=e.message) def collection_post(self): + """ + Receive new Article document package which must contain a XML file. + """ try: xml_file_field = self.request.POST.get('xml_file') - xml_file = self._get_file_property(xml_file_field) - managers.post_article( + xml_id = Path(xml_file_field.filename).name + xml_file = xml_file_field.file.read() + article_url = managers.post_article( article_id=self.request.POST['article_id'], + xml_id=xml_id, xml_file=xml_file, **self.request.db_settings ) - body = { - 'url': '/rawfiles/7ca9f9b2687cb/' + xml_file_field.filename - } - return Response(status_code=201, json=body) + return Response(status_code=201, json={'url': article_url}) + except managers.exceptions.ManagerFileError as e: + raise HTTPBadRequest(detail=e.message) except managers.article_manager.ArticleManagerException as e: raise HTTPInternalServerError(detail=e.message) diff --git a/managers/__init__.py b/managers/__init__.py index 1514abd..a8c415c 100644 --- a/managers/__init__.py +++ b/managers/__init__.py @@ -1,6 +1,6 @@ from managers.article_manager import ArticleManager from managers.exceptions import ManagerFileError -from managers.models.article_model import ArticleDocument +from managers.models.article_model import ArticleDocument, InvalidXMLContent from managers.models.file import File from persistence.databases import CouchDBManager from persistence.services import ( @@ -35,9 +35,12 @@ def _get_article_manager(**db_settings): database_config = db_settings articles_database_config = database_config.copy() articles_database_config['database_name'] = "articles" + files_database_config = database_config.copy() + files_database_config['database_name'] = "files" return ArticleManager( CouchDBManager(**articles_database_config), + CouchDBManager(**files_database_config), _get_changes_services(db_settings) ) @@ -57,11 +60,33 @@ def create_file(filename, content): return File(file_name=filename, content=content) -def post_article(xml_file, **db_settings): - """""" - article_manager = _get_article_manager(**db_settings) - article_manager.add_document() - return xml_file.get_version() +def post_article(article_id, xml_id, xml_file, **db_settings): + """ + Registra novo documento de artigo em banco de dados informado, persistindo + a versão codificada em XML recebida e um manifesto do artigo contendo a + referência para recuperar o arquivo XML. + + :param article_id: ID do Documento do tipo Artigo, para identificação + referencial + :param xml_id: identificação do arquivo + :param xml_file: objeto File-like conteúdo do XML + :param db_settings: dicionário com as configurações do banco de dados. + Deve conter: + - database_uri: URI do banco de dados (host:porta) + - database_username: usuário do banco de dados + - database_password: senha do banco de dados + + :returns: URL pública para recuperar a versão registrada do artigo + codificado em XML + :rtype: str + """ + try: + article_document = ArticleDocument(article_id) + article_manager = _get_article_manager(**db_settings) + article_document.add_version(xml_id, xml_file) + return article_manager.add_document(article_document) + except InvalidXMLContent as e: + raise ManagerFileError(message=e.message) def put_article(article_id, xml_file, assets_files=[], **db_settings): @@ -165,7 +190,8 @@ def set_assets_public_url(article_id, xml_content, assets_filenames, :returns: conteúdo do XML atualizado """ xml_file = File(file_name="xml_file.xml", content=xml_content) - article = ArticleDocument(article_id, xml_file) + article = ArticleDocument(article_id) + article.xml_file = xml_file for name in article.assets: if name in assets_filenames: article.assets[name].href = public_url.format(article_id, diff --git a/managers/article_manager.py b/managers/article_manager.py index 3eadba8..b97826f 100644 --- a/managers/article_manager.py +++ b/managers/article_manager.py @@ -27,9 +27,12 @@ class ArticleManagerMissingAssetFileException(Exception): class ArticleManager: - def __init__(self, articles_db_manager, changes_services): + def __init__(self, articles_db_manager, files_db_manager, + changes_services): self.article_db_service = DatabaseService( articles_db_manager, changes_services) + self.file_db_service = DatabaseService( + files_db_manager, changes_services) def receive_package(self, id, xml_file, files=None): article = self.receive_xml_file(id, xml_file) @@ -37,7 +40,8 @@ def receive_package(self, id, xml_file, files=None): return article.unexpected_files_list, article.missing_files_list def receive_xml_file(self, id, xml_file): - article = ArticleDocument(id, xml_file) + article = ArticleDocument(id) + article.xml_file = xml_file article_record = Record( document_id=article.id, @@ -72,7 +76,14 @@ def receive_asset_file(self, article, file): ) def add_document(self, article_document): - pass + added_file_url = self.file_db_service.add_file( + file_id=article_document.xml_file_id, + content=article_document.xml_tree.content + ) + article_document.update_version(added_file_url) + article_record = article_document.get_record() + self.article_db_service.register(article_document.id, article_record) + return "/rawfile/" + article_document.xml_file_id def get_article_data(self, article_id): try: diff --git a/managers/models/article_model.py b/managers/models/article_model.py index 7d9d8ba..083d8f9 100644 --- a/managers/models/article_model.py +++ b/managers/models/article_model.py @@ -1,8 +1,19 @@ # coding=utf-8 +import hashlib +from enum import Enum from ..xml.article_xml_tree import ArticleXMLTree +class InvalidXMLContent(Exception): + message = "Invalid XML Content" + + +class DocumentType(Enum): + DOCUMENT = 'DOC' + ARTICLE = 'ART' + + class AssetDocument: """Metadados de um documento do tipo Ativo Digital. Um Ativo Digital é um arquivo associado a um documento do tipo Artigo @@ -34,20 +45,34 @@ def href(self, value): class ArticleDocument: - """Metadados de um documento do tipo Artigo. - - Os metadados contam com uma referência ao Artigo codificado em XML e - referências aos seus ativos digitais. - - Exemplo de uso: - - >>> doc = ArticleDocument('art01', ) - """ - def __init__(self, article_id, xml_file): + """Metadados de um documento do tipo Artigo.""" + def __init__(self, article_id): self.id = article_id + self.versions = [] self.assets = {} self.unexpected_files_list = [] - self.xml_file = xml_file + + def add_version(self, file_id, xml_content): + """Adiciona nova versão de artigo codificado em XML em :attr:`versions` + e cria nova referência para atualizar os dados do manifesto do artigo. + Caso o conteúdo do XML for inválido, a exceção + :class:`InvalidXMLContent` é lançada. + """ + self.xml_tree = ArticleXMLTree(xml_content) + if self.xml_tree.xml_error: + raise InvalidXMLContent + checksum = hashlib.sha1(xml_content).hexdigest() + self.xml_file_id = '/'.join([checksum[:13], file_id]) + self.versions.append({ + 'data': self.xml_file_id, + 'assets': [] + }) + + def update_version(self, added_file_url): + """ + Atualiza referência do artigo codificado em XML em :attr:`versions`. + """ + self.versions[-1].update({'data': added_file_url}) @property def xml_file(self): @@ -67,7 +92,9 @@ def xml_file(self): @xml_file.setter def xml_file(self, xml_file): self._xml_file = xml_file - self.xml_tree = ArticleXMLTree(self._xml_file.content) + self.xml_tree = ArticleXMLTree(xml_file.content) + if self.xml_tree.xml_error: + raise InvalidXMLContent self.assets = { name: AssetDocument(node) for name, node in self.xml_tree.asset_nodes.items() @@ -116,6 +143,18 @@ def get_record_content(self): ] return record_content + def get_record(self): + """Obtém um dicionário que descreve a instância de + :class:`ArticleDocument` da seguinte maneira: chave ``id``, contendo o + ID do artigo e chave ``versions``, contendo uma lista de versões do + artigo, com a URI da respectiva codificação XML e seus assets. + """ + record_content = {} + record_content['document_id'] = self.id + record_content['document_type'] = DocumentType.ARTICLE.value + record_content['versions'] = self.versions + return record_content + @property def missing_files_list(self): """Obtém uma lista com os nomes dos arquivos dos ativos digitais do diff --git a/managers/tests/conftest.py b/managers/tests/conftest.py index 0f82c85..294e97d 100644 --- a/managers/tests/conftest.py +++ b/managers/tests/conftest.py @@ -24,13 +24,11 @@ def test_fixture_dir(): def read_file(fixture_dir, dir_path, filename): - xml_file = File(filename) file_path = os.path.join(fixture_dir, dir_path, filename) if os.path.isfile(file_path): with open(file_path, 'rb') as fb: - xml_file.content = fb.read() - xml_file.size = os.stat(file_path).st_size - return xml_file + xml_file = File(filename, fb.read()) + return xml_file @pytest.fixture(scope="module") @@ -100,9 +98,10 @@ def functional_config(request): @pytest.fixture def change_service(functional_config): + db_host = 'http://inmemory' return ( - InMemoryDBManager(database_name='articles'), - InMemoryDBManager(database_name='changes') + InMemoryDBManager(database_uri=db_host, database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='changes') ) @@ -120,8 +119,9 @@ def xml_test(): @pytest.fixture def seqnumber_generator(request): + db_host = 'http://inmemory' s = SeqNumGenerator( - InMemoryDBManager(database_name='test3'), + InMemoryDBManager(database_uri=db_host, database_name='test3'), 'CHANGE' ) @@ -134,16 +134,18 @@ def fin(): @pytest.fixture def changes_service(request, seqnumber_generator): + db_host = 'http://inmemory' return ChangesService( - InMemoryDBManager(database_name='test2'), + InMemoryDBManager(database_uri=db_host, database_name='test2'), seqnumber_generator ) @pytest.fixture def databaseservice_params(functional_config, changes_service): + db_host = 'http://inmemory' return ( - InMemoryDBManager(database_name='test1'), + InMemoryDBManager(database_uri=db_host, database_name='test1'), changes_service ) @@ -161,13 +163,20 @@ def fin(): @pytest.fixture -def inmemory_receive_package(databaseservice_params, test_package_A): - article_manager = ArticleManager( - databaseservice_params[0], +def set_inmemory_article_manager(setup, databaseservice_params): + db_host = 'http://inmemory' + return ArticleManager( + InMemoryDBManager(database_uri=db_host, database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='files'), databaseservice_params[1]) - return article_manager.receive_package(id='ID', - xml_file=test_package_A[0], - files=test_package_A[1:]) + + +@pytest.fixture +def inmemory_receive_package(set_inmemory_article_manager, test_package_A): + return set_inmemory_article_manager.receive_package( + id='ID', + xml_file=test_package_A[0], + files=test_package_A[1:]) @pytest.fixture diff --git a/managers/tests/test_article_manager.py b/managers/tests/test_article_manager.py index 5266339..18056df 100644 --- a/managers/tests/test_article_manager.py +++ b/managers/tests/test_article_manager.py @@ -2,23 +2,22 @@ import pytest -from persistence.databases import DocumentNotFound +from persistence.databases import DocumentNotFound, InMemoryDBManager from persistence.services import DatabaseService from persistence.models import RecordType from managers.article_manager import ( ArticleManager, ArticleManagerException ) +from managers.models.article_model import ArticleDocument from managers.xml.xml_tree import ( XMLTree ) -def test_receive_xml_file(databaseservice_params, test_package_A, +def test_receive_xml_file(set_inmemory_article_manager, test_package_A, test_packA_filenames): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) + article_manager = set_inmemory_article_manager expected = { 'attachments': [test_packA_filenames[0]], 'content': { @@ -37,10 +36,8 @@ def test_receive_xml_file(databaseservice_params, test_package_A, assert sorted(got['attachments']) == sorted(expected['attachments']) -def test_receive_package(databaseservice_params, test_package_A): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) +def test_receive_package(set_inmemory_article_manager, test_package_A): + article_manager = set_inmemory_article_manager unexpected, missing = article_manager.receive_package( id='ID', xml_file=test_package_A[0], @@ -50,17 +47,137 @@ def test_receive_package(databaseservice_params, test_package_A): assert missing == [] +def test_article_manager(databaseservice_params): + db_host = 'http://inmemory' + article_manager = ArticleManager( + InMemoryDBManager(database_uri=db_host, database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='files'), + databaseservice_params[1] + ) + assert article_manager.article_db_service is not None + assert isinstance(article_manager.article_db_service, DatabaseService) + assert article_manager.file_db_service is not None + assert isinstance(article_manager.file_db_service, DatabaseService) + + +@patch.object(DatabaseService, 'add_file', side_effect=Exception()) +def test_add_document_add_file_error(mocked_register, + set_inmemory_article_manager, + test_package_A): + article_document = ArticleDocument(test_package_A[0].name) + article_manager = set_inmemory_article_manager + pytest.raises( + Exception, + article_manager.add_document, + article_document + ) + + +@patch.object(DatabaseService, 'register') +def test_add_document_add_file_with_file_name_and_content( + mocked_register, + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + with patch.object(article_manager.file_db_service, 'add_file') \ + as mocked_add_file: + article_manager.add_document(article_document) + mocked_add_file.assert_called_once_with( + file_id=article_document.xml_file_id, + content=article_document.xml_tree.content + ) + + +@patch.object(DatabaseService, 'add_file') +@patch.object(DatabaseService, 'register') +def test_add_document_updates_article_document_xml_file_id( + mocked_register, + mocked_add_file, + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + added_file_url = '/rawfile/' + article_document.xml_file_id + mocked_add_file.return_value = added_file_url + article_manager = set_inmemory_article_manager + article_manager.add_document(article_document) + assert article_document.versions[-1]['data'] == added_file_url + + +def test_add_document_article_get_record(set_inmemory_article_manager, + test_package_A): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + with patch.object(article_document, 'get_record'): + article_manager = set_inmemory_article_manager + article_manager.add_document(article_document) + article_document.get_record.assert_called_once() + + +@patch.object(DatabaseService, 'register', side_effect=Exception()) +def test_add_document_register_to_database_error(mocked_register, + set_inmemory_article_manager, + test_package_A): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + pytest.raises( + Exception, + article_manager.add_document, + article_document + ) + + +@patch.object(DatabaseService, 'register') +def test_add_document_register_with_article_record( + mocked_register, + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + fake_article_record = { + 'document_id': '1234', + 'content': b'acbdet' + } + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + with patch.object(article_document, 'get_record'): + article_document.get_record.return_value = fake_article_record + article_manager = set_inmemory_article_manager + article_manager.add_document(article_document) + mocked_register.assert_called_with(xml_file.name, fake_article_record) + + +def test_add_document_register_to_database_ok_returns_article_url( + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + article_document = ArticleDocument('ID') + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + article_url = article_manager.add_document(article_document) + assert article_url is not None + assert article_url.endswith('/' + xml_file.name) + + @patch.object(DatabaseService, 'read') def test_get_article_in_database(mocked_dataservices_read, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): article_id = 'ID' mocked_dataservices_read.return_value = {'document_id': article_id} - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_check = article_manager.get_article_data(article_id) assert article_check is not None assert isinstance(article_check, dict) @@ -70,15 +187,12 @@ def test_get_article_in_database(mocked_dataservices_read, @patch.object(DatabaseService, 'read', side_effect=DocumentNotFound) def test_get_article_in_database_not_found(mocked_dataservices_read, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): article_id = 'ID' mocked_dataservices_read.return_value = {'document_id': article_id} - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( ArticleManagerException, article_manager.get_article_data, @@ -87,12 +201,9 @@ def test_get_article_in_database_not_found(mocked_dataservices_read, def test_get_article_record(setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_id = 'ID' article_check = article_manager.get_article_data(article_id) assert article_check is not None @@ -110,16 +221,13 @@ def test_get_article_record(setup, @patch.object(DatabaseService, 'get_attachment') def test_get_article_file_in_database(mocked_get_attachment, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package, xml_test, test_packA_filenames): mocked_get_attachment.return_value = xml_test.encode('utf-8') article_id = 'ID' - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_manager.get_article_file(article_id) mocked_get_attachment.assert_called_with( document_id=article_id, @@ -130,12 +238,9 @@ def test_get_article_file_in_database(mocked_get_attachment, @patch.object(DatabaseService, 'get_attachment', side_effect=DocumentNotFound) def test_get_article_file_not_found(mocked_get_attachment, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( ArticleManagerException, article_manager.get_article_file, @@ -144,13 +249,10 @@ def test_get_article_file_not_found(mocked_get_attachment, def test_get_article_file(setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package, test_package_A): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_check = article_manager.get_article_file('ID') assert article_check is not None xml_tree = XMLTree(test_package_A[0].content) @@ -160,12 +262,9 @@ def test_get_article_file(setup, @patch.object(DatabaseService, 'get_attachment', side_effect=DocumentNotFound) def test_get_asset_file_not_found(mocked_get_attachment, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( ArticleManagerException, article_manager.get_asset_file, @@ -174,12 +273,10 @@ def test_get_asset_file_not_found(mocked_get_attachment, ) -def test_get_asset_file(databaseservice_params, +def test_get_asset_file(set_inmemory_article_manager, test_package_A, test_packA_filenames): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) + article_manager = set_inmemory_article_manager article_manager.receive_package(id='ID', xml_file=test_package_A[0], files=test_package_A[1:]) @@ -189,11 +286,9 @@ def test_get_asset_file(databaseservice_params, assert file.content == content -def test_get_asset_files(databaseservice_params, test_package_A): +def test_get_asset_files(set_inmemory_article_manager, test_package_A): files = test_package_A[1:] - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) + article_manager = set_inmemory_article_manager article_manager.receive_package(id='ID', xml_file=test_package_A[0], files=test_package_A[1:]) diff --git a/managers/tests/test_article_model.py b/managers/tests/test_article_model.py index cbd8551..9be288d 100644 --- a/managers/tests/test_article_model.py +++ b/managers/tests/test_article_model.py @@ -1,65 +1,65 @@ +import hashlib + +import pytest from managers.models.article_model import ( ArticleDocument, + InvalidXMLContent ) -from managers.xml.xml_tree import ( - XMLTree, -) +from managers.xml.article_xml_tree import ArticleXMLTree -def test_article(test_package_A, test_packA_filenames): - article = ArticleDocument('ID', test_package_A[0]) - article.update_asset_files(test_package_A[1:]) - expected = { - 'assets': [asset for asset in test_packA_filenames[1:]], - 'xml': test_packA_filenames[0], +def test_article_document(test_package_A, test_packA_filenames): + article_document = ArticleDocument('ID') + assert article_document.id == 'ID' + assert article_document.versions == [] + assert article_document.get_record() == { + 'document_id': 'ID', + 'document_type': 'ART', + 'versions': [] } - assert article.xml_file.name == test_packA_filenames[0] - assert article.xml_file.content == test_package_A[0].content - assert article.xml_tree.xml_error is None - assert article.xml_tree.compare(test_package_A[0].content) - assert article.get_record_content() == expected -def test_missing_files_list(test_package_B): - article = ArticleDocument('ID', test_package_B[0]) - article.update_asset_files(test_package_B[1:]) +def test_article_invalid_xml(): + article_document = ArticleDocument('ID') + with pytest.raises(InvalidXMLContent): + article_document.add_version( + 'file.xml', + b'
\n\n
' + ) - assert len(article.assets) == 3 - assert sorted(article.assets.keys()) == sorted( - [ - '0034-8910-rsp-S01518-87872016050006741-gf01.jpg', - '0034-8910-rsp-S01518-87872016050006741-gf01-pt.jpg', - '0034-8910-rsp-S01518-87872016050006741-gf31.jpg', - ] - ) - assert article.unexpected_files_list == [] - assert article.missing_files_list == [ - '0034-8910-rsp-S01518-87872016050006741-gf31.jpg', - ] - -def test_unexpected_files_list(test_package_C, test_packC_filenames): - article = ArticleDocument('ID', test_package_C[0]) - article.update_asset_files(test_package_C[1:]) - - assert len(article.assets) == 2 - assert sorted(article.assets.keys()) == sorted( - [ - '0034-8910-rsp-S01518-87872016050006741-gf01-pt.jpg', - '0034-8910-rsp-S01518-87872016050006741-gf01.jpg' +def test_article_document_add_version(test_package_A, test_packA_filenames): + xml_file = test_package_A[0] + checksum = hashlib.sha1(xml_file.content).hexdigest() + filename = '/'.join([checksum[:13], xml_file.name]) + expected = { + 'document_id': 'ID', + 'document_type': 'ART', + 'versions': [ + { + 'data': filename, + 'assets': [] + } ] - ) - assert article.unexpected_files_list == [ - test_packC_filenames[3] - ] - assert article.missing_files_list == [] + } + article_document = ArticleDocument('ID') + article_document.add_version(file_id=xml_file.name, + xml_content=xml_file.content) + assert article_document.xml_file_id == filename + assert article_document.xml_tree is not None + assert isinstance(article_document.xml_tree, ArticleXMLTree) + assert article_document.xml_tree.xml_error is None + assert article_document.xml_tree.compare(xml_file.content) + assert article_document.get_record() == expected def test_update_href(test_package_A, test_packA_filenames): new_href = 'novo href' filename = '0034-8910-rsp-S01518-87872016050006741-gf01.jpg' - article = ArticleDocument('ID', test_package_A[0]) + xml_file = test_package_A[0] + article = ArticleDocument('ID') + article.xml_file = xml_file article.update_asset_files(test_package_A[1:]) content = article.xml_tree.content asset = article.assets.get(filename) diff --git a/managers/tests/test_managers.py b/managers/tests/test_managers.py index 3915462..baae9eb 100644 --- a/managers/tests/test_managers.py +++ b/managers/tests/test_managers.py @@ -1,7 +1,8 @@ -import hashlib from unittest.mock import patch +import pytest + import managers from persistence.services import DatabaseService from managers.article_manager import ArticleManager @@ -26,15 +27,16 @@ def test_list_changes_return_from_changeservice_list_changes( assert changes == list_changes_expected -@patch.object(ArticleManager, 'add_document') -def test_post_article( - mocked_article_manager_add, - test_package_A, - database_config -): - checksum = hashlib.sha1(test_package_A[0].content).hexdigest() - expected = '/'.join([checksum[:13], test_package_A[0].name]) - db_settings = { +def test_create_file_succeeded(test_package_A): + xml_file = test_package_A[0] + file = managers.create_file(filename=xml_file.name, + content=xml_file.content) + assert file is not None + assert isinstance(file, managers.models.file.File) + + +def test_get_article_manager(database_config): + db_config = { 'database_uri': '{}:{}'.format( database_config['db_host'], database_config['db_port'] @@ -42,10 +44,75 @@ def test_post_article( 'database_username': database_config['username'], 'database_password': database_config['password'], } - result = managers.post_article( - xml_file=test_package_A[0], - **db_settings + article_manager = managers._get_article_manager( + **db_config + ) + assert article_manager is not None + assert isinstance(article_manager, managers.article_manager.ArticleManager) + assert article_manager.article_db_service is not None + assert isinstance(article_manager.article_db_service, DatabaseService) + assert article_manager.file_db_service is not None + assert isinstance(article_manager.file_db_service, DatabaseService) + + +@patch.object(managers, '_get_article_manager') +@patch.object(managers.models.article_model.ArticleDocument, '__init__') +def test_post_article_file_error(mocked_article_model, + mocked__get_article_manager, + test_package_A, + set_inmemory_article_manager): + mocked__get_article_manager.return_value = set_inmemory_article_manager + error_msg = 'Invalid XML Content' + mocked_article_model.side_effect = \ + managers.models.article_model.InvalidXMLContent() + with pytest.raises(managers.exceptions.ManagerFileError) as excinfo: + managers.post_article( + article_id='ID-post-article-123', + xml_id=test_package_A[0].name, + xml_file=test_package_A[0].content, + **{} + ) + assert excinfo.value.message == error_msg + + +@patch.object(managers, '_get_article_manager') +@patch.object(ArticleManager, 'add_document') +def test_post_article_insert_file_to_database_error( + mocked_article_manager_add, + mocked__get_article_manager, + test_package_A, + set_inmemory_article_manager +): + mocked__get_article_manager.return_value = set_inmemory_article_manager + error_msg = 'Database Error' + mocked_article_manager_add.side_effect = \ + managers.article_manager.ArticleManagerException(message=error_msg) + + with pytest.raises(managers.article_manager.ArticleManagerException) \ + as excinfo: + managers.post_article( + article_id='ID-post-article-123', + xml_id=test_package_A[0].name, + xml_file=test_package_A[0].content, + **{} + ) + assert excinfo.value.message == error_msg + + +@patch.object(managers, '_get_article_manager') +def test_post_article_insert_file_to_database_ok( + mocked__get_article_manager, + test_package_A, + set_inmemory_article_manager +): + xml_file = test_package_A[0] + mocked__get_article_manager.return_value = set_inmemory_article_manager + + article_url = managers.post_article( + article_id='ID-post-article-123', + xml_id=xml_file.name, + xml_file=xml_file.content, + **{} ) - mocked_article_manager_add.assert_called_once() - assert result is not None - assert result == expected + assert article_url is not None + assert article_url.endswith(xml_file.name) diff --git a/persistence/tests/test_changes.py b/persistence/tests/test_changes.py index 58e7917..f13fb0e 100644 --- a/persistence/tests/test_changes.py +++ b/persistence/tests/test_changes.py @@ -69,6 +69,7 @@ def test_register_change_must_keep_sequential_order(database_service): isinstance(change_list['change_id'], int) for change_list in changes_list ]) + # XXX: Usar sorted(list) ao invés de gerar uma lista ordenada assert all([ changes_list[i]['change_id'] < changes_list[i + 1]['change_id'] for i in range(len(changes_list) - 1) diff --git a/tests/test_functional.py b/tests/test_functional.py index f326383..9b4abf3 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -7,6 +7,46 @@ from lxml import etree +def test_post_article_register_change(testapp, test_package_A): + article_id = 'ID-post-article-123' + url = '/articles' + + # Um documento é registrado no módulo de persistencia. Ex: Artigo + xml_file_path = test_package_A[0] + article_url = os.path.basename(xml_file_path) + changes_expected = { + 'results': [{ + 'change_id': 1, + 'document_id': article_url, + 'type': 'CREATE' + }], + 'latest': 1 + } + params = OrderedDict([ + ("article_id", article_id), + ("xml_file", webtest.forms.Upload(xml_file_path)) + ]) + result = testapp.post(url, + params=params, + content_type='multipart/form-data') + assert result.status_code == 201 + assert result.json is not None + assert result.json.get('url').endswith(article_url) + + # Deve ser possível recuperar o registro de mudança do documento de + # acordo com os parâmetros informados no serviço + last_sequence = '' + limit = 10 + result = testapp.get('/changes?since={}&limit={}'.format(last_sequence, + limit)) + assert result.status_code == 200 + assert result.json is not None + assert len(result.json) > len(changes_expected['results']) + for resp_result, expected in zip(result.json, changes_expected['results']): + assert resp_result['document_id'].endswith(expected['document_id']) + assert resp_result['type'] == expected['type'] + + def test_add_article_register_change(testapp, test_package_A): article_id = 'ID-post-article-123' url = '/articles/{}'.format(article_id) From 76896daed1fb599e2df4c71aa527aa5e625dd6ad Mon Sep 17 00:00:00 2001 From: Patricia Morimoto Date: Wed, 20 Jun 2018 14:23:32 -0300 Subject: [PATCH 6/6] =?UTF-8?q?Ajustes=20relacionados=20aos=20coment=C3=A1?= =?UTF-8?q?rios=20do=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ajuste do `managers._get_article_manager` - Ajuste em `post_article` no tratamento de exceção e nos testes - Criada property `ArticleXMLTree.checksum` --- managers/__init__.py | 12 ++++++------ managers/models/article_model.py | 4 +--- managers/tests/test_article_model.py | 3 ++- managers/tests/test_managers.py | 6 +++--- managers/xml/article_xml_tree.py | 5 +++++ 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/managers/__init__.py b/managers/__init__.py index 894212d..764e56d 100644 --- a/managers/__init__.py +++ b/managers/__init__.py @@ -32,10 +32,9 @@ def _get_changes_services(db_settings): def _get_article_manager(**db_settings): - database_config = db_settings - articles_database_config = database_config.copy() + articles_database_config = db_settings.copy() articles_database_config['database_name'] = "articles" - files_database_config = database_config.copy() + files_database_config = db_settings.copy() files_database_config['database_name'] = "files" return ArticleManager( @@ -80,13 +79,14 @@ def post_article(article_id, xml_id, xml_file, **db_settings): codificado em XML :rtype: str """ + article_document = ArticleDocument(article_id) + article_manager = _get_article_manager(**db_settings) try: - article_document = ArticleDocument(article_id) - article_manager = _get_article_manager(**db_settings) article_document.add_version(xml_id, xml_file) - return article_manager.add_document(article_document) except InvalidXMLContent as e: raise ManagerFileError(message=e.message) + else: + return article_manager.add_document(article_document) def put_article(article_id, xml_file, assets_files=[], **db_settings): diff --git a/managers/models/article_model.py b/managers/models/article_model.py index 5e92fc8..f177ca8 100644 --- a/managers/models/article_model.py +++ b/managers/models/article_model.py @@ -1,5 +1,4 @@ # coding=utf-8 -import hashlib from enum import Enum import os @@ -74,8 +73,7 @@ def add_version(self, file_id, xml_content): self.xml_tree = ArticleXMLTree(xml_content) if self.xml_tree.xml_error: raise InvalidXMLContent - checksum = hashlib.sha1(xml_content).hexdigest() - self.xml_file_id = '/'.join([checksum[:13], file_id]) + self.xml_file_id = '/'.join([self.xml_tree.checksum[:13], file_id]) self.versions.append({ 'data': self.xml_file_id, 'assets': [] diff --git a/managers/tests/test_article_model.py b/managers/tests/test_article_model.py index 18fbbb0..bc7df10 100644 --- a/managers/tests/test_article_model.py +++ b/managers/tests/test_article_model.py @@ -31,7 +31,8 @@ def test_article_invalid_xml(): def test_article_document_add_version(test_package_A, test_packA_filenames): xml_file = test_package_A[0] - checksum = hashlib.sha1(xml_file.content).hexdigest() + checksum = hashlib.sha1( + ArticleXMLTree(xml_file.content).content).hexdigest() filename = '/'.join([checksum[:13], xml_file.name]) expected = { 'document_id': 'ID', diff --git a/managers/tests/test_managers.py b/managers/tests/test_managers.py index baae9eb..6f1dfb4 100644 --- a/managers/tests/test_managers.py +++ b/managers/tests/test_managers.py @@ -56,14 +56,14 @@ def test_get_article_manager(database_config): @patch.object(managers, '_get_article_manager') -@patch.object(managers.models.article_model.ArticleDocument, '__init__') -def test_post_article_file_error(mocked_article_model, +@patch.object(managers.models.article_model.ArticleDocument, 'add_version') +def test_post_article_file_error(mocked_article_add_version, mocked__get_article_manager, test_package_A, set_inmemory_article_manager): mocked__get_article_manager.return_value = set_inmemory_article_manager error_msg = 'Invalid XML Content' - mocked_article_model.side_effect = \ + mocked_article_add_version.side_effect = \ managers.models.article_model.InvalidXMLContent() with pytest.raises(managers.exceptions.ManagerFileError) as excinfo: managers.post_article( diff --git a/managers/xml/article_xml_tree.py b/managers/xml/article_xml_tree.py index b2c8feb..6684d8c 100644 --- a/managers/xml/article_xml_tree.py +++ b/managers/xml/article_xml_tree.py @@ -1,4 +1,5 @@ # coding=utf-8 +import hashlib from .xml_tree import ( XMLTree, @@ -49,3 +50,7 @@ def nodes_which_has_xlink_href(self): if self.tree is not None: return self.tree.findall( './/*[@{http://www.w3.org/1999/xlink}href]') + + @property + def checksum(self): + return hashlib.sha1(self.content).hexdigest()