From 5ec7a48ea725be9df13cee1f1b1f9981a44e7707 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 31 Jan 2023 21:44:32 +0000 Subject: [PATCH 01/19] make sure `LazyTableReference` isn't resolved until referenced table has been initialised --- piccolo/columns/column_types.py | 8 ++++++-- piccolo/columns/reference.py | 14 ++++++++++++++ piccolo/table.py | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 5adabe3d3..7aaff6452 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2097,8 +2097,12 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: except AttributeError: pass else: - if _foreign_key_meta.proxy_columns == [] and isinstance( - _foreign_key_meta.references, LazyTableReference + if ( + _foreign_key_meta.proxy_columns == [] + and isinstance( + _foreign_key_meta.references, LazyTableReference + ) + and _foreign_key_meta.references.ready ): object.__getattribute__(self, "set_proxy_columns")() diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 841545eeb..105bbb7f5 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -44,6 +44,10 @@ def __post_init__(self): raise ValueError( "Specify either app_name or module_path - not both." ) + # We should only try resolving it once it is ready. We know it's ready + # because the table metaclass sets this to `True` when the + # corresponding table has been initialised. + self.ready = False def resolve(self) -> t.Type[Table]: if self.app_name is not None: @@ -87,8 +91,18 @@ def __str__(self): @dataclass class LazyColumnReferenceStore: + # Foreign key columns which use LazyTableReference. foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) + def set_ready(self, table_class_name: str): + for foreign_key_column in self.foreign_key_columns: + references = t.cast( + LazyTableReference, + foreign_key_column._foreign_key_meta.references, + ) + if references.table_class_name == table_class_name: + references.ready = True + def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: return [ i diff --git a/piccolo/table.py b/piccolo/table.py index 2982180bd..d6966aead 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -324,6 +324,7 @@ def __init_subclass__( ) TABLE_REGISTRY.append(cls) + LAZY_COLUMN_REFERENCES.set_ready(table_class_name=cls.__name__) def __init__( self, From 0afc5a953c880d9ecae4727b36062fc10b944dc7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 08:43:07 +0000 Subject: [PATCH 02/19] raise exception if `LazyTableReference` is resolved before ready --- piccolo/columns/base.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index adbfb6248..de3dbe404 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -95,7 +95,14 @@ def resolved_references(self) -> t.Type[Table]: from piccolo.table import Table if isinstance(self.references, LazyTableReference): - return self.references.resolve() + if self.references.ready: + return self.references.resolve() + else: + raise ValueError( + "The tables haven't fully initialised yet - please " + "refactor your code so all tables have imported before " + "using them." + ) elif inspect.isclass(self.references) and issubclass( self.references, Table ): From 074cf4a7d4bf0059614f8f83ab79e933e396e060 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 08:43:46 +0000 Subject: [PATCH 03/19] more robust checks for table readiness - using app name and module, as well as table class name --- piccolo/columns/reference.py | 20 +++++++++++++++++--- piccolo/conf/apps.py | 3 +++ piccolo/table.py | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 105bbb7f5..8e2bb70a4 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -94,14 +94,28 @@ class LazyColumnReferenceStore: # Foreign key columns which use LazyTableReference. foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) - def set_ready(self, table_class_name: str): + def set_ready(self, table_class: t.Type[Table]): + """ + The ``Table`` metaclass calls this once a ``Table`` has been imported. + It tells each ``LazyTableReference`` which references that table that + it's ready. + """ for foreign_key_column in self.foreign_key_columns: references = t.cast( LazyTableReference, foreign_key_column._foreign_key_meta.references, ) - if references.table_class_name == table_class_name: - references.ready = True + if references.table_class_name == table_class.__class__.__name__: + if ( + references.module_path + and references.module_path == table_class.__module__ + ): + references.ready = True + elif ( + references.app_name + and references.app_name == table_class._meta.app_name + ): + references.ready = True def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: return [ diff --git a/piccolo/conf/apps.py b/piccolo/conf/apps.py index 6176224af..282b3733e 100644 --- a/piccolo/conf/apps.py +++ b/piccolo/conf/apps.py @@ -156,6 +156,9 @@ def __post_init__(self): i if isinstance(i, Command) else Command(i) for i in self.commands ] + for table_class in self.table_classes: + table_class._meta.app_name = self.app_name + if isinstance(self.migrations_folder_path, pathlib.Path): self.migrations_folder_path = str(self.migrations_folder_path) diff --git a/piccolo/table.py b/piccolo/table.py index d6966aead..757e7d2cd 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -69,6 +69,7 @@ class TableMeta: """ tablename: str = "" + app_name: t.Optiona[str] = None columns: t.List[Column] = field(default_factory=list) default_columns: t.List[Column] = field(default_factory=list) non_default_columns: t.List[Column] = field(default_factory=list) @@ -324,7 +325,7 @@ def __init_subclass__( ) TABLE_REGISTRY.append(cls) - LAZY_COLUMN_REFERENCES.set_ready(table_class_name=cls.__name__) + LAZY_COLUMN_REFERENCES.set_ready(table_class=cls) def __init__( self, From b5715809f7b704a029ba27d2c096c2f4a9644e05 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 09:40:13 +0000 Subject: [PATCH 04/19] fix tests --- piccolo/columns/column_types.py | 14 +++++++++++++- piccolo/columns/reference.py | 31 ++++++++++++++++++++----------- piccolo/table.py | 2 +- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7aaff6452..4c55e6e9d 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1895,13 +1895,19 @@ def __init__( target_column=target_column, ) - def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: + def _setup( + self, + table_class: t.Type[Table], + loaded_table_classes: t.List[t.Type[Table]] = [], + ) -> ForeignKeySetupResponse: """ This is called by the ``TableMetaclass``. A ``ForeignKey`` column can only be completely setup once it's parent ``Table`` is known. :param table_class: The parent ``Table`` class for this column. + :param loaded_table_classes: + Any ``Table`` classes which have already been loaded. """ from piccolo.table import Table @@ -1932,6 +1938,12 @@ def _setup(self, table_class: t.Type[Table]) -> ForeignKeySetupResponse: ) is_lazy = isinstance(references, LazyTableReference) + + if is_lazy: + # We should check if the referenced tables are already + # available. + references.set_ready(table_classes=loaded_table_classes) + is_table_class = inspect.isclass(references) and issubclass( references, Table ) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 8e2bb70a4..00df3a291 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -49,6 +49,25 @@ def __post_init__(self): # corresponding table has been initialised. self.ready = False + def set_ready(self, table_classes: t.List[t.Type[Table]]): + if self.ready: + return + else: + for table_class in table_classes: + if self.table_class_name == table_class.__name__: + if ( + self.module_path + and self.module_path == table_class.__module__ + ): + self.ready = True + break + elif ( + self.app_name + and self.app_name == table_class._meta.app_name + ): + self.ready = True + break + def resolve(self) -> t.Type[Table]: if self.app_name is not None: from piccolo.conf.apps import Finder @@ -105,17 +124,7 @@ def set_ready(self, table_class: t.Type[Table]): LazyTableReference, foreign_key_column._foreign_key_meta.references, ) - if references.table_class_name == table_class.__class__.__name__: - if ( - references.module_path - and references.module_path == table_class.__module__ - ): - references.ready = True - elif ( - references.app_name - and references.app_name == table_class._meta.app_name - ): - references.ready = True + references.set_ready(table_classes=[table_class]) def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: return [ diff --git a/piccolo/table.py b/piccolo/table.py index 757e7d2cd..24812f160 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -317,7 +317,7 @@ def __init_subclass__( # ForeignKey columns require additional setup based on their # parent Table. foreign_key_setup_response = foreign_key_column._setup( - table_class=cls + table_class=cls, loaded_table_classes=TABLE_REGISTRY ) if foreign_key_setup_response.is_lazy: LAZY_COLUMN_REFERENCES.foreign_key_columns.append( From 889edb3f40c0be4b7e11b641f8a979d6715e5c99 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 09:49:18 +0000 Subject: [PATCH 05/19] reduce repetition --- piccolo/columns/reference.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 00df3a291..f588fcd6f 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -58,10 +58,7 @@ def set_ready(self, table_classes: t.List[t.Type[Table]]): if ( self.module_path and self.module_path == table_class.__module__ - ): - self.ready = True - break - elif ( + ) or ( self.app_name and self.app_name == table_class._meta.app_name ): From e20f9556a1d8ddcdd4964dcf4e99a3274f5d57f0 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 09:50:33 +0000 Subject: [PATCH 06/19] fix linter error --- piccolo/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/table.py b/piccolo/table.py index 24812f160..74621074c 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -69,7 +69,7 @@ class TableMeta: """ tablename: str = "" - app_name: t.Optiona[str] = None + app_name: t.Optional[str] = None columns: t.List[Column] = field(default_factory=list) default_columns: t.List[Column] = field(default_factory=list) non_default_columns: t.List[Column] = field(default_factory=list) From 976b9a54a6ff986a277b6db0661623f5c2026b8a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 11:33:00 +0000 Subject: [PATCH 07/19] improve assertions --- tests/columns/foreign_key/test_attribute_access.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index c658d7d98..716ab0636 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -37,7 +37,7 @@ def test_attribute_access(self): references. """ for band_table in (BandA, BandB, BandC, BandD): - self.assertTrue(isinstance(band_table.manager.name, Varchar)) + self.assertIsInstance(band_table.manager.name, Varchar) def test_recursion_limit(self): """ @@ -47,7 +47,7 @@ def test_recursion_limit(self): # Should be fine: column: Column = Manager.manager.name self.assertTrue(len(column._meta.call_chain), 1) - self.assertTrue(isinstance(column, Varchar)) + self.assertIsInstance(column, Varchar) with self.assertRaises(Exception): Manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.manager.name # noqa @@ -59,4 +59,4 @@ def test_recursion_time(self): start = time.time() Manager.manager.manager.manager.manager.manager.manager.name end = time.time() - self.assertTrue(end - start < 1.0) + self.assertLess(end - start, 1.0) From 02602ad7116b2d201b6324e195192e429cd949a1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 12:15:48 +0000 Subject: [PATCH 08/19] `set_ready` now returns the status --- piccolo/columns/reference.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index f588fcd6f..61b3c3e3a 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -13,6 +13,12 @@ from piccolo.table import Table +@dataclass +class SetReadyResponse: + is_ready: bool + was_changed: bool + + @dataclass class LazyTableReference: """ @@ -49,9 +55,15 @@ def __post_init__(self): # corresponding table has been initialised. self.ready = False - def set_ready(self, table_classes: t.List[t.Type[Table]]): + def set_ready( + self, table_classes: t.List[t.Type[Table]] + ) -> SetReadyResponse: + """ + Checks to see if the table being referenced is in the provided list, + meaning it has successfully loaded. + """ if self.ready: - return + return SetReadyResponse(is_ready=True, was_changed=False) else: for table_class in table_classes: if self.table_class_name == table_class.__name__: @@ -121,7 +133,8 @@ def set_ready(self, table_class: t.Type[Table]): LazyTableReference, foreign_key_column._foreign_key_meta.references, ) - references.set_ready(table_classes=[table_class]) + if references.set_ready(table_classes=[table_class]).was_changed: + foreign_key_column._on_ready() def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: return [ From a49c9019abbcd830dc8368fe70193ff16663f8ad Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 12:52:20 +0000 Subject: [PATCH 09/19] check_ready --- piccolo/columns/column_types.py | 44 +++++++++++++-------------------- piccolo/columns/reference.py | 12 ++++++--- piccolo/table.py | 3 ++- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 4c55e6e9d..a9c70a6b6 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1895,6 +1895,18 @@ def __init__( target_column=target_column, ) + def _on_ready(self): + """ + Once we know the table being referred to has been loaded, we call this + method. + """ + # Record the reverse relationship on the target table. + referenced_table = self._foreign_key_meta.resolved_references + referenced_table._meta._foreign_key_references.append(self) + # Allow columns on the referenced table to be accessed via + # auto completion. + self.set_proxy_columns() + def _setup( self, table_class: t.Type[Table], @@ -1942,7 +1954,10 @@ def _setup( if is_lazy: # We should check if the referenced tables are already # available. - references.set_ready(table_classes=loaded_table_classes) + if references.check_ready( + table_classes=loaded_table_classes + ).was_changed: + self._on_ready() is_table_class = inspect.isclass(references) and issubclass( references, Table @@ -1957,12 +1972,7 @@ def _setup( ) if is_table_class: - # Record the reverse relationship on the target table. - references._meta._foreign_key_references.append(self) - - # Allow columns on the referenced table to be accessed via - # auto completion. - self.set_proxy_columns() + self._on_ready() return ForeignKeySetupResponse(is_lazy=is_lazy) @@ -2098,26 +2108,6 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: case a copy is returned with an updated call_chain (which records the joins required). """ - # If the ForeignKey is using a lazy reference, we need to set the - # attributes here. Attributes starting with a double underscore are - # unlikely to be column names. - if not name.startswith("__"): - try: - _foreign_key_meta = object.__getattribute__( - self, "_foreign_key_meta" - ) - except AttributeError: - pass - else: - if ( - _foreign_key_meta.proxy_columns == [] - and isinstance( - _foreign_key_meta.references, LazyTableReference - ) - and _foreign_key_meta.references.ready - ): - object.__getattribute__(self, "set_proxy_columns")() - try: value = object.__getattribute__(self, name) except AttributeError: diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 61b3c3e3a..937ae5af1 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -55,7 +55,7 @@ def __post_init__(self): # corresponding table has been initialised. self.ready = False - def set_ready( + def check_ready( self, table_classes: t.List[t.Type[Table]] ) -> SetReadyResponse: """ @@ -75,7 +75,11 @@ def set_ready( and self.app_name == table_class._meta.app_name ): self.ready = True - break + return SetReadyResponse( + is_ready=True, was_changed=True + ) + + return SetReadyResponse(is_ready=False, was_changed=False) def resolve(self) -> t.Type[Table]: if self.app_name is not None: @@ -122,7 +126,7 @@ class LazyColumnReferenceStore: # Foreign key columns which use LazyTableReference. foreign_key_columns: t.List[ForeignKey] = field(default_factory=list) - def set_ready(self, table_class: t.Type[Table]): + def check_ready(self, table_class: t.Type[Table]): """ The ``Table`` metaclass calls this once a ``Table`` has been imported. It tells each ``LazyTableReference`` which references that table that @@ -133,7 +137,7 @@ def set_ready(self, table_class: t.Type[Table]): LazyTableReference, foreign_key_column._foreign_key_meta.references, ) - if references.set_ready(table_classes=[table_class]).was_changed: + if references.check_ready(table_classes=[table_class]).was_changed: foreign_key_column._on_ready() def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: diff --git a/piccolo/table.py b/piccolo/table.py index 74621074c..d7e1def24 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -319,13 +319,14 @@ def __init_subclass__( foreign_key_setup_response = foreign_key_column._setup( table_class=cls, loaded_table_classes=TABLE_REGISTRY ) + if foreign_key_setup_response.is_lazy: LAZY_COLUMN_REFERENCES.foreign_key_columns.append( foreign_key_column ) TABLE_REGISTRY.append(cls) - LAZY_COLUMN_REFERENCES.set_ready(table_class=cls) + LAZY_COLUMN_REFERENCES.check_ready(table_class=cls) def __init__( self, From 39c7d3da4b6708dea7eb102a4d465a77da90bdfa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 14:33:30 +0000 Subject: [PATCH 10/19] fix tests --- piccolo/columns/base.py | 2 +- piccolo/columns/column_types.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index de3dbe404..5971a694c 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -79,7 +79,7 @@ def __repr__(self): @dataclass class ForeignKeyMeta: - references: t.Union[t.Type[Table], LazyTableReference] + references: t.Union[t.Type[Table], LazyTableReference, str] on_delete: OnDelete on_update: OnUpdate target_column: t.Union[Column, str, None] diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index a9c70a6b6..0efdae163 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1889,7 +1889,7 @@ def __init__( # The ``TableMetaclass``` sets the actual value for # ``ForeignKeyMeta.references``, if the user passed in a string. self._foreign_key_meta = ForeignKeyMeta( - references=Table if isinstance(references, str) else references, + references=references, on_delete=on_delete, on_update=on_update, target_column=target_column, @@ -1902,7 +1902,8 @@ def _on_ready(self): """ # Record the reverse relationship on the target table. referenced_table = self._foreign_key_meta.resolved_references - referenced_table._meta._foreign_key_references.append(self) + if self not in referenced_table._meta._foreign_key_references: + referenced_table._meta._foreign_key_references.append(self) # Allow columns on the referenced table to be accessed via # auto completion. self.set_proxy_columns() @@ -1949,6 +1950,8 @@ def _setup( module_path=module_path, ) + self._foreign_key_meta.references = references + is_lazy = isinstance(references, LazyTableReference) if is_lazy: @@ -1963,9 +1966,7 @@ def _setup( references, Table ) - if is_lazy or is_table_class: - self._foreign_key_meta.references = references - else: + if not (is_lazy or is_table_class): raise ValueError( "Error - ``references`` must be a ``Table`` subclass, or " "a ``LazyTableReference`` instance." From 4ff57c043e17d9fadd52b473e7a7e5d1bc22c166 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 15:15:15 +0000 Subject: [PATCH 11/19] try getting table from TABLE_REGISTRY instead of using importlib Using importlib could mean we get partially initialised modules --- piccolo/columns/reference.py | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index 937ae5af1..c182d2633 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -3,8 +3,6 @@ """ from __future__ import annotations -import importlib -import inspect import typing as t from dataclasses import dataclass, field @@ -54,6 +52,7 @@ def __post_init__(self): # because the table metaclass sets this to `True` when the # corresponding table has been initialised. self.ready = False + self._resolved: t.Optional[t.Table] = None def check_ready( self, table_classes: t.List[t.Type[Table]] @@ -82,33 +81,34 @@ def check_ready( return SetReadyResponse(is_ready=False, was_changed=False) def resolve(self) -> t.Type[Table]: + if self._resolved is not None: + return self._resolved + if self.app_name is not None: from piccolo.conf.apps import Finder finder = Finder() - return finder.get_table_with_name( + table_class = finder.get_table_with_name( app_name=self.app_name, table_class_name=self.table_class_name ) + self._resolved = table_class + return table_class if self.module_path: - module = importlib.import_module(self.module_path) - table: t.Optional[t.Type[Table]] = getattr( - module, self.table_class_name, None - ) + from piccolo.table import TABLE_REGISTRY + + for table_class in TABLE_REGISTRY: + if ( + table_class.__name__ == self.table_class_name + and table_class.__module__ == self.module_path + ): + self._resolved = table_class + return table_class - from piccolo.table import Table - - if ( - table is not None - and inspect.isclass(table) - and issubclass(table, Table) - ): - return table - else: - raise ValueError( - "Can't find a Table subclass called " - f"{self.table_class_name} in {self.module_path}" - ) + raise ValueError( + "Can't find a Table subclass called " + f"{self.table_class_name} in {self.module_path}" + ) raise ValueError("You must specify either app_name or module_path.") From 6f58db473b6072ce56e867a0813cdd2e674d3b68 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 15:24:49 +0000 Subject: [PATCH 12/19] more explicit error message --- piccolo/columns/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 5971a694c..6767d8bf7 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -99,9 +99,9 @@ def resolved_references(self) -> t.Type[Table]: return self.references.resolve() else: raise ValueError( - "The tables haven't fully initialised yet - please " - "refactor your code so all tables have imported before " - "using them." + f"The {self.references.table_class_name} table hasn't " + "fully initialised yet - please refactor your code so all " + "tables have imported before using them." ) elif inspect.isclass(self.references) and issubclass( self.references, Table From d586190cbcef1e0a9c669728ad3bb392cc3ad080 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 19:43:43 +0000 Subject: [PATCH 13/19] clean up some tests --- tests/columns/foreign_key/test_attribute_access.py | 3 ++- tests/columns/foreign_key/test_foreign_key_string.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index 716ab0636..8537e301a 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -21,7 +21,8 @@ class BandB(Table): class BandC(Table): manager = ForeignKey( references=LazyTableReference( - table_class_name="Manager", module_path=__name__ + table_class_name="Manager", + module_path=__module__, # type: ignore ) ) diff --git a/tests/columns/foreign_key/test_foreign_key_string.py b/tests/columns/foreign_key/test_foreign_key_string.py index 38d9e6bee..1b7549fe5 100644 --- a/tests/columns/foreign_key/test_foreign_key_string.py +++ b/tests/columns/foreign_key/test_foreign_key_string.py @@ -16,13 +16,13 @@ class BandB(Table): manager = ForeignKey( references=LazyTableReference( table_class_name="Manager", - module_path=__name__, + module_path=__module__, # type: ignore ) ) class BandC(Table, tablename="band"): - manager = ForeignKey(references=f"{__name__}.Manager") + manager = ForeignKey(references=f"{__module__}.Manager") # type: ignore class TestForeignKeyString(TestCase): @@ -33,7 +33,7 @@ class TestForeignKeyString(TestCase): def test_foreign_key_string(self): for band_table in (BandA, BandB, BandC): - self.assertEqual( + self.assertIs( band_table.manager._foreign_key_meta.resolved_references, Manager, ) @@ -66,4 +66,4 @@ def test_lazy_reference_to_app(self): table_class_name="Manager", app_name="music" ) - self.assertTrue(reference.resolve() is Manager) + self.assertIs(reference.resolve(), Manager) From 78af24aa2e1c8525589bb3262d45ce942bb6be66 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 20:00:11 +0000 Subject: [PATCH 14/19] change variable name --- piccolo/columns/reference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index c182d2633..e6acd771e 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -133,11 +133,11 @@ def check_ready(self, table_class: t.Type[Table]): it's ready. """ for foreign_key_column in self.foreign_key_columns: - references = t.cast( + reference = t.cast( LazyTableReference, foreign_key_column._foreign_key_meta.references, ) - if references.check_ready(table_classes=[table_class]).was_changed: + if reference.check_ready(table_classes=[table_class]).was_changed: foreign_key_column._on_ready() def for_table(self, table: t.Type[Table]) -> t.List[ForeignKey]: From af3d33d90ad5d62a179195a1bffc62d6010f815c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 22:40:01 +0000 Subject: [PATCH 15/19] improve tests --- .../auto/integration/test_migrations.py | 60 ++++++++++++++----- .../migrations/auto/test_serialisation.py | 13 +--- .../foreign_key/test_attribute_access.py | 2 +- .../foreign_key/test_foreign_key_string.py | 4 +- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index 23387c192..e8dbc50d9 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -123,6 +123,7 @@ def _test_migrations( """ temp_directory_path = tempfile.gettempdir() + print(temp_directory_path) migrations_folder_path = os.path.join( temp_directory_path, "piccolo_migrations" ) @@ -900,25 +901,56 @@ def test_m2m(self): ############################################################################### -@engines_only("postgres", "cockroach") -class TestForeignKeys(MigrationTestCase): - def setUp(self): - class TableA(Table): - pass +class TableA(Table): + pass - class TableB(Table): - fk = ForeignKey(TableA) - class TableC(Table): - fk = ForeignKey(TableB) +class TableB(Table): + fk = ForeignKey(TableA) + + +class TableC(Table): + fk = ForeignKey(TableB) + + +class TableD(Table): + fk = ForeignKey(TableC) + - class TableD(Table): - fk = ForeignKey(TableC) +class TableE(Table): + fk_1 = ForeignKey(TableD) + fk_2 = ForeignKey(LazyTableReference("TableF", module_path=__name__)) + fk_3 = ForeignKey(LazyTableReference("TableG", module_path=__name__)) - class TableE(Table): - fk = ForeignKey(TableD) - self.table_classes = [TableA, TableB, TableC, TableD, TableE] +class TableF(Table): + """ + A table with a custom PK. + """ + + id = UUID(primary_key=True) + + +class TableG(Table): + """ + A table with the default Serial PK. + """ + + pass + + +@engines_only("postgres", "cockroach") +class TestForeignKeys(MigrationTestCase): + def setUp(self): + self.table_classes = [ + TableA, + TableB, + TableC, + TableD, + TableE, + TableF, + TableG, + ] def tearDown(self): drop_db_tables_sync(Migration, *self.table_classes) diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 688bdf09e..66d6065af 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -243,18 +243,7 @@ def test_lazy_table_reference(self): self.assertTrue(len(serialised.extra_definitions) == 1) - if engine_is("postgres"): - self.assertEqual( - serialised.extra_definitions[0].__str__(), - ( - 'class Manager(Table, tablename="manager"): ' - "id = Serial(null=False, primary_key=True, unique=False, " # noqa: E501 - "index=False, index_method=IndexMethod.btree, " - "choices=None, db_column_name='id', secret=False)" - ), - ) - - if engine_is("cockroach"): + if engine_is("postgres") or engine_is("cockroach"): self.assertEqual( serialised.extra_definitions[0].__str__(), ( diff --git a/tests/columns/foreign_key/test_attribute_access.py b/tests/columns/foreign_key/test_attribute_access.py index 8537e301a..87caeb78e 100644 --- a/tests/columns/foreign_key/test_attribute_access.py +++ b/tests/columns/foreign_key/test_attribute_access.py @@ -22,7 +22,7 @@ class BandC(Table): manager = ForeignKey( references=LazyTableReference( table_class_name="Manager", - module_path=__module__, # type: ignore + module_path=__name__, ) ) diff --git a/tests/columns/foreign_key/test_foreign_key_string.py b/tests/columns/foreign_key/test_foreign_key_string.py index 1b7549fe5..1dd8e3aee 100644 --- a/tests/columns/foreign_key/test_foreign_key_string.py +++ b/tests/columns/foreign_key/test_foreign_key_string.py @@ -16,13 +16,13 @@ class BandB(Table): manager = ForeignKey( references=LazyTableReference( table_class_name="Manager", - module_path=__module__, # type: ignore + module_path=__name__, ) ) class BandC(Table, tablename="band"): - manager = ForeignKey(references=f"{__module__}.Manager") # type: ignore + manager = ForeignKey(references=f"{__name__}.Manager") class TestForeignKeyString(TestCase): From 3787b7dd8b82de88742d9646166bb71630d595f5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 23:32:56 +0000 Subject: [PATCH 16/19] improve serialisation of lazy table references in migrations --- .../migrations/auto/integration/test_migrations.py | 9 ++++++++- tests/apps/migrations/auto/test_serialisation.py | 11 +++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/apps/migrations/auto/integration/test_migrations.py b/tests/apps/migrations/auto/integration/test_migrations.py index e8dbc50d9..b8ad12f50 100644 --- a/tests/apps/migrations/auto/integration/test_migrations.py +++ b/tests/apps/migrations/auto/integration/test_migrations.py @@ -918,9 +918,9 @@ class TableD(Table): class TableE(Table): - fk_1 = ForeignKey(TableD) fk_2 = ForeignKey(LazyTableReference("TableF", module_path=__name__)) fk_3 = ForeignKey(LazyTableReference("TableG", module_path=__name__)) + fk_5 = ForeignKey("TableG") class TableF(Table): @@ -972,6 +972,13 @@ def test_foreign_keys(self): for table_class in table_classes: self.assertTrue(table_class.table_exists().run_sync()) + def test_subset(self): + """ + Test a smaller subset, just in case there's a bug with importing + dependencies. + """ + self._test_migrations(table_snapshots=[[TableE, TableF, TableG]]) + @engines_only("postgres", "cockroach") class TestTargetColumn(MigrationTestCase): diff --git a/tests/apps/migrations/auto/test_serialisation.py b/tests/apps/migrations/auto/test_serialisation.py index 66d6065af..a2bb9a079 100644 --- a/tests/apps/migrations/auto/test_serialisation.py +++ b/tests/apps/migrations/auto/test_serialisation.py @@ -235,10 +235,13 @@ def test_lazy_table_reference(self): serialised.params["references"].__repr__() == "Manager" ) - self.assertTrue(len(serialised.extra_imports) == 1) - self.assertEqual( - serialised.extra_imports[0].__str__(), - "from piccolo.table import Table", + self.assertListEqual( + sorted([i.__str__() for i in serialised.extra_imports]), + [ + "from piccolo.columns.column_types import Serial", + "from piccolo.columns.indexes import IndexMethod", + "from piccolo.table import Table", + ], ) self.assertTrue(len(serialised.extra_definitions) == 1) From 61cb43b839a048caaebcf78ec05276315f264916 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 23:39:36 +0000 Subject: [PATCH 17/19] missing code --- piccolo/apps/migrations/auto/serialisation.py | 76 ++++++++++++++----- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/piccolo/apps/migrations/auto/serialisation.py b/piccolo/apps/migrations/auto/serialisation.py index 2b45dfba7..7f3350701 100644 --- a/piccolo/apps/migrations/auto/serialisation.py +++ b/piccolo/apps/migrations/auto/serialisation.py @@ -15,7 +15,7 @@ from piccolo.columns import Column from piccolo.columns.defaults.base import Default from piccolo.columns.reference import LazyTableReference -from piccolo.table import Table +from piccolo.table import Table, create_table_class from piccolo.utils.repr import repr_class_instance from .serialisation_legacy import deserialise_legacy_params @@ -477,6 +477,31 @@ def __repr__(self): ############################################################################### +def _primary_key_params(primary_key: Column) -> SerialisedParams: + """ + Returns the extra imports and definitions required for a primary column. + """ + primary_key_class = primary_key.__class__ + + pk_serialised_params: SerialisedParams = serialise_params( + params=primary_key._meta.params + ) + + pk_serialised_params.extra_imports.append( + Import( + module=primary_key_class.__module__, + target=primary_key_class.__name__, + expect_conflict_with_global_name=getattr( + UniqueGlobalNames, + f"COLUMN_{primary_key_class.__name__.upper()}", + None, + ), + ) + ) + + return pk_serialised_params + + def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: """ When writing column params to a migration file, we need to serialise some @@ -658,12 +683,24 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: expect_conflict_with_global_name=UniqueGlobalNames.TABLE, ) ) + pk_serialised_params = _primary_key_params( + table_type._meta.primary_key + ) + extra_imports.extend(pk_serialised_params.extra_imports) + extra_definitions.extend(pk_serialised_params.extra_definitions) continue - # Replace any Table class values into class and table names - if inspect.isclass(value) and issubclass(value, Table): - params[key] = SerialisedCallable(callable_=value) - extra_definitions.append(SerialisedTableType(table_type=value)) + if key == "references" and isinstance(value, str): + # This isn't perfect - it's better if the user uses + # LazyTableReference, but do the best we can. + + referenced_table_class = create_table_class( + class_name=value.rsplit(".")[-1] + ) + + extra_definitions.append( + SerialisedTableType(table_type=referenced_table_class) + ) extra_imports.append( Import( module=Table.__module__, @@ -672,23 +709,26 @@ def serialise_params(params: t.Dict[str, t.Any]) -> SerialisedParams: ) ) - primary_key_class = value._meta.primary_key.__class__ + pk_serialised_params = _primary_key_params( + referenced_table_class._meta.primary_key + ) + extra_imports.extend(pk_serialised_params.extra_imports) + extra_definitions.extend(pk_serialised_params.extra_definitions) + continue + + # Replace any Table class values into class and table names + if inspect.isclass(value) and issubclass(value, Table): + params[key] = SerialisedCallable(callable_=value) + extra_definitions.append(SerialisedTableType(table_type=value)) extra_imports.append( Import( - module=primary_key_class.__module__, - target=primary_key_class.__name__, - expect_conflict_with_global_name=getattr( - UniqueGlobalNames, - f"COLUMN_{primary_key_class.__name__.upper()}", - None, - ), + module=Table.__module__, + target=UniqueGlobalNames.TABLE, + expect_conflict_with_global_name=UniqueGlobalNames.TABLE, ) ) - # Include the extra imports and definitions required for the - # primary column params. - pk_serialised_params: SerialisedParams = serialise_params( - params=value._meta.primary_key._meta.params - ) + + pk_serialised_params = _primary_key_params(value._meta.primary_key) extra_imports.extend(pk_serialised_params.extra_imports) extra_definitions.extend(pk_serialised_params.extra_definitions) From 7e31608d93d992c7b9248de5f02b85020ad4c1ed Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 1 Feb 2023 23:54:42 +0000 Subject: [PATCH 18/19] update the docs for `LazyTableReference` --- docs/src/piccolo/api_reference/index.rst | 1 - piccolo/columns/column_types.py | 35 ++++++++++++++++-------- piccolo/columns/reference.py | 1 + 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/src/piccolo/api_reference/index.rst b/docs/src/piccolo/api_reference/index.rst index 1e1395174..33e1616cd 100644 --- a/docs/src/piccolo/api_reference/index.rst +++ b/docs/src/piccolo/api_reference/index.rst @@ -27,7 +27,6 @@ LazyTableReference .. currentmodule:: piccolo.columns .. autoclass:: LazyTableReference - :members: ------------------------------------------------------------------------------- diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 0efdae163..ec190d45a 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -1718,15 +1718,27 @@ class Musician(Table): In certain situations, you may be unable to reference a ``Table`` class if it causes a circular dependency. Try and avoid these by refactoring - your code. If unavoidable, you can specify a lazy reference. If the - ``Table`` is defined in the same file: + your code. If unavoidable, you can specify a lazy reference using + :class:`LazyTableReference `. + If the ``Table`` is defined in the same file: .. code-block:: python + from piccolo.columns.reference import LazyTableReference + class Band(Table): - manager = ForeignKey(references='Manager') + manager = ForeignKey( + references=LazyTableReference( + table_class_name="Manager", + module_path=__name__, + ) + # Alternatively, Piccolo will interpret this string the + # same as above: + # references="Manager" + ) - If the ``Table`` is defined in a Piccolo app: + Or if the ``Table`` is defined in a different file: + Python module: .. code-block:: python @@ -1735,12 +1747,15 @@ class Band(Table): class Band(Table): manager = ForeignKey( references=LazyTableReference( - table_class_name="Manager", app_name="my_app", + table_class_name="Manager", + module_path="some_module.tables", ) + # Alternatively, Piccolo will interpret this string the + # same as above: + # references="some_module.tables.Manager" ) - If you aren't using Piccolo apps, you can specify a ``Table`` in any - Python module: + If the ``Table`` is defined in a Piccolo app, you can also use this: .. code-block:: python @@ -1749,12 +1764,8 @@ class Band(Table): class Band(Table): manager = ForeignKey( references=LazyTableReference( - table_class_name="Manager", - module_path="some_module.tables", + table_class_name="Manager", app_name="my_app", ) - # Alternatively, Piccolo will interpret this string as - # the same as above: - # references="some_module.tables.Manager" ) :param on_delete: diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index e6acd771e..c21f18afe 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -23,6 +23,7 @@ class LazyTableReference: Holds a reference to a :class:`Table ` subclass. Used to avoid circular dependencies in the ``references`` argument of :class:`ForeignKey ` columns. + Pass in either ``app_name`` OR ``module_path``. :param table_class_name: The name of the ``Table`` subclass. For example, ``'Manager'``. From 0bd1f1245c8b257b86a0b7f21f3d95028503e64d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 26 Jul 2023 20:34:24 +0100 Subject: [PATCH 19/19] rename `SetReadyResponse` -> `CheckReadyResponse` --- piccolo/columns/reference.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piccolo/columns/reference.py b/piccolo/columns/reference.py index c21f18afe..85fa87453 100644 --- a/piccolo/columns/reference.py +++ b/piccolo/columns/reference.py @@ -12,7 +12,7 @@ @dataclass -class SetReadyResponse: +class CheckReadyResponse: is_ready: bool was_changed: bool @@ -57,13 +57,13 @@ def __post_init__(self): def check_ready( self, table_classes: t.List[t.Type[Table]] - ) -> SetReadyResponse: + ) -> CheckReadyResponse: """ Checks to see if the table being referenced is in the provided list, meaning it has successfully loaded. """ if self.ready: - return SetReadyResponse(is_ready=True, was_changed=False) + return CheckReadyResponse(is_ready=True, was_changed=False) else: for table_class in table_classes: if self.table_class_name == table_class.__name__: @@ -75,11 +75,11 @@ def check_ready( and self.app_name == table_class._meta.app_name ): self.ready = True - return SetReadyResponse( + return CheckReadyResponse( is_ready=True, was_changed=True ) - return SetReadyResponse(is_ready=False, was_changed=False) + return CheckReadyResponse(is_ready=False, was_changed=False) def resolve(self) -> t.Type[Table]: if self._resolved is not None: