From 6a0f2c10186c825dacc48268bbb2b36038c7aa8b Mon Sep 17 00:00:00 2001 From: Tony Ramos Date: Fri, 27 Aug 2021 16:38:10 -0500 Subject: [PATCH 1/7] imported errors --- datajoint/external.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datajoint/external.py b/datajoint/external.py index 5e4c7ff0c..a74587212 100644 --- a/datajoint/external.py +++ b/datajoint/external.py @@ -8,7 +8,7 @@ from .declare import EXTERNAL_TABLE_ROOT from . import s3 from .utils import safe_write, safe_copy - +from . import errors CACHE_SUBFOLDING = (2, 2) # (2, 2) means "0123456789abcd" will be saved as "01/23/0123456789abcd" SUPPORT_MIGRATED_BLOBS = True # support blobs migrated from datajoint 0.11.* @@ -125,7 +125,10 @@ def _download_buffer(self, external_path): if self.spec['protocol'] == 's3': return self.s3.get(external_path) if self.spec['protocol'] == 'file': - return Path(external_path).read_bytes() + try: + return Path(external_path).read_bytes() + except FileNotFoundError: + raise errors.MissingExternalFile('Missing external file %s' % external_path) from None assert False def _remove_external_file(self, external_path): From f722fefa9d66651e3a9176a68267b792851c6461 Mon Sep 17 00:00:00 2001 From: Tony Ramos Date: Fri, 3 Sep 2021 13:54:28 -0500 Subject: [PATCH 2/7] changed datajoint migrate script to be compatible --- datajoint/migrate.py | 48 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/datajoint/migrate.py b/datajoint/migrate.py index 445bc317c..68e80dada 100644 --- a/datajoint/migrate.py +++ b/datajoint/migrate.py @@ -33,6 +33,13 @@ def migrate_dj011_external_blob_storage_to_dj012(migration_schema, store): def _migrate_dj011_blob(schema, default_store): + warning_string = """External storage was lost on 2/8/2020, as a result the external for this stack is unavailable. + The message you are seeing now is a result of a migration to dj 0.12.9 on 8/25/2021. If you would like this stack + repopulated, please contact the pipeline engineer """ + warning_string_arr = np.array([warning_string]) + packed_warning_string = blob.pack(warning_string_arr) + + query = schema.connection.query LEGACY_HASH_SIZE = 43 @@ -42,16 +49,17 @@ def _migrate_dj011_blob(schema, default_store): '`{db}`.`~external`'.format(db=schema.database)) # get referencing tables - refs = [{k.lower(): v for k, v in elem.items()} for elem in query(""" + refs = query(""" SELECT concat('`', table_schema, '`.`', table_name, '`') as referencing_table, column_name, constraint_name FROM information_schema.key_column_usage WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}" """.format( tab=legacy_external.table_name, - db=legacy_external.database), as_dict=True).fetchall()] + db=legacy_external.database), as_dict=True).fetchall() for ref in refs: + print(ref) # get comment column = query( 'SHOW FULL COLUMNS FROM {referencing_table}' @@ -59,13 +67,18 @@ def _migrate_dj011_blob(schema, default_store): **ref), as_dict=True).fetchone() store, comment = re.match( - r':external(-(?P.+))?:(?P.*)', + r':external(-(?P.+?))?:(?P.*)', column['Comment']).group('store', 'comment') + + print(store) + print(comment) + # get all the hashes from the reference hashes = {x[0] for x in query( 'SELECT `{column_name}` FROM {referencing_table}'.format( **ref))} + # sanity check make sure that store suffixes match if store is None: @@ -75,6 +88,7 @@ def _migrate_dj011_blob(schema, default_store): # create new-style external table ext = schema.external[store or default_store] + # add the new-style reference field temp_suffix = 'tempsub' @@ -91,6 +105,16 @@ def _migrate_dj011_blob(schema, default_store): print('Column already added') pass + # Copy references into the new external table + # No Windows! Backslashes will cause problems + + contents_hash_function = { + 'file': lambda ext, relative_path: dj.hash.uuid_from_file( + str(Path(ext.spec['location'], relative_path))), + 's3': lambda ext, relative_path: dj.hash.uuid_from_buffer( + ext.s3.get(relative_path)) + } + for _hash, size in zip(*legacy_external.fetch('hash', 'size')): if _hash in hashes: relative_path = str(Path(schema.database, _hash).as_posix()) @@ -99,7 +123,18 @@ def _migrate_dj011_blob(schema, default_store): if ext.spec['protocol'] == 's3': contents_hash = dj.hash.uuid_from_buffer(ext._download_buffer(external_path)) else: - contents_hash = dj.hash.uuid_from_file(external_path) + try: + contents_hash = dj.hash.uuid_from_file(external_path) + except FileNotFoundError: + ## save new content under old filename + print(f"{external_path} not found") + print("inserting warning message in-place") + contents_hash = dj.hash.uuid_from_buffer(init_string=warning_string) + safe_write(external_path,packed_warning_string) + + + + ext.insert1(dict( filepath=relative_path, size=size, @@ -142,16 +177,15 @@ def _migrate_dj011_blob(schema, default_store): # Drop the old external table but make sure it's no longer referenced # get referencing tables - refs = [{k.lower(): v for k, v in elem.items()} for elem in query(""" + refs = query(""" SELECT concat('`', table_schema, '`.`', table_name, '`') as referencing_table, column_name, constraint_name FROM information_schema.key_column_usage WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}" """.format( tab=legacy_external.table_name, - db=legacy_external.database), as_dict=True).fetchall()] + db=legacy_external.database), as_dict=True).fetchall() assert not refs, 'Some references still exist' - # drop old external table legacy_external.drop_quick() From 11b0aa1f924248f9be41df6d032ae1002b835f8f Mon Sep 17 00:00:00 2001 From: Anthony Ramos Date: Sat, 4 Sep 2021 11:24:09 -0700 Subject: [PATCH 3/7] removed drop of legacy external --- datajoint/migrate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datajoint/migrate.py b/datajoint/migrate.py index 68e80dada..d618c49cb 100644 --- a/datajoint/migrate.py +++ b/datajoint/migrate.py @@ -188,4 +188,3 @@ def _migrate_dj011_blob(schema, default_store): assert not refs, 'Some references still exist' # drop old external table - legacy_external.drop_quick() From 273695b8a31293c9037cd32277c949366a318fea Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 19 Jan 2022 12:05:11 -0600 Subject: [PATCH 4/7] bugfix in unite_master_parts - handles `dj.Lookup` part tables --- datajoint/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/dependencies.py b/datajoint/dependencies.py index a4e1afb0c..75301c8c5 100644 --- a/datajoint/dependencies.py +++ b/datajoint/dependencies.py @@ -18,7 +18,7 @@ def unite_master_parts(lst): """ for i in range(2, len(lst)): name = lst[i] - match = re.match(r'(?P`\w+`.`\w+)__\w+`', name) + match = re.match(r'(?P`\w+`.`#?\w+)__\w+`', name) if match: # name is a part table master = match.group('master') for j in range(i-1, -1, -1): From 361a20124666ed50ba3c035f03af71aeee39ecb7 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Thu, 20 Jan 2022 16:43:49 -0600 Subject: [PATCH 5/7] - Removed error throw for uniting parts with missing masters - Added test case for #927 --- datajoint/dependencies.py | 2 -- tests/test_declare.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/datajoint/dependencies.py b/datajoint/dependencies.py index 75301c8c5..32b729550 100644 --- a/datajoint/dependencies.py +++ b/datajoint/dependencies.py @@ -26,8 +26,6 @@ def unite_master_parts(lst): # move from the ith position to the (j+1)th position lst[j+1:i+1] = [name] + lst[j+1:i] break - else: - raise DataJointError("Found a part table {name} without its master table.".format(name=name)) return lst diff --git a/tests/test_declare.py b/tests/test_declare.py index cce2d6366..f0128e2b5 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -160,6 +160,37 @@ def test_dependencies(): assert_set_equal(set(s.full_table_name for s in channel.parents(primary=True, as_objects=True)), {ephys.full_table_name}) + @staticmethod + def test_descendants_only_contain_part_table(): + """issue #927""" + + @schema + class A(dj.Manual): + definition = """ + a: int + """ + + @schema + class B(dj.Manual): + definition = """ + -> A + b: int + """ + + @schema + class Master(dj.Manual): + definition = """ + table_master: int + """ + + class Part(dj.Part): + definition = """ + -> master + -> B + """ + + assert A.descendants() == ['`djtest_test1`.`a`', '`djtest_test1`.`b`', '`djtest_test1`.`master__part`'] + @staticmethod @raises(dj.DataJointError) def test_bad_attribute_name(): From 4a75ae106d1c164cb7bdc99b01b472f6cb1dfe17 Mon Sep 17 00:00:00 2001 From: Zhuokun Ding Date: Wed, 18 May 2022 14:41:05 -0500 Subject: [PATCH 6/7] Remove declare_context assertion when creating headings for FreeTable with adapters. --- datajoint/heading.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datajoint/heading.py b/datajoint/heading.py index ca637912f..f7f725883 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -227,7 +227,6 @@ def init_from_database(self, conn, database, table_name, context): attr.update(special) # process adapted attribute types if special and TYPE_PATTERN['ADAPTED'].match(attr['type']): - assert context is not None, 'Declaration context is not set' adapter_name = special['type'] try: attr.update(adapter=get_adapter(context, adapter_name)) From 6f605192fc14feea6946d01d03f4ffa5e343ba09 Mon Sep 17 00:00:00 2001 From: Zhuokun Ding Date: Wed, 18 May 2022 22:06:09 -0500 Subject: [PATCH 7/7] Improve error handeling of tables with adapters --- datajoint/heading.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datajoint/heading.py b/datajoint/heading.py index f7f725883..51fa72013 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -229,9 +229,10 @@ def init_from_database(self, conn, database, table_name, context): if special and TYPE_PATTERN['ADAPTED'].match(attr['type']): adapter_name = special['type'] try: + assert context is not None attr.update(adapter=get_adapter(context, adapter_name)) - except DataJointError: - # if no adapter, then delay the error until the first invocation + except (DataJointError, AssertionError): + # if no adapter or no context, then delay the error until the first invocation attr.update(adapter=AttributeAdapter()) else: attr.update(type=attr['adapter'].attribute_type)