From cd351127bb9d63c6f3b9713998e6a1a84c1f9bbd Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Tue, 10 Jun 2025 10:53:10 +0200 Subject: [PATCH 1/5] Add enhancement export tenant layer --- asab/library/service.py | 103 +++++++++++++++++++++++++++------------- 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/asab/library/service.py b/asab/library/service.py index 5d97e73e0..fa637bd95 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -637,58 +637,95 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: L.info("Item '{}' not found in directory '{}'.".format(filename, directory)) return None - - async def export(self, path: str = "/", remove_path: bool = False) -> typing.IO: + async def export( + self, + path: str = "/", + remove_path: bool = False, + tenant: bool = False + ) -> typing.IO: """ - Return a file-like stream containing a gzipped tar archive of the library contents of the path. + Produce a gzipped tar of global and, optionally, tenant layers. Args: - path: The path to export. - tenant (str | None ): The tenant to use for the operation. - remove_path: If `True`, the path will be removed from the tar file. - + path: Directory path to export (must end with '/'). + remove_path: If True, strip `path` prefix from archived names. + tenant: If True, append each tenant's layer under `.tenants//`. Returns: - A file object containing a gzipped tar archive. + A file-like object with the tar.gz archive. """ - _validate_path_directory(path) + provider = self.Libraries[0] fileobj = tempfile.TemporaryFile() tarobj = tarfile.open(name=None, mode='w:gz', fileobj=fileobj) - items = await self._list(path, providers=self.Libraries[:1]) - recitems = list(items[:]) - - while len(recitems) > 0: - - item = recitems.pop(0) - if item.type != 'dir': - continue - - child_items = await self._list(item.name, providers=item.providers) - items.extend(child_items) - recitems.extend(child_items) - + # -- Global layer -- + items = await self._collect_items(path, providers=[provider]) for item in items: if item.type != 'item': continue - my_data = await self.Libraries[0].read(item.name) - if remove_path: - assert item.name.startswith(path) - tar_name = item.name[len(path):] - else: - tar_name = item.name - info = tarfile.TarInfo(tar_name) - my_data.seek(0, io.SEEK_END) - info.size = my_data.tell() - my_data.seek(0, io.SEEK_SET) + data = await provider.read(item.name) + if data is None: + continue + name = item.name[len(path):] if remove_path else item.name + info = tarfile.TarInfo(name) + data.seek(0, io.SEEK_END) + info.size = data.tell() + data.seek(0) info.mtime = time.time() - tarobj.addfile(tarinfo=info, fileobj=my_data) + tarobj.addfile(tarinfo=info, fileobj=data) + + # -- Tenant layers -- + if tenant: + try: + tenants = await provider._get_tenants() + except Exception: + tenants = [] + + for t in tenants: + token = Tenant.set(t) + try: + t_items = await self._collect_items(path, providers=[provider]) + finally: + Tenant.reset(token) + + for item in t_items: + if item.type != 'item': + continue + data = await provider.read(item.name) + if data is None: + continue + rel = item.name[len(path):] if remove_path else item.name + archive_name = ".tenants/{0}/{1}".format(t, rel.lstrip("/")) + info = tarfile.TarInfo(archive_name) + data.seek(0, io.SEEK_END) + info.size = data.tell() + data.seek(0) + info.mtime = time.time() + tarobj.addfile(tarinfo=info, fileobj=data) tarobj.close() fileobj.seek(0) return fileobj + async def _collect_items( + self, + path: str, + providers: typing.List[LibraryProviderABC] + ) -> typing.List[LibraryItem]: + """ + Helper to recursively collect all LibraryItem objects under `path` for given providers. + """ + items = await self._list(path, providers=providers) + rec = list(items) + while rec: + node = rec.pop(0) + if node.type != 'dir': + continue + children = await self._list(node.name, providers=node.providers) + items.extend(children) + rec.extend(children) + return items async def subscribe( self, From 24b176f87f229182a7ea8801dce5146930d117f4 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Tue, 10 Jun 2025 15:21:27 +0200 Subject: [PATCH 2/5] Solve flake8 --- asab/library/service.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/asab/library/service.py b/asab/library/service.py index fa637bd95..f5a1080e5 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -638,10 +638,9 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: return None async def export( - self, - path: str = "/", - remove_path: bool = False, - tenant: bool = False + self, + path: str = "/", + remove_path: bool = False, ) -> typing.IO: """ Produce a gzipped tar of global and, optionally, tenant layers. @@ -676,42 +675,41 @@ async def export( tarobj.addfile(tarinfo=info, fileobj=data) # -- Tenant layers -- - if tenant: + try: + tenants = await provider._get_tenants() + except Exception: + tenants = [] + + for t in tenants: + token = Tenant.set(t) try: - tenants = await provider._get_tenants() - except Exception: - tenants = [] - - for t in tenants: - token = Tenant.set(t) - try: - t_items = await self._collect_items(path, providers=[provider]) - finally: - Tenant.reset(token) - - for item in t_items: - if item.type != 'item': - continue - data = await provider.read(item.name) - if data is None: - continue - rel = item.name[len(path):] if remove_path else item.name - archive_name = ".tenants/{0}/{1}".format(t, rel.lstrip("/")) - info = tarfile.TarInfo(archive_name) - data.seek(0, io.SEEK_END) - info.size = data.tell() - data.seek(0) - info.mtime = time.time() - tarobj.addfile(tarinfo=info, fileobj=data) + t_items = await self._collect_items(path, providers=[provider]) + finally: + Tenant.reset(token) + + for item in t_items: + if item.type != 'item': + continue + data = await provider.read(item.name) + if data is None: + continue + rel = item.name[len(path):] if remove_path else item.name + archive_name = ".tenants/{0}/{1}".format(t, rel.lstrip("/")) + info = tarfile.TarInfo(archive_name) + data.seek(0, io.SEEK_END) + info.size = data.tell() + data.seek(0) + info.mtime = time.time() + tarobj.addfile(tarinfo=info, fileobj=data) tarobj.close() fileobj.seek(0) return fileobj async def _collect_items( - self, - path: str, - providers: typing.List[LibraryProviderABC] + self, + path: str, + providers: typing.List[LibraryProviderABC] ) -> typing.List[LibraryItem]: """ Helper to recursively collect all LibraryItem objects under `path` for given providers. From a69b033d491b6485e0f1d1e70f60c6850fec2104 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Fri, 13 Jun 2025 14:06:32 +0200 Subject: [PATCH 3/5] Store under tenants --- asab/library/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asab/library/service.py b/asab/library/service.py index f5a1080e5..69379d1d9 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -694,7 +694,7 @@ async def export( if data is None: continue rel = item.name[len(path):] if remove_path else item.name - archive_name = ".tenants/{0}/{1}".format(t, rel.lstrip("/")) + archive_name = "tenants/{0}/{1}".format(t, rel.lstrip("/")) info = tarfile.TarInfo(archive_name) data.seek(0, io.SEEK_END) info.size = data.tell() From 08e9b37b4cbcc0431f364dfb4f4f770cecba2999 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Mon, 30 Jun 2025 08:46:08 +0200 Subject: [PATCH 4/5] Export per tenant --- asab/library/service.py | 48 ++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/asab/library/service.py b/asab/library/service.py index 69379d1d9..2d5ca2303 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -638,17 +638,17 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: return None async def export( - self, - path: str = "/", - remove_path: bool = False, + self, + path: str = "/", + remove_path: bool = False ) -> typing.IO: """ - Produce a gzipped tar of global and, optionally, tenant layers. + Produce a gzipped tar of global and, if a tenant context is set, that tenant layer. Args: path: Directory path to export (must end with '/'). remove_path: If True, strip `path` prefix from archived names. - tenant: If True, append each tenant's layer under `.tenants//`. + Returns: A file-like object with the tar.gz archive. """ @@ -674,19 +674,14 @@ async def export( info.mtime = time.time() tarobj.addfile(tarinfo=info, fileobj=data) - # -- Tenant layers -- + # -- Tenant-specific layer -- try: - tenants = await provider._get_tenants() - except Exception: - tenants = [] - - for t in tenants: - token = Tenant.set(t) - try: - t_items = await self._collect_items(path, providers=[provider]) - finally: - Tenant.reset(token) + tenant_id = Tenant.get() + except LookupError: + tenant_id = None + if tenant_id: + t_items = await self._collect_items(path, providers=[provider]) for item in t_items: if item.type != 'item': continue @@ -694,7 +689,7 @@ async def export( if data is None: continue rel = item.name[len(path):] if remove_path else item.name - archive_name = "tenants/{0}/{1}".format(t, rel.lstrip("/")) + archive_name = "tenants/{0}/{1}".format(tenant_id, rel.lstrip("/")) info = tarfile.TarInfo(archive_name) data.seek(0, io.SEEK_END) info.size = data.tell() @@ -706,6 +701,25 @@ async def export( fileobj.seek(0) return fileobj + async def _collect_items( + self, + path: str, + providers: typing.List[LibraryProviderABC] + ) -> typing.List[LibraryItem]: + """ + Helper to recursively collect all LibraryItem objects under `path` for given providers. + """ + items = await self._list(path, providers=providers) + rec = list(items) + while rec: + node = rec.pop(0) + if node.type != 'dir': + continue + children = await self._list(node.name, providers=node.providers) + items.extend(children) + rec.extend(children) + return items + async def _collect_items( self, path: str, From a16fd27ae95a167ad3e1140c836408bcca9fe733 Mon Sep 17 00:00:00 2001 From: Mithun Shivashankar Date: Mon, 30 Jun 2025 08:51:49 +0200 Subject: [PATCH 5/5] Flake8 --- asab/library/service.py | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/asab/library/service.py b/asab/library/service.py index 2d5ca2303..b1184dbdd 100644 --- a/asab/library/service.py +++ b/asab/library/service.py @@ -638,9 +638,9 @@ async def get_item_metadata(self, path: str) -> typing.Optional[dict]: return None async def export( - self, - path: str = "/", - remove_path: bool = False + self, + path: str = "/", + remove_path: bool = False ) -> typing.IO: """ Produce a gzipped tar of global and, if a tenant context is set, that tenant layer. @@ -701,25 +701,6 @@ async def export( fileobj.seek(0) return fileobj - async def _collect_items( - self, - path: str, - providers: typing.List[LibraryProviderABC] - ) -> typing.List[LibraryItem]: - """ - Helper to recursively collect all LibraryItem objects under `path` for given providers. - """ - items = await self._list(path, providers=providers) - rec = list(items) - while rec: - node = rec.pop(0) - if node.type != 'dir': - continue - children = await self._list(node.name, providers=node.providers) - items.extend(children) - rec.extend(children) - return items - async def _collect_items( self, path: str,