From a57c2c6bf632c4a7b565dc3ad2198bf45473d4bd Mon Sep 17 00:00:00 2001 From: BG Date: Mon, 20 Aug 2018 23:03:35 +0300 Subject: [PATCH 01/11] adding backup option --- rbd2qcow2/main.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index 21472e5..0a7a431 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -147,9 +147,9 @@ async def do_transfer( raise -def get_latest_backup(xxx: str, image_name: str) -> int: +def get_latest_backup(xxx: str, image_name: str)->dict: log.debug('Searching for previous images in backup dir.') - ts = 0 + ret = dict() for filename in os.listdir(xxx): mtch = re.fullmatch(r'([^@]+)@([0-9]+)\.qcow2', filename) if mtch is None: @@ -157,10 +157,8 @@ def get_latest_backup(xxx: str, image_name: str) -> int: if mtch.group(1) != image_name: log.warning('Unexpected filename %r in dir %r.', filename, xxx) continue - timestamp = int(mtch.group(2)) - if timestamp > ts: - ts = timestamp - return ts + ret[int(mtch.group(2))] = filename + return ret def check_skip_backup(rbd_image) -> bool: @@ -196,7 +194,14 @@ async def do_backup(rbd_image_name: str, loop, ioctx): if not os.path.isdir(xxx): log.debug('Creating directory %s.', xxx) os.makedirs(xxx) - latest_ts = get_latest_backup(xxx, rbd_image_name) + itms = get_latest_backup(xxx, rbd_image_name) + cnt=len(itms) + srt=sorted(itms) + latest_ts=srt.last() + if cnt >=4: + args = ['qemu-img', 'rebase', '-b',os.path.join(xxx, itms[srt[1]]),os.path.join(xxx, itms[srt[3]]) ]; + subprocess.check_call(args) + os.remove(os.path.join(xxx,itms[srt[2]])) if latest_ts == 0: log.info('Did not found previous backup for image %s.', rbd_image_name) empty_image_path = os.path.join(xxx, 'empty.qcow2') From cbfb49732ff56ea34c764c4c2cacfe14abfc1cc6 Mon Sep 17 00:00:00 2001 From: BG Date: Tue, 21 Aug 2018 06:28:48 +0300 Subject: [PATCH 02/11] fix stupid error --- rbd2qcow2/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index 0a7a431..c8d5018 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -197,7 +197,7 @@ async def do_backup(rbd_image_name: str, loop, ioctx): itms = get_latest_backup(xxx, rbd_image_name) cnt=len(itms) srt=sorted(itms) - latest_ts=srt.last() + latest_ts=srt[-1] if cnt >=4: args = ['qemu-img', 'rebase', '-b',os.path.join(xxx, itms[srt[1]]),os.path.join(xxx, itms[srt[3]]) ]; subprocess.check_call(args) From e1cd86afa83fe67cda1cb913ee01b3b3f201e2a6 Mon Sep 17 00:00:00 2001 From: bg Date: Tue, 4 Sep 2018 14:00:08 +1000 Subject: [PATCH 03/11] fix permissions --- rbd2qcow2/main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index c8d5018..9421baa 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -130,7 +130,7 @@ async def do_transfer( os.fsync(xxx.fileno()) log.debug('Fsyncing complete.') - os.chmod(tmp_filename, 0o400) + #os.chmod(tmp_filename, 0o400) os.rename(tmp_filename, qcow2_name) # Safe rename fd = os.open(qcow2_directory, os.O_DIRECTORY | os.O_RDONLY) @@ -198,10 +198,13 @@ async def do_backup(rbd_image_name: str, loop, ioctx): cnt=len(itms) srt=sorted(itms) latest_ts=srt[-1] + print(srt) + print(latest_ts) if cnt >=4: - args = ['qemu-img', 'rebase', '-b',os.path.join(xxx, itms[srt[1]]),os.path.join(xxx, itms[srt[3]]) ]; + args = ['qemu-img', 'rebase', '-b',os.path.join(xxx, itms[srt[1]]),os.path.join(xxx, itms[srt[3]]) ]; + print(args); subprocess.check_call(args) - os.remove(os.path.join(xxx,itms[srt[2]])) + if latest_ts == 0: log.info('Did not found previous backup for image %s.', rbd_image_name) empty_image_path = os.path.join(xxx, 'empty.qcow2') @@ -225,6 +228,7 @@ async def do_backup(rbd_image_name: str, loop, ioctx): qcow2_name, backing_store_filename, ) + os.remove(os.path.join(xxx,itms[srt[2]])) except Exception as e: log.info('Removing RBD snapshot %s@%s due to error %r.', rbd_image_name, rbd_new_snapshot_name, e) From f2498b687452e0d4d8eed5812fe1d14e0dc82df4 Mon Sep 17 00:00:00 2001 From: BG Date: Fri, 7 Sep 2018 13:10:39 +0300 Subject: [PATCH 04/11] add command line option --- rbd2qcow2/main.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index 9421baa..a3e79f1 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -198,13 +198,10 @@ async def do_backup(rbd_image_name: str, loop, ioctx): cnt=len(itms) srt=sorted(itms) latest_ts=srt[-1] - print(srt) - print(latest_ts) - if cnt >=4: + if cnt >=options.bk_count: args = ['qemu-img', 'rebase', '-b',os.path.join(xxx, itms[srt[1]]),os.path.join(xxx, itms[srt[3]]) ]; - print(args); subprocess.check_call(args) - + os.remove(os.path.join(xxx,itms[srt[2]])) if latest_ts == 0: log.info('Did not found previous backup for image %s.', rbd_image_name) empty_image_path = os.path.join(xxx, 'empty.qcow2') @@ -228,7 +225,7 @@ async def do_backup(rbd_image_name: str, loop, ioctx): qcow2_name, backing_store_filename, ) - os.remove(os.path.join(xxx,itms[srt[2]])) + except Exception as e: log.info('Removing RBD snapshot %s@%s due to error %r.', rbd_image_name, rbd_new_snapshot_name, e) @@ -336,6 +333,14 @@ def main(): help='Path to a directory where backups should be placed.' ) + parser.add_argument( + '--bk_count', + metavar='BK_COUNT', + type=int, + help='Count of backups to store.' + default=4 + ) + parser.add_argument( 'images', metavar='IMAGE_NAME', @@ -348,6 +353,9 @@ def main(): if not (1 <= options.parallel <= 100): raise ValueError('Wrong parallel count.') + if not (options.bk_count >= 4): + raise ValueError('Wrong backups count. Minimum is 4.') + logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO) log.info('Starting backup process.') From 9ed531e0010402292eef60e42b237033ed104a56 Mon Sep 17 00:00:00 2001 From: BG Date: Fri, 7 Sep 2018 13:23:53 +0300 Subject: [PATCH 05/11] fix stupid miss --- rbd2qcow2/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index a3e79f1..af608dd 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -337,7 +337,7 @@ def main(): '--bk_count', metavar='BK_COUNT', type=int, - help='Count of backups to store.' + help='Count of backups to store.', default=4 ) From 162356c41b3c57aa49794ebe3f6e655bc310260e Mon Sep 17 00:00:00 2001 From: BG Date: Fri, 7 Sep 2018 13:34:29 +0300 Subject: [PATCH 06/11] add deletion of extra snaps --- rbd2qcow2/main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index af608dd..d988031 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -199,9 +199,16 @@ async def do_backup(rbd_image_name: str, loop, ioctx): srt=sorted(itms) latest_ts=srt[-1] if cnt >=options.bk_count: - args = ['qemu-img', 'rebase', '-b',os.path.join(xxx, itms[srt[1]]),os.path.join(xxx, itms[srt[3]]) ]; + args = ['qemu-img', + 'rebase', + '-b', + os.path.join(xxx, itms[srt[1]]), + os.path.join(xxx, itms[srt[options.bk_count-1]]) + ]; subprocess.check_call(args) - os.remove(os.path.join(xxx,itms[srt[2]])) + for idx in range(2, options.bk_count-2): + print(os.path.join(xxx,itms[srt[idx]])) + os.remove(os.path.join(xxx,itms[srt[idx]])) if latest_ts == 0: log.info('Did not found previous backup for image %s.', rbd_image_name) empty_image_path = os.path.join(xxx, 'empty.qcow2') From 54b0f84b70298faee795246b80cc7787ae01112d Mon Sep 17 00:00:00 2001 From: BG Date: Fri, 7 Sep 2018 18:07:24 +0300 Subject: [PATCH 07/11] add deletion of extra snaps fix --- rbd2qcow2/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index d988031..12719fa 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -197,7 +197,10 @@ async def do_backup(rbd_image_name: str, loop, ioctx): itms = get_latest_backup(xxx, rbd_image_name) cnt=len(itms) srt=sorted(itms) - latest_ts=srt[-1] + if cnt>0: + latest_ts=srt[-1] + else: + latest_ts=0 if cnt >=options.bk_count: args = ['qemu-img', 'rebase', From effe1a6a7197936e460df7684b83346c901b42c0 Mon Sep 17 00:00:00 2001 From: BG Date: Sat, 8 Sep 2018 12:06:22 +0300 Subject: [PATCH 08/11] add debug print --- rbd2qcow2/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index 12719fa..40da504 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -208,6 +208,7 @@ async def do_backup(rbd_image_name: str, loop, ioctx): os.path.join(xxx, itms[srt[1]]), os.path.join(xxx, itms[srt[options.bk_count-1]]) ]; + print(args) subprocess.check_call(args) for idx in range(2, options.bk_count-2): print(os.path.join(xxx,itms[srt[idx]])) From d932b97406ab4a5c19aacefb77547889e96c5b66 Mon Sep 17 00:00:00 2001 From: BG Date: Sat, 8 Sep 2018 20:38:05 +0300 Subject: [PATCH 09/11] fix logic --- rbd2qcow2/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index 40da504..63bb8e0 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -201,16 +201,17 @@ async def do_backup(rbd_image_name: str, loop, ioctx): latest_ts=srt[-1] else: latest_ts=0 - if cnt >=options.bk_count: + if cnt >= options.bk_count: + removingCount = 1 + cnt - options.bk_count; args = ['qemu-img', 'rebase', '-b', os.path.join(xxx, itms[srt[1]]), - os.path.join(xxx, itms[srt[options.bk_count-1]]) + os.path.join(xxx, itms[srt[removingCount+2]]) ]; print(args) subprocess.check_call(args) - for idx in range(2, options.bk_count-2): + for idx in range(2, removingCount+2): print(os.path.join(xxx,itms[srt[idx]])) os.remove(os.path.join(xxx,itms[srt[idx]])) if latest_ts == 0: From 940789f39fbfe1d0d9b88e67db391b911a27ff43 Mon Sep 17 00:00:00 2001 From: BG Date: Sat, 8 Sep 2018 22:48:13 +0300 Subject: [PATCH 10/11] =?UTF-8?q?fix=20log=D0=BF=D1=88=D1=82=D0=BF=D1=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rbd2qcow2/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index 63bb8e0..e416975 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -209,10 +209,10 @@ async def do_backup(rbd_image_name: str, loop, ioctx): os.path.join(xxx, itms[srt[1]]), os.path.join(xxx, itms[srt[removingCount+2]]) ]; - print(args) + log.info('Rebasing image %s.', args) subprocess.check_call(args) for idx in range(2, removingCount+2): - print(os.path.join(xxx,itms[srt[idx]])) + log.info('Removing old image %s.',os.path.join(xxx,itms[srt[idx]])) os.remove(os.path.join(xxx,itms[srt[idx]])) if latest_ts == 0: log.info('Did not found previous backup for image %s.', rbd_image_name) From 7f5d74ff2da5544277128e534ad84347a1238692 Mon Sep 17 00:00:00 2001 From: BG Date: Sun, 9 Sep 2018 02:52:42 +0300 Subject: [PATCH 11/11] fix defaults --- rbd2qcow2/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rbd2qcow2/main.py b/rbd2qcow2/main.py index e416975..954b914 100644 --- a/rbd2qcow2/main.py +++ b/rbd2qcow2/main.py @@ -13,6 +13,7 @@ import rados import rbd +import sys from rbd2qcow2.libc_wrappers import fallocate, FALLOC_FL_KEEP_SIZE from rbd2qcow2.nbd_client import open_image @@ -350,7 +351,7 @@ def main(): metavar='BK_COUNT', type=int, help='Count of backups to store.', - default=4 + default=sys.maxsize ) parser.add_argument(