From 59e0428653b44acd8b1f5fed6bba34b63d1f6a0b Mon Sep 17 00:00:00 2001 From: Lars Kiesow Date: Sat, 4 Feb 2023 23:47:08 +0100 Subject: [PATCH 1/5] Sync via git repository This patch allows Watson to sync via a simple git repository, making it unnecessary to run a specific backend. Instead, any git repository with write access will do. You can configure a git backend by setting something like: watson config backend.repo git@github.com:user/repo.git If y repository is set, the `sync` command will try using git. If not, the old backend server is used if `server` and `token` are set. --- docs/user-guide/commands.md | 3 + docs/user-guide/configuration.md | 19 +++- watson/cli.py | 7 +- watson/watson.py | 147 ++++++++++++++++++++++--------- 4 files changed, 131 insertions(+), 45 deletions(-) diff --git a/docs/user-guide/commands.md b/docs/user-guide/commands.md index 387639f..09dba25 100644 --- a/docs/user-guide/commands.md +++ b/docs/user-guide/commands.md @@ -739,6 +739,9 @@ Example: $ watson config backend.url http://localhost:4242 $ watson config backend.token 7e329263e329 + or + $ watson config backend.repo git@github.com:user/repo.git + $ watson sync Received 42 frames from the server Pushed 23 frames to the server diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index be163b8..fbccd93 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -86,7 +86,21 @@ $ watson config -e ### Backend -At this time there is no official backend for Watson. We are working on it. But in a near future, you will be able to synchronize Watson with a public (or your private) repository via the [`sync`](./commands.md#sync) command. To configure your repository please set up the `[backend]` section. +You will be able to synchronize Watson with a public (or your private) repository via the [`sync`](./commands.md#sync) command. +To configure your repository please set up the `[backend]` section. +You have two options for synchronization: + +- The [crick](https://github.com/TailorDev/crick) server +- A git repository + +If using crick, set `backend.url` and `backend.token`. +If using a git repository, set `backend.repo`. + + +#### `backend.repo` (default: empty) + +The remote URL of a git repository to clone. +Something like `git@github.com:user/repo.git`. #### `backend.url` (default: empty) @@ -225,8 +239,7 @@ A basic configuration file looks like the following: # Watson configuration [backend] -url = https://api.crick.fr -token = yourapitoken +repo = git@github.com:user/repo.git [options] stop_on_start = true diff --git a/watson/cli.py b/watson/cli.py index ac406b5..8af0b3e 100644 --- a/watson/cli.py +++ b/watson/cli.py @@ -1545,16 +1545,19 @@ def sync(watson): \b $ watson config backend.url http://localhost:4242 $ watson config backend.token 7e329263e329 + or + $ watson config backend.repo git@github.com:user/repo.git + \b $ watson sync Received 42 frames from the server Pushed 23 frames to the server """ last_pull = arrow.utcnow() pulled = watson.pull() - click.echo("Received {} frames from the server".format(len(pulled))) + click.echo("Received {} frames from the server".format(pulled)) pushed = watson.push(last_pull) - click.echo("Pushed {} frames to the server".format(len(pushed))) + click.echo("Pushed {} frames to the server".format(pushed)) watson.last_sync = arrow.utcnow() watson.save() diff --git a/watson/watson.py b/watson/watson.py index 7ab7da1..f31a387 100644 --- a/watson/watson.py +++ b/watson/watson.py @@ -7,6 +7,7 @@ from configparser import Error as CFGParserError import arrow import click +import subprocess from .config import ConfigParser from .frames import Frames @@ -54,6 +55,7 @@ def __init__(self, **kwargs): self.frames_file = os.path.join(self._dir, 'frames') self.state_file = os.path.join(self._dir, 'state') self.last_sync_file = os.path.join(self._dir, 'last_sync') + self.sync_dir = os.path.join(self._dir, 'sync_repo') if 'frames' in kwargs: self.frames = kwargs['frames'] @@ -371,43 +373,63 @@ def _get_remote_projects(self): return self._remote_projects['projects'] def pull(self): - import requests - dest, headers = self._get_request_info('frames') - try: - response = requests.get( - dest, params={'last_sync': self.last_sync}, headers=headers - ) - assert response.status_code == 200 - except requests.ConnectionError: - raise WatsonError("Unable to reach the server.") - except AssertionError: - raise WatsonError( - "An error occurred with the remote " - "server: {}".format(response.json()) - ) + repo = self.config.get('backend', 'repo') + if repo: + # clone git repository if necessary + if not os.path.isdir(self.sync_dir): + sync_dir = self.sync_dir + subprocess.run(['git', 'clone', repo, sync_dir], check=True) - frames = response.json() or () + # git pull + subprocess.run(['git', 'pull'], cwd=self.sync_dir, check=True) + sync_file = os.path.join(self.sync_dir, 'frames') + try: + with open(sync_file, 'r') as f: + frames = json.load(f) + except FileNotFoundError: + frames = [] + else: + import requests + dest, headers = self._get_request_info('frames') + try: + response = requests.get( + dest, params={'last_sync': self.last_sync}, headers=headers + ) + assert response.status_code == 200 + except requests.ConnectionError: + raise WatsonError("Unable to reach the server.") + except AssertionError: + raise WatsonError( + "An error occurred with the remote " + "server: {}".format(response.json()) + ) + + frames = response.json() or () + + updated_frames = 0 for frame in frames: frame_id = uuid.UUID(frame['id']).hex - self.frames[frame_id] = ( - frame['project'], - frame['begin_at'], - frame['end_at'], - frame['tags'] - ) + try: + self.frames[frame_id] + except KeyError: + updated_frames += 1 + self.frames[frame_id] = ( + frame['project'], + frame['begin_at'], + frame['end_at'], + frame['tags'] + ) - return frames + return updated_frames def push(self, last_pull): - import requests - dest, headers = self._get_request_info('frames/bulk') frames = [] - - for frame in self.frames: - if last_pull > frame.updated_at > self.last_sync: + repo = self.config.get('backend', 'repo') + if repo: + for frame in self.frames: frames.append({ 'id': uuid.UUID(frame.id).urn, 'begin_at': str(frame.start.to('utc')), @@ -416,21 +438,66 @@ def push(self, last_pull): 'tags': frame.tags }) - try: - response = requests.post(dest, json.dumps(frames), headers=headers) - assert response.status_code == 201 - except requests.ConnectionError: - raise WatsonError("Unable to reach the server.") - except AssertionError: - raise WatsonError( - "An error occurred with the remote server (status: {}). " - "Response was:\n{}".format( - response.status_code, - response.text + # Get number of synced frames + sync_file = os.path.join(self.sync_dir, 'frames') + try: + with open(sync_file, 'r') as f: + n_frames = len(json.load(f)) + except FileNotFoundError: + n_frames = 0 + + # Write frames to repo + with open(sync_file, 'w') as f: + json.dump(frames, f, indent=2) + + # Check if anything has changed + try: + command = ['git', 'diff', '--quiet'] + subprocess.run(command, cwd=self.sync_dir, check=True) + return 0 + except subprocess.CalledProcessError: + pass + + # git push + cwd = self.sync_dir + updated = len(frames) - n_frames + msg = f'Add {updated} frames ({datetime.datetime.now()})' + subprocess.run(['git', 'add', 'frames'], cwd=cwd, check=True) + subprocess.run(['git', 'commit', '-m', msg], cwd=cwd, check=True) + subprocess.run(['git', 'push'], cwd=cwd, check=True) + + return updated + + else: + import requests + dest, headers = self._get_request_info('frames/bulk') + + for frame in self.frames: + if last_pull > frame.updated_at > self.last_sync: + frames.append({ + 'id': uuid.UUID(frame.id).urn, + 'begin_at': str(frame.start.to('utc')), + 'end_at': str(frame.stop.to('utc')), + 'project': frame.project, + 'tags': frame.tags + }) + + try: + body = json.dumps(frames) + response = requests.post(dest, body, headers=headers) + assert response.status_code == 201 + except requests.ConnectionError: + raise WatsonError("Unable to reach the server.") + except AssertionError: + raise WatsonError( + "An error occurred with the remote server (status: {}). " + "Response was:\n{}".format( + response.status_code, + response.text + ) ) - ) - return frames + return len(frames) def merge_report(self, frames_with_conflict): conflict_file_frames = Frames(self._load_json_file( From 130973bbbca304c7531d8590949f4b816383c567 Mon Sep 17 00:00:00 2001 From: Lars Kiesow Date: Sun, 23 Apr 2023 19:12:39 +0200 Subject: [PATCH 2/5] Use colors for sync To make the stats more legible, this patch uses color for the summary lines printed out so they stand out more from the git logs. --- watson/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/watson/cli.py b/watson/cli.py index 8af0b3e..31b5887 100644 --- a/watson/cli.py +++ b/watson/cli.py @@ -1554,10 +1554,10 @@ def sync(watson): """ last_pull = arrow.utcnow() pulled = watson.pull() - click.echo("Received {} frames from the server".format(pulled)) + click.secho(f'Received {pulled} frames from the server.', fg='green') pushed = watson.push(last_pull) - click.echo("Pushed {} frames to the server".format(pushed)) + click.secho(f'Pushed {pushed} frames to the server.', fg='green') watson.last_sync = arrow.utcnow() watson.save() From f9a976142290e65dcf6a36b358934c433014c575 Mon Sep 17 00:00:00 2001 From: Lars Kiesow Date: Fri, 23 Jun 2023 10:52:31 +0200 Subject: [PATCH 3/5] Sort frames stored in git This should minimize the changes when different clients commit to the repository. --- watson/watson.py | 1 + 1 file changed, 1 insertion(+) diff --git a/watson/watson.py b/watson/watson.py index f31a387..52460f4 100644 --- a/watson/watson.py +++ b/watson/watson.py @@ -437,6 +437,7 @@ def push(self, last_pull): 'project': frame.project, 'tags': frame.tags }) + frames.sort(key=lambda frame: frame['begin_at']) # Get number of synced frames sync_file = os.path.join(self.sync_dir, 'frames') From 8cb766689d75f3d0d7185ac09d8a481f615cc197 Mon Sep 17 00:00:00 2001 From: teutat3s <10206665+teutat3s@users.noreply.github.com> Date: Sat, 17 Aug 2024 21:17:07 +0200 Subject: [PATCH 4/5] test: fix previously empty sync test E AssertionError: assert 4003 == 4001 --- tests/test_watson.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_watson.py b/tests/test_watson.py index b44ee17..0261835 100644 --- a/tests/test_watson.py +++ b/tests/test_watson.py @@ -698,9 +698,9 @@ def json(self): assert watson.frames[0].id == '1c006c6e6cc14c80ab22b51c857c0b06' assert watson.frames[0].project == 'foo' - assert watson.frames[0].start.int_timestamp == 4003 - assert watson.frames[0].stop.int_timestamp == 4004 - assert watson.frames[0].tags == ['A'] + assert watson.frames[0].start.int_timestamp == 4001 + assert watson.frames[0].stop.int_timestamp == 4002 + assert watson.frames[0].tags == ['A', 'B'] assert watson.frames[1].id == 'c44aa8154d774a58bddd1afa95562141' assert watson.frames[1].project == 'bar' From 919a1e591987280f654d0f7c3a022c6a14f96d19 Mon Sep 17 00:00:00 2001 From: teutat3s <10206665+teutat3s@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:21:05 +0200 Subject: [PATCH 5/5] sync: fix initial sync --- watson/watson.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/watson/watson.py b/watson/watson.py index 52460f4..3f39be2 100644 --- a/watson/watson.py +++ b/watson/watson.py @@ -452,18 +452,18 @@ def push(self, last_pull): json.dump(frames, f, indent=2) # Check if anything has changed + cwd = self.sync_dir try: - command = ['git', 'diff', '--quiet'] - subprocess.run(command, cwd=self.sync_dir, check=True) + subprocess.run(['git', 'add', 'frames'], cwd=cwd, check=True) + command = ['git', 'diff', '--staged', '--quiet'] + subprocess.run(command, cwd=cwd, check=True) return 0 except subprocess.CalledProcessError: pass # git push - cwd = self.sync_dir updated = len(frames) - n_frames msg = f'Add {updated} frames ({datetime.datetime.now()})' - subprocess.run(['git', 'add', 'frames'], cwd=cwd, check=True) subprocess.run(['git', 'commit', '-m', msg], cwd=cwd, check=True) subprocess.run(['git', 'push'], cwd=cwd, check=True)