Merge pull request #13 from provenance-io/log-tailing

Unit testing

Author: wbaker-figure

.gitignore

@@ -4,3 +4,5 @@ build
 dist
 venv
 .DS_Store
+.coverage
+htmlcov

README.md

@@ -25,6 +25,15 @@ forge interactive
 
 * The repo would then checkout to the release version and the binaries would be built, genesis file downloaded/constructed, and a command to run the node would be output to the console.
 
+### Optional
+
+Forge uses the GitHub Api to pull information about the provenance repo for you to use when spinning up a node. If you use forge a lot in a short time, you could hit the 60 calls per hour limit. You can add a GitHub Api Token to your environment in order to increase this to 5000 calls per hour.
+
+You can follow the instructions [Here](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) to setup a personal access token.
+Once that is complete, add the token generated to your environment:
+```sh
+GITHUB_API_TOKEN=token_value
+```
 
 ### Forge comes with command line tools that can speed up the process.
 

forgepb/builder.py

@@ -22,7 +22,7 @@ def build(environment, network, config, provenance_branch=None, version=None, ar
     elif not provenance_branch and version:
         if version and version not in utils.get_versions():
             print(
-                "The version entered doesn't exist in provenance. Please run 'forge -lsv' to list all versions")
+                "The version entered doesn't exist in provenance. Please run 'forge provenance tags' to list all versions")
             return
         repo = git.Repo(provenance_path)
         repo.git.reset('--hard')
@@ -88,7 +88,7 @@ def build(environment, network, config, provenance_branch=None, version=None, ar
         utils.persist_localnet_information(
             build_path, config, version, validator_info)
 
-        run_command = "{}/bin/provenanced start --home {}".format(
+        run_command = "{}/bin/provenanced start --home {} -t".format(
             build_path, build_path).split(" ")
         log_path = '{}/logs/{}.txt'.format(build_path,
                                            str(datetime.datetime.now()).replace(' ', '-'))
@@ -170,27 +170,27 @@ def build(environment, network, config, provenance_branch=None, version=None, ar
 
 
 def populate_genesis(build_path, moniker, chain_id):
-    command1 = "{}/bin/provenanced --home {} init {} --chain-id {};".format(
+    command1 = "{}/bin/provenanced --home {} -t init {} --chain-id {};".format(
         build_path, build_path, moniker, chain_id)
-    command2 = "{}/bin/provenanced --home {} keys add validator --keyring-backend test 2>&1;".format(
+    command2 = "{}/bin/provenanced --home {} -t keys add validator --keyring-backend test 2>&1;".format(
         build_path, build_path, build_path)
-    command3 = "{}/bin/provenanced --home {} add-genesis-root-name validator pio --keyring-backend test 2>&- || echo pio root name already exists, skipping...;".format(
+    command3 = "{}/bin/provenanced --home {} -t add-genesis-root-name validator pio --keyring-backend test 2>&- || echo pio root name already exists, skipping...;".format(
         build_path, build_path)
-    command3 += "{}/bin/provenanced --home {} add-genesis-root-name validator pb --restrict=false --keyring-backend test 2>&- || echo pb root name already exists, skipping...;".format(
+    command3 += "{}/bin/provenanced --home {} -t add-genesis-root-name validator pb --restrict=false --keyring-backend test 2>&- || echo pb root name already exists, skipping...;".format(
         build_path, build_path)
-    command3 += "{}/bin/provenanced --home {} add-genesis-root-name validator io --restrict --keyring-backend test 2>&- || echo io root name already exists, skipping...;".format(
+    command3 += "{}/bin/provenanced --home {} -t add-genesis-root-name validator io --restrict --keyring-backend test 2>&- || echo io root name already exists, skipping...;".format(
         build_path, build_path)
-    command3 += "{}/bin/provenanced --home {} add-genesis-root-name validator provenance --keyring-backend test 2>&- || echo validator root name already exists, skipping...;".format(
+    command3 += "{}/bin/provenanced --home {} -t add-genesis-root-name validator provenance --keyring-backend test 2>&- || echo validator root name already exists, skipping...;".format(
         build_path, build_path)
-    command3 += "{}/bin/provenanced --home {} add-genesis-account validator 100000000000000000000nhash --keyring-backend test 2>&-;".format(
+    command3 += "{}/bin/provenanced --home {} -t add-genesis-account validator 100000000000000000000nhash --keyring-backend test 2>&-;".format(
         build_path, build_path)
-    command3 += "{}/bin/provenanced --home {} gentx validator 1000000000000000nhash --keyring-backend test --chain-id={} 2>&- || echo gentx file already exists, skipping;".format(
+    command3 += "{}/bin/provenanced --home {} -t gentx validator 1000000000000000nhash --keyring-backend test --chain-id={} 2>&- || echo gentx file already exists, skipping;".format(
         build_path, build_path, chain_id)
-    command3 += "{}/bin/provenanced --home {} add-genesis-marker 100000000000000000000nhash --manager validator --access mint,burn,admin,withdraw,deposit --activate --keyring-backend test 2>&- || echo existing address, skipping;".format(
+    command3 += "{}/bin/provenanced --home {} -t add-genesis-marker 100000000000000000000nhash --manager validator --access mint,burn,admin,withdraw,deposit --activate --keyring-backend test 2>&- || echo existing address, skipping;".format(
         build_path, build_path)
-    command3 += "{}/bin/provenanced --home {} collect-gentxs".format(
+    command3 += "{}/bin/provenanced --home {} -t collect-gentxs".format(
         build_path, build_path)
-    validator_check_command = "{}/bin/provenanced --home {} keys show validator".format(
+    validator_check_command = "{}/bin/provenanced --home {} -t keys show validator".format(
         build_path, build_path)
     os.system(command1)
     validator_check_process = subprocess.Popen(
@@ -231,25 +231,25 @@ def spawnDaemon(node_command, version, network, config, log_path):
 
 def start_node(node_command, version, network, config, log_path):
     try:
-        log = open(log_path, 'w+')
-        print('Running {}'.format(' '.join(node_command)))
-        print('You can view the logs here: {}'.format(log_path))
-        process = subprocess.Popen(
-            node_command, shell=False, stdout=log, stderr=log)
-        if network == 'localnet':
-            config[network][version]['run-command'] = node_command
-            config[network][version]['log-path'] = log_path
-        else:
-            config[network]['run-command'] = node_command
-            config[network]['log-path'] = log_path
-        config['running-node'] = True
-        config['running-node-info'] = {
-            "pid": process.pid,
-            "version": version,
-            "network": network
-        }
-        utils.save_config(config)
-        process.wait()
+        with open(log_path, 'w+') as log:
+            print('Running {}'.format(' '.join(node_command)))
+            print('You can view the logs here: {}'.format(log_path))
+            process = subprocess.Popen(
+                node_command, shell=False, stdout=log, stderr=log)
+            if network == 'localnet':
+                config[network][version]['run-command'] = node_command
+                config[network][version]['log-path'] = log_path
+            else:
+                config[network]['run-command'] = node_command
+                config[network]['log-path'] = log_path
+            config['running-node'] = True
+            config['running-node-info'] = {
+                "pid": process.pid,
+                "version": version,
+                "network": network
+            }
+            utils.save_config(config)
+            process.wait()
     except FileNotFoundError:
         print("It looks like a node was initialized and deleted.\nForge is removing this from the config so you can run the same command again and the node will be initialized and started.".format())
         config.pop(network)

forgepb/cmd/node.py

@@ -195,7 +195,8 @@ def node_init_cmd(network, tag, moniker, chain_id, provenance_branch, boot_args)
             os.path.expanduser('~'))['config']
         provenance_path = config['saveDir'] + "forge" + "/provenance"
         print("Cloning repo, this can take a few seconds...")
-        git.Repo.clone_from(global_.PROVENANCE_REPO, provenance_path)
+        if not os.path.exists(provenance_path):
+            git.Repo.clone_from(global_.PROVENANCE_REPO, provenance_path)
     if not tag and not provenance_branch and not network:
         provenance_branch = 'main'
     if not moniker:
@@ -228,8 +229,7 @@ def node_status_cmd():
         print(json.dumps(node_status, indent=4))
     else:
         print(message)
-    exit()
-
+    return
 
 @click.command(
     'mnemonic',
@@ -259,15 +259,15 @@ def node_list_mnemonic_cmd(tag, provenance_branch):
         if tag:
             if tag not in utils.get_versions():
                 print(
-                    "The version entered doesn't exist in provenance. Please run 'forge -lsv' to list all versions")
+                    "The tag entered doesn't exist in provenance. Please run 'forge provenance tags' to list all tags")
             else:
                 print(" ".join(config['localnet'][tag]['mnemonic']))
         elif provenance_branch:
             if provenance_branch not in utils.get_remote_branches():
                 print(
-                    "The version entered doesn't exist in provenance. Please run 'forge -lsv' to list all versions")
+                    "The branch entered doesn't exist in provenance. Please run 'forge provenance branches' to list all branches")
             else:
-                print(" ".join([provenance_branch]['mnemonic']))
+                print(" ".join(config['localnet'][provenance_branch]['mnemonic']))
         else:
             for version in config['localnet'].keys():
                 print("Localnet version: {}\nMnemonic: {}".format(version, " ".join(config['localnet'][version]['mnemonic'])))

forgepb/utils.py

@@ -13,8 +13,8 @@
 
 # Pull existing config from file
 def load_config():
-    config_file = open(global_.CONFIG_PATH + "/config.json")
-    return json.load(config_file)
+    with open(global_.CONFIG_PATH + "/config.json") as config_file:
+        return json.load(config_file)
 
 
 def exists_config():
@@ -38,25 +38,29 @@ def get_version_info(network, environment, provenance_path):
 
     if network == 'localnet':
         filtered_tags = get_versions()
+        branches = get_remote_branches()
         for index, version_tag in zip(range(5), filtered_tags):
             print(version_tag)
     repo = git.Repo(provenance_path)
 
     # In case of localnet, list release versions for user to select
     while version == None:
         try:
-            version = input("Enter a release version from above. Run forge -v for full list of versions [{}]:\n".format(
+            version = input("Enter a release version from above or a proveance branch. Run 'forge provenance tags' for full list of versions or 'forge provenance branches' for a full list of branches [{}]:\n".format(
                 filtered_tags[0]))
         except ValueError:
             continue
         if not version:
             version = filtered_tags[0]
-        if not version in filtered_tags:
+        if not version in filtered_tags and not version in branches:
             version = None
 
     # Checkout to branch for obtaining provenance binary
     # Version should be integer or sanitized
     try:
+        repo.git.reset('--hard')
+        repo.git.checkout('main')
+        repo.remotes.origin.pull()
         repo.git.checkout("-f", "tags/{}".format(version), "-b", version)
     except git.exc.GitCommandError:
         repo.git.checkout("-f", version)
@@ -81,12 +85,12 @@ def select_network():
         except ValueError:
             continue
         if network == len(global_.NETWORK_STRINGS) + 1:
-            exit()
+            return
         if network > len(global_.NETWORK_STRINGS) or network < 1:
             continue
         builder.build(global_.CHAIN_ID_STRINGS[global_.NETWORK_STRINGS[network - 1]],
                       global_.NETWORK_STRINGS[network - 1], config)
-        exit()
+        return
 
 
 # Collect moniker and chain id for a localnet node
@@ -216,6 +220,42 @@ def follow_logs(log_path):
         if p.poll(1):
             print(f.stdout.readline().decode('utf-8').strip())
 
+# Print last 1000 lines from the log file given
+def print_logs(log_path):
+    list_of_lines = []
+    with open(log_path, 'rb') as read_obj:
+
+        read_obj.seek(0, os.SEEK_END)
+        buffer = bytearray()
+        pointer_location = read_obj.tell()
+        while pointer_location >= 0 and len(list_of_lines) < 1000:
+            read_obj.seek(pointer_location)
+            pointer_location = pointer_location - 1
+            new_byte = read_obj.read(1)
+            if new_byte == b'\n':
+                list_of_lines.append(buffer.decode('unicode_escape')[::-1])
+                buffer = bytearray()
+            else:
+                buffer.extend(new_byte)
+        if len(buffer) > 0:
+            list_of_lines.append(buffer.decode()[::-1])
+    for line in reversed(list_of_lines):
+        print(line)
+        time.sleep(.04)
+
+
+# Tail the logs
+def follow_logs(log_path):
+    f = subprocess.Popen(['tail', '-F', log_path],
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    p = select.poll()
+    p.register(f.stdout)
+
+    while True:
+        if p.poll(1):
+            print(f.stdout.readline().decode('utf-8').strip())
+
+
 # Fetch info stored for currently executing process.
 def view_running_node_info():
     if not exists_config():
@@ -284,7 +324,7 @@ def start_node():
         builder.spawnDaemon(
             node_info['run-command'], version, network, config, node_info['log-path'])
     except Exception:
-        print("You haven't initialized a node. Try running 'forge' to start the wizard.")
+        print("You haven't initialized a node. Try running 'forge i' to start the wizard.")
 
 
 def handle_running_node(process_information):
@@ -303,31 +343,53 @@ def handle_running_node(process_information):
 
 # Returns a list of version tags for localnet to use
 def get_versions():
+    github_api_token = os.getenv('GITHUB_API_TOKEN')
+    if not github_api_token == None:
+        headers = {'Authorization': 'token ' + github_api_token}
+    else:
+        headers = {}
     try:
-        res=requests.get(global_.GITHUB_URL + 'tags?simple=yes&per_page=100&page=1')
-        tag_info=res.json()
+        res = requests.get(global_.GITHUB_URL +
+                           'tags?simple=yes&per_page=100&page=1', headers=headers)
+        tag_info = res.json()
         while 'next' in res.links.keys():
-            res=requests.get(res.links['next']['url'])
+            res = requests.get(res.links['next']['url'])
             tag_info.extend(res.json())
         return [tag['name'] for tag in tag_info]
     except:
-        print("Something went wrong reaching out to {}".format(
-            global_.GITHUB_URL + 'branches'))
+        if github_api_token == None:
+            print("Something went wrong reaching out to {}\nTry adding GITHUB_API_TOKEN to your environment to increase number of times you can call the github api.".format(
+                global_.GITHUB_URL + 'tags'))
+        
+        else:
+            print("Something went wrong reaching out to {}\nYou may be out of api requests which is limited to 5000 per hour".format(
+                global_.GITHUB_URL + 'tags'))
         return False
 
 
 # Returns a list of all remote branches
 def get_remote_branches():
+    github_api_token = os.getenv('GITHUB_API_TOKEN')
+    if not github_api_token == None:
+        headers = {'Authorization': 'token ' + github_api_token}
+    else:
+        headers = {}
     try:
-        res=requests.get(global_.GITHUB_URL + 'branches?simple=yes&per_page=100&page=1')
-        branch_info=res.json()
+        res = requests.get(global_.GITHUB_URL +
+                           'branches?simple=yes&per_page=100&page=1', headers=headers)
+        branch_info = res.json()
         while 'next' in res.links.keys():
-            res=requests.get(res.links['next']['url'])
+            res = requests.get(res.links['next']['url'])
             branch_info.extend(res.json())
         return [branch['name'] for branch in branch_info]
     except:
-        print("Something went wrong reaching out to {}".format(
-            global_.GITHUB_URL + 'branches'))
+        if github_api_token == None:
+            print("Something went wrong reaching out to {}\nTry adding GITHUB_API_TOKEN to your environment to increase number of times you can call the github api.".format(
+                global_.GITHUB_URL + 'branches'))
+        
+        else:
+            print("Something went wrong reaching out to {}\nYou may be out of api requests which is limited to 5000 per hour".format(
+                global_.GITHUB_URL + 'branches'))
         return False
 
 

setup.py

@@ -20,6 +20,7 @@
         ]
     },
     keyword="provenance, node, bootstrap",
+    description="Node manager for the Provenance Blockchain",
     long_description=long_description,
     long_description_content_type="text/markdown",
 )