Integrate CLI recommendation into VS Code extension

service/azservice/__main__.py

@@ -5,6 +5,7 @@
 # --------------------------------------------------------------------------------------------
 from __future__ import print_function
 
+from concurrent.futures import ThreadPoolExecutor
 from sys import stdin, stdout, stderr
 import json
 import time
@@ -16,6 +17,8 @@
 
 from azservice.tooling import GLOBAL_ARGUMENTS, initialize, load_command_table, get_help, get_current_subscription, get_configured_defaults, get_defaults, is_required, run_argument_value_completer, get_arguments, load_arguments, arguments_loaded
 
+from azservice.recommend_tooling import request_recommend_service, init as recommend_init
+
 NO_AZ_PREFIX_COMPLETION_ENABLED = True # Adds proposals without 'az' as prefix to trigger, 'az' is then inserted as part of the completion.
 AUTOMATIC_SNIPPETS_ENABLED = True # Adds snippet proposals derived from the command table
 TWO_SEGMENTS_COMPLETION_ENABLED = False # Adds 'webapp create', 'appservice plan', etc. as proposals.
@@ -29,6 +32,7 @@
 
 def get_group_index(command_table):
     index = { '': [], '-': [] }
+    # build subgroup tree
     for command in command_table.values():
         parts = command.name.split()
         len_parts = len(parts)
@@ -116,14 +120,18 @@ def add_command_documentation(completion, command):
 
 def get_completions(group_index, command_table, snippets, query, verbose=False):
     if 'argument' in query:
+        # input commands finished and begin to input argument
         return get_argument_value_completions(command_table, query, verbose)
     if 'subcommand' not in query:
+        # provide subgroup and required arguments completions
         return get_snippet_completions(command_table, snippets) + get_prefix_command_completions(group_index, command_table) + [AZ_COMPLETION]
     command_name = query['subcommand']
     if command_name in command_table:
+        # input commands finished
         return get_argument_name_completions(command_table, query) + \
             get_global_argument_name_completions(query)
     if command_name in group_index:
+        # input commands not finished
         return get_command_completions(group_index, command_table, command_name)
     if verbose: print('Subcommand not found ({})'.format(command_name), file=stderr)
     return []
@@ -304,12 +312,16 @@ def get_options(options):
         option if isinstance(option, str) else
         option.target if hasattr(option, 'target') else
         None
-    for option in options ] if option ]
+    for option in options ] if option ] 
+
+def log(message):
+    print(message, file=stderr)
 
 def main():
-    timings = False
+    timings = True
     start = time.time()
     initialize()
+    recommend_init()
     if timings: print('initialize {} s'.format(time.time() - start), file=stderr)
 
     start = time.time()
@@ -327,22 +339,29 @@ def main():
     def enqueue_output(input, queue):
         for line in iter(input.readline, b''):
             queue.put(line)
+            log('put to queue: size - {}'.format(queue.qsize()))
 
+    # create a thread to put requests in a queue
     queue = Queue()
     thread = Thread(target=enqueue_output, args=(stdin, queue))
     thread.daemon = True
     thread.start()
 
+    recommend_executor = ThreadPoolExecutor(max_workers=1)
+
     bkg_start = time.time()
     keep_loading = True
-    while True:
-
+    while True:  
+        # loading all arguments is time consuming 
+        # load 10 arguments per loop until all are loaded finished
         if keep_loading:
             keep_loading = load_arguments(command_table, 10)
             if not keep_loading and timings: print('load_arguments {} s'.format(time.time() - bkg_start), file=stderr)
 
         try:
+            # non-blocking way to get request from the queue if keep loading
             line = queue.get_nowait() if keep_loading else queue.get()
+            log('line: {}'.format(line))
         except Empty:
             continue
 
@@ -353,11 +372,17 @@ def enqueue_output(input, queue):
             response_data = get_status()
             if timings: print('get_status {} s'.format(time.time() - start), file=stderr)
         elif request['data'].get('request') == 'hover':
+            # display highlight
             response_data = get_hover_text(group_index, command_table, request['data']['command'])
             if timings: print('get_hover_text {} s'.format(time.time() - start), file=stderr)
+        elif request['data'].get('request') == 'recommendation':
+            recommend_executor.submit(request_recommend_service, (request))
+            if timings: print('submit {} s'.format(time.time() - start), file=stderr)
+            continue
         else:
             response_data = get_completions(group_index, command_table, snippets, request['data'], True)
             if timings: print('get_completions {} s'.format(time.time() - start), file=stderr)
+
         response = {
             'sequence': request['sequence'],
             'data': response_data

service/azservice/recommend_tooling.py

@@ -0,0 +1,133 @@
+import hashlib
+import json
+import time
+from enum import Enum
+from sys import stdout, stderr
+
+from azure.cli.core import __version__ as version
+from azure.cli.core import telemetry
+from azure.cli.core.azclierror import RecommendationError
+
+from azservice.tooling2 import cli_ctx
+
+class RecommendType(int, Enum):
+    All = 1
+    Solution = 2
+    Command = 3
+    Scenario = 4
+
+cli_ctx = None
+def init():
+    global cli_ctx
+    from azservice.tooling2 import cli_ctx
+
+def request_recommend_service(request):
+    start = time.time()
+
+    command_list = request['data']['commandList']
+    recommends = []
+    from azure.cli.core.azclierror import RecommendationError
+    try:
+        recommends = get_recommends(command_list)
+    except RecommendationError as e:
+        print(e.error_msg, file=stderr)
+
+    response = {
+            'sequence': request['sequence'],
+            'data': recommends
+    }
+    output = json.dumps(response)
+    stdout.write(output + '\n')
+    stdout.flush()
+    stderr.flush()
+    print('request_recommend_service {} s'.format(time.time() - start), file=stderr)
+
+
+def get_recommends(command_list):
+    print('cli_ctx - {}'.format(cli_ctx), file=stderr)
+    api_recommends = get_recommends_from_api(command_list, cli_ctx.config.getint('next', 'num_limit', fallback=5))
+    recommends = get_scenarios_info(api_recommends)
+    return recommends
+
+
+def get_recommends_from_api(command_list, top_num=5):
+    """query next command from web api"""
+    import requests
+    url = "https://cli-recommendation.azurewebsites.net/api/RecommendationService"
+    debug_url = "http://localhost:7071/api/RecommendationService"
+
+    user_id = telemetry._get_user_azure_id()  # pylint: disable=protected-access
+    hashed_user_id = hashlib.sha256(user_id.encode('utf-8')).hexdigest()
+
+    type = RecommendType.All
+
+    payload = {
+        "command_list": command_list,
+        "type": type,
+        "top_num": top_num,
+        'cli_version': version,
+        'user_id': hashed_user_id
+    }
+
+    correlation_id = telemetry._session.correlation_id
+    subscription_id = telemetry._get_azure_subscription_id()
+    if telemetry.is_telemetry_enabled():
+        if correlation_id:
+            payload['correlation_id'] = correlation_id
+        if subscription_id:
+            payload['subscription_id'] = subscription_id
+
+    print('request body - {}'.format(payload), file=stderr)
+
+    try:
+        request_body = json.dumps(payload)
+        start = time.time()
+        response = requests.post(url, request_body, timeout=2)
+        print('request recommendation service {} s'.format(time.time() - start), file=stderr)
+        response.raise_for_status()
+    except requests.ConnectionError as e:
+        raise RecommendationError(f'Network Error: {e}') from e
+    except requests.exceptions.HTTPError as e:
+        raise RecommendationError(f'{e}') from e
+    except requests.RequestException as e:
+        raise RecommendationError(f'Request Error: {e}') from e
+
+    recommends = []
+    if 'data' in response.json():
+        recommends = response.json()['data']
+
+    return recommends
+
+
+def get_scenarios_info(recommends):
+    scenarios = get_scenarios(recommends) or []
+    scenarios_info = []
+    print('scenarios size - {}'.format(len(scenarios)), file=stderr)
+    for idx, s in enumerate(scenarios):
+        scenarios_info.append(get_info_of_one_scenario(s, idx))
+    return scenarios_info
+
+
+def get_info_of_one_scenario(s, index):
+    idx_display = f'[{index + 1}]'
+    scenario_desc = f'{s["scenario"]}'
+    command_size = f'{len(s["nextCommandSet"])} Commands'
+    description = f'{idx_display} {scenario_desc} ({command_size})'
+
+    next_command_set = []
+    for next_command in s['nextCommandSet']:
+        command_info = {
+            'reason': next_command['reason'],
+            'example': next_command['example']
+        }
+        next_command_set.append(command_info)
+
+    return {
+        'description': description,
+        'executeIndex': s['executeIndex'],
+        'nextCommandSet': next_command_set
+    }
+
+
+def get_scenarios(recommends):
+    return [rec for rec in recommends if rec['type'] == RecommendType.Scenario]

service/azservice/tooling.py

@@ -9,4 +9,4 @@
 if LooseVersion(__version__) < LooseVersion('2.0.24'):
     from azservice.tooling1 import GLOBAL_ARGUMENTS, initialize, load_command_table, get_help, get_current_subscription, get_configured_defaults, get_defaults, is_required, run_argument_value_completer, get_arguments, load_arguments, arguments_loaded
 else:
-    from azservice.tooling2 import GLOBAL_ARGUMENTS, initialize, load_command_table, get_help, get_current_subscription, get_configured_defaults, get_defaults, is_required, run_argument_value_completer, get_arguments, load_arguments, arguments_loaded
+    from azservice.tooling2 import GLOBAL_ARGUMENTS, initialize, load_command_table, get_help, get_current_subscription, get_configured_defaults, get_defaults, is_required, run_argument_value_completer, get_arguments, load_arguments, arguments_loaded

service/azservice/tooling2.py

@@ -156,13 +156,13 @@ def run_argument_value_completer(command, argument, cli_arguments):
         args = _to_argument_object(command, cli_arguments)
         _add_defaults(command, args)
         return argument.completer(prefix='', action=None, parsed_args=args)
-    except TypeError:
+    except Exception:
         try:
             return argument.completer(prefix='')
-        except TypeError:
+        except Exception:
             try:
                 return argument.completer()
-            except TypeError:
+            except Exception:
                 return None
 
 

service/start.py

@@ -2,5 +2,6 @@
 import sys
 
 sys.path.insert(0, os.path.dirname(__file__))
+print(sys.executable, file=sys.stderr)
 
-import azservice.__main__
+import azservice.__main__ 

src/azService.ts

@@ -91,7 +91,7 @@ export class AzService {
         }, onCancel);
     }
 
-    private async send<T, R>(data: T, onCancel?: (handle: () => void) => void): Promise<R> {
+    async send<T, R>(data: T, onCancel?: (handle: () => void) => void): Promise<R> {
         const process = await this.getProcess();
         return new Promise<R>((resolve, reject) => {
             if (onCancel) {
@@ -117,16 +117,21 @@ export class AzService {
 
     private async getProcess(): Promise<ChildProcess> {
         if (this.process) {
+            console.log("process exists already");
             return this.process;
         }
         return this.process = (async () => {
+            console.log("begin to create process");
+
             const { stdout } = await exec('az --version');
             let version = (
                 /azure-cli\s+\(([^)]+)\)/m.exec(stdout)
                 || /azure-cli\s+(\S+)/m.exec(stdout)
                 || []
             )[1];
             if (version) {
+                console.log("version: " + version);
+
                 const r = /[^-][a-z]/ig;
                 if (r.exec(version)) {
                     version = version.substr(0, r.lastIndex - 1) + '-' + version.substr(r.lastIndex - 1);
@@ -136,6 +141,7 @@ export class AzService {
                 throw 'wrongVersion';
             }
             const pythonLocation = (/^Python location '([^']*)'/m.exec(stdout) || [])[1];
+            console.log('pythonLocation: ' + pythonLocation)
             const processOptions = await this.getSpawnProcessOptions();
             return this.spawn(pythonLocation, processOptions);
         })().catch(err => {