[_499] authenticate native and pam_password schemes using iRODS 4.3+ auth flow

remove redundant scripts

new write_pam_irodsA_file function

experimental - auth options/write_pam_irodsA related

auth_options per connection (in support of writing PAM .irodsA using new auth-fwk)

review-related and experimental changes

fixing stuff.

fix pam free fn

import io in free fn

irods.auth import at head leads to circular import loop.

 when importing irods.connection before other modules.

pam free fn to write irodsA works

time-to-live

addl script login pam w/o env

check_ssl, proper reading of pw

possible ""

error if no env.

chgs to pam write irodsA w/ raw_server_version

name changes for (PAM and native) irodsA creator functions

also change vector

keep password in mem

variable names, doc, refactor

corrections fri 2/14 am

corrections client_init.py

force_legacy_auth setting

preliminary test that inline pam_password (ie w/o env files) doesn't write .irodsA

Author: d-w-moore

irods/__init__.py

@@ -29,8 +29,10 @@ def env_filename_from_keyword_args(kwargs):
 def derived_auth_filename(env_filename):
     if not env_filename:
         return ""
-    default_irods_authentication_file = os.path.join(
-        os.path.dirname(env_filename), ".irodsA"
+    default_irods_authentication_file = (
+        ##->https://github.com/irods/python-irodsclient/issues/686
+        #os.path.join(os.path.dirname(env_filename), ".irodsA")
+        os.path.expanduser("~/.irods/.irodsA")
     )
     return os.environ.get(
         "IRODS_AUTHENTICATION_FILE", default_irods_authentication_file

irods/account.py

@@ -28,6 +28,11 @@ def __init__(
                 irods_host = v
 
         self.env_file = env_file
+
+        # The '_auth_file' attribute will be written in the call to iRODSSession.configure,
+        # if an .irodsA file from the client environment is used to load password information.
+        self._auth_file = ""
+
         tuplify = lambda _: _ if isinstance(_, (list, tuple)) else (_,)
         schemes = [_.lower() for _ in tuplify(irods_authentication_scheme)]
 

irods/api_number.py

@@ -179,4 +179,5 @@
     "REPLICA_CLOSE_APN": 20004,
     "TOUCH_APN": 20007,
     "AUTH_PLUG_REQ_AN": 1201,
+    "AUTHENTICATION_APN": 110000
 }

irods/auth/__init__.py

@@ -1,8 +1,85 @@
+import importlib
+import logging
+import weakref
+from irods.api_number import api_number
+from irods.message import iRODSMessage, JSON_Message
+import irods.password_obfuscation as obf
+import irods.session
+
+
 __all__ = ["pam_password", "native"]
 
+
 AUTH_PLUGIN_PACKAGE = "irods.auth"
 
-import importlib
+
+NoneType = type(None)
+
+
+class AuthStorage:
+
+    @staticmethod
+    def get_env_password(filename = None):
+        options = dict(irods_authentication_file = filename) if filename else {}
+        return irods.session.iRODSSession.get_irods_password(**options)
+
+    @staticmethod
+    def get_env_password_file():
+        return irods.session.iRODSSession.get_irods_password_file()
+
+    @staticmethod
+    def set_env_password(unencoded_pw, filename = None):
+        if filename is None:
+            filename = AuthStorage.get_env_password_file()
+        from ..client_init import _open_file_for_protected_contents
+        with _open_file_for_protected_contents(filename,'w') as irodsA:
+            irodsA.write(obf.encode(unencoded_pw))
+        return filename
+
+    @staticmethod
+    def get_temp_pw_storage(conn):
+        return getattr(conn,'auth_storage',lambda:None)()
+
+    @staticmethod
+    def create_temp_pw_storage(conn):
+        """A reference to the value returned by this call should be stored for the duration of the
+           authentication exchange.
+        """
+        store = getattr(conn,'auth_storage',None)
+        if store is None:
+            store = AuthStorage(conn)
+            # So that the connection object doesn't hold on to password data too long:
+            conn.auth_storage = weakref.ref(store)
+        return store
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.pw = ''
+        self._auth_file = ''
+
+    @property
+    def auth_file(self):
+        if self._auth_file is None:
+            return ''
+        return self._auth_file or self.conn.account.derived_auth_file
+
+    def use_client_auth_file(self, auth_file):
+        if isinstance(auth_file, (str, NoneType)):
+            self._auth_file = auth_file
+        else:
+            msg = f"Invalid object in {self.__class__}._auth_file"
+            raise RuntimeError(msg)
+
+    def store_pw(self,pw):
+        if self.auth_file:
+            self.set_env_password(pw, filename = self.auth_file)
+        else:
+            self.pw = pw
+
+    def retrieve_pw(self):
+        if self.auth_file:
+            return self.get_env_password(filename = self.auth_file)
+        return self.pw
 
 
 def load_plugins(subset=set(), _reload=False):
@@ -18,9 +95,66 @@ def load_plugins(subset=set(), _reload=False):
     return dir_
 
 
-# TODO(#499): X models a class which we could define here as a base for various server or client state machines
-#             as appropriate for the various authentication types.
+class REQUEST_IS_MISSING_KEY(Exception): pass
+
+
+def throw_if_request_message_is_missing_key( request, required_keys ):
+  for key in required_keys:
+    if not key in request:
+      raise REQUEST_IS_MISSING_KEY(f"key = {key}")
+
+
+def _auth_api_request(conn, data):
+    message_body = JSON_Message(data, conn.server_version)
+    message = iRODSMessage('RODS_API_REQ', msg=message_body,
+        int_info=api_number['AUTHENTICATION_APN']
+    )
+    conn.send(message)
+    response = conn.recv()
+    return response.get_json_encoded_struct()
+
+
+__FLOW_COMPLETE__ = "authentication_flow_complete"
+__NEXT_OPERATION__ = "next_operation"
+
+
+CLIENT_GET_REQUEST_RESULT = 'client_get_request_result'
+FORCE_PASSWORD_PROMPT = "force_password_prompt"
+STORE_PASSWORD_IN_MEMORY = "store_password_in_memory"
+
+class authentication_base:
+
+    def __init__(self, connection, scheme):
+        self.conn = connection
+        self.loggedIn = 0
+        self.scheme = scheme
+
+    def call(self, next_operation, request):
+        logging.info('next operation = %r', next_operation)
+        old_func = func = next_operation
+        while isinstance(func, str):
+            old_func, func = (func, getattr(self, func, None))
+        func = (func or old_func)
+        if not func:
+            raise RuntimeError("client request contains no callable 'next_operation'")
+        resp = func(request)
+        logging.info('resp = %r',resp)
+        return resp
+
+    def authenticate_client(self, next_operation = "auth_client_start", initial_request = {}):
+
+        to_send = initial_request.copy()
+        to_send["scheme"] = self.scheme
 
+        while True:
+            resp = self.call(next_operation, to_send)
+            if self.loggedIn:
+                break
+            next_operation = resp.get(__NEXT_OPERATION__)
+            if next_operation is None:
+              raise ClientAuthError("next_operation key missing; cannot determine next operation")
+            if next_operation in (__FLOW_COMPLETE__,""):
+              raise ClientAuthError(f"authentication flow stopped without success: scheme = {self.scheme}")
+            to_send = resp
 
-class X:
-    pass
+        logging.info("fully authenticated")

irods/auth/native.py

@@ -1,11 +1,110 @@
-def login(conn):
-    conn._login_native()
+import base64
+import logging
+import hashlib
+import struct
 
+from irods import MAX_PASSWORD_LENGTH
 
-# TODO (#499): Here, we could define client & server auth_state classes (ie state machines mimicking the mechanics
-#              of 4.3+ iCommands/iRods-runtime authentication framework), using this pattern for an inheritance hook.
-from . import X as X_base
+from . import (__NEXT_OPERATION__, __FLOW_COMPLETE__,
+    AuthStorage,
+    authentication_base, _auth_api_request,
+    throw_if_request_message_is_missing_key)
 
 
-class X(X_base):
-    pass
+def login(conn, **extra_opt):
+    opt = {'user_name': conn.account.proxy_user,
+           'zone_name': conn.account.proxy_zone}
+    opt.update(extra_opt)
+    authenticate_native(conn, req = opt)
+
+
+_scheme = 'native'
+
+
+def authenticate_native(conn, req):
+
+    logging.info('----------- %s (begin)', _scheme)
+
+    native_ClientAuthState(
+        conn,
+        scheme = _scheme
+    ).authenticate_client(
+        # initial_request is called context (or ctx for short) in iRODS core library code.
+        initial_request = req
+    )
+
+    logging.info('----------- %s (end)', _scheme)
+
+
+class native_ClientAuthState(authentication_base):
+
+    def auth_client_start(self, request):
+        resp = request.copy()
+        # user_name and zone_name keys injected by authenticate_client() method
+        resp[__NEXT_OPERATION__] = self.AUTH_CLIENT_AUTH_REQUEST # native_auth_client_request
+        return resp
+
+    # Client defines. These strings should match instance method names within the class namespace.
+    AUTH_AGENT_START = 'native_auth_agent_start'
+    AUTH_CLIENT_AUTH_REQUEST = 'native_auth_client_request'
+    AUTH_ESTABLISH_CONTEXT = 'native_auth_establish_context'
+    AUTH_CLIENT_AUTH_RESPONSE = 'native_auth_client_response'
+
+    # Server defines.
+    AUTH_AGENT_AUTH_REQUEST = "auth_agent_auth_request"
+    AUTH_AGENT_AUTH_RESPONSE = "auth_agent_auth_response"
+
+    def native_auth_client_request(self, request):
+        server_req = request.copy()
+        server_req[__NEXT_OPERATION__] = self.AUTH_AGENT_AUTH_REQUEST
+
+        resp = _auth_api_request(self.conn, server_req)
+
+        resp[__NEXT_OPERATION__] = self.AUTH_ESTABLISH_CONTEXT
+        return resp
+
+    def native_auth_establish_context(self, request):
+        throw_if_request_message_is_missing_key(request,
+            ["user_name", "zone_name", "request_result"])
+        request = request.copy()
+
+        password = ''
+        depot = AuthStorage.get_temp_pw_storage(self.conn)
+        if depot:
+            # The following is how pam_password communicates a server-generated password.
+            password = depot.retrieve_pw()
+
+        if not password:
+            password = self.conn.account.password or ''
+
+        challenge = request["request_result"].encode('utf-8')
+        self.conn._client_signature = "".join("{:02x}".format(c) for c in challenge[:16])
+
+        padded_pwd = struct.pack(
+            "%ds" % MAX_PASSWORD_LENGTH, password.encode(
+                'utf-8').strip())
+
+        m = hashlib.md5()
+        m.update(challenge)
+        m.update(padded_pwd)
+
+        encoded_pwd = m.digest()
+        if b'\x00' in encoded_pwd:
+            encoded_pwd_array = bytearray(encoded_pwd)
+            encoded_pwd = bytes(encoded_pwd_array.replace(b'\0', b'\1'))
+        request['digest'] = base64.encodebytes(encoded_pwd).strip().decode('utf-8')
+
+        request[__NEXT_OPERATION__] = self.AUTH_CLIENT_AUTH_RESPONSE
+        return request
+
+    def native_auth_client_response (self,request):
+        throw_if_request_message_is_missing_key(request,
+            ["user_name", "zone_name", "digest"])
+
+        server_req = request.copy()
+        server_req[__NEXT_OPERATION__] = self.AUTH_AGENT_AUTH_RESPONSE
+        resp = _auth_api_request(self.conn, server_req)
+
+        self.loggedIn = 1;
+        resp [__NEXT_OPERATION__] = __FLOW_COMPLETE__
+        return resp