Merge pull request #405 from stephenhky/develop

Code cleaning up in categorization.py

Author: stephenhky

.readthedocs.yml

@@ -12,7 +12,7 @@ sphinx:
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.12"
+    python: "3.10"
 
 # Build documentation with MkDocs
 #mkdocs:

README.md

@@ -81,7 +81,8 @@ If you would like to contribute, feel free to submit the pull requests. You can
 
 ## News
 
-* 05/29/2025: `shorttext` 2.2.0 released.
+* 06/02/2025: `shorttext` 2.2.1 released. (Acknowledgement:  [Minseo Kim](https://kmingseo.github.io/))
+* 05/29/2025: `shorttext` 2.2.0 released. (Acknowledgement:  [Minseo Kim](https://kmingseo.github.io/))
 * 05/08/2025: `shorttext` 2.1.1 released.
 * 12/14/2024: `shorttext` 2.1.0 released.
 * 07/12/2024: `shorttext` 2.0.0 released.

docs/conf.py

@@ -58,7 +58,7 @@
 # The short X.Y version.
 version = u'2.2'
 # The full version, including alpha/beta/rc tags.
-release = u'2.2.0'
+release = u'2.2.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/news.rst

@@ -1,6 +1,7 @@
 News
 ====
 
+* 06/02/2025: `shorttext` 2.2.1 released.
 * 05/29/2025: `shorttext` 2.2.0 released.
 * 05/08/2025: `shorttext` 2.1.1 released.
 * 12/14/2024: `shorttext` 2.1.0 released.
@@ -84,10 +85,16 @@ News
 What's New
 ----------
 
-Release 2.2.0 (May 20, 2025)
+Release 2.2.1 (June 2, 2025)
 ----------------------------
 
-* Update `keras` to `tensorflow.keras`.
+* Code cleanup for token categorization. (Acknowledgements: Minseo Kim)
+
+
+Release 2.2.0 (May 29, 2025)
+----------------------------
+
+* Update `keras` to `tensorflow.keras`. (Acknowledgements: Minseo Kim)
 
 
 Release 2.1.1 (May 8, 2025)

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "shorttext"
-version = "2.2.0"
+version = "2.2.1"
 authors = [
     {name = "Kwan Yuet Stephen Ho", email = "stephenhky@yahoo.com.hk"}
 ]

shorttext/cli/categorization.py

@@ -2,39 +2,48 @@
 import os
 from functools import partial
 import argparse
+import logging
 
 from ..utils.compactmodel_io import get_model_classifier_name
 from ..utils.classification_exceptions import AlgorithmNotExistException, WordEmbeddingModelNotExistException
 from ..utils import load_word2vec_model, load_fasttext_model, load_poincare_model
 from ..smartload import smartload_compact_model
 from ..classifiers import TopicVectorCosineDistanceClassifier
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-allowed_classifiers = ['ldatopic', 'lsitopic', 'rptopic', 'kerasautoencoder', 'topic_sklearn',
-                       'nnlibvec', 'sumvec', 'maxent']
+allowed_classifiers = [
+    'ldatopic', 'lsitopic', 'rptopic', 'kerasautoencoder',
+    'topic_sklearn', 'nnlibvec', 'sumvec', 'maxent'
+]
 needembedded_classifiers = ['nnlibvec', 'sumvec']
 topicmodels = ['ldatopic', 'lsitopic', 'rptopic', 'kerasautoencoder']
 
 load_word2vec_nonbinary_model = partial(load_word2vec_model, binary=False)
 load_poincare_binary_model = partial(load_poincare_model, binary=True)
 
-typedict = {'word2vec': load_word2vec_model,
-            'word2vec_nonbinary': load_word2vec_nonbinary_model,
-            'fasttext': load_fasttext_model,
-            'poincare': load_poincare_model,
-            'poincare_binary': load_poincare_binary_model}
+typedict = {
+    'word2vec': load_word2vec_model,
+    'word2vec_nonbinary': load_word2vec_nonbinary_model,
+    'fasttext': load_fasttext_model,
+    'poincare': load_poincare_model,
+    'poincare_binary': load_poincare_binary_model
+}
 
 
 def get_argparser():
-    argparser = argparse.ArgumentParser(description='Perform prediction on short text with a given trained model.')
-    argparser.add_argument('model_filepath', help='Path of the trained (compact) model.')
-    argparser.add_argument('--wv', default='', help='Path of the pre-trained Word2Vec model. (None if not needed)')
-    argparser.add_argument('--vecsize', default=300, type=int, help='Vector dimensions. (Default: 300)')
-    argparser.add_argument('--topn', type=int, default=10, help='Number of top-scored results displayed. (Default: 10)')
-    argparser.add_argument('--inputtext', default=None, help='single input text for classification. Run console if set to None. (Default: None)')
-    argparser.add_argument('--type', default='word2vec',
-                           help='Type of word-embedding model (default: "word2vec"; other options: "fasttext", "poincare", "word2vec_nonbinary", "poincare_binary")')
-    return argparser
+    parser = argparse.ArgumentParser(
+        description='Perform prediction on short text with a given trained model.'
+    )
+    parser.add_argument('model_filepath', help='Path of the trained (compact) model.')
+    parser.add_argument('--wv', default='', help='Path of the pre-trained Word2Vec model.')
+    parser.add_argument('--vecsize', default=300, type=int, help='Vector dimensions. (Default: 300)')
+    parser.add_argument('--topn', type=int, default=10, help='Number of top results to show.')
+    parser.add_argument('--inputtext', default=None, help='Single input text for classification. If omitted, will enter console mode.')
+    parser.add_argument('--type', default='word2vec', choices=typedict.keys(),
+                        help='Type of word-embedding model (default: word2vec)')
+    return parser
 
 # main block
 def main():
@@ -43,51 +52,52 @@ def main():
 
     # check if the model file is given
     if not os.path.exists(args.model_filepath):
-        raise IOError('Model file '+args.model_filepath+' not found!')
-
+        raise IOError(f'Model file "{args.model_filepath}" not found!')
+    
     # get the name of the classifier
-    print('Retrieving classifier name...')
+    logger.info('Retrieving classifier name...')
     classifier_name = get_model_classifier_name(args.model_filepath)
-    if not (classifier_name in allowed_classifiers):
+
+    if classifier_name not in allowed_classifiers:
         raise AlgorithmNotExistException(classifier_name)
 
     # load the Word2Vec model if necessary
     wvmodel = None
     if classifier_name in needembedded_classifiers:
-        # check if thw word embedding model is available
+        # check if the word embedding model is available
         if not os.path.exists(args.wv):
             raise WordEmbeddingModelNotExistException(args.wv)
         # if there, load it
-        print('Loading word-embedding model: '+args.wv)
+        logger.info(f'Loading word-embedding model from {args.wv}...')
         wvmodel = typedict[args.type](args.wv)
 
     # load the classifier
-    print('Initializing the classifier...')
-    classifier = None
+    logger.info('Initializing the classifier...')
     if classifier_name in topicmodels:
         topicmodel = smartload_compact_model(args.model_filepath, wvmodel, vecsize=args.vecsize)
         classifier = TopicVectorCosineDistanceClassifier(topicmodel)
     else:
         classifier = smartload_compact_model(args.model_filepath, wvmodel, vecsize=args.vecsize)
 
-
-    if args.inputtext != None:
-        if len(args.inputtext) > 0:
-            scoredict = classifier.score(args.inputtext)
-            for label, score in sorted(scoredict.items(), key=lambda s: s[1], reverse=True)[:args.topn]:
-                print(label, ' : ', score)
-        else:
-            print('No input text available!')
+    # predict single input or run in console mode
+    if args.inputtext is not None:
+        if len(args.inputtext.strip()) == 0:
+            print('No input text provided.')
+            return
+        scoredict = classifier.score(args.inputtext)
+        for label, score in sorted(scoredict.items(), key=lambda x: x[1], reverse=True)[:args.topn]:
+            print(f'{label} : {score:.4f}')
     else:
-        # Console
-        run = True
-        while run:
-            shorttext = input('text> ')
-            if len(shorttext) > 0:
-                scoredict = classifier.score(shorttext)
-                for label, score in sorted(scoredict.items(), key=lambda s: s[1], reverse=True)[:args.topn]:
-                    print(label+' : '+'%.4f' % (score))
-            else:
-                run = False
-
+        # Console 
+        print('Enter text to classify (empty input to quit):')
+        while True:
+            shorttext = input('text> ').strip()
+            if not shorttext:
+                break
+            scoredict = classifier.score(shorttext)
+            for label, score in sorted(scoredict.items(), key=lambda x: x[1], reverse=True)[:args.topn]:
+                print(f'{label} : {score:.4f}')
         print('Done.')
+
+if __name__ == "__main__":
+    main()