32-bit Windows porting + disable libmagic

fulltext/data/winmake.py

@@ -16,6 +16,7 @@
 import glob
 import functools
 import os
+import platform
 import shutil
 import site
 import subprocess
@@ -80,7 +81,15 @@ def safe_print(text, file=sys.stdout, flush=False):
     file.write("\n")
 
 
-def sh(cmd, nolog=False):
+def sh(cmd):
+    safe_print("cmd: " + cmd)
+    env = os.environ.copy()
+    ret = subprocess.call(cmd, shell=True, env=env)
+    if ret != 0:
+        raise RuntimeError("exit code = %s" % ret)
+
+
+def shout(cmd, nolog=False):
     if not nolog:
         safe_print("cmd: " + cmd)
     p = subprocess.Popen(cmd, shell=True, env=os.environ, cwd=os.getcwd(),
@@ -304,8 +313,17 @@ def set_python(s):
             "can't find any python installation matching %r" % orig)
 
 
-def is_windows64():
-    return 'PROGRAMFILES(X86)' in os.environ
+def is_python_64():
+    """
+    Determine if this is Python 64 bit.
+    """
+    arch = platform.architecture()
+    if arch[0] == '64bit':
+        return True
+    elif arch[0] == '32bit':
+        return False
+    else:
+        raise ValueError("can't determine bitness from %s" % str(arch))
 
 
 def venv():
@@ -337,7 +355,7 @@ def install_deps():
     def run_pyinstaller():
         rm(os.path.join(ROOT_DIR, "dist"))
         bindir = os.path.join(
-            DATA_DIR, "bin64" if is_windows64() else "bin32")
+            DATA_DIR, "bin64" if is_python_64() else "bin32")
         assert os.path.exists(bindir), bindir
         sh("venv\\Scripts\\pyinstaller --upx-dir=%s pyinstaller.spec" % bindir)
 
@@ -346,10 +364,10 @@ def test_exe():
         exe = os.path.join(ROOT_DIR, "dist", "%s.exe" % PRJNAME)
         assert os.path.exists(exe), exe
         # Test those extensions for which we know we rely on external exes.
-        out = sh("%s extract %s" % (
+        out = shout("%s extract %s" % (
             exe, os.path.join(ROOT_DIR, "fulltext/test/files/test.pdf")))
         assertMultiLineEqual(out.strip(), TEXT.strip())
-        out = sh("%s extract %s" % (
+        out = shout("%s extract %s" % (
             exe, os.path.join(ROOT_DIR, "fulltext/test/files/test.rtf")))
         assertMultiLineEqual(out.strip(), TEXT.strip())
 

fulltext/test/__init__.py

@@ -225,7 +225,8 @@ def test_check(self):
                 # On Windows we expect a bunch of backends not to work.
                 # XXX maybe this is too strict.
                 lines = [x.split(':')[0] for x in
-                         sorted(err.decode().splitlines())]
+                         sorted(err.decode().splitlines())
+                         if x.split(':')[0].startswith('fulltext.')]
                 self.assertEqual(
                     lines,
                     ['fulltext.backends.__doc',
@@ -302,6 +303,7 @@ class TestInstalledDeps(BaseTestCase):
     """Make sure certain deps are installed."""
 
     @unittest.skipIf(APPVEYOR, "AppVeyor can't detect magic")
+    @unittest.skipIf(WINDOWS, "libmagic not supported on win")
     def test_magic(self):
         self.assertIsNotNone(magic)
 

fulltext/util.py

@@ -10,6 +10,7 @@
 import functools
 import tempfile
 import shutil
+import platform
 
 from os.path import join as pathjoin
 
@@ -138,25 +139,32 @@ def is_windows():
     return 'win' in sys.platform
 
 
-def is_windows64():
+def is_python_64():
     """
-    Determine if platform is 64 bit Windows.
+    Determine if this is Python 64 bit.
     """
-    return is_windows() and 'PROGRAMFILES(X86)' in os.environ
+    arch = platform.architecture()
+    if arch[0] == '64bit':
+        return True
+    elif arch[0] == '32bit':
+        return False
+    else:
+        raise ValueError("can't determine bitness from %s" % str(arch))
 
 
-def get_data_dir():
+def get_bin_dir():
     # When running under PyInstaller things are a bit different.
+    basename = 'bin64' if is_python_64() else 'bin32'
     if hasattr(sys, '_MEIPASS'):
-        path = pathjoin(sys._MEIPASS, 'fulltext', 'data')
+        path = pathjoin(sys._MEIPASS, 'fulltext', 'data', basename)
         # XXX: this absolutely ugly hack is needed in order to build
         # duster with pyinstaller.
         if not os.path.isdir(path):
             print(">>> WARN: assuming you're using pyinstaller from duster",
                   file=sys.stderr)
-            path = pathjoin(sys._MEIPASS, 'duster', 'data')
+            path = pathjoin(sys._MEIPASS, 'duster', 'data', basename)
     else:
-        path = pathjoin(HERE, 'data')
+        path = pathjoin(HERE, 'data', basename)
 
     assert os.path.isdir(path), path
     return path
@@ -175,36 +183,35 @@ def assert_cmd_exists(cmd):
     def _set_binpath():
         # Help the magic wrapper locate magic1.dll, we include it in
         # bin/bin{32,64}.
-        bindir = 'bin64' if is_windows64() else 'bin32'
-        path = pathjoin(get_data_dir(), bindir)
+        path = get_bin_dir()
         os.environ['PATH'] += os.pathsep + path
         assert_cmd_exists("pdftotext")
         assert_cmd_exists("unrtf")
         assert_cmd_exists("exiftool")
         assert_cmd_exists("unrar")
 
     _set_binpath()
+    magic = None
 
-    def _import_magic():
-        # Instantiate our own Magic instance so we can tell it where the
-        # magic file lives.
-        from magic import Magic as _Magic
-
-        class Magic(_Magic):
-            # Overridden because differently from the UNIX version
-            # the Windows version does not provide mime kwarg.
-            def from_file(self, filename, mime=True):
-                return _Magic.from_file(self, filename)
+    # def _import_magic():
+    #     # Instantiate our own Magic instance so we can tell it where the
+    #     # magic file lives.
+    #     from magic import Magic as _Magic
 
-            def from_buffer(self, buf, mime=True):
-                return _Magic.from_buffer(self, buf)
+    #     class Magic(_Magic):
+    #         # Overridden because differently from the UNIX version
+    #         # the Windows version does not provide mime kwarg.
+    #         def from_file(self, filename, mime=True):
+    #             return _Magic.from_file(self, filename)
 
-        path = pathjoin(get_data_dir(), 'magic')
-        assert os.path.isfile(path), path
-        return Magic(mime=True, magic_file=path)
+    #         def from_buffer(self, buf, mime=True):
+    #             return _Magic.from_buffer(self, buf)
 
-    magic = _import_magic()
+    #     path = pathjoin(get_bin_dir(), 'magic')
+    #     assert os.path.isfile(path), path
+    #     return Magic(mime=True, magic_file=path)
 
+    # magic = _import_magic()
 
 def memoize(fun):
     """A simple memoize decorator for functions supporting (hashable)

make.bat

@@ -1,7 +1,8 @@
-@echo off
-
 rem ==========================================================================
 rem Shortcuts for various tasks, emulating UNIX "make" on Windows.
 rem ==========================================================================
 
+set HOME=%USERPROFILE%
+set PYTHON=C:\Python36-32\python.exe
+
 %PYTHON% fulltext\data\winmake.py %1 %2 %3 %4 %5 %6

pyinstaller.spec

@@ -8,9 +8,7 @@ a = Analysis(['fulltext\\__main__.py'],
              datas=[
                 ('fulltext/*', 'fulltext/'),
                 ('fulltext/backends/*', 'fulltext/backends'),
-                ('fulltext/data/magic*', 'fulltext/data/'),
                 ('fulltext/data/bin32', 'fulltext/data/bin32'),
-                ('fulltext/data/bin64', 'fulltext/data/bin64'),
              ],
              hiddenimports=[],
              hookspath=[],
@@ -31,5 +29,5 @@ exe = EXE(pyz,
           name='fulltext',
           debug=False,
           strip=False,
-          upx=True,
+          upx=False,
           console=True )

requirements.txt

@@ -9,12 +9,12 @@ pytesseract
 Pillow
 pycodestyle
 pyflakes
-python-magic
 ebooklib
 contextlib2
 mock
 rarfile
 flake8
+python-magic; os_name != 'nt'
 
 git+https://github.com/mattgwwalker/msg-extractor.git@2a24c9950d34932ed5979693cb70d758d78715df#egg=ExtractMsg
 

setup.py

@@ -1,7 +1,5 @@
 #!/bin/env python
 
-import os
-import sys
 from os.path import dirname
 from os.path import join as pathjoin
 from setuptools import find_packages
@@ -10,9 +8,6 @@
 
 NAME = 'fulltext'
 VERSION = '0.7'
-if os.name == 'nt' and not sys.maxsize > 2 ** 32:
-    # https://github.com/btimby/fulltext/issues/79
-    raise RuntimeError("Python 32 bit is not supported")
 
 
 with open(pathjoin(dirname(__file__), 'README.rst')) as f: