diff --git a/fulltext/data/bin32/exiftool.exe b/fulltext/data/bin32/exiftool.exe new file mode 100755 index 0000000..962b26b Binary files /dev/null and b/fulltext/data/bin32/exiftool.exe differ diff --git a/fulltext/data/magic b/fulltext/data/bin32/magic similarity index 100% rename from fulltext/data/magic rename to fulltext/data/bin32/magic diff --git a/fulltext/data/bin32/magic.mgc b/fulltext/data/bin32/magic.mgc new file mode 100644 index 0000000..435f5bd Binary files /dev/null and b/fulltext/data/bin32/magic.mgc differ diff --git a/fulltext/data/bin32/unrar.exe b/fulltext/data/bin32/unrar.exe new file mode 100644 index 0000000..86d5a5f Binary files /dev/null and b/fulltext/data/bin32/unrar.exe differ diff --git a/fulltext/data/bin32/unrtf.exe b/fulltext/data/bin32/unrtf.exe new file mode 100755 index 0000000..db50f6d Binary files /dev/null and b/fulltext/data/bin32/unrtf.exe differ diff --git a/fulltext/data/generate_manifest.py b/fulltext/data/generate_manifest.py deleted file mode 100644 index 3dfa67e..0000000 --- a/fulltext/data/generate_manifest.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -""" -Generate MANIFEST.in file. -""" - -import os -import subprocess - - -IGNORED_EXTS = [] -IGNORED_FILES = ['.travis.yml', 'appveyor.yml'] -IGNORED_PREFIXES = ["files/"] - - -def sh(cmd): - return subprocess.check_output( - cmd, shell=True, universal_newlines=True).strip() - - -def main(): - files = sh("git ls-files").split('\n') - for file in files: - if file.startswith('.ci/') or \ - os.path.splitext(file)[1].lower() in IGNORED_EXTS or \ - file in IGNORED_FILES: - continue - - for prefix in IGNORED_PREFIXES: - if file.startswith(prefix): - break - else: - print("include " + file) - - -if __name__ == '__main__': - main() diff --git a/fulltext/data/magic.mgc b/fulltext/data/magic.mgc deleted file mode 100644 index 3410230..0000000 Binary files a/fulltext/data/magic.mgc and /dev/null differ diff --git a/fulltext/data/winmake.py b/fulltext/data/winmake.py index 5073a0b..0fbd6d0 100644 --- a/fulltext/data/winmake.py +++ b/fulltext/data/winmake.py @@ -16,6 +16,7 @@ import glob import functools import os +import platform import shutil import site import subprocess @@ -80,7 +81,15 @@ def safe_print(text, file=sys.stdout, flush=False): file.write("\n") -def sh(cmd, nolog=False): +def sh(cmd): + safe_print("cmd: " + cmd) + env = os.environ.copy() + ret = subprocess.call(cmd, shell=True, env=env) + if ret != 0: + raise RuntimeError("exit code = %s" % ret) + + +def shout(cmd, nolog=False): if not nolog: safe_print("cmd: " + cmd) p = subprocess.Popen(cmd, shell=True, env=os.environ, cwd=os.getcwd(), @@ -304,8 +313,17 @@ def set_python(s): "can't find any python installation matching %r" % orig) -def is_windows64(): - return 'PROGRAMFILES(X86)' in os.environ +def is_python_64(): + """ + Determine if this is Python 64 bit. + """ + arch = platform.architecture() + if arch[0] == '64bit': + return True + elif arch[0] == '32bit': + return False + else: + raise ValueError("can't determine bitness from %s" % str(arch)) def venv(): @@ -337,7 +355,7 @@ def install_deps(): def run_pyinstaller(): rm(os.path.join(ROOT_DIR, "dist")) bindir = os.path.join( - DATA_DIR, "bin64" if is_windows64() else "bin32") + DATA_DIR, "bin64" if is_python_64() else "bin32") assert os.path.exists(bindir), bindir sh("venv\\Scripts\\pyinstaller --upx-dir=%s pyinstaller.spec" % bindir) @@ -346,10 +364,10 @@ def test_exe(): exe = os.path.join(ROOT_DIR, "dist", "%s.exe" % PRJNAME) assert os.path.exists(exe), exe # Test those extensions for which we know we rely on external exes. - out = sh("%s extract %s" % ( + out = shout("%s extract %s" % ( exe, os.path.join(ROOT_DIR, "fulltext/test/files/test.pdf"))) assertMultiLineEqual(out.strip(), TEXT.strip()) - out = sh("%s extract %s" % ( + out = shout("%s extract %s" % ( exe, os.path.join(ROOT_DIR, "fulltext/test/files/test.rtf"))) assertMultiLineEqual(out.strip(), TEXT.strip()) diff --git a/fulltext/test/__init__.py b/fulltext/test/__init__.py index c050300..ec4c4af 100755 --- a/fulltext/test/__init__.py +++ b/fulltext/test/__init__.py @@ -225,7 +225,8 @@ def test_check(self): # On Windows we expect a bunch of backends not to work. # XXX maybe this is too strict. lines = [x.split(':')[0] for x in - sorted(err.decode().splitlines())] + sorted(err.decode().splitlines()) + if x.split(':')[0].startswith('fulltext.')] self.assertEqual( lines, ['fulltext.backends.__doc', @@ -302,6 +303,7 @@ class TestInstalledDeps(BaseTestCase): """Make sure certain deps are installed.""" @unittest.skipIf(APPVEYOR, "AppVeyor can't detect magic") + @unittest.skipIf(WINDOWS, "libmagic not supported on win") def test_magic(self): self.assertIsNotNone(magic) diff --git a/fulltext/util.py b/fulltext/util.py index a6e3602..93995ab 100644 --- a/fulltext/util.py +++ b/fulltext/util.py @@ -10,6 +10,7 @@ import functools import tempfile import shutil +import platform from os.path import join as pathjoin @@ -138,25 +139,32 @@ def is_windows(): return 'win' in sys.platform -def is_windows64(): +def is_python_64(): """ - Determine if platform is 64 bit Windows. + Determine if this is Python 64 bit. """ - return is_windows() and 'PROGRAMFILES(X86)' in os.environ + arch = platform.architecture() + if arch[0] == '64bit': + return True + elif arch[0] == '32bit': + return False + else: + raise ValueError("can't determine bitness from %s" % str(arch)) -def get_data_dir(): +def get_bin_dir(): # When running under PyInstaller things are a bit different. + basename = 'bin64' if is_python_64() else 'bin32' if hasattr(sys, '_MEIPASS'): - path = pathjoin(sys._MEIPASS, 'fulltext', 'data') + path = pathjoin(sys._MEIPASS, 'fulltext', 'data', basename) # XXX: this absolutely ugly hack is needed in order to build # duster with pyinstaller. if not os.path.isdir(path): print(">>> WARN: assuming you're using pyinstaller from duster", file=sys.stderr) - path = pathjoin(sys._MEIPASS, 'duster', 'data') + path = pathjoin(sys._MEIPASS, 'duster', 'data', basename) else: - path = pathjoin(HERE, 'data') + path = pathjoin(HERE, 'data', basename) assert os.path.isdir(path), path return path @@ -175,8 +183,7 @@ def assert_cmd_exists(cmd): def _set_binpath(): # Help the magic wrapper locate magic1.dll, we include it in # bin/bin{32,64}. - bindir = 'bin64' if is_windows64() else 'bin32' - path = pathjoin(get_data_dir(), bindir) + path = get_bin_dir() os.environ['PATH'] += os.pathsep + path assert_cmd_exists("pdftotext") assert_cmd_exists("unrtf") @@ -184,27 +191,27 @@ def _set_binpath(): assert_cmd_exists("unrar") _set_binpath() + magic = None - def _import_magic(): - # Instantiate our own Magic instance so we can tell it where the - # magic file lives. - from magic import Magic as _Magic - - class Magic(_Magic): - # Overridden because differently from the UNIX version - # the Windows version does not provide mime kwarg. - def from_file(self, filename, mime=True): - return _Magic.from_file(self, filename) + # def _import_magic(): + # # Instantiate our own Magic instance so we can tell it where the + # # magic file lives. + # from magic import Magic as _Magic - def from_buffer(self, buf, mime=True): - return _Magic.from_buffer(self, buf) + # class Magic(_Magic): + # # Overridden because differently from the UNIX version + # # the Windows version does not provide mime kwarg. + # def from_file(self, filename, mime=True): + # return _Magic.from_file(self, filename) - path = pathjoin(get_data_dir(), 'magic') - assert os.path.isfile(path), path - return Magic(mime=True, magic_file=path) + # def from_buffer(self, buf, mime=True): + # return _Magic.from_buffer(self, buf) - magic = _import_magic() + # path = pathjoin(get_bin_dir(), 'magic') + # assert os.path.isfile(path), path + # return Magic(mime=True, magic_file=path) + # magic = _import_magic() def memoize(fun): """A simple memoize decorator for functions supporting (hashable) diff --git a/make.bat b/make.bat index b567e90..2b0123e 100644 --- a/make.bat +++ b/make.bat @@ -1,7 +1,8 @@ -@echo off - rem ========================================================================== rem Shortcuts for various tasks, emulating UNIX "make" on Windows. rem ========================================================================== +set HOME=%USERPROFILE% +set PYTHON=C:\Python36-32\python.exe + %PYTHON% fulltext\data\winmake.py %1 %2 %3 %4 %5 %6 diff --git a/pyinstaller.spec b/pyinstaller.spec index d79367b..780cf94 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -8,9 +8,7 @@ a = Analysis(['fulltext\\__main__.py'], datas=[ ('fulltext/*', 'fulltext/'), ('fulltext/backends/*', 'fulltext/backends'), - ('fulltext/data/magic*', 'fulltext/data/'), ('fulltext/data/bin32', 'fulltext/data/bin32'), - ('fulltext/data/bin64', 'fulltext/data/bin64'), ], hiddenimports=[], hookspath=[], @@ -31,5 +29,5 @@ exe = EXE(pyz, name='fulltext', debug=False, strip=False, - upx=True, + upx=False, console=True ) diff --git a/requirements.txt b/requirements.txt index 202dbe3..abcd9b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,12 +9,12 @@ pytesseract Pillow pycodestyle pyflakes -python-magic ebooklib contextlib2 mock rarfile flake8 +python-magic; os_name != 'nt' git+https://github.com/mattgwwalker/msg-extractor.git@2a24c9950d34932ed5979693cb70d758d78715df#egg=ExtractMsg diff --git a/setup.py b/setup.py index a815a73..26fdce8 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/bin/env python -import os -import sys from os.path import dirname from os.path import join as pathjoin from setuptools import find_packages @@ -10,9 +8,6 @@ NAME = 'fulltext' VERSION = '0.7' -if os.name == 'nt' and not sys.maxsize > 2 ** 32: - # https://github.com/btimby/fulltext/issues/79 - raise RuntimeError("Python 32 bit is not supported") with open(pathjoin(dirname(__file__), 'README.rst')) as f: