Global lock on haskell side to complement GIL

.github/workflows/ci.yml

@@ -70,4 +70,4 @@ jobs:
     # ----------------
     - name: Test
       run: |
-        if [ "${{ matrix.skip-test }}" == "" ]; then cabal test all; fi
+        if [ "${{ matrix.skip-test }}" == "" ]; then timeout 3m cabal test all; fi

README.md

@@ -18,7 +18,8 @@ main = withPython $ do
   let input = [1..10] :: [Int]
   let square :: Int -> IO Int
       square x = pure (x * x)
-  print =<< fromPy' @[Int] =<< [pye| [ square_hs(x) for x in input_hs ] |]
+  print =<< runPy $ do
+    fromPy' @[Int] =<< [pye| [ square_hs(x) for x in input_hs ] |]
 ```
 
 it would output:

inline-python.cabal

@@ -53,6 +53,7 @@ Library
                    , process
                    , transformers     >=0.4
                    , inline-c         >=0.9.1
+                   , stm              >=2.4
                    , template-haskell -any
                    , text             >=2
                    , bytestring
@@ -95,10 +96,12 @@ library test
     TST.Roundtrip
     TST.Util
 
+-- Running tests using several threads does very good job at finding threading
+-- bugs. Especially deadlocks
 test-suite inline-python-tests
   import:           language
   type:             exitcode-stdio-1.0
-  Ghc-options:      -threaded
+  Ghc-options:      -threaded -with-rtsopts=-N2
   hs-source-dirs:   test/exe
   main-is:          main.hs
   build-depends:    base

src/Python/Inline.hs

@@ -1,15 +1,51 @@
--- |
+-- | This library allows to embed as quasiquotes and execute arbitrary
+-- python code in haskell programs. Take for example following program:
+--
+-- > {-# LANGUAGE QuasiQuotes #-}
+-- > import Python.Inline
+-- > import Python.Inline.QQ
+-- >
+-- > main :: IO ()
+-- > main = withPython $ do
+-- >   let input = [1..10] :: [Int]
+-- >   let square :: Int -> Py Int
+-- >       square x = pure (x * x)
+-- >   print =<< runPy $ do
+-- >     fromPy' @[Int] =<< [pye| [ square_hs(x) for x in input_hs ] |]
+--
+-- Quasiquotation 'Python.Inline.QQ.pye' captures variables @input@
+-- and @square@ from environment and produces python object which
+-- `fromPy'` converts to haskell list. As one expect it would output:
+--
+-- > [1,4,9,16,25,36,49,64,81,100]
+--
+-- Module "Python.Inline.QQ" provides several quasiquoters with
+-- different semantics but general rules are:
+--
+--  1. All python variables ending with @_hs@ are captured from
+--     environment and converted to python objects according to their
+--     'ToPy' instance.
+--
+--  2. Syntax errors in embedded python will be caught during
+--     compilation.
+--
+--  3. All code interacting with python must be in 'Py' monad which
+--     could be run using 'runPy'.
+--
+--  4. Python interpreter must be initialized before calling any
+--     python code.
 module Python.Inline
   ( -- * Interpreter initialization
+    -- $initialization
     initializePython
   , finalizePython
   , withPython
     -- * Core data types
   , Py
   , runPy
   , PyObject
-  , PyError(..)
     -- * Conversion between haskell and python
+    -- $conversion
   , toPy
   , fromPyEither
   , fromPy
@@ -18,8 +54,42 @@ module Python.Inline
   , FromPy
   ) where
 
-
 import Python.Types
 import Python.Inline.Literal
-
 import Python.Internal.Eval
+
+
+-- $initialization
+--
+-- Python supports being initialized and shut down multiple times. 
+-- This however has caveats. Quoting it documentation:
+--
+-- >  Bugs and caveats: The destruction of modules and objects in
+-- >  modules is done in random order; this may cause destructors
+-- >  (__del__() methods) to fail when they depend on other objects
+-- >  (even functions) or modules. Dynamically loaded extension
+-- >  modules loaded by Python are not unloaded. Small amounts of
+-- >  memory allocated by the Python interpreter may not be freed (if
+-- >  you find a leak, please report it). Memory tied up in circular
+-- >  references between objects is not freed. Some memory allocated
+-- >  by extension modules may not be freed. Some extensions may not
+-- >  work properly if their initialization routine is called more
+-- >  than once.
+--
+-- More importantly for this library. All pointers held by 'PyObject'
+-- becomes invalid after interpreter is shut down. If GC tries to run
+-- finalizers after interpreter is intialized again program will
+-- surely segfault.
+--
+-- For that reason it's only possible to initialize python once and
+-- attempts to initialize python after is was shut down will raise
+-- exceptions.
+
+
+-- $conversion
+--
+-- Python objects are opaque blobs and accessing them may involve
+-- running arbitrary python code. Most notable iteration protocol or
+-- any of dunder methods. For that reason conversion from python to
+-- haskell must happen in 'Py' monad. Conversion also always performs
+-- full copy. Conversion from haskell to python is stateful as well.

src/Python/Inline/Literal.hs

@@ -294,6 +294,7 @@ instance (ToPy a, ToPy b) => ToPy (a,b) where
     p_b <- takeOwnership =<< checkNull (basicToPy b)
     liftIO [CU.exp| PyObject* { PyTuple_Pack(2, $(PyObject* p_a), $(PyObject* p_b)) } |]
 
+-- | Will accept any iterable
 instance (FromPy a, FromPy b) => FromPy (a,b) where
   basicFromPy p_tup = evalContT $ do
     -- Unpack 2-tuple.
@@ -318,6 +319,7 @@ instance (ToPy a, ToPy b, ToPy c) => ToPy (a,b,c) where
     liftIO [CU.exp| PyObject* {
       PyTuple_Pack(3, $(PyObject *p_a), $(PyObject *p_b), $(PyObject *p_c)) } |]
 
+-- | Will accept any iterable
 instance (FromPy a, FromPy b, FromPy c) => FromPy (a,b,c) where
   basicFromPy p_tup = evalContT $ do
     -- Unpack 3-tuple.
@@ -345,6 +347,7 @@ instance (ToPy a, ToPy b, ToPy c, ToPy d) => ToPy (a,b,c,d) where
     liftIO [CU.exp| PyObject* {
       PyTuple_Pack(4, $(PyObject *p_a), $(PyObject *p_b), $(PyObject *p_c), $(PyObject *p_d)) } |]
 
+-- | Will accept any iterable
 instance (FromPy a, FromPy b, FromPy c, FromPy d) => FromPy (a,b,c,d) where
   basicFromPy p_tup = evalContT $ do
     -- Unpack 3-tuple.
@@ -368,6 +371,7 @@ instance (FromPy a, FromPy b, FromPy c, FromPy d) => FromPy (a,b,c,d) where
 instance (ToPy a) => ToPy [a] where
   basicToPy = basicListToPy
 
+-- | Will accept any iterable
 instance (FromPy a) => FromPy [a] where
   basicFromPy p_list = do
     p_iter <- Py [CU.block| PyObject* {
@@ -427,6 +431,7 @@ instance (FromPy a) => FromPy [a] where
 --        with async exception out of the blue
 
 
+-- | Converted to 0-ary function
 instance (ToPy b) => ToPy (IO b) where
   basicToPy f = Py $ do
     --
@@ -436,6 +441,7 @@ instance (ToPy b) => ToPy (IO b) where
     [CU.exp| PyObject* { inline_py_callback_METH_NOARGS($(PyCFunction f_ptr)) } |]
 
 
+-- | Only accepts positional parameters
 instance (FromPy a, Show a, ToPy b) => ToPy (a -> IO b) where
   basicToPy f = Py $ do
     --
@@ -445,7 +451,7 @@ instance (FromPy a, Show a, ToPy b) => ToPy (a -> IO b) where
     --
     [CU.exp| PyObject* { inline_py_callback_METH_O($(PyCFunction f_ptr)) } |]
 
-
+-- | Only accepts positional parameters
 instance (FromPy a1, FromPy a2, ToPy b) => ToPy (a1 -> a2 -> IO b) where
   basicToPy f = Py $ do
     --
@@ -464,7 +470,7 @@ instance (FromPy a1, FromPy a2, ToPy b) => ToPy (a1 -> a2 -> IO b) where
 
 -- | Execute haskell callback function
 pyCallback :: Program (Ptr PyObject) (Ptr PyObject) -> IO (Ptr PyObject)
-pyCallback io = unPy $ ensureGIL $ evalContT io `catch` convertHaskell2Py
+pyCallback io = callbackEnsurePyLock $ unPy $ ensureGIL $ evalContT io `catch` convertHaskell2Py
 
 -- | Load argument from python object for haskell evaluation
 loadArg
@@ -507,7 +513,6 @@ raiseBadNArgs expected got = Py [CU.block| PyObject* {
   } |]
 
 
-
 type FunWrapper a = a -> IO (FunPtr a)
 
 foreign import ccall "wrapper" wrapCFunction

src/Python/Inline/QQ.hs

@@ -13,7 +13,6 @@ module Python.Inline.QQ
 import Language.Haskell.TH.Quote
 
 import Python.Internal.EvalQQ
-import Python.Internal.Eval
 
 
 -- | Evaluate python code in context of main module. All variables