Changes to .janno and .ssf columns for Poseidon v3.0.0

CHANGELOG.md

@@ -1,3 +1,21 @@
+- V X.X.X.X:
+    - Introduced smart .janno field construction based on the relevant Poseidon version.
+    - Changes to .janno columns according to Poseidon v3.0.0:
+        - Replaced column `Source_Tissue`  with column `Source_Material`.
+        - New column `Individual_ID`.
+        - New column `Species`.
+        - New column `Alternative_IDs_Context` linked to `Alternative_IDs`.
+        - New column `Custodian_Institution`.
+        - New columns `Cultural_Era` + `Cultural_Era_URL` and `Archaeological_Culture` + `Archaeological_Culture_URL`.
+        - New column `Chromosomal_Anomalies`.
+        - Made column `Collection_ID` a list column.
+        - Soft-retired the option `ReferenceGenome` in the column `Capture_Type`.
+        - Added rescaling feature for the columns `Endogenous` and `Damage` for packages below Poseidon v3.0.0..
+        - Made column `Damage` a list column.
+        - Added the option `WISC2013` to the column `Capture_Type`.
+        - Changed the handling of `_Note` columns. Previously they were explicitly specified and part of the `JannoRow` record type. Now they are just treated as arbitrary additional columns that get algorithmically sorted in when writing .janno files (e.g. in `forge`). See `makeHeaderWithAdditionalColumns`.
+    - Changes to .ssf columns according to Poseidon v3.0.0:
+        - New column `submitted_md5`.
 - V 1.6.8.0:
     - Added a mechanism to check for the presence and completeness of usually optional .janno and .ssf columns. It is exclusively used in `validate`, where a user can set one or multiple of these additional mandatory columns with `-j,--mandatoryJannoColumn` and `-s,--mandatorySSFColumn`.
     - Fixed the golden tests for `validate`. They had become ineffective, because `validate` does not generate stdout any more.

src/Poseidon/CLI/Jannocoalesce.hs

@@ -3,28 +3,29 @@
 
 module Poseidon.CLI.Jannocoalesce where
 
-import           Poseidon.Janno         (JannoRow (..), JannoRows (..),
-                                         parseJannoRowFromNamedRecord,
-                                         readJannoFile, writeJannoFile)
-import           Poseidon.Package       (PackageReadOptions (..),
-                                         defaultPackageReadOptions,
-                                         getJointJanno,
-                                         readPoseidonPackageCollection)
-import           Poseidon.Utils         (PoseidonException (..), PoseidonIO,
-                                         logDebug, logInfo, logWarning)
+import           Poseidon.Janno           (JannoRow (..), JannoRows (..),
+                                           parseJannoRowFromNamedRecord,
+                                           readJannoFile, writeJannoFile)
+import           Poseidon.Package         (PackageReadOptions (..),
+                                           defaultPackageReadOptions,
+                                           getJointJanno,
+                                           readPoseidonPackageCollection)
+import           Poseidon.Utils           (PoseidonException (..), PoseidonIO,
+                                           logDebug, logInfo, logWarning)
 
-import           Control.Monad          (filterM, forM_, when)
-import           Control.Monad.Catch    (MonadThrow, throwM)
-import           Control.Monad.IO.Class (liftIO)
-import qualified Data.ByteString.Char8  as BSC
-import qualified Data.Csv               as Csv
-import qualified Data.HashMap.Strict    as HM
-import qualified Data.IORef             as R
-import           Data.List              ((\\))
-import           Data.Text              (pack, replace, unpack)
-import           System.Directory       (createDirectoryIfMissing)
-import           System.FilePath        (takeDirectory)
-import           Text.Regex.TDFA        ((=~))
+import           Control.Monad            (filterM, forM_, when)
+import           Control.Monad.Catch      (MonadThrow, throwM)
+import           Control.Monad.IO.Class   (liftIO)
+import qualified Data.ByteString.Char8    as BSC
+import qualified Data.Csv                 as Csv
+import qualified Data.HashMap.Strict      as HM
+import qualified Data.IORef               as R
+import           Data.List                ((\\))
+import           Data.Text                (pack, replace, unpack)
+import           Poseidon.PoseidonVersion (latestPoseidonVersion)
+import           System.Directory         (createDirectoryIfMissing)
+import           System.FilePath          (takeDirectory)
+import           Text.Regex.TDFA          ((=~))
 
 -- the source can be a single janno file, or a set of base directories as usual.
 data JannoSourceSpec = JannoSourceSingle FilePath | JannoSourceBaseDirs [FilePath]
@@ -48,7 +49,7 @@ data JannoCoalesceOptions = JannoCoalesceOptions
 runJannocoalesce :: JannoCoalesceOptions -> PoseidonIO ()
 runJannocoalesce (JannoCoalesceOptions sourceSpec target outSpec fields overwrite sKey tKey maybeStrip) = do
     JannoRows sourceRows <- case sourceSpec of
-        JannoSourceSingle sourceFile -> readJannoFile [] sourceFile
+        JannoSourceSingle sourceFile -> readJannoFile latestPoseidonVersion [] sourceFile
         JannoSourceBaseDirs sourceDirs -> do
             let pacReadOpts = defaultPackageReadOptions {
                       _readOptIgnoreChecksums      = True
@@ -57,7 +58,7 @@ runJannocoalesce (JannoCoalesceOptions sourceSpec target outSpec fields overwrit
                     , _readOptOnlyLatest           = True
                 }
             getJointJanno <$> readPoseidonPackageCollection pacReadOpts sourceDirs
-    JannoRows targetRows <- readJannoFile [] target
+    JannoRows targetRows <- readJannoFile latestPoseidonVersion [] target
 
     newJanno <- makeNewJannoRows sourceRows targetRows fields overwrite sKey tKey maybeStrip
 
@@ -123,7 +124,7 @@ mergeRow cp targetRow sourceRow fields overwrite sKey tKey = do
         -- fill in the target row with dummy values for desired fields that might not be present yet
         targetComplete    = HM.union targetRowRecord (HM.fromList $ map (, BSC.empty) sourceKeysDesired)
         newRowRecord      = HM.mapWithKey fillFromSource targetComplete
-        parseResult       = Csv.runParser . parseJannoRowFromNamedRecord [] $ newRowRecord
+        parseResult       = Csv.runParser . parseJannoRowFromNamedRecord latestPoseidonVersion [] $ newRowRecord
     logInfo $ "matched target " ++ BSC.unpack (targetComplete  HM.! BSC.pack tKey) ++
               " with source "   ++ BSC.unpack (sourceRowRecord HM.! BSC.pack sKey)
     case parseResult of

src/Poseidon/CLI/Rectify.hs

@@ -4,32 +4,34 @@ module Poseidon.CLI.Rectify (
     runRectify, RectifyOptions (..), PackageVersionUpdate (..), ChecksumsToRectify (..)
     ) where
 
-import           Poseidon.Contributor   (ContributorSpec (..))
-import           Poseidon.EntityTypes   (HasNameAndVersion (..),
-                                         PacNameAndVersion (..),
-                                         renderNameWithVersion)
-import           Poseidon.GenotypeData  (GenotypeDataSpec (..),
-                                         GenotypeFileSpec (..))
-import           Poseidon.Janno         (writeJannoFileWithoutEmptyCols)
-import           Poseidon.Package       (PackageReadOptions (..),
-                                         PoseidonPackage (..),
-                                         defaultPackageReadOptions,
-                                         readPoseidonPackageCollection,
-                                         writePoseidonPackage)
-import           Poseidon.Utils         (PoseidonIO, getChecksum, logDebug,
-                                         logInfo, logWarning)
-import           Poseidon.Version       (VersionComponent (..),
-                                         updateThreeComponentVersion)
+import           Poseidon.Contributor     (ContributorSpec (..))
+import           Poseidon.EntityTypes     (HasNameAndVersion (..),
+                                           PacNameAndVersion (..),
+                                           renderNameWithVersion)
+import           Poseidon.GenotypeData    (GenotypeDataSpec (..),
+                                           GenotypeFileSpec (..))
+import           Poseidon.Janno           (writeJannoFileWithoutEmptyCols)
+import           Poseidon.Package         (PackageReadOptions (..),
+                                           PoseidonPackage (..),
+                                           defaultPackageReadOptions,
+                                           readPoseidonPackageCollection,
+                                           writePoseidonPackage)
+import           Poseidon.PoseidonVersion (PoseidonVersion (..))
+import           Poseidon.Utils           (PoseidonIO, getChecksum, logDebug,
+                                           logInfo, logWarning)
+import           Poseidon.Version         (VersionComponent (..),
+                                           updateThreeComponentVersion)
 
-import           Control.DeepSeq        ((<$!!>))
-import           Control.Monad          (when)
-import           Control.Monad.IO.Class (MonadIO, liftIO)
-import           Data.List              (nub)
-import           Data.Maybe             (fromJust)
-import           Data.Time              (UTCTime (..), getCurrentTime)
-import           Data.Version           (Version (..), makeVersion, showVersion)
-import           System.Directory       (doesFileExist, removeFile)
-import           System.FilePath        ((</>))
+import           Control.DeepSeq          ((<$!!>))
+import           Control.Monad            (when)
+import           Control.Monad.IO.Class   (MonadIO, liftIO)
+import           Data.List                (nub)
+import           Data.Maybe               (fromJust)
+import           Data.Time                (UTCTime (..), getCurrentTime)
+import           Data.Version             (Version (..), makeVersion,
+                                           showVersion)
+import           System.Directory         (doesFileExist, removeFile)
+import           System.FilePath          ((</>))
 
 data RectifyOptions = RectifyOptions
     { _rectifyBaseDirs              :: [FilePath]
@@ -96,7 +98,7 @@ updatePoseidonVersion :: Maybe Version -> PoseidonPackage -> PoseidonIO Poseidon
 updatePoseidonVersion Nothing    pac = return pac
 updatePoseidonVersion (Just ver) pac = do
     logDebug "Updating Poseidon version"
-    return pac { posPacPoseidonVersion = ver }
+    return pac { posPacPoseidonVersion = PoseidonVersion ver }
 
 addContributors :: Maybe [ContributorSpec] -> PoseidonPackage -> PoseidonIO PoseidonPackage
 addContributors Nothing pac = return pac

src/Poseidon/CLI/Validate.hs

@@ -27,6 +27,7 @@ import qualified Data.ByteString.Char8     as Bchs
 import           Data.List                 (groupBy, intercalate, sortOn)
 import           Data.Yaml                 (decodeEither')
 import           Poseidon.EntityTypes      (IndividualInfo (..))
+import           Poseidon.PoseidonVersion  (latestPoseidonVersion)
 import           System.Exit               (exitFailure, exitSuccess)
 
 -- | A datatype representing command line options for the validate command
@@ -106,12 +107,12 @@ runValidate (ValidateOptions (ValPlanGeno geno) _ _ noExitCode _) = do
     conclude True noExitCode
 runValidate (ValidateOptions (ValPlanJanno path) mandatoryJannoCols _ noExitCode _) = do
     logInfo $ "Validating: " ++ path
-    (JannoRows entries) <- readJannoFile mandatoryJannoCols path
+    (JannoRows entries) <- readJannoFile latestPoseidonVersion mandatoryJannoCols path
     logInfo $ "All " ++ show (length entries) ++ " entries are valid"
     conclude True noExitCode
 runValidate (ValidateOptions (ValPlanSSF path) _ mandatorySSFCols noExitCode _) = do
     logInfo $ "Validating: " ++ path
-    (SeqSourceRows entries) <- readSeqSourceFile mandatorySSFCols path
+    (SeqSourceRows entries) <- readSeqSourceFile latestPoseidonVersion mandatorySSFCols path
     logInfo $ "All " ++ show (length entries) ++ " entries are valid"
     conclude True noExitCode
 runValidate (ValidateOptions (ValPlanBib path) _ _ noExitCode _) = do