feat: add DatumCache for memory optimization of inline datums

.github/actions/nix-cachix-setup/action.yml

@@ -9,6 +9,21 @@ runs:
   using: composite
 
   steps:
+  - name: 🧹 Free disk space
+    if: runner.os == 'Linux'
+    shell: bash
+    run: |
+      echo "Disk space before cleanup:"
+      df -h /
+      # Remove unnecessary tools to free up disk space
+      sudo rm -rf /usr/share/dotnet
+      sudo rm -rf /usr/local/lib/android
+      sudo rm -rf /opt/ghc
+      sudo rm -rf /opt/hostedtoolcache/CodeQL
+      sudo docker image prune --all --force || true
+      echo "Disk space after cleanup:"
+      df -h /
+
   - name: ❄ Prepare nix
     uses: cachix/install-nix-action@v30
     with:

.github/workflows/docker.yaml

@@ -13,6 +13,8 @@ on:
   push:
     branches: [ "master" ]
     tags: [ "*.*.*" ]
+  pull_request:
+    branches: [ "master" ]
   workflow_dispatch:
     inputs:
       ref_name:
@@ -73,6 +75,12 @@ jobs:
         # And the version as the git commit.
         VERSION=${{github.sha}}
 
+        # For PRs, tag as pr-<number>
+        if [[ "${{github.event_name}}" == "pull_request" ]]; then
+          IMAGE_LABEL=pr-${{github.event.pull_request.number}}
+          VERSION=${{github.event.pull_request.head.sha}}
+        fi
+
         # Determine whether we are building a tag and if yes, set the label
         # name to be the tag name, and the version to be the tag.
         BUILDING_TAG=${{github.ref_type == 'tag'}}

hydra-cluster/src/HydraNode.hs

@@ -421,6 +421,7 @@ prepareHydraNode chainConfig workDir hydraNodeId hydraSKey hydraVKeys allNodeIds
             { cardanoLedgerProtocolParametersFile
             }
       , apiTransactionTimeout = 100000
+      , datumHotCacheSize = 0
       }
  where
   port = fromIntegral $ 5_000 + hydraNodeId

hydra-node/golden/Greetings/Greetings.json

@@ -5,6 +5,7 @@
             "env": {
                 "configuredPeers": "",
                 "contestationPeriod": 43200,
+                "datumHotCacheSize": 0,
                 "depositPeriod": 78099,
                 "otherParties": [],
                 "participants": [],

hydra-node/golden/RunOptions.json

@@ -42,6 +42,7 @@
                 },
                 "tag": "CardanoChainConfig"
             },
+            "datumHotCacheSize": 100,
             "hydraSigningKey": "b/c.sk",
             "hydraVerificationKeys": [
                 "b/a.vk",
@@ -93,6 +94,7 @@
                 "offlineHeadSeed": "605a4786e76db4b28c3668751db34615",
                 "tag": "OfflineChainConfig"
             },
+            "datumHotCacheSize": 100,
             "hydraSigningKey": "a/c.sk",
             "hydraVerificationKeys": [
                 "c.vk",
@@ -179,6 +181,7 @@
                 },
                 "tag": "CardanoChainConfig"
             },
+            "datumHotCacheSize": 100,
             "hydraSigningKey": "a.sk",
             "hydraVerificationKeys": [
                 "a/b/c.vk",
@@ -233,6 +236,7 @@
                 "offlineHeadSeed": "329e98b0b3a9efc003e42da13950a34f",
                 "tag": "OfflineChainConfig"
             },
+            "datumHotCacheSize": 100,
             "hydraSigningKey": "b/c.sk",
             "hydraVerificationKeys": [
                 "b/a.vk"
@@ -297,6 +301,7 @@
                 "startChainFrom": null,
                 "tag": "CardanoChainConfig"
             },
+            "datumHotCacheSize": 100,
             "hydraSigningKey": "b/c/c/b/a/b.sk",
             "hydraVerificationKeys": [
                 "b.vk"

hydra-node/hydra-node.cabal

@@ -69,6 +69,7 @@ library
     Hydra.Chain.Direct.Wallet
     Hydra.Chain.Offline
     Hydra.Chain.ScriptRegistry
+    Hydra.DatumCache
     Hydra.Events
     Hydra.Events.FileBased
     Hydra.Events.Rotation

hydra-node/json-schemas/api.yaml

@@ -3113,6 +3113,7 @@ components:
       - participants
       - contestationPeriod
       - depositPeriod
+      - datumHotCacheSize
       - configuredPeers
       additionalProperties: false
       properties:
@@ -3132,6 +3133,9 @@ components:
           $ref: "api.yaml#/components/schemas/ContestationPeriod"
         depositPeriod:
           $ref: "api.yaml#/components/schemas/DepositPeriod"
+        datumHotCacheSize:
+          type: integer
+          description: Maximum number of datums to keep in the hot cache (0 = unlimited)
         configuredPeers:
           type: string
 
@@ -3472,6 +3476,7 @@ components:
         - chainState
         - headId
         - headSeed
+        - datumCache
       properties:
         parameters:
           $ref: "api.yaml#/components/schemas/HeadParameters"
@@ -3483,6 +3488,38 @@ components:
           $ref: "api.yaml#/components/schemas/HeadId"
         headSeed:
           $ref: "api.yaml#/components/schemas/HeadSeed"
+        datumCache:
+          $ref: "api.yaml#/components/schemas/DatumCache"
+
+    DatumCache:
+      type: object
+      additionalProperties: false
+      description: |
+        A cache for inline datums extracted from UTxOs to reduce memory usage.
+        Contains a list of datum entries, each with a hash and the CBOR-encoded datum.
+      required:
+        - entries
+      properties:
+        entries:
+          type: array
+          description: List of cached datum entries
+          items:
+            $ref: "api.yaml#/components/schemas/DatumCacheEntry"
+
+    DatumCacheEntry:
+      type: object
+      additionalProperties: false
+      description: A single entry in the datum cache
+      required:
+        - hash
+        - datum
+      properties:
+        hash:
+          type: string
+          description: The hash of the script data (hex-encoded)
+        datum:
+          type: string
+          description: The CBOR-encoded script data (hex-encoded)
 
     ClosedState:
       type: object

hydra-node/src/Hydra/API/HTTPServer.hs

@@ -538,7 +538,7 @@ handleSubmitL2Tx ::
 handleSubmitL2Tx putClientInput apiTransactionTimeout responseChannel body = do
   case Aeson.eitherDecode' @(SubmitL2TxRequest tx) body of
     Left err ->
-      pure $ responseLBS status400 jsonContent (Aeson.encode $ Aeson.String $ pack err)
+      pure $ responseLBS status400 jsonContent (Aeson.encode $ SubmitTxRejectedResponse $ pack err)
     Right SubmitL2TxRequest{submitL2Tx} -> do
       -- Duplicate the channel to avoid consuming messages from other consumers.
       dupChannel <- atomically $ dupTChan responseChannel
@@ -581,8 +581,9 @@ handleSubmitL2Tx putClientInput apiTransactionTimeout responseChannel body = do
     go = do
       event <- atomically $ readTChan dupChannel
       case event of
-        Right (RejectedInput{clientInput = NewTx{}, reason}) -> do
-          pure $ SubmitTxRejectedResponse reason
+        Right (RejectedInput{clientInput = NewTx{transaction}, reason})
+          | txId transaction == txid ->
+              pure $ SubmitTxRejectedResponse reason
         Left (TimedServerOutput{output}) -> case output of
           TxValid{transactionId}
             | transactionId == txid ->

hydra-node/src/Hydra/API/Server.hs

@@ -32,6 +32,7 @@ import Hydra.Cardano.Api (LedgerEra)
 import Hydra.Chain (Chain (..))
 import Hydra.Chain.ChainState (IsChainState)
 import Hydra.Chain.Direct.State ()
+import Hydra.DatumCache (HasDatumCache)
 import Hydra.Events (EventSink (..), EventSource (..))
 import Hydra.HeadLogic (
   HeadState (..),
@@ -81,7 +82,7 @@ data APIServerConfig = APIServerConfig
 
 withAPIServer ::
   forall tx.
-  IsChainState tx =>
+  (IsChainState tx, HasDatumCache (UTxOType tx)) =>
   APIServerConfig ->
   Environment ->
   FilePath ->
@@ -99,7 +100,8 @@ withAPIServer config env stateFile party eventSource tracer chain pparams server
     responseChannel <- newBroadcastTChanIO
     -- Initialize our read models from stored events
     -- NOTE: we do not keep the stored events around in memory
-    nodeStateP <- mkProjection "nodeStateP" (initNodeState mkChainState) aggregateNodeState
+    -- Use 0 (unlimited) for API projections since we're just tracking state for display
+    nodeStateP <- mkProjection "nodeStateP" (initNodeState mkChainState) (aggregateNodeState 0)
     -- XXX: We never subscribe to changes of commitInfoP et al directly so a
     -- single read model and normal functions mapping from HeadState ->
     -- CommitInfo etc. would suffice and are less fragile