refactor(learningpath-api): move learningsteps into learningpath document

common/src/main/scala/no/ndla/common/model/domain/learningpath/LearningPath.scala

@@ -17,6 +17,7 @@ import no.ndla.common.model.domain.Priority
 import sttp.tapir.Schema
 import no.ndla.common.DeriveHelpers
 import no.ndla.common.model.domain.RevisionMeta
+import no.ndla.common.model.domain.learningpath.StepStatus.ACTIVE
 
 case class LearningPath(
     id: Option[Long],
@@ -35,7 +36,7 @@ case class LearningPath(
     owner: String,
     copyright: LearningpathCopyright,
     isMyNDLAOwner: Boolean,
-    learningsteps: Option[Seq[LearningStep]] = None,
+    learningsteps: Seq[LearningStep],
     message: Option[Message] = None,
     madeAvailable: Option[NDLADate] = None,
     responsible: Option[Responsible],
@@ -47,27 +48,25 @@ case class LearningPath(
 ) extends Content {
 
   def supportedLanguages: Seq[String] = {
-    val stepLanguages = learningsteps.getOrElse(Seq.empty).flatMap(_.supportedLanguages)
-
-    (
-      getSupportedLanguages(title, description, tags) ++ stepLanguages
-    ).distinct
+    val stepLanguages         = learningsteps.flatMap(_.supportedLanguages)
+    val allSupportedLanguages = getSupportedLanguages(title, description, tags) ++ stepLanguages
+    allSupportedLanguages.distinct
   }
 
   def isPrivate: Boolean   = Seq(LearningPathStatus.PRIVATE, LearningPathStatus.READY_FOR_SHARING).contains(status)
   def isPublished: Boolean = status == LearningPathStatus.PUBLISHED
   def isDeleted: Boolean   = status == LearningPathStatus.DELETED
 
+  def withOnlyActiveSteps: LearningPath = {
+    val activeSteps = learningsteps.filter(_.status == ACTIVE)
+    this.copy(learningsteps = activeSteps)
+  }
+
 }
 
 object LearningPath {
-  // NOTE: We remove learningsteps from the JSON object before decoding it since it is stored in a separate table
-  implicit val encoder: Encoder[LearningPath] = deriveEncoder[LearningPath].mapJsonObject(_.remove("learningsteps"))
-  implicit val decoder: Decoder[LearningPath] = deriveDecoder[LearningPath].prepare { obj =>
-    val learningsteps = obj.downField("learningsteps")
-    if (learningsteps.succeeded) learningsteps.delete
-    else obj
-  }
+  implicit val encoder: Encoder[LearningPath] = deriveEncoder[LearningPath]
+  implicit val decoder: Decoder[LearningPath] = deriveDecoder[LearningPath]
 
   import sttp.tapir.generic.auto.*
   implicit def schema: Schema[LearningPath] = DeriveHelpers.getSchema

integration-tests/src/test/scala/no/ndla/integrationtests/searchapi/learningpathapi/LearningpathApiClientTest.scala

@@ -83,6 +83,7 @@ class LearningpathApiClientTest
         .insert(
           learningpathapi.TestData.sampleDomainLearningPath.copy(id = Some(id), lastUpdated = NDLADate.fromUnixTime(0))
         )
+        .get
     })
   }
 

learningpath-api/src/main/scala/no/ndla/learningpathapi/controller/InternController.scala

@@ -144,9 +144,7 @@ class InternController(using
     .in(jsonBody[commonDomain.LearningPath])
     .out(jsonBody[commonDomain.LearningPath])
     .errorOut(errorOutputsFor(404))
-    .serverLogicPure { dumpToInsert =>
-      updateService.insertDump(dumpToInsert).asRight
-    }
+    .serverLogicPure(dumpToInsert => updateService.insertDump(dumpToInsert))
 
   private def learningPathStats: ServerEndpoint[Any, Eff] = endpoint
     .get

learningpath-api/src/main/scala/no/ndla/learningpathapi/db/migration/V39__MadeAvailableForThePublished.scala

@@ -8,23 +8,6 @@
 
 package no.ndla.learningpathapi.db.migration
 
-import no.ndla.common.CirceUtil
-import no.ndla.common.model.domain.learningpath.LearningPath
-import no.ndla.common.model.domain.learningpath.LearningPathStatus.{PUBLISHED, UNLISTED}
-import no.ndla.database.DocumentMigration
+import no.ndla.database.FinishedMigration
 
-class V39__MadeAvailableForThePublished extends DocumentMigration {
-  override val columnName: String = "document"
-  override val tableName: String  = "learningpaths"
-
-  def convertColumn(document: String): String = {
-    val oldLp         = CirceUtil.unsafeParseAs[LearningPath](document)
-    val madeAvailable = oldLp.status match {
-      case UNLISTED | PUBLISHED => Some(oldLp.lastUpdated)
-      case _                    => None
-    }
-
-    val newLearningPath = oldLp.copy(madeAvailable = madeAvailable)
-    CirceUtil.toJsonString(newLearningPath)
-  }
-}
+class V39__MadeAvailableForThePublished extends FinishedMigration

learningpath-api/src/main/scala/no/ndla/learningpathapi/db/migration/V60__MoveLearningStepsToLearningPathDocument.scala

@@ -0,0 +1,125 @@
+/*
+ * Part of NDLA learningpath-api
+ * Copyright (C) 2025 NDLA
+ *
+ * See LICENSE
+ *
+ */
+
+package no.ndla.learningpathapi.db.migration
+
+import com.typesafe.scalalogging.StrictLogging
+import io.circe.Json
+import no.ndla.common.CirceUtil
+import org.flywaydb.core.api.migration.{BaseJavaMigration, Context}
+import org.postgresql.util.PGobject
+import org.slf4j.MDC
+import scalikejdbc.*
+
+case class LpDocumentRowWithId(learningPathId: Long, learningPathDocument: String)
+case class StepDocumentRowWithMeta(
+    learningStepId: Long,
+    learningPathId: Long,
+    revision: Int,
+    externalId: Option[String],
+    learningStepDocument: String,
+)
+
+class V60__MoveLearningStepsToLearningPathDocument extends BaseJavaMigration with StrictLogging {
+  private val chunkSize = 1000
+
+  override def migrate(context: Context): Unit = DB(context.getConnection)
+    .autoClose(false)
+    .withinTx { session =>
+      migrateRows(using session)
+    }
+
+  private def countAllRows(implicit session: DBSession): Option[Long] = {
+    sql"select count(*) from learningpaths where document is not null".map(rs => rs.long("count")).single()
+  }
+
+  private def allLearningPaths(offset: Long)(implicit session: DBSession): List[LpDocumentRowWithId] = {
+    sql"select id, document from learningpaths where document is not null order by id limit $chunkSize offset $offset"
+      .map(rs => LpDocumentRowWithId(rs.long("id"), rs.string("document")))
+      .list()
+  }
+
+  private def getStepDatas(learningPathId: Long)(using session: DBSession): List[StepDocumentRowWithMeta] = {
+    sql"""
+      select id, learning_path_id, revision, external_id, document
+      from learningsteps
+      where learning_path_id = $learningPathId and document is not null
+      order by id
+    """
+      .map { rs =>
+        StepDocumentRowWithMeta(
+          learningStepId = rs.long("id"),
+          learningPathId = rs.long("learning_path_id"),
+          revision = rs.int("revision"),
+          externalId = rs.stringOpt("external_id"),
+          learningStepDocument = rs.string("document"),
+        )
+      }
+      .list()
+  }
+
+  private def updateLp(
+      row: LpDocumentRowWithId,
+      steps: List[StepDocumentRowWithMeta],
+  )(using session: DBSession): Unit = {
+    val updatedLpJson = CirceUtil.tryParse(mergeLearningSteps(row.learningPathDocument, steps)).get
+
+    val dataObject = new PGobject()
+    dataObject.setType("jsonb")
+    dataObject.setValue(updatedLpJson.noSpaces)
+
+    val updated = sql"update learningpaths set document = $dataObject where id = ${row.learningPathId}".update()
+    if (updated != 1)
+      throw new RuntimeException(s"Failed to update learning path document for id ${row.learningPathId}")
+  }
+
+  private[migration] def mergeLearningSteps(
+      learningPathDocument: String,
+      steps: List[StepDocumentRowWithMeta],
+  ): String = {
+    val oldLp        = CirceUtil.tryParse(learningPathDocument).get
+    val updatedSteps = steps.sortBy(stepSeqNo).map(enrichStep)
+    oldLp.mapObject(_.remove("learningsteps").add("learningsteps", Json.fromValues(updatedSteps))).noSpaces
+  }
+
+  private def enrichStep(step: StepDocumentRowWithMeta): Json = {
+    val json = CirceUtil.tryParse(step.learningStepDocument).get
+    json.mapObject { obj =>
+      val withIds = obj
+        .add("id", Json.fromLong(step.learningStepId))
+        .add("revision", Json.fromInt(step.revision))
+        .add("learningPathId", Json.fromLong(step.learningPathId))
+      step.externalId match {
+        case Some(value) => withIds.add("externalId", Json.fromString(value))
+        case None        => withIds.remove("externalId")
+      }
+    }
+  }
+
+  private def stepSeqNo(step: StepDocumentRowWithMeta): Int = {
+    val json = CirceUtil.tryParse(step.learningStepDocument).get
+    json.hcursor.get[Int]("seqNo").toOption.getOrElse(Int.MaxValue)
+  }
+
+  private def migrateRows(using session: DBSession): Unit = {
+    val count        = countAllRows.get
+    var numPagesLeft = (count / chunkSize) + 1
+    var offset       = 0L
+
+    MDC.put("migrationName", this.getClass.getSimpleName): Unit
+    while (numPagesLeft > 0) {
+      allLearningPaths(offset * chunkSize).foreach { lpData =>
+        val steps = getStepDatas(lpData.learningPathId)(using session)
+        updateLp(lpData, steps)(using session)
+      }
+      numPagesLeft -= 1
+      offset += 1
+    }
+    MDC.remove("migrationName"): Unit
+  }
+}

learningpath-api/src/main/scala/no/ndla/learningpathapi/model/domain/ImplicitLearningPath.scala

@@ -47,11 +47,10 @@ extension (learningPath: LearningPath) {
 
   def canEditPath(userInfo: CombinedUser): Boolean = canEditLearningPath(userInfo).isSuccess
 
-  private def lsLength: Int           = learningPath.learningsteps.map(_.length).getOrElse(0)
   def validateSeqNo(seqNo: Int): Unit = {
-    if (seqNo < 0 || seqNo > lsLength - 1) {
+    if (seqNo < 0 || seqNo > learningPath.learningsteps.length - 1) {
       throw new ValidationException(errors =
-        List(ValidationMessage("seqNo", s"seqNo must be between 0 and ${lsLength - 1}"))
+        List(ValidationMessage("seqNo", s"seqNo must be between 0 and ${learningPath.learningsteps.length - 1}"))
       )
     }
   }