check used resources on all envs for disk and ram

Author: dnsi0

src/components/c2d/compute_engine_base.ts

@@ -317,7 +317,7 @@ export abstract class C2DEngine {
       CORE_LOGGER.error('Failed to get running jobs:' + e.message)
     }
 
-    const envResourceIds = new Set((env.resources || []).map((r) => r.id))
+    const envResourceMap = new Map((env.resources || []).map((r) => [r.id, r]))
 
     let totalJobs = 0
     let totalFreeJobs = 0
@@ -354,9 +354,12 @@ export abstract class C2DEngine {
 
       if (isRunning) {
         for (const resource of job.resources) {
-          if (envResourceIds.has(resource.id)) {
-            const isCpu = resource.id === 'cpu'
-            if (isCpu && !isThisEnv) continue // CPU is partitioned, skip other envs
+          const envRes = envResourceMap.get(resource.id)
+          if (envRes) {
+            // GPUs are shared-exclusive: inUse tracked globally across all envs
+            // Everything else (cpu, ram, disk) is per-env exclusive
+            const isSharedExclusive = envRes.type === 'gpu'
+            if (!isSharedExclusive && !isThisEnv) continue
             if (!(resource.id in usedResources)) usedResources[resource.id] = 0
             usedResources[resource.id] += resource.amount
             if (job.isFree) {
@@ -381,12 +384,41 @@ export abstract class C2DEngine {
     }
   }
 
+  private checkGlobalResourceAvailability(
+    allEnvironments: ComputeEnvironment[],
+    resourceId: string,
+    amount: number,
+    isFree: boolean
+  ) {
+    let globalUsed = 0
+    let globalTotal = 0
+    for (const e of allEnvironments) {
+      const res = isFree
+        ? e.free
+          ? this.getResource(e.free.resources, resourceId)
+          : null
+        : this.getResource(e.resources, resourceId)
+      if (res) {
+        globalTotal += res.total || 0
+        globalUsed += res.inUse || 0
+      }
+    }
+    const globalRemainder = globalTotal - globalUsed
+    if (globalRemainder < amount) {
+      const suffix = isFree ? ' for free' : ''
+      throw new Error(
+        `Not enough available ${resourceId}${suffix} globally (remaining: ${globalRemainder}, requested: ${amount})`
+      )
+    }
+  }
+
   // overridden by each engine if required
   // eslint-disable-next-line require-await
   public async checkIfResourcesAreAvailable(
     resourcesRequest: ComputeResourceRequest[],
     env: ComputeEnvironment,
-    isFree: boolean
+    isFree: boolean,
+    allEnvironments?: ComputeEnvironment[]
   ) {
     // Filter out resources with amount 0 as they're not actually being requested
     const activeResources = resourcesRequest.filter((r) => r.amount > 0)
@@ -396,12 +428,33 @@ export abstract class C2DEngine {
       if (!envResource) throw new Error(`No such resource ${request.id}`)
       if (envResource.total - envResource.inUse < request.amount)
         throw new Error(`Not enough available ${request.id}`)
+
+      // Global check for non-GPU resources (cpu, ram, disk are per-env exclusive)
+      // GPUs are shared-exclusive so their inUse already reflects global usage
+      if (allEnvironments && envResource.type !== 'gpu') {
+        this.checkGlobalResourceAvailability(
+          allEnvironments,
+          request.id,
+          request.amount,
+          false
+        )
+      }
+
       if (isFree) {
         if (!env.free) throw new Error(`No free resources`)
         envResource = this.getResource(env.free?.resources, request.id)
         if (!envResource) throw new Error(`No such free resource ${request.id}`)
         if (envResource.total - envResource.inUse < request.amount)
           throw new Error(`Not enough available ${request.id} for free`)
+
+        if (allEnvironments && envResource.type !== 'gpu') {
+          this.checkGlobalResourceAvailability(
+            allEnvironments,
+            request.id,
+            request.amount,
+            true
+          )
+        }
       }
     }
     if ('maxJobs' in env && env.maxJobs && env.runningJobs + 1 > env.maxJobs) {

src/components/c2d/compute_engine_docker.ts

@@ -1553,12 +1553,11 @@ export class C2DEngineDocker extends C2DEngine {
       }
       // check if resources are available now
       try {
-        const env = await this.getComputeEnvironment(
-          job.payment && job.payment.chainId ? job.payment.chainId : null,
-          job.environment,
-          null
-        )
-        await this.checkIfResourcesAreAvailable(job.resources, env, job.isFree)
+        const chainId = job.payment && job.payment.chainId ? job.payment.chainId : null
+        const allEnvs = await this.getComputeEnvironments(chainId)
+        const env = allEnvs.find((e) => e.id === job.environment)
+        if (!env) throw new Error(`Environment ${job.environment} not found`)
+        await this.checkIfResourcesAreAvailable(job.resources, env, job.isFree, allEnvs)
       } catch (err) {
         // resources are still not available
         return

src/components/core/compute/startCompute.ts

@@ -29,6 +29,7 @@ import {
 import { EncryptMethod } from '../../../@types/fileObject.js'
 import {
   ComputeAccessList,
+  ComputeEnvironment,
   ComputeResourceRequestWithPrice
 } from '../../../@types/C2D/C2D.js'
 // import { verifyProviderFees } from '../utils/feesHandler.js'
@@ -119,8 +120,10 @@ export class PaidComputeStartHandler extends CommonComputeHandler {
         }
       }
 
+      let allEnvs: ComputeEnvironment[]
       try {
-        env = await engine.getComputeEnvironment(null, task.environment)
+        allEnvs = await engine.getComputeEnvironments()
+        env = allEnvs.find((e) => e.id === task.environment)
         if (!env) {
           return {
             stream: null,
@@ -150,7 +153,7 @@ export class PaidComputeStartHandler extends CommonComputeHandler {
         }
       }
       try {
-        await engine.checkIfResourcesAreAvailable(task.resources, env, false)
+        await engine.checkIfResourcesAreAvailable(task.resources, env, false, allEnvs)
       } catch (e) {
         if (task.queueMaxWaitTime > 0) {
           CORE_LOGGER.verbose(
@@ -885,7 +888,8 @@ export class FreeComputeStartHandler extends CommonComputeHandler {
           }
         }
       }
-      const env = await engine.getComputeEnvironment(null, task.environment)
+      const allFreeEnvs = await engine.getComputeEnvironments()
+      const env = allFreeEnvs.find((e) => e.id === task.environment)
       if (!env) {
         return {
           stream: null,
@@ -931,7 +935,7 @@ export class FreeComputeStartHandler extends CommonComputeHandler {
         }
       }
       try {
-        await engine.checkIfResourcesAreAvailable(task.resources, env, true)
+        await engine.checkIfResourcesAreAvailable(task.resources, env, true, allFreeEnvs)
       } catch (e) {
         if (task.queueMaxWaitTime > 0) {
           CORE_LOGGER.verbose(