update

README.md

@@ -100,4 +100,9 @@ GPU is occupied by which job(s)
 **DisableBatchAllocation**
 ```
 ?action=pool_disable_batch
+```
+
+**PoolDump**
+```
+?action=debug_pool_dump
 ```

src/main.go

@@ -309,6 +309,13 @@ func serverAPI(w http.ResponseWriter, r *http.Request) {
 		w.Write(js)
 		break
 
+	case "debug_pool_dump":
+		log.Debug("debug_pool_dump")
+		js, _ := json.Marshal(InstanceOfResourcePool().DebugDump())
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(js)
+		break
+
 	default:
 		http.Error(w, "Not Found", http.StatusNotFound)
 		break

src/resource_pool.go

@@ -150,6 +150,7 @@ func (pool *ResourcePool) init(conf Configuration) {
 				if len(nodes) == 0 {
 					left = append(left, pool.batchJobs[0])
 					pool.batchJobs = pool.batchJobs[1:]
+					log.Info("cannot find a valid allocation, remove a job randomly")
 					continue
 				}
 				for i, task := range job.Tasks {
@@ -688,6 +689,9 @@ func (pool *ResourcePool) acquireResource(job Job) []NodeStatus {
 		}
 		return pool.doAcquireResource(job)
 	}
+	pool.batchMu.Lock()
+	pool.batchJobs = append(pool.batchJobs, job)
+	pool.batchMu.Unlock()
 	for {
 		if _, ok := pool.batchAllocations[job.Name]; ok {
 			break
@@ -1030,3 +1034,10 @@ func (pool *ResourcePool) SetPreScheduleRatio(ratio float64) bool {
 	log.Info("enablePreScheduleRatio is updated to ", ratio)
 	return true
 }
+
+func (pool *ResourcePool) DebugDump() map[string]interface{} {
+	res := map[string]interface{}{}
+	res["batchJobs"] = pool.batchJobs
+	res["pools"] = pool.pools
+	return res
+}

src/scheduler_fair.go

@@ -261,8 +261,8 @@ func (scheduler *SchedulerFair) Start() {
 					Memorytmp += task.Memory
 				}
 
-				log.Info("schedulingJobs are ", scheduler.schedulingJobs)
+				log.Debug("schedulingJobs are ", scheduler.schedulingJobs)
-				log.Info("Before ")
+				log.Debug("Before ")
 				for queue, quota := range scheduler.queuesQuota {
 					log.Info("Queue<->", queue)
 					log.Info("GPU:", quota.NumberGPU)