2019-03-20 03:14:07 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
type AllocatorFIFO struct {
|
2019-03-25 07:36:30 +00:00
|
|
|
history []*Job
|
|
|
|
queue []Job
|
|
|
|
mu sync.Mutex
|
2019-03-20 03:14:07 +00:00
|
|
|
scheduling sync.Mutex
|
2019-03-25 07:36:30 +00:00
|
|
|
|
|
|
|
jobs map[string]*JobManager
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (allocator *AllocatorFIFO) start() {
|
2019-03-25 07:36:30 +00:00
|
|
|
allocator.jobs = map[string]*JobManager{}
|
|
|
|
allocator.history = []*Job{}
|
|
|
|
|
2019-03-20 03:14:07 +00:00
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
//fmt.Print("Scheduling ")
|
|
|
|
time.Sleep(time.Second * 5)
|
|
|
|
allocator.scheduling.Lock()
|
|
|
|
allocator.mu.Lock()
|
|
|
|
if len(allocator.queue) > 0 {
|
|
|
|
|
|
|
|
jm := JobManager{}
|
|
|
|
jm.job = allocator.queue[0]
|
|
|
|
allocator.queue = allocator.queue[1:]
|
|
|
|
jm.allocator = allocator
|
2019-03-25 07:36:30 +00:00
|
|
|
allocator.jobs[jm.job.Name] = &jm
|
|
|
|
|
|
|
|
for i := range allocator.history {
|
|
|
|
if allocator.history[i].Name == jm.job.Name {
|
|
|
|
allocator.history[i].Status = Starting
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-20 03:14:07 +00:00
|
|
|
go func() {
|
|
|
|
jm.start()
|
|
|
|
}()
|
|
|
|
} else {
|
|
|
|
allocator.scheduling.Unlock()
|
|
|
|
}
|
|
|
|
allocator.mu.Unlock()
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2019-03-25 07:36:30 +00:00
|
|
|
func (allocator *AllocatorFIFO) ack(job *Job) {
|
2019-03-20 03:14:07 +00:00
|
|
|
allocator.scheduling.Unlock()
|
2019-03-25 07:36:30 +00:00
|
|
|
for i := range allocator.history {
|
|
|
|
if allocator.history[i].Name == job.Name {
|
|
|
|
allocator.history[i].Status = Running
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (allocator *AllocatorFIFO) finish(job *Job) {
|
|
|
|
for i := range allocator.history {
|
|
|
|
if allocator.history[i].Name == job.Name {
|
|
|
|
allocator.history[i].Status = Finished
|
|
|
|
}
|
|
|
|
}
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (allocator *AllocatorFIFO) schedule(job Job) {
|
|
|
|
allocator.mu.Lock()
|
|
|
|
defer allocator.mu.Unlock()
|
|
|
|
|
|
|
|
allocator.queue = append(allocator.queue, job)
|
2019-03-25 07:36:30 +00:00
|
|
|
allocator.history = append(allocator.history, &job)
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (allocator *AllocatorFIFO) requestResource(task Task) MsgAgent {
|
|
|
|
pool.mu.Lock()
|
|
|
|
defer pool.mu.Unlock()
|
|
|
|
|
|
|
|
res := MsgAgent{}
|
|
|
|
for id, node := range pool.nodes {
|
|
|
|
var available []NodeStatus
|
|
|
|
for _, status := range node {
|
|
|
|
if status.MemoryAllocated == 0 {
|
|
|
|
available = append(available, status)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(available) >= task.NumberGPU {
|
|
|
|
res.ClientID = id
|
|
|
|
res.Status = available[0:task.NumberGPU]
|
|
|
|
|
|
|
|
for i := range res.Status {
|
|
|
|
for j := range node {
|
|
|
|
if res.Status[i].UUID == node[j].UUID {
|
|
|
|
node[j].MemoryAllocated = task.MemoryGPU
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
|
|
|
func (allocator *AllocatorFIFO) returnResource(agent MsgAgent) {
|
|
|
|
pool.mu.Lock()
|
|
|
|
defer pool.mu.Unlock()
|
|
|
|
nodes := pool.nodes[agent.ClientID]
|
|
|
|
for _, gpu := range agent.Status {
|
|
|
|
for j := range nodes {
|
|
|
|
if gpu.UUID == nodes[j].UUID {
|
|
|
|
nodes[j].MemoryAllocated = 0
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (allocator *AllocatorFIFO) status(jobName string) MsgJobStatus {
|
2019-03-25 07:36:30 +00:00
|
|
|
jm, ok := allocator.jobs[jobName]
|
|
|
|
if !ok {
|
|
|
|
return MsgJobStatus{Code: 1, Error: "Job not exist!"}
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|
2019-03-25 07:36:30 +00:00
|
|
|
return jm.status()
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|
|
|
|
|
2019-03-25 07:36:30 +00:00
|
|
|
func (allocator *AllocatorFIFO) logs(jobName string, taskName string) MsgLog {
|
|
|
|
jm, ok := allocator.jobs[jobName]
|
|
|
|
if !ok {
|
|
|
|
return MsgLog{Code: 1, Error: "Job not exist!"}
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|
2019-03-25 07:36:30 +00:00
|
|
|
return jm.logs(taskName)
|
|
|
|
}
|
2019-03-20 03:14:07 +00:00
|
|
|
|
2019-03-25 07:36:30 +00:00
|
|
|
func (allocator *AllocatorFIFO) listJobs() MsgJobList {
|
|
|
|
return MsgJobList{Code: 0, Jobs: allocator.history}
|
2019-03-20 03:14:07 +00:00
|
|
|
}
|