| | 1 | | using LGDXRobotCloud.API.Services.Common; |
| | 2 | | using LGDXRobotCloud.API.Services.Navigation; |
| | 3 | | using LGDXRobotCloud.Data.Contracts; |
| | 4 | | using LGDXRobotCloud.Data.DbContexts; |
| | 5 | | using LGDXRobotCloud.Data.Entities; |
| | 6 | | using LGDXRobotCloud.Protos; |
| | 7 | | using LGDXRobotCloud.Utilities.Enums; |
| | 8 | | using LGDXRobotCloud.Utilities.Helpers; |
| | 9 | | using MassTransit; |
| | 10 | | using Microsoft.EntityFrameworkCore; |
| | 11 | | using Microsoft.Extensions.Caching.Distributed; |
| | 12 | | using Microsoft.Extensions.Caching.Memory; |
| | 13 | |
|
| | 14 | | namespace LGDXRobotCloud.API.Services.Automation; |
| | 15 | |
|
| | 16 | | public interface IAutoTaskSchedulerService |
| | 17 | | { |
| | 18 | | void ResetIgnoreRobot(int realmId); |
| | 19 | | Task<RobotClientsAutoTask?> GetAutoTaskAsync(Guid robotId); |
| | 20 | | Task<RobotClientsAutoTask?> AutoTaskAbortAsync(Guid robotId, int taskId, string token, AutoTaskAbortReason autoTaskAbo |
| | 21 | | Task<bool> AutoTaskAbortApiAsync(int taskId); |
| | 22 | | Task<RobotClientsAutoTask?> AutoTaskNextAsync(Guid robotId, int taskId, string token); |
| | 23 | | Task<AutoTask?> AutoTaskNextApiAsync(Guid robotId, int taskId, string token); |
| | 24 | | Task<RobotClientsAutoTask?> AutoTaskNextConstructAsync(AutoTask autoTask); |
| | 25 | | } |
| | 26 | |
|
| 20 | 27 | | public class AutoTaskSchedulerService( |
| 20 | 28 | | IAutoTaskPathPlannerService autoTaskPathPlanner, |
| 20 | 29 | | IBus bus, |
| 20 | 30 | | IEmailService emailService, |
| 20 | 31 | | IMemoryCache memoryCache, |
| 20 | 32 | | IOnlineRobotsService onlineRobotsService, |
| 20 | 33 | | IRobotService robotService, |
| 20 | 34 | | ITriggerService triggerService, |
| 20 | 35 | | LgdxContext context |
| 20 | 36 | | ) : IAutoTaskSchedulerService |
| | 37 | | { |
| 20 | 38 | | private readonly IAutoTaskPathPlannerService _autoTaskPathPlanner = autoTaskPathPlanner ?? throw new ArgumentNullExcep |
| 20 | 39 | | private readonly IBus _bus = bus ?? throw new ArgumentNullException(nameof(bus)); |
| 20 | 40 | | private readonly IEmailService _emailService = emailService ?? throw new ArgumentNullException(nameof(emailService)); |
| 20 | 41 | | private readonly IMemoryCache _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); |
| 20 | 42 | | private readonly IOnlineRobotsService _onlineRobotsService = onlineRobotsService ?? throw new ArgumentNullException(na |
| 20 | 43 | | private readonly IRobotService _robotService = robotService ?? throw new ArgumentNullException(nameof(robotService)); |
| 20 | 44 | | private readonly ITriggerService _triggerService = triggerService ?? throw new ArgumentNullException(nameof(triggerSer |
| 20 | 45 | | private readonly LgdxContext _context = context ?? throw new ArgumentNullException(nameof(context)); |
| 11 | 46 | | private static string GetIgnoreRobotsKey(int realmId) => $"AutoTaskSchedulerService_IgnoreRobot_{realmId}"; |
| | 47 | |
|
| | 48 | | private async Task AddAutoTaskJourney(AutoTask autoTask) |
| 7 | 49 | | { |
| 7 | 50 | | var autoTaskJourney = new AutoTaskJourney |
| 7 | 51 | | { |
| 7 | 52 | | AutoTaskId = autoTask.Id, |
| 7 | 53 | | CurrentProgressId = autoTask.CurrentProgressId |
| 7 | 54 | | }; |
| 7 | 55 | | await _context.AutoTasksJourney.AddAsync(autoTaskJourney); |
| 7 | 56 | | await _context.SaveChangesAsync(); |
| 7 | 57 | | } |
| | 58 | |
|
| | 59 | | private async Task<RobotClientsAutoTask?> GenerateTaskDetail(AutoTask? task, bool continueAutoTask = false) |
| 13 | 60 | | { |
| 13 | 61 | | if (task == null) |
| 3 | 62 | | { |
| 3 | 63 | | return null; |
| | 64 | | } |
| | 65 | |
|
| 10 | 66 | | var progress = await _context.Progresses.AsNoTracking() |
| 10 | 67 | | .Where(p => p.Id == task.CurrentProgressId) |
| 10 | 68 | | .Select(p => new { p.Name }) |
| 10 | 69 | | .FirstOrDefaultAsync(); |
| 10 | 70 | | if (!continueAutoTask) |
| 6 | 71 | | { |
| | 72 | | // Notify the updated task |
| 6 | 73 | | var flowName = await _context.Flows.AsNoTracking() |
| 6 | 74 | | .Where(f => f.Id == task.FlowId) |
| 6 | 75 | | .Select(f => f.Name) |
| 6 | 76 | | .FirstOrDefaultAsync(); |
| 6 | 77 | | string? robotName = null; |
| 6 | 78 | | if (task.AssignedRobotId != null) |
| 6 | 79 | | { |
| 6 | 80 | | robotName = await _context.Robots.AsNoTracking() |
| 6 | 81 | | .Where(r => r.Id == task.AssignedRobotId) |
| 6 | 82 | | .Select(r => r.Name) |
| 6 | 83 | | .FirstOrDefaultAsync(); |
| 6 | 84 | | } |
| 6 | 85 | | await _bus.Publish(new AutoTaskUpdateContract |
| 6 | 86 | | { |
| 6 | 87 | | Id = task.Id, |
| 6 | 88 | | Name = task.Name, |
| 6 | 89 | | Priority = task.Priority, |
| 6 | 90 | | FlowId = task.FlowId ?? 0, |
| 6 | 91 | | FlowName = flowName ?? "Deleted Flow", |
| 6 | 92 | | RealmId = task.RealmId, |
| 6 | 93 | | AssignedRobotId = task.AssignedRobotId, |
| 6 | 94 | | AssignedRobotName = robotName, |
| 6 | 95 | | CurrentProgressId = task.CurrentProgressId, |
| 6 | 96 | | CurrentProgressName = progress!.Name, |
| 6 | 97 | | }); |
| 6 | 98 | | } |
| | 99 | |
|
| 10 | 100 | | if (task.CurrentProgressId == (int)ProgressState.Completed || task.CurrentProgressId == (int)ProgressState.Aborted) |
| 2 | 101 | | { |
| | 102 | | // Return immediately if the task is completed / aborted |
| 2 | 103 | | return new RobotClientsAutoTask |
| 2 | 104 | | { |
| 2 | 105 | | TaskId = task.Id, |
| 2 | 106 | | TaskName = task.Name ?? string.Empty, |
| 2 | 107 | | TaskProgressId = task.CurrentProgressId, |
| 2 | 108 | | TaskProgressName = progress!.Name ?? string.Empty, |
| 2 | 109 | | Paths = { }, |
| 2 | 110 | | NextToken = string.Empty, |
| 2 | 111 | | }; |
| | 112 | | } |
| | 113 | |
|
| 8 | 114 | | var flowDetail = await _context.FlowDetails.AsNoTracking() |
| 8 | 115 | | .Where(fd => fd.FlowId == task.FlowId && fd.Order == (int)task.CurrentProgressOrder!) |
| 8 | 116 | | .FirstOrDefaultAsync(); |
| 8 | 117 | | if (!continueAutoTask && flowDetail!.TriggerId != null) |
| 1 | 118 | | { |
| | 119 | | // Fire the trigger |
| 1 | 120 | | await _triggerService.InitialiseTriggerAsync(task, flowDetail); |
| 1 | 121 | | } |
| | 122 | |
|
| 8 | 123 | | List<RobotClientsPath> paths = []; |
| | 124 | | try |
| 8 | 125 | | { |
| 8 | 126 | | paths = await _autoTaskPathPlanner.GeneratePath(task); |
| 8 | 127 | | } |
| 0 | 128 | | catch (Exception) |
| 0 | 129 | | { |
| 0 | 130 | | await AutoTaskAbortSqlAsync(task.Id); |
| 0 | 131 | | await _emailService.SendAutoTaskAbortEmailAsync(task.Id, AutoTaskAbortReason.PathPlanner); |
| 0 | 132 | | await AddAutoTaskJourney(task); |
| 0 | 133 | | return null; |
| | 134 | | } |
| | 135 | |
|
| 8 | 136 | | string nextToken = task.NextToken ?? string.Empty; |
| 8 | 137 | | if (flowDetail?.AutoTaskNextControllerId != (int)AutoTaskNextController.Robot) |
| 1 | 138 | | { |
| | 139 | | // API has the control |
| 1 | 140 | | nextToken = string.Empty; |
| 1 | 141 | | } |
| | 142 | |
|
| 8 | 143 | | return new RobotClientsAutoTask |
| 8 | 144 | | { |
| 8 | 145 | | TaskId = task.Id, |
| 8 | 146 | | TaskName = task.Name ?? string.Empty, |
| 8 | 147 | | TaskProgressId = task.CurrentProgressId, |
| 8 | 148 | | TaskProgressName = progress!.Name ?? string.Empty, |
| 8 | 149 | | Paths = { paths }, |
| 8 | 150 | | NextToken = nextToken, |
| 8 | 151 | | }; |
| 13 | 152 | | } |
| | 153 | |
|
| | 154 | | public void ResetIgnoreRobot(int realmId) |
| 1 | 155 | | { |
| 1 | 156 | | _memoryCache.Remove(GetIgnoreRobotsKey(realmId)); |
| 1 | 157 | | } |
| | 158 | |
|
| | 159 | | private async Task<AutoTask?> GetRunningAutoTaskSqlAsync(Guid robotId) |
| 8 | 160 | | { |
| 8 | 161 | | return await _context.AutoTasks.AsNoTracking() |
| 8 | 162 | | .Include(t => t.AutoTaskDetails) |
| 8 | 163 | | .Where(t => t.AssignedRobotId == robotId) |
| 8 | 164 | | .Where(t => !LgdxHelper.AutoTaskStaticStates.Contains(t.CurrentProgressId)) |
| 8 | 165 | | .OrderByDescending(t => t.Priority) // In case the robot has multiple running task by mistake |
| 8 | 166 | | .ThenByDescending(t => t.AssignedRobotId) |
| 8 | 167 | | .ThenBy(t => t.Id) |
| 8 | 168 | | .FirstOrDefaultAsync(); |
| 8 | 169 | | } |
| | 170 | |
|
| | 171 | | private async Task<AutoTask?> AssignAutoTaskSqlAsync(Guid robotId) |
| 3 | 172 | | { |
| 3 | 173 | | AutoTask? task = null; |
| 3 | 174 | | using var transaction = await _context.Database.BeginTransactionAsync(); |
| | 175 | | try |
| 3 | 176 | | { |
| | 177 | | // Get waiting task |
| 3 | 178 | | task = await _context.AutoTasks.FromSql( |
| 3 | 179 | | $@"SELECT * FROM ""Automation.AutoTasks"" AS T |
| 3 | 180 | | WHERE T.""CurrentProgressId"" = {(int)ProgressState.Waiting} AND (T.""AssignedRobotId"" = {robotId} OR T.""A |
| 3 | 181 | | ORDER BY T.""Priority"" DESC, T.""AssignedRobotId"" DESC, T.""Id"" |
| 3 | 182 | | LIMIT 1 FOR UPDATE SKIP LOCKED" |
| 3 | 183 | | ).FirstOrDefaultAsync(); |
| | 184 | |
|
| | 185 | | // Get flow detail |
| 3 | 186 | | var flowDetail = await _context.FlowDetails |
| 3 | 187 | | .Where(f => f.FlowId == task!.FlowId) |
| 3 | 188 | | .Select(f => new { |
| 3 | 189 | | f.ProgressId, |
| 3 | 190 | | f.Order |
| 3 | 191 | | }) |
| 3 | 192 | | .OrderBy(f => f.Order) |
| 3 | 193 | | .FirstOrDefaultAsync(); |
| | 194 | |
|
| | 195 | | // Update task |
| 2 | 196 | | task!.AssignedRobotId = robotId; |
| 2 | 197 | | task.CurrentProgressId = flowDetail!.ProgressId; |
| 2 | 198 | | task.CurrentProgressOrder = flowDetail.Order; |
| 2 | 199 | | task.NextToken = LgdxHelper.GenerateMd5Hash($"{robotId} {task.Id} {task.CurrentProgressId} {DateTime.UtcNow}"); |
| 2 | 200 | | await _context.SaveChangesAsync(); |
| 2 | 201 | | await transaction.CommitAsync(); |
| 2 | 202 | | return task; |
| | 203 | | } |
| 1 | 204 | | catch (Exception) |
| 1 | 205 | | { |
| 1 | 206 | | await transaction.RollbackAsync(); |
| 1 | 207 | | } |
| 1 | 208 | | return null; |
| 3 | 209 | | } |
| | 210 | |
|
| | 211 | | public async Task<RobotClientsAutoTask?> GetAutoTaskAsync(Guid robotId) |
| 9 | 212 | | { |
| 9 | 213 | | var realmId = await _robotService.GetRobotRealmIdAsync(robotId) ?? 0; |
| 9 | 214 | | _memoryCache.TryGetValue(GetIgnoreRobotsKey(realmId), out HashSet<Guid>? ignoreRobotIds); |
| 9 | 215 | | if (ignoreRobotIds != null && (ignoreRobotIds?.Contains(robotId) ?? false)) |
| 1 | 216 | | { |
| 1 | 217 | | return null; |
| | 218 | | } |
| | 219 | |
|
| 8 | 220 | | var currentTask = await GetRunningAutoTaskSqlAsync(robotId); |
| 8 | 221 | | bool continueAutoTask = currentTask != null; |
| 8 | 222 | | if (currentTask == null) |
| 4 | 223 | | { |
| 4 | 224 | | if (!_onlineRobotsService.GetPauseAutoTaskAssignment(robotId)) |
| 3 | 225 | | { |
| 3 | 226 | | currentTask = await AssignAutoTaskSqlAsync(robotId); |
| 3 | 227 | | if (currentTask != null) |
| 2 | 228 | | { |
| 2 | 229 | | await AddAutoTaskJourney(currentTask); |
| 2 | 230 | | } |
| 3 | 231 | | } |
| | 232 | | else |
| 1 | 233 | | { |
| | 234 | | // If pause auto task assignment is true, new task will not be assigned. |
| 1 | 235 | | return null; |
| | 236 | | } |
| 3 | 237 | | } |
| | 238 | |
|
| 7 | 239 | | if (currentTask == null) |
| 1 | 240 | | { |
| | 241 | | // No task for this robot, pause database access. |
| 1 | 242 | | ignoreRobotIds ??= []; |
| 1 | 243 | | ignoreRobotIds.Add(robotId); |
| 1 | 244 | | _memoryCache.Set(GetIgnoreRobotsKey(realmId), ignoreRobotIds, TimeSpan.FromMinutes(5)); |
| 1 | 245 | | } |
| 7 | 246 | | return await GenerateTaskDetail(currentTask, continueAutoTask); |
| 9 | 247 | | } |
| | 248 | |
|
| | 249 | | private async Task<AutoTask?> AutoTaskAbortSqlAsync(int taskId, Guid? robotId = null, string? token = null) |
| 4 | 250 | | { |
| 4 | 251 | | AutoTask? task = null; |
| 4 | 252 | | using var transaction = await _context.Database.BeginTransactionAsync(); |
| | 253 | | try |
| 4 | 254 | | { |
| | 255 | | // Get task |
| 4 | 256 | | if (robotId == null && string.IsNullOrWhiteSpace(token)) |
| 2 | 257 | | { |
| | 258 | | // From API |
| 2 | 259 | | task = await _context.AutoTasks.FromSql( |
| 2 | 260 | | $@"SELECT * FROM ""Automation.AutoTasks"" AS T |
| 2 | 261 | | WHERE T.""Id"" = {taskId} |
| 2 | 262 | | LIMIT 1 FOR UPDATE NOWAIT" |
| 2 | 263 | | ).FirstOrDefaultAsync(); |
| 2 | 264 | | } |
| | 265 | | else |
| 2 | 266 | | { |
| 2 | 267 | | task = await _context.AutoTasks.FromSql( |
| 2 | 268 | | $@"SELECT * FROM ""Automation.AutoTasks"" AS T |
| 2 | 269 | | WHERE T.""Id"" = {taskId} AND T.""AssignedRobotId"" = {robotId} AND T.""NextToken"" = {token} |
| 2 | 270 | | LIMIT 1 FOR UPDATE NOWAIT" |
| 2 | 271 | | ).FirstOrDefaultAsync(); |
| 2 | 272 | | } |
| | 273 | |
|
| | 274 | | // Update task |
| 4 | 275 | | task!.CurrentProgressId = (int)ProgressState.Aborted; |
| 2 | 276 | | task.CurrentProgressOrder = null; |
| 2 | 277 | | task.NextToken = null; |
| 2 | 278 | | await _context.SaveChangesAsync(); |
| 2 | 279 | | await transaction.CommitAsync(); |
| 2 | 280 | | } |
| 2 | 281 | | catch (Exception) |
| 2 | 282 | | { |
| 2 | 283 | | await transaction.RollbackAsync(); |
| 2 | 284 | | } |
| 4 | 285 | | return task; |
| 4 | 286 | | } |
| | 287 | |
|
| | 288 | | private async Task DeleteTriggerRetries(int taskId) |
| 2 | 289 | | { |
| 2 | 290 | | var count = await _context.TriggerRetries.Where(tr => tr.AutoTaskId == taskId).CountAsync(); |
| 2 | 291 | | if (count > 0) |
| 0 | 292 | | { |
| 0 | 293 | | await _context.TriggerRetries.Where(tr => tr.AutoTaskId == taskId).ExecuteDeleteAsync(); |
| 0 | 294 | | } |
| 2 | 295 | | } |
| | 296 | |
|
| | 297 | | public async Task<RobotClientsAutoTask?> AutoTaskAbortAsync(Guid robotId, int taskId, string token, AutoTaskAbortReaso |
| 2 | 298 | | { |
| 2 | 299 | | var task = await AutoTaskAbortSqlAsync(taskId, robotId, token); |
| 2 | 300 | | if (task != null) |
| 1 | 301 | | { |
| 1 | 302 | | await DeleteTriggerRetries(taskId); |
| 1 | 303 | | await _emailService.SendAutoTaskAbortEmailAsync(taskId, autoTaskAbortReason); |
| 1 | 304 | | await AddAutoTaskJourney(task); |
| 1 | 305 | | } |
| 2 | 306 | | return await GenerateTaskDetail(task); |
| 2 | 307 | | } |
| | 308 | |
|
| | 309 | | public async Task<bool> AutoTaskAbortApiAsync(int taskId) |
| 2 | 310 | | { |
| 2 | 311 | | var task = await AutoTaskAbortSqlAsync(taskId); |
| 2 | 312 | | if (task == null) |
| 1 | 313 | | return false; |
| | 314 | |
|
| 1 | 315 | | await DeleteTriggerRetries(taskId); |
| 1 | 316 | | await _emailService.SendAutoTaskAbortEmailAsync(taskId, AutoTaskAbortReason.UserApi); |
| 1 | 317 | | await AddAutoTaskJourney(task); |
| 1 | 318 | | return true; |
| 2 | 319 | | } |
| | 320 | |
|
| | 321 | | private async Task<AutoTask?> AutoTaskNextSqlAsync(Guid robotId, int taskId, string token) |
| 5 | 322 | | { |
| 5 | 323 | | AutoTask? task = null; |
| 5 | 324 | | using var transaction = await _context.Database.BeginTransactionAsync(); |
| | 325 | | try |
| 5 | 326 | | { |
| | 327 | | // Get waiting task |
| 5 | 328 | | task = await _context.AutoTasks.FromSql( |
| 5 | 329 | | $@"SELECT * FROM ""Automation.AutoTasks"" AS T |
| 5 | 330 | | WHERE T.""Id"" = {taskId} AND T.""AssignedRobotId"" = {robotId} AND T.""NextToken"" = {token} |
| 5 | 331 | | LIMIT 1 FOR UPDATE NOWAIT" |
| 5 | 332 | | ).FirstOrDefaultAsync(); |
| | 333 | |
|
| | 334 | | // Get flow detail |
| 5 | 335 | | var flowDetail = await _context.FlowDetails.AsNoTracking() |
| 5 | 336 | | .Where(f => f.FlowId == task!.FlowId) |
| 5 | 337 | | .Where(f => f.Order > task!.CurrentProgressOrder) |
| 5 | 338 | | .Select(f => new { |
| 5 | 339 | | f.ProgressId, |
| 5 | 340 | | f.Order |
| 5 | 341 | | }) |
| 5 | 342 | | .OrderBy(f => f.Order) |
| 5 | 343 | | .FirstOrDefaultAsync(); |
| | 344 | |
|
| | 345 | | // Update task |
| 3 | 346 | | if (flowDetail != null) |
| 2 | 347 | | { |
| 2 | 348 | | task!.CurrentProgressId = flowDetail!.ProgressId; |
| 2 | 349 | | task.CurrentProgressOrder = flowDetail.Order; |
| 2 | 350 | | task.NextToken = LgdxHelper.GenerateMd5Hash($"{robotId} {task.Id} {task.CurrentProgressId} {DateTime.UtcNow}"); |
| 2 | 351 | | } |
| | 352 | | else |
| 1 | 353 | | { |
| 1 | 354 | | task!.CurrentProgressId = (int)ProgressState.Completed; |
| 1 | 355 | | task.CurrentProgressOrder = null; |
| 1 | 356 | | task.NextToken = null; |
| 1 | 357 | | } |
| | 358 | |
|
| 3 | 359 | | await _context.SaveChangesAsync(); |
| 3 | 360 | | await transaction.CommitAsync(); |
| 3 | 361 | | } |
| 2 | 362 | | catch (Exception) |
| 2 | 363 | | { |
| 2 | 364 | | await transaction.RollbackAsync(); |
| 2 | 365 | | } |
| 5 | 366 | | return task; |
| 5 | 367 | | } |
| | 368 | |
|
| | 369 | | public async Task<RobotClientsAutoTask?> AutoTaskNextAsync(Guid robotId, int taskId, string token) |
| 3 | 370 | | { |
| 3 | 371 | | var task = await AutoTaskNextSqlAsync(robotId, taskId, token); |
| 3 | 372 | | if (task != null) |
| 2 | 373 | | { |
| 2 | 374 | | await AddAutoTaskJourney(task); |
| 2 | 375 | | } |
| 3 | 376 | | return await GenerateTaskDetail(task); |
| 3 | 377 | | } |
| | 378 | |
|
| | 379 | | public async Task<AutoTask?> AutoTaskNextApiAsync(Guid robotId, int taskId, string token) |
| 2 | 380 | | { |
| 2 | 381 | | var task = await AutoTaskNextSqlAsync(robotId, taskId, token); |
| 2 | 382 | | if (task != null) |
| 1 | 383 | | { |
| 1 | 384 | | await AddAutoTaskJourney(task); |
| 1 | 385 | | } |
| 2 | 386 | | return task; |
| 2 | 387 | | } |
| | 388 | |
|
| | 389 | | public async Task<RobotClientsAutoTask?> AutoTaskNextConstructAsync(AutoTask autoTask) |
| 1 | 390 | | { |
| 1 | 391 | | return await GenerateTaskDetail(autoTask); |
| 1 | 392 | | } |
| | 393 | | } |