From bfb76b9dc023909a8fbfcf1b3b817d6dd20061f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A4=D1=80=D0=BE=D0=BB=D0=BE=D0=B2?= Date: Wed, 10 Nov 2021 14:23:53 +0500 Subject: [PATCH] Add request tracker. it also track users activity. --- AsbCloudApp/Data/RequestLogDto.cs | 19 +++ AsbCloudApp/Data/RequestLogUserDto.cs | 19 +++ AsbCloudApp/Services/IRequestTracker.cs | 17 +++ AsbCloudDb/Readme.md | 2 +- AsbCloudInfrastructure/DependencyInjection.cs | 3 +- .../Services/RequerstTrackerService.cs | 133 ++++++++++++++++++ .../Controllers/RequerstTrackerController.cs | 54 +++++++ AsbCloudWebApi/Startup.cs | 39 +++++ 8 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 AsbCloudApp/Data/RequestLogDto.cs create mode 100644 AsbCloudApp/Data/RequestLogUserDto.cs create mode 100644 AsbCloudApp/Services/IRequestTracker.cs create mode 100644 AsbCloudInfrastructure/Services/RequerstTrackerService.cs create mode 100644 AsbCloudWebApi/Controllers/RequerstTrackerController.cs diff --git a/AsbCloudApp/Data/RequestLogDto.cs b/AsbCloudApp/Data/RequestLogDto.cs new file mode 100644 index 00000000..ffbc89ac --- /dev/null +++ b/AsbCloudApp/Data/RequestLogDto.cs @@ -0,0 +1,19 @@ +using System; + +namespace AsbCloudApp.Data +{ + public class RequestLogDto + { + public string UserLogin { get; set; } + public int UserId { get; set; } + public string UserIp { get; set; } + public string RequestMethod { get; set; } + public string RequestPath { get; set; } + public string Referer { get; set; } + public long ElapsedMilliseconds { get; set; } + public int Status { get; set; } + public string ExceptionMessage { get; set; } + public DateTime Date { get; set; } + public string ExceptionStack { get; set; } + } +} \ No newline at end of file diff --git a/AsbCloudApp/Data/RequestLogUserDto.cs b/AsbCloudApp/Data/RequestLogUserDto.cs new file mode 100644 index 00000000..151a4b57 --- /dev/null +++ b/AsbCloudApp/Data/RequestLogUserDto.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace AsbCloudApp.Data +{ + public class RequestLogUserDto + { + public int Id { get; set; } + public string Login { get; set; } + public string Ip { get; set; } + public long ElapsedMs { get; set; } + public DateTime LastDate { get; set; } + public long Requests { get; set; } + public long Errors { get; set; } + } +} diff --git a/AsbCloudApp/Services/IRequestTracker.cs b/AsbCloudApp/Services/IRequestTracker.cs new file mode 100644 index 00000000..f18d8b9f --- /dev/null +++ b/AsbCloudApp/Services/IRequestTracker.cs @@ -0,0 +1,17 @@ +using AsbCloudApp.Data; +using System; +using System.Collections.Generic; + +namespace AsbCloudApp.Services +{ + public interface IRequerstTrackerService + { + void RegisterRequest(RequestLogDto requestLog); + void RegisterRequestError(RequestLogDto requestLog, Exception ex); + IEnumerable GetAll(int take = -1); + IEnumerable GetFast(int take = -1); + IEnumerable GetSlow(int take = -1); + IEnumerable GetError(int take = -1); + IEnumerable GetUsersStat(int take = -1); + } +} diff --git a/AsbCloudDb/Readme.md b/AsbCloudDb/Readme.md index e86ce827..e2826da2 100644 --- a/AsbCloudDb/Readme.md +++ b/AsbCloudDb/Readme.md @@ -16,7 +16,7 @@ dotnet ef migrations remvoe --project AsbCloudDb #backup ``` - sudo -u postgres pg_dump -U postgres postgres -W | gzip > 2021-09-27_dump.sql.gz +sudo -u postgres pg_dump -U postgres postgres -W | gzip > 2021-09-27_dump.sql.gz ``` #restore diff --git a/AsbCloudInfrastructure/DependencyInjection.cs b/AsbCloudInfrastructure/DependencyInjection.cs index bcc361ea..92805779 100644 --- a/AsbCloudInfrastructure/DependencyInjection.cs +++ b/AsbCloudInfrastructure/DependencyInjection.cs @@ -19,6 +19,7 @@ namespace AsbCloudInfrastructure options.UseNpgsql(configuration.GetConnectionString("DefaultConnection")), ServiceLifetime.Scoped); services.AddScoped(provider => provider.GetService()); + services.AddScoped(); services.AddHostedService(); services.AddHostedService(); @@ -26,6 +27,7 @@ namespace AsbCloudInfrastructure services.AddSingleton(new CacheDb()); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddTransient(); services.AddTransient(); @@ -45,7 +47,6 @@ namespace AsbCloudInfrastructure services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); // admin crud services: services.AddTransient, CrudServiceBase>(); diff --git a/AsbCloudInfrastructure/Services/RequerstTrackerService.cs b/AsbCloudInfrastructure/Services/RequerstTrackerService.cs new file mode 100644 index 00000000..fda74784 --- /dev/null +++ b/AsbCloudInfrastructure/Services/RequerstTrackerService.cs @@ -0,0 +1,133 @@ +using AsbCloudApp.Data; +using AsbCloudApp.Services; +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace AsbCloudInfrastructure.Services +{ + public class RequerstTrackerService : IRequerstTrackerService + { + const int fastRequestsCount = 1000; + const int slowRequestsCount = 1000; + const int errorRequestsCount = 1000; + const int span = 100; + + const int fastLimitMs = 500; + static readonly char[] stackTraceSeparators = "\r\n".ToCharArray(); + + ConcurrentQueue fastRequests = new ConcurrentQueue(); + ConcurrentQueue slowRequests = new ConcurrentQueue(); + ConcurrentQueue errorRequests = new ConcurrentQueue(); + ConcurrentDictionary users = new ConcurrentDictionary(); + + private IEnumerable Get(IEnumerable list, int take = -1) + { + IEnumerable orderedlist = list.OrderByDescending(r => r.Date); + if (take > 0) + orderedlist = orderedlist.Take(take); + return orderedlist; + } + + public IEnumerable GetUsersStat(int take = -1) + { + IEnumerable result = users.Values.OrderByDescending(u => u.LastDate); + if (take > 0) + result = result.Take(take); + return result; + } + + public IEnumerable GetAll(int take = -1) + { + var result = fastRequests + .Union(slowRequests) + .Union(errorRequests); + + return Get(result, take); + } + + public IEnumerable GetFast(int take = -1) + => Get(fastRequests, take); + + public IEnumerable GetSlow(int take = -1) + => Get(slowRequests, take); + + public IEnumerable GetError(int take = -1) + => Get(errorRequests, take); + + public void RegisterRequest(RequestLogDto requestLog) + { + if (requestLog.Status < 200) + return; + requestLog.Date = DateTime.Now; + if (requestLog.ElapsedMilliseconds > fastLimitMs) + RegisterSlowRequest(requestLog); + else + RegisterFastRequest(requestLog); + + UpdateUserStat(requestLog); + } + + private void RegisterFastRequest(RequestLogDto requestLog) + { + fastRequests.Enqueue(requestLog); + if (fastRequests.Count > fastRequestsCount + span) + while (fastRequests.Count > fastRequestsCount) + fastRequests.TryDequeue(out _); + } + + private void RegisterSlowRequest(RequestLogDto requestLog) + { + slowRequests.Enqueue(requestLog); + if (slowRequests.Count > slowRequestsCount + span) + while (slowRequests.Count > slowRequestsCount) + slowRequests.TryDequeue(out _); + } + + public void RegisterRequestError(RequestLogDto requestLog, Exception ex) + { + requestLog.Date = DateTime.Now; + requestLog.ExceptionMessage = ex.InnerException?.InnerException?.Message + ?? ex.InnerException?.Message + ?? ex.Message; + requestLog.ExceptionStack = ex.StackTrace?.Split(stackTraceSeparators)[0]; + errorRequests.Enqueue(requestLog); + if (errorRequests.Count > errorRequestsCount + span) + while (errorRequests.Count > errorRequestsCount) + errorRequests.TryDequeue(out _); + + UpdateUserStat(requestLog); + } + + private void UpdateUserStat(RequestLogDto requestLog) + { + if (!string.IsNullOrEmpty(requestLog.UserLogin)) + { + var key = $"{requestLog?.UserId}>{requestLog?.UserIp}"; + if (!users.ContainsKey(key)) + users[key] = new RequestLogUserDto + { + Id = requestLog.UserId, + Ip = requestLog.UserIp, + Login = requestLog.UserLogin, + }; + users[key].ElapsedMs += requestLog.ElapsedMilliseconds; + users[key].LastDate = requestLog.Date; + users[key].Requests++; + if(!string.IsNullOrEmpty(requestLog.ExceptionMessage)) + users[key].Errors++; + + if(users.Count > 1000) + { + var count = 900 - users.Count; + var toRemove = users.OrderBy(kv => kv.Value.LastDate).Take(count); + foreach (var kv in toRemove) + users.TryRemove(kv); + } + } + } + } +} diff --git a/AsbCloudWebApi/Controllers/RequerstTrackerController.cs b/AsbCloudWebApi/Controllers/RequerstTrackerController.cs new file mode 100644 index 00000000..d8ada861 --- /dev/null +++ b/AsbCloudWebApi/Controllers/RequerstTrackerController.cs @@ -0,0 +1,54 @@ +using AsbCloudApp.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace AsbCloudWebApi.Controllers +{ + [Route("api/[controller]")] + [ApiController] + [Authorize] + public class RequerstTrackerController : ControllerBase + { + private readonly IRequerstTrackerService service; + + public RequerstTrackerController(IRequerstTrackerService service) + { + this.service = service; + } + + [HttpGet] + public IActionResult GetAll(int take = 512) + { + var result = service.GetAll(take); + return Ok(result); + } + + [HttpGet("fast")] + public IActionResult GetFast(int take = 512) + { + var result = service.GetFast(take); + return Ok(result); + } + + [HttpGet("slow")] + public IActionResult GetSlow(int take = 512) + { + var result = service.GetSlow(take); + return Ok(result); + } + + [HttpGet("error")] + public IActionResult GetError(int take = 512) + { + var result = service.GetError(take); + return Ok(result); + } + + [HttpGet("users")] + public IActionResult GetUsersStat(int take = 512) + { + var result = service.GetUsersStat(take); + return Ok(result); + } + } +} diff --git a/AsbCloudWebApi/Startup.cs b/AsbCloudWebApi/Startup.cs index 69325559..d34649fd 100644 --- a/AsbCloudWebApi/Startup.cs +++ b/AsbCloudWebApi/Startup.cs @@ -73,9 +73,48 @@ namespace AsbCloudWebApi app.UseStaticFiles(); app.UseCors("ClientPermission"); app.UseRouting(); + + app.UseResponseCaching(); + //app.UseResponseCompression(); + app.UseAuthentication(); app.UseAuthorization(); + app.Use(async (context, next) => { + + var service = context.RequestServices.GetRequiredService(); + var requestLog = new AsbCloudApp.Data.RequestLogDto + { + UserLogin = context.User?.Identity.Name, + UserIp = context.Connection.RemoteIpAddress.ToString(), + RequestMethod = context.Request.Method, + RequestPath = context.Request.Path.Value, + Referer = context.Request.Headers["Referer"].ToString(), + }; + { + var userIdString = context.User?.FindFirst("id")?.Value; + if (!string.IsNullOrEmpty(userIdString) && int.TryParse(userIdString, out int userId)) + requestLog.UserId = userId; + } + var sw = System.Diagnostics.Stopwatch.StartNew(); + try + { + await next?.Invoke(); + sw.Stop(); + requestLog.ElapsedMilliseconds = sw.ElapsedMilliseconds; + requestLog.Status = context.Response.StatusCode; + service.RegisterRequest(requestLog); + } + catch(System.Exception ex) + { + sw.Stop(); + requestLog.ElapsedMilliseconds = sw.ElapsedMilliseconds; + requestLog.Status = context.Response.StatusCode; + service.RegisterRequestError(requestLog, ex); + throw; + } + }); + app.UseEndpoints(endpoints => { endpoints.MapControllers();