2
\$\begingroup\$

I have this hosted services which garbs the data from the API and save it in the database. But before saving it checks if the data already exists in the DB if not it creates the new row and if it already exists it compares updated_at from new data with DB if they are equal it ignores it if not it updates it.

Additionally, I have this deleted_at check where it needs to compare existing data with new data from API, my current logic for this process is as below but I'm sure there will be another approach to this.

This hosted services works just fine for me but sometimes (once or twice a day) I'm getting Timeout error at deleted_at logic.

I need it to be reviewed and looking for some inputs on efficiency on my overall code and how can I do this in a better way.

public class DisasteRecoveryServices : DelegatingHandler, IHostedService
{
    public IConfiguration Configuration { get; }
    protected IMemoryCache _cache;
    private Timer _timer;
    public IHttpClientFactory _clientFactory;
    protected HttpClient _client_DR;
    private readonly IServiceScopeFactory _scopeFactory;

    public DisasteRecoveryServices(IConfiguration configuration, IMemoryCache memoryCache, IHttpClientFactory clientFactory, IServiceScopeFactory scopeFactory)
    {
        Configuration = configuration;
        _cache = memoryCache;
        _clientFactory = clientFactory;
        _scopeFactory = scopeFactory;

        // NamedClients foreach Env.
        _client_DR = _clientFactory.CreateClient("DisasteRecoveryEnv");
    }

    public Task StartAsync(CancellationToken cancellationToken)
    {
        _timer = new Timer(GetAccessToken, null, 0, 3300000);
        // Thread.Sleep(2000);
        _timer = new Timer(Heartbeat, null, 1000, 1000);
        return Task.CompletedTask;
    }

    public Task StopAsync(CancellationToken cancellationToken)
    {
        //Timer does not have a stop. 
        _timer?.Change(Timeout.Infinite, 0);
        return Task.CompletedTask;
    }

    public async Task<Token> GetToken(Uri authenticationUrl, Dictionary<string, string> authenticationCredentials)
    {
        HttpClient client = new HttpClient();
        FormUrlEncodedContent content = new FormUrlEncodedContent(authenticationCredentials);
        HttpResponseMessage response = await client.PostAsync(authenticationUrl, content);

        if (response.StatusCode != System.Net.HttpStatusCode.OK)
        {
            string message = String.Format("POST failed. Received HTTP {0}", response.StatusCode);
            throw new ApplicationException(message);
        }

        string responseString = await response.Content.ReadAsStringAsync();
        Token token = JsonConvert.DeserializeObject<Token>(responseString);

        return token;
    }

    private void GetAccessToken(object state)
    {
        Dictionary<string, string> authenticationCredentials_np = Configuration.GetSection("DisasteRecoveryEnvironment:Credentials").GetChildren().Select(x => new KeyValuePair<string, string>(x.Key, x.Value)).ToDictionary(x => x.Key, x => x.Value);
        Token token_dr = GetToken(new Uri(Configuration["DisasteRecoveryEnvironment:URL"]), authenticationCredentials_np).Result;
        _client_DR.DefaultRequestHeaders.Add("Authorization", $"Bearer {token_dr.AccessToken}");
    }

    public void Heartbeat(object state)
    {
        // Discard the result
        _ = GetOrg();
        _ = GetSpace();
        _ = GetApps();
    }

    public async Task GetOrg()
    {
        var request = new HttpRequestMessage(HttpMethod.Get, "organizations");
        var response = await _client_DR.SendAsync(request);
        var json = await response.Content.ReadAsStringAsync();
        OrganizationsClass.OrgsRootObject model = JsonConvert.DeserializeObject<OrganizationsClass.OrgsRootObject>(json);

        using (var scope = _scopeFactory.CreateScope())
        {
            var _DBcontext = scope.ServiceProvider.GetRequiredService<DBContexts>();

            foreach (var item in model.resources)
            {
                var g = Guid.Parse(item.guid);
                var x = _DBcontext.Organizations.FirstOrDefault(o => o.OrgGuid == g);
                if (x == null)
                {
                    _DBcontext.Organizations.Add(new Organizations
                    {
                        OrgGuid = g,
                        Name = item.name,
                        CreatedAt = item.created_at,
                        UpdatedAt = item.updated_at,
                        Timestamp = DateTime.Now,
                        Foundation = 5
                    });
                }
                else if (x.UpdatedAt != item.updated_at)
                {
                    x.CreatedAt = item.created_at;
                    x.UpdatedAt = item.updated_at;
                    x.Timestamp = DateTime.Now;
                }
            }

            await _DBcontext.SaveChangesAsync();
        }
    }

    public async Task GetSpace()
    {
        var request = new HttpRequestMessage(HttpMethod.Get, "spaces");
        var response = await _client_DR.SendAsync(request);
        var json = await response.Content.ReadAsStringAsync();
        SpacesClass.SpaceRootObject model = JsonConvert.DeserializeObject<SpacesClass.SpaceRootObject>(json);

        using (var scope = _scopeFactory.CreateScope())
        {
            var _DBcontext = scope.ServiceProvider.GetRequiredService<DBContexts>();

            foreach (var item in model.resources)
            {
                var g = Guid.Parse(item.guid);
                var x = _DBcontext.Spaces.FirstOrDefault(o => o.SpaceGuid == g);
                if (x == null)
                {
                    _DBcontext.Spaces.Add(new Spaces
                    {
                        SpaceGuid = Guid.Parse(item.guid),
                        Name = item.name,
                        CreatedAt = item.created_at,
                        UpdatedAt = item.updated_at,
                        OrgGuid = Guid.Parse(item.relationships.organization.data.guid),
                        Foundation = 5,
                        Timestamp = DateTime.Now
                    });
                }

                else if (x.UpdatedAt != item.updated_at)
                {
                    x.CreatedAt = item.created_at;
                    x.UpdatedAt = item.updated_at;
                    x.Timestamp = DateTime.Now;
                }
            }

            await _DBcontext.SaveChangesAsync();
        }
    }

    public async Task GetApps()
    {
        var request = new HttpRequestMessage(HttpMethod.Get, "apps?per_page=200");
        var response = await _client_DR.SendAsync(request);
        var json = await response.Content.ReadAsStringAsync();
        AppsClass.AppsRootobject model = JsonConvert.DeserializeObject<AppsClass.AppsRootobject>(json);

        using (var scope = _scopeFactory.CreateScope())
        {
            var _DBcontext = scope.ServiceProvider.GetRequiredService<DBContexts>();

            foreach (var item in model.resources)
            {
                var g = Guid.Parse(item.guid);
                var x = _DBcontext.Apps.FirstOrDefault(o => o.AppGuid == g);

                if (x == null)
                {
                    _DBcontext.Apps.Add(new Apps
                    {
                        AppGuid = Guid.Parse(item.guid),
                        Name = item.name,
                        State = item.state,
                        CreatedAt = item.created_at,
                        UpdatedAt = item.updated_at,
                        SpaceGuid = Guid.Parse(item.relationships.space.data.guid),
                        Foundation = 5,
                        Timestamp = DateTime.Now
                    });
                }

                else if (x.UpdatedAt != item.updated_at)
                {
                    x.State = item.state;
                    x.CreatedAt = item.created_at;
                    x.UpdatedAt = item.updated_at;
                    x.Timestamp = DateTime.Now;
                }
            }

            var guids = model.resources.Select(r => Guid.Parse(r.guid));
            var apps = _DBcontext.Apps.Where(o => guids.Contains(o.AppGuid) == false && o.Foundation == 5 && o.DeletedAt == null);
            foreach (var app in apps)
            {
                app.DeletedAt = DateTime.Now;
            }

            await _DBcontext.SaveChangesAsync();
        }
    }
}
\$\endgroup\$
3
1
\$\begingroup\$

Interesting problem, hard to solve with so Little information but perhaps test if you're data management is not as efficient as you hope or you might even have a locking issue on the database where you're blocking yourself.

Normally what I would do in a situation like this is the following.

  1. Have the webservice write the data in a staging table and return as efficient as possible.

  2. use a stored procedure to process the data on the server, Ideally use a Merge statement if you need to do update, insert and deletes.

I am guessing that your dbcontext is getting to much data in memory an that this causes to much server side processing. Have a look at this post if you have large volumes.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.