2
\$\begingroup\$

I am creating a website for users to track their Magic the Gathering inventory. I have created some functionality to import a csv to easily add cards to their inventory. The functionality works as expected however when you start to get to the larger amounts of records, in the tens of thousands, it starts to struggle and sometimes crashes the system.

I'm wondering if there is a more efficient and performant way to import data.

// ======================
//  CSV Import Function
// ======================
let Upload = document.getElementById("upload");
Upload.onchange = function () {
  var file = this.files[0];
  var reader = new FileReader();
  reader.onload = function (e) {
    var text = e.target.result;
    json = csvJSON(text);
    $.ajax({
      url: "/account/inventory/import-data/?product-type=" + product_type,
      type: "POST",
      data: {
        csrfmiddlewaretoken: csrf_token,
        data: JSON.stringify(json),
      },
      dataType: "json",
      error: function (response) {
        console.log(response);
      },
    });
  };
  reader.readAsText(file);
  location.reload();
};

function csvJSON(text) {
  let p = "", row = [""], ret = [row], i = 0, r = 0, s = !0, l;
  for (l of text) {
    if ('"' === l) {
      if (s && l === p) row[i] += l;
      s = !s;
    } else if ("," === l && s) l = row[++i] = "";
    else if ("\n" === l && s) {
      if ("\r" === p) row[i] = row[i].slice(0, -1);
      row = ret[++r] = [(l = "")];
      i = 0;
    } else row[i] += l;
    p = l;
  }
  return ret;
}
# =================================
#  CSV Import Functionality
# =================================
@login_required()
def import_data(request):
    current_user = request.user
    json_data = json.loads(request.POST['data'])

    if "code" not in json_data[0]:
        code_index = ''
    else:
        code_index = json_data[0].index("code")

    if "number" not in json_data[0]:
        number_index = ''
    else:
        number_index = json_data[0].index("number")

    if "name" not in json_data[0]:
        name_index = ''
    else:
        name_index = json_data[0].index("name")

    if "non_foil" not in json_data[0]:
        nonfoil_index = ''
    else:
        nonfoil_index = json_data[0].index("non_foil")

    if "foil" not in json_data[0]:
        foil_index = ''
    else:
        foil_index = json_data[0].index("foil")

    if "quantity" not in json_data[0]:
        quantity_index = ''
    else:
        quantity_index = json_data[0].index("quantity")

    json_data.pop(0)
    json_data.pop(-1)

    if request.GET.get('product-type') == 'Cards':
        for x in json_data:
            card_id = magic_sets_cards.objects.filter(set_id__code=x[code_index]).exclude(side='b').filter(number=x[number_index]).filter(name=x[name_index]).values_list('id', flat=True).get()
            if x[nonfoil_index] == '0' and x[foil_index] == '0':
                magic_sets_cards_inventory.objects.filter(user_id=current_user.id).filter(card_id=card_id).delete()
            else:
                magic_sets_cards_inventory.objects.update_or_create(user_id=current_user.id, card_id=card_id, defaults={'quantity_non_foil': x[nonfoil_index], 'quantity_foil': x[foil_index]})
        return HttpResponse(status=201)
    elif request.GET.get('product-type') == 'Tokens':
        for x in json_data:
            token_id = magic_sets_tokens.objects.filter(set_id__code=x[code_index]).exclude(side='b').filter(number=x[number_index]).filter(name=x[name_index]).values_list('id', flat=True).get()
            if x[nonfoil_index] == '0' and x[foil_index] == '0':
                magic_sets_tokens_inventory.objects.filter(user_id=current_user.id).filter(token_id=token_id).delete()
            else:
                magic_sets_tokens_inventory.objects.update_or_create(user_id=current_user.id, token_id=token_id, defaults={'quantity_non_foil': x[nonfoil_index], 'quantity_foil': x[foil_index]})
        return HttpResponse(status=201)
    elif request.GET.get('product-type') == 'Sealed':
        for x in json_data:
            sealed_product_id = magic_sets_sealed_products.objects.filter(set_id__code=x[code_index]).filter(name=x[name_index]).values_list('id', flat=True).get()
            if x[quantity_index] == '0':
                magic_sets_sealed_products.objects.filter(user_id=current_user.id).filter(sealed_product_id=sealed_product_id).delete()
            else:
                magic_sets_sealed_products.objects.update_or_create(user_id=current_user.id, sealed_product_id=sealed_product_id, defaults={'quantity': x[quantity_index]})
        return HttpResponse(status=201)
\$\endgroup\$
2
  • \$\begingroup\$ So the first JS segment is the client, and the second Python segment is the server? \$\endgroup\$
    – Reinderien
    Dec 10, 2021 at 17:26
  • \$\begingroup\$ Yeah that is correct \$\endgroup\$
    – Ross
    Dec 10, 2021 at 18:17

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Browse other questions tagged or ask your own question.