I am creating a website for users to track their Magic the Gathering inventory. I have created some functionality to import a csv to easily add cards to their inventory. The functionality works as expected however when you start to get to the larger amounts of records, in the tens of thousands, it starts to struggle and sometimes crashes the system.
I'm wondering if there is a more efficient and performant way to import data.
// ======================
// CSV Import Function
// ======================
let Upload = document.getElementById("upload");
Upload.onchange = function () {
var file = this.files[0];
var reader = new FileReader();
reader.onload = function (e) {
var text = e.target.result;
json = csvJSON(text);
$.ajax({
url: "/account/inventory/import-data/?product-type=" + product_type,
type: "POST",
data: {
csrfmiddlewaretoken: csrf_token,
data: JSON.stringify(json),
},
dataType: "json",
error: function (response) {
console.log(response);
},
});
};
reader.readAsText(file);
location.reload();
};
function csvJSON(text) {
let p = "", row = [""], ret = [row], i = 0, r = 0, s = !0, l;
for (l of text) {
if ('"' === l) {
if (s && l === p) row[i] += l;
s = !s;
} else if ("," === l && s) l = row[++i] = "";
else if ("\n" === l && s) {
if ("\r" === p) row[i] = row[i].slice(0, -1);
row = ret[++r] = [(l = "")];
i = 0;
} else row[i] += l;
p = l;
}
return ret;
}
# =================================
# CSV Import Functionality
# =================================
@login_required()
def import_data(request):
current_user = request.user
json_data = json.loads(request.POST['data'])
if "code" not in json_data[0]:
code_index = ''
else:
code_index = json_data[0].index("code")
if "number" not in json_data[0]:
number_index = ''
else:
number_index = json_data[0].index("number")
if "name" not in json_data[0]:
name_index = ''
else:
name_index = json_data[0].index("name")
if "non_foil" not in json_data[0]:
nonfoil_index = ''
else:
nonfoil_index = json_data[0].index("non_foil")
if "foil" not in json_data[0]:
foil_index = ''
else:
foil_index = json_data[0].index("foil")
if "quantity" not in json_data[0]:
quantity_index = ''
else:
quantity_index = json_data[0].index("quantity")
json_data.pop(0)
json_data.pop(-1)
if request.GET.get('product-type') == 'Cards':
for x in json_data:
card_id = magic_sets_cards.objects.filter(set_id__code=x[code_index]).exclude(side='b').filter(number=x[number_index]).filter(name=x[name_index]).values_list('id', flat=True).get()
if x[nonfoil_index] == '0' and x[foil_index] == '0':
magic_sets_cards_inventory.objects.filter(user_id=current_user.id).filter(card_id=card_id).delete()
else:
magic_sets_cards_inventory.objects.update_or_create(user_id=current_user.id, card_id=card_id, defaults={'quantity_non_foil': x[nonfoil_index], 'quantity_foil': x[foil_index]})
return HttpResponse(status=201)
elif request.GET.get('product-type') == 'Tokens':
for x in json_data:
token_id = magic_sets_tokens.objects.filter(set_id__code=x[code_index]).exclude(side='b').filter(number=x[number_index]).filter(name=x[name_index]).values_list('id', flat=True).get()
if x[nonfoil_index] == '0' and x[foil_index] == '0':
magic_sets_tokens_inventory.objects.filter(user_id=current_user.id).filter(token_id=token_id).delete()
else:
magic_sets_tokens_inventory.objects.update_or_create(user_id=current_user.id, token_id=token_id, defaults={'quantity_non_foil': x[nonfoil_index], 'quantity_foil': x[foil_index]})
return HttpResponse(status=201)
elif request.GET.get('product-type') == 'Sealed':
for x in json_data:
sealed_product_id = magic_sets_sealed_products.objects.filter(set_id__code=x[code_index]).filter(name=x[name_index]).values_list('id', flat=True).get()
if x[quantity_index] == '0':
magic_sets_sealed_products.objects.filter(user_id=current_user.id).filter(sealed_product_id=sealed_product_id).delete()
else:
magic_sets_sealed_products.objects.update_or_create(user_id=current_user.id, sealed_product_id=sealed_product_id, defaults={'quantity': x[quantity_index]})
return HttpResponse(status=201)