I recently worked with some pickle files which had a really strange format where the keys are sometimes delimiter separated with colons and placed in the same depth but also mixed with ordinary key value pairs.
This is a basic example of this mixed format:
{
"objects": {
"list::1": {
"attr1": "foo"
"attr2": "bar"
},
"list::2": {
"attr1": "foo"
"attr2": "bar"
},
"list::3": {
"attr1": "foo"
"attr2": "bar"
"nested::data::inner": {
"test": 1
}
}
},
"dates": {
"2018::11::01": true,
"2018::11::02": false,
"2018::10::02": false,
}
}
The challenge was to convert this to a .mat file so it can be analyzed using matlab. However, because the length of the fields are sometimes > 63 (the delimiter separated string) the conversion was unsuccessful. This is my implementation on normalizing these mixed format nested dictionaries.
Maybe there is an inverse to json_normalize() from pandas library to do this?
import json
mydict = {
"a": {
"test::test2::test3::test4": {
"name": "ok",
"age": 1
},
"test::test2::test4::test4": {
"name": "ok1",
"age": 2
},
"test::test2::test4::test5": {
"name": "ok1",
"body::head::foot": {
"age": 2,
"thing": "test"
}
}
},
"b": {
"name": "ok2",
"age": "test2"
}
}
def set_nested(data, args, new_val):
if args and data:
element = args[0]
if element:
value = data.get(element)
if len(args) == 1:
data[element] = new_val
else:
set_nested(value, args[1:], new_val)
from collections import MutableMapping
# see https://stackoverflow.com/questions/7204805/dictionaries-of-dictionaries-merge/24088493#24088493
def rec_merge(d1, d2):
'''
Update two dicts of dicts recursively,
if either mapping has leaves that are non-dicts,
the second's leaf overwrites the first's.
'''
for k, v in d1.items(): # in Python 2, use .iteritems()!
if k in d2:
# this next check is the only difference!
if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
d2[k] = rec_merge(v, d2[k])
# we could further check types and merge as appropriate here.
d3 = d1.copy()
d3.update(d2)
return d3
def normalize(old_dict, delim="::"):
new_dict = { }
for key in old_dict.keys():
splitted = key.split(delim)
is_split = len(splitted) > 1
if is_split:
new_key = splitted[0]
x = new_dict.get(new_key)
if not x or not isinstance(x, dict):
x = {}
y = {}
for s in reversed(splitted[1:]):
y = {s: y}
set_nested(y, splitted[1:], old_dict[key])
new_val = rec_merge(x, y)
new_dict[new_key] = new_val
if isinstance(old_dict[key], dict):
new_dict[new_key] = normalize(new_dict[new_key])
else:
if isinstance(old_dict[key], dict):
new_dict[key] = normalize(old_dict[key])
else:
new_dict[key] = old_dict[key]
return new_dict
print (json.dumps(normalize(mydict), indent=2))
This is the expected output after normalizing:
{
"a": {
"test": {
"test2": {
"test3": {
"test4": {
"name": "ok",
"age": 1
}
},
"test4": {
"test4": {
"name": "ok1",
"age": 2
},
"test5": {
"name": "ok1",
"body": {
"head": {
"foot": {
"age": 2,
"thing": "test"
}
}
}
}
}
}
}
},
"b": {
"name": "ok2",
"age": "test2"
}
}
Is there a way to simplify the logic in normalize function?
"a::b": {"c": 2}, "a::b::c": {"d": 3}
a possibility and how to deal with it? \$\endgroup\$ – Graipher Nov 26 '19 at 12:56