# Extracting data from dict and processing it

Consider the following

 def __genereate_initial_guess(dictonary) -> tuple[dict, dict]:
joined_lst = [dictonary["main"], dictonary["background"]]
config_input_params = {}
config_input_params_limits = {}
for key, kind in zip(dictonary,joined_lst):
tmp_lst_2 = []
tmp_lst_3 = []
for line in kind:
tmp_lst = []
tmp_lst_4 = []
user_input_params = [k for k in line if k["use"] == True]
for dic in user_input_params:
tmp_dict = {}
tmp_dict2 = {}
del dic["use"]
tmp = dic.pop("limits")
pair = dic.popitem()
tmp_dict[pair[0]] = pair[1]
tmp_dict2[pair[0]] = tuple(tmp)
tmp_lst.append(tmp_dict)
tmp_lst_4.append(tmp_dict2)
tmp_lst_2.append(dict(ChainMap(*tmp_lst)))
tmp_lst_3.append(dict(ChainMap(*tmp_lst_4)))
config_input_params[key] = tmp_lst_2
config_input_params_limits[key] = tmp_lst_3
return config_input_params, config_input_params_limits

example = {
"main" : [
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1},
{"C"    : 0,        "limits" : [0, 1],      "use" : 0}
],
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1},
{"C"    : 0,        "limits" : [0, 1],      "use" : 1}
]
],
"background" : [
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1}
]
]
}

b, c = __genereate_initial_guess(example)


The code does what it's supposed to, but I find it really hard to read... Also generalizing this seems a bit difficult (e.g. later on I might need to add another keyword like "main" and "background" to example). Is there an easier way to get this done? Any module package, class, function can be freely used.

• Why do you extract the values by key and later zip them with the keys? Use dict.items() instead. This will also scale for additional keys. Jan 18 at 12:12

# Naming

The name of __genereate_initial_guess() starts with a double underscore. Since it is not a method, it will not get mangled with a class name anyway. I'd suggest to remove the underscores.

Then the name of your function does not convey in any way what it's doing. It is not generating a guess in any way. It's extracting parameters and limits.

# Documentation

The function's name is a segue to good documentation. Use docstrings to briefly describe what your functions are doing.

# Divide and conquer

Your function __genereate_initial_guess() does way too much, that can be divided into separate functions, each dealing with part of the issue at hand.

# Know your datatypes

The built-in dict has a method items() to iterate over its key-value pairs. Use it.

# Consistency

Be consistent. If you use type hints, why only for the return value?

# Generalize

On the bottom level of your dict processing, the only difference lies between extracting the paramters and their limits. You can use a callback function to process the leaves of your dict, so you don't have to write the superordinate iteration functions twice.

# Suggested change

from typing import Callable, Iterator, NamedTuple, Union

Number = Union[int, float]
Limits = tuple[Number, Number]
T = Union[Number, Limits]

class ParamsAndLimits(NamedTuple):
"""Parameters and limits of the data structure."""

params: dict[str, Number]
limits: dict[str, Limits]

def process_kind(kind: list, callback: Callable) -> Iterator[dict[str, T]]:
"""Yield dicts for each kind."""

for line in kind:
yield dict(process_line(line, callback))

def process_line(line: list, callback: Callable) -> Iterator[tuple[str, T]]:
"""Yield items for each line."""

for item in filter(lambda item: item.get('use'), line):
yield from callback(item)

def get_params(item: dict) -> Iterator[tuple[str, Number]]:
"""Yield parameters of the item."""

for key in filter(lambda key: key not in {'limits', 'use'}, item):
yield key, item[key]

def get_limits(item: dict) -> Iterator[tuple[str, tuple[Number, Number]]]:
"""Yield limits of the item."""

for key in filter(lambda key: key not in {'limits', 'use'}, item):
yield key, tuple(item['limits'])

def get_parameters_and_limits(dataset: dict) -> ParamsAndLimits:
"""Return one dict with parameters and one with limits."""

params = {}
limits = {}

for key, kind in dataset.items():
params[key] = list(process_kind(kind, get_params))
limits[key] = list(process_kind(kind, get_limits))

return ParamsAndLimits(params, limits)

example = {
"main" : [
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1},
{"C"    : 0,        "limits" : [0, 1],      "use" : 0}
],
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1},
{"C"    : 0,        "limits" : [0, 1],      "use" : 1}
]
],
"background" : [
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1}
]
]
}

params, limits = get_parameters_and_limits(example)
print(params, limits)


# Alternative without callbacks

Since the above code still has some duplication and iterates over the source dict twice, you can also replace the callback approach by extracting the parameters and limits in one go. The disadvantage of this is, that you, in the end, still have to loop over the results twice due to the nature of your desired output datastructure. It also makes the functions a bit more complex, since the extraction function will do two things at once.

from typing import Iterator, NamedTuple, Union

Number = Union[int, float]
Limits = tuple[Number, Number]

class ExtractedParams(NamedTuple):
"""Extracted parameters."""

key: str
value: Number
limits: Limits

class ParamsAndLimits(NamedTuple):
"""Parameters and limits of the data structure."""

params: dict[str, Number]
limits: dict[str, Limits]

def process_kind(kind: list) -> Iterator[ParamsAndLimits]:
"""Yield params and limits for each kind."""

for line in kind:
params = {}
limits = {}

for extracted_params in process_line(line):
params[extracted_params.key] = extracted_params.value
limits[extracted_params.key] = extracted_params.limits

yield ParamsAndLimits(params, limits)

def process_line(line: list) -> Iterator[ExtractedParams]:
"""Yield items for each line."""

for item in filter(lambda item: item.get('use'), line):
for key in filter(lambda key: key not in {'limits', 'use'}, item):
yield ExtractedParams(key, item[key], tuple(item['limits']))

def get_parameters_and_limits(dataset: dict) -> ParamsAndLimits:
"""Return one dict with parameters and one with limits."""

params = {}
limits = {}

for key, kind in dataset.items():
params_and_limits = list(process_kind(kind))
params[key] = [p.params for p in params_and_limits]
limits[key] = [p.limits for p in params_and_limits]

return ParamsAndLimits(params, limits)

example = {
"main" : [
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1},
{"C"    : 0,        "limits" : [0, 1],      "use" : 0}
],
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1},
{"C"    : 0,        "limits" : [0, 1],      "use" : 1}
]
],
"background" : [
[
{"A"    : 0,     "limits" : [0, 0.1],      "use" : 1},
{"B"    : 0.01,    "limits" : [-0.1, 0.1],   "use" : 1}
]
]
}

params, limits = get_parameters_and_limits(example)
print(params, limits)

• Thanks a lot for the feedback and suggestions! I'm not really familiar with the yield-keyword and this "callback"-approach, so this turned out to be a good learning oportunity!
– Sito
Jan 18 at 15:17

Functions should not modify their inputs silently or needlessly. If someone calls a function with arguments, they do not not expect the function to modify the arguments unless it was advertised as such. By applying del to the input data, you are destroying the input rather than merely extracting information from it.

Your current data structure seems questionable. The innermost dicts are of particular concern. You seem to want to extract the keys/values from those dicts in ways that suggest that the dicts themselves are modeled incorrectly. Sometimes you want to link the letter key (A, B, C) to its numeric value (that is consistent with the dict structure, because you are grabbing a key and its associated value). But sometimes you want to link the letter key to an entirely different value from the dict, namely the limits tuple (that is inconsistent with the dict structure). The inconsistency makes it awkward to extract data from these dicts in a natural way while iterating over the key-value pairs of each dict. My primary recommendation is that you convert your data so that the innermost dicts are more flexible, given your extraction needs.

def converted(d):
# Takes an inner dict. Returns a converted dict.

# We start by getting our own copy of the dict so that
# we don't destroy the input data.
d = dict(d)

# Grab the needed info.
# This code assumes the input dict has only 3 key-value pairs.
limits = tuple(d.pop('limits'))
use = d.pop('use')
key, val = tuple(d.items())[0]

# You could also consider returning an instance of a dataclass.
# For this demo, we'll use ordinary dicts.
return {
'key': key,
'val': val,
'limits': limits,
'use': use,
}


Converting the whole data structure. With that conversion function in place, we can write a simple function to apply the conversion across the entire data structure. Here and elsewhere I'm using fairly generic terminology (data, key, val, outer, inner, and so forth) because we don't know your domain. You should try to use more specific/meaningful names where possible.

def converted_data(data):
return {
ko : [
[converted(d) for d in inner]
for inner in outer
]
for ko, outer in data.items()
}


Extracting information from the new structure. The new arrangement of information is easier to work with. A single function can extract the needed values or limits:

def extract_parameters(data, param):
return {
ko : [
{
cd['key'] : cd[param]
for cd in converted_dicts
if cd['use']
}
for converted_dicts in outer
]
for ko, outer in data.items()
}

def main():
ex = converted_data(example)
vals = extract_parameters(ex, 'val')
limits = extract_parameters(ex, 'limits')
print(vals)
print(limits)

if __name__ == '__main__':
main()