# Change formatted list of KV tuples, or dict into structured dict

I made this and left it up on a gist a while back broken. I don't like broken code, so today I spent a while getting tests working, using more sensible names, and making tests pass. Its goal is to take inputs in the form of:

[
('root[subnode][name][]','that guy'),
('root[subnode][attribute][]','somevalue'),
('root[subnode][attribute][]','another value')
]


Then it would transform into:

{
'root': {
'subnode': {
'name': {
0: 'that guy'
},
'attribute': { # there is no way to guarantee order AFAIK
0: 'somevalue',
1: 'another value'
}
}
}
}


I'm looking for feedback to improve upon it. It passes PEP8 check and has inline doctests. The way it's designed at the moment from outside it can be made immutable by import deepcopy from copy in the app / lib code using and passing deepcopy(input).

# FormData / Flat-to-Structured Dictionary Accessories For Python
# Copyright (C) 2015 Lewis Cowles
#
# This program is free software: you can redistribute it and/or modify
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re

def _int_if_poss(s):
"""
Returns an integer Number if non-NaN input provided

>>> _int_if_poss(99)
99

>>> _int_if_poss(12.0)
12

>>> _int_if_poss("999999")
999999

>>> _int_if_poss("0")
0

>>> _int_if_poss("tudor dave")
'tudor dave'

>>> _int_if_poss("0tudor dave")
'0tudor dave'

>>> _int_if_poss("tudor dave22")
'tudor dave22'

>>> _int_if_poss("tudor99dave")
'tudor99dave'
"""
try:
return int(s)
except ValueError:
pass
return s

def _autoKey(collection):
"""
Generates an auto-key based upon length of collection

>>> _autoKey({})
0

>>> _autoKey({"name":"robert poulsen"})
1

>>> _autoKey({"name":"robert poulsen","rules":[
...     "do not talk about fight club",
...     "do not talk about fight club",
...     "do not talk about fight club"]})
2
"""
return len(collection)

def _parse_to_dict_val(key, value, dictin):
"""
Parses depth, encoded names into a JSON'ish disctionary structure

>>> _parse_to_dict_val(
...     "name[]",
...     "starlord",
...     {})['name'][0]
'starlord'

>>> _parse_to_dict_val(
...     "name[accusor]",
...     "ronin",
...     {})['name']['accusor']
'ronin'

>>> _parse_to_dict_val(
...     "characters[by name][accusor]",
...     "ronin",
...     {})['characters']['by name']['accusor']
'ronin'

>>> _parse_to_dict_val("name", "bob", {})
{'name': 'bob'}
"""
patt = re.compile(r'(?P<name>.*?)?[$](?P<key>.*?)[$](?P<leftover>.*?)\$')
matcher = patt.match(key)
matched = matcher is not None

# Guard clause for non-nested
if not matched:
dictin[key] = value
return dictin

tmp = matcher.groupdict()

basename = _int_if_poss(str(tmp['name']))
subkey = str(tmp['key'])
leftover = str(tmp['leftover'])

if len(str(basename)) > 0:
# First off, we should ALWAYS have a matched name

dictin.setdefault(basename, {})

if len(leftover) == 0:
# Is this deeply nested or not
if len(subkey) == 0:
# For standard flat values and when no more remains (easy)
if "[" in key and "]" in key:
# in this case the key needs auto-key
autoKey = _autoKey(dictin[basename])
dictin[basename][autoKey] = value
else:
# no list / dict initialiser pair present
dictin[basename] = value
elif len(subkey) > 0:
# if nothing remains to be done, but we have a key, set a value
dictin[basename][subkey] = value
else:
# This is definitely deeply nested
if len(subkey) > 0:
# For n-length nesting by name (leftover becomes nested)
dictin[basename].setdefault(subkey, {})
_parse_to_dict_val((subkey+leftover), value, dictin[basename])
else:
# For n-length nesting without name (auto-key)
autoKey = _autoKey(dictin[basename])
dictin[basename].setdefault(autoKey, {})
_parse_to_dict_val(
(str(autoKey)+leftover),
value,
dictin[basename])

return dictin

def parse_to_dict_vals(listin):
"""
Parses dictionary for encoded keys signifying depth

>>> parse_to_dict_vals([
...     ("name[]","starlord"),
...     ("name[accusor]","ronin")
... ])['name'][0]
'starlord'

>>> parse_to_dict_vals({
...     "name[]":"starlord",
...     "name[accusor]":"ronin"
... })['name']['accusor']
'ronin'

>>> parse_to_dict_vals({
...     "goodies[starlord][age]":"29",
...     "goodies[starlord][planet]":"Earth"
... })['goodies']['starlord']['age']
'29'

>>> parse_to_dict_vals({"name":"bob"})
{'name': 'bob'}

>>> 'abilities' in parse_to_dict_vals([
...     ("goodies[starlord][age]","29"),
...     ("goodies[starlord][planet]","Earth"),
...     ("goodies[Yondu][species]","Centaurian"),
...     ("goodies[Yondu][abilities][]","Magic Arrow"),
...     ("goodies[Yondu][abilities][]","Not Eating starlord")
... ])['goodies']['Yondu']
True

>>> parsed = parse_to_dict_vals([
...     ("goodies[starlord][age]","29"),
...     ("goodies[starlord][planet]","Earth"),
...     ("goodies[Yondu][species]","Centaurian"),
...     ("goodies[Yondu][abilities][]","Magic Arrow"),
...     ("goodies[Yondu][abilities][]","Not Eating starlord")
... ])
>>> len(parsed['goodies']['Yondu']['abilities'])
2

>>> parse_to_dict_vals({"root[subnode][attribute][][]":"value"})
{'root': {'subnode': {'attribute': {0: {0: 'value'}}}}}

>>> parse_to_dict_vals({"root[subnode][attribute][][me]":"value"})
{'root': {'subnode': {'attribute': {0: {'me': 'value'}}}}}
"""
dictout = {}
if isinstance(listin, dict):
listin = listin.items()
for key, value in listin:
_parse_to_dict_val(key, value, dictout)

return dictout

if __name__ == "__main__":
import doctest
doctest.testmod()


Earlier on the #python IRC someone mentioned using defaultdict, which I didn't understand. Then my wife wanted to enjoy the sunshine, so I probably missed some feedback. Be as brutal as necessary.

I'm looking for ways to use more of the standard library, or alternative existing Python code, maybe ways to allow this to take and output different data structures if it becomes necessary.

For the fight-club and guardians of galaxy test data, I was just looking for something to structure.

Some have said I should just send in JSON. Whilst that is true, I'd prefer to have this work with form-data that can be sent by lots of utilities easily, CURL, web-browsers from IE5 - chrome nightly. JSON just isn't what this is designed for.

A living copy of the code is available here.

• I believe your example output should contain 'name': {0: 'that guy'} rather than 'name': 'that guy'? – 200_success May 6 '18 at 3:40
• That is correct because of an unfortunate use of singular for a group I probably confused myself. The entry has now been amended. – MrMesees May 6 '18 at 8:48

I don't think that there is much benefit to using defaultdict; calling .setdefault() is not that cumbersome. Furthermore, if you use defaultdict, then the resulting data structure would also consist of defaultdicts, which might not be desirable.

I have no idea why your _parse_to_dict_val() helper function is so complicated. I also don't think that it is smart to have it work recursively — that just makes you have to execute the regex over and over again.

With a little bit of renaming in parse_to_dict_vals() and vast simplification of _parse_to_dict_val(), this solution also passes your parse_to_dict_vals() tests:

import re

def _maybe_int(s):
try:
return int(s)
except ValueError:
return s

def _assign(struct, path, value):
components = [
name or key for name, key in re.findall(r'(^[^[]*)|$(.*?)$', path)
]
for comp in components[:-1]:
struct = struct.setdefault(_maybe_int(comp or len(struct)), {})
struct[_maybe_int(components[-1] or len(struct))] = value

def parse_to_dict_vals(assignments):
struct = {}
if isinstance(assignments, dict):
assignments = assignments.items()
for path, value in assignments:
_assign(struct, path, value)
return struct

• Because you renamed everything, even where it made less sense to do so it's taking longer to understand the changes. Thank you for this, I'm sure once I understand why the _assign method works, i'll upvote. – MrMesees May 6 '18 at 8:54
• on for comp in components[:-1]:. When I run this in python, nothing happens to struct. How is this creating the deep structure within struct? – MrMesees May 6 '18 at 9:25
• Note that in the loop, struct is being redefined, such that it tracks the movement along the path. If you set orig_struct = struct before the loop, you'll see that orig_struct changes. – 200_success May 6 '18 at 10:08
• Oh gosh, so by using setdefault it's simultaneously creating the child, then setting as a pointer / reference for the struct local arg var, returning the created child in the local argument struct. I <3 you! Thanks – MrMesees May 6 '18 at 10:11
• We both missed something node.replace(']', '') for node in key.split('[') works a lot better and is probably a lot more efficient than regex. – MrMesees May 7 '18 at 3:49