Take the 2-minute tour ×
Code Review Stack Exchange is a question and answer site for peer programmer code reviews. It's 100% free, no registration required.

This is a framework for a JSON parser I put together last night.

Any comments appreciated.

  • JsonLexer.l: Breaks the input into lexemes
  • JsonParser.y: Understands the language syntax
  • JsonParser.h: Header file to bring it all togeter
  • main.cpp: Test harness so it can be tested.
  • Makefile: Sinmple build script.

Testing:

echo '{ "Plop": "Test" }' | ./test

JsonLexer.l

%option c++
%option noyywrap

%{
#define  IN_LEXER
#include "JsonParser.tab.hpp"
%}

DIGIT           [0-9]
DIGIT1          [1-9]
INTNUM          {DIGIT1}{DIGIT}*
FRACT           "."{DIGIT}+
FLOAT           ({INTNUM}|0){FRACT}?
EXP             [eE][+-]?{DIGIT}+
NUMBER          -?{FLOAT}{EXP}?

UNICODE         \\u[A-Fa-f0-9]{4}
ESCAPECHAR      \\["\\/bfnrt]
CHAR            [^"\\]|{ESCAPECHAR}|{UNICODE}
STRING          \"{CHAR}*\"

WHITESPACE      [ \t\n]


%%

\{              {LOG("LEX({)");     return '{';}
\}              {LOG("LEX(})");     return '}';}
\[              {LOG("LEX([)");     return '[';}
\]              {LOG("LEX(])");     return ']';}
,               {LOG("LEX(,)");     return ',';}
:               {LOG("LEX(:)");     return ':';}
true            {LOG("LEX(true)");  return yy::JsonParser::token::JSON_TRUE;}
false           {LOG("LEX(false)"); return yy::JsonParser::token::JSON_FALSE;}
null            {LOG("LEX(null)");  return yy::JsonParser::token::JSON_NULL;}
{STRING}        {LOG("LEX(String)");return yy::JsonParser::token::JSON_STRING;}
{NUMBER}        {LOG("LEX(Number)");return yy::JsonParser::token::JSON_NUMBER;}

{WHITESPACE}    {/*IGNORE*/}

%%

JsonParser.y

%skeleton "lalr1.cc"
%require "2.1a"
%defines
%define "parser_class_name" "JsonParser"

%{

#include "JsonParser.h"
#include <stdexcept>

%}

%parse-param {FlexLexer& lexer}
%lex-param   {FlexLexer& lexer}

%token  JSON_STRING
%token  JSON_NUMBER
%token  JSON_TRUE
%token  JSON_FALSE
%token  JSON_NULL


%%

JsonObject              :   JsonMap                                 {LOG("JsonObject: JsonMap");}
                        |   JsonArray                               {LOG("JsonObject: JsonArray");}

JsonMap                 :   '{' JsonMapValueListOpt '}'             {LOG("JsonMap: { JsonMapValueListOpt }");}
JsonMapValueListOpt     :                                           {LOG("JsonMapValueListOpt: EMPTY");}
                        |   JsonMapValueList                        {LOG("JsonMapValueListOpt: JsonMapValueList");}
JsonMapValueList        :   JsonMapValue                            {LOG("JsonMapValueList: JsonMapValue");}
                        |   JsonMapValueList ',' JsonMapValue       {LOG("JsonMapValueList: JsonMapValueList , JsonMapValue");}
JsonMapValue            :   JSON_STRING ':' JsonValue               {LOG("JsonMapValue: JSON_STRING : JsonValue");}    

JsonArray               :   '[' JsonArrayValueListOpt ']'           {LOG("JsonArray: [ JsonArrayValueListOpt ]");}
JsonArrayValueListOpt   :                                           {LOG("JsonArrayValueListOpt: EMPTY");}
                        |   JsonArrayValueList                      {LOG("JsonArrayValueListOpt: JsonArrayValueList");}
JsonArrayValueList      :   JsonValue                               {LOG("JsonArrayValueList: JsonValue");}
                        |   JsonArrayValueList ',' JsonValue        {LOG("JsonArrayValueList: JsonArrayValueList , JsonValue");}

JsonValue               :   JsonMap                                 {LOG("JsonValue: JsonMap");}
                        |   JsonArray                               {LOG("JsonValue: JsonArray");}
                        |   JSON_STRING                             {LOG("JsonValue: JSON_STRING");}
                        |   JSON_NUMBER                             {LOG("JsonValue: JSON_NUMBER");}
                        |   JSON_TRUE                               {LOG("JsonValue: JSON_TRUE");}
                        |   JSON_FALSE                              {LOG("JsonValue: JSON_FALSE");}
                        |   JSON_NULL                               {LOG("JsonValue: JSON_NULL");}


%%

int yylex(int*, FlexLexer& lexer)
{
    return lexer.yylex();
}

void yy::JsonParser::error(yy::location const&, std::string const& msg)
{
    throw std::runtime_error(msg);
}

main.cpp

#include "JsonParser.tab.hpp"
#include <iostream>

int main()
{
    try
    {
        yyFlexLexer         lexer(&std::cin, &std::cout);
        yy::JsonParser      parser(lexer);

        std::cout << (parser.parse() == 0 ? "OK" : "FAIL") << "\n";
    }
    catch(std::exception const& e)
    {
        std::cout << "Exception: " << e.what() << "\n";
    }
}

JsonParser.h

#ifndef THORSANVIL_JSON_PARSER_H
#define THORSANVIL_JSON_PARSER_H

#ifndef IN_LEXER
#include <FlexLexer.h>
#endif

int yylex(int*, FlexLexer& lexer);

#ifdef DEBUG_LOG
#include <iostream>
#define LOG(x)      std::cout << x << "\n"
#else
#define LOG(x)      0 /*Empty Statement that will be optimized out*/
#endif

#endif

Makefile

YACC            = bison
LEX             = flex
CXX             = g++

CXXFLAGS        = -DDEBUG_LOG

TARGET          = json
LEX_SRC         = $(wildcard *.l)
YACC_SRC        = $(wildcard *.y)
CPP_SRC         = $(filter-out %.lex.cpp %.tab.cpp,$(wildcard *.cpp))
SRC             = $(patsubst %.y,%.tab.cpp,$(YACC_SRC)) $(patsubst %.l,%.lex.cpp,$(LEX_SRC)) $(CPP_SRC)
OBJ             = $(patsubst %.cpp,%.o,$(SRC))


all: $(OBJ)
    $(CXX) -o test $(OBJ) -lFl

clean:
    rm $(OBJ) $(patsubst %.y,%.tab.cpp,$(YACC_SRC)) $(patsubst %.l,%.lex.cpp,$(LEX_SRC))


$(TARGET):  $(OBJ)
    $(CXX)  -o $* $(OBJ)

.PRECIOUS: %.tab.cpp

%.tab.cpp: %.y
    $(YACC) -o $@ -d $<

.PRECIOUS: %.lex.cpp

%.lex.cpp: %.l
    $(LEX) -t $< > $@
share|improve this question
    
What is C++ or object oriented about this? Flex & Bison generate C code. The only C++ code that is see is the test code written in main.cpp. Either you call it a C parser or use flex and bison equivalent for C++ –  apeirogon Oct 8 '12 at 8:30
3  
@apeirogon: Actually flex and bison both generate C++ classes see %option c++ and %skeleton "lalr1.cc". But this is not really relevant. It is the interface the lexer and scanner present to the world that make the code OO. The current interface is OO allowing you to use parser/lexer independently and in a very OO manor. –  Loki Astari Oct 8 '12 at 15:53
1  
@apeirogon: Also the link you provide actually states that those tools have now atrophied while the real flex and bison have continued to be developed. <quote> But releases ended in 1993 and flex++ and bison++ fell out of date as C++, gcc, flex,and bison all evolved </quote>. So I think I will stick to using real development tools rather than somebodies abandoned project. –  Loki Astari Oct 8 '12 at 15:57
    
The real reason that I put C++ rather than C down is that I don't write C anymore. So the glue code is C++. –  Loki Astari Oct 8 '12 at 16:00
    
@LokiAstari: May I ask, why do you not write C anymore? The topic interests me, as I see multiple different people both criticize C++ for its weaknesses and praise it for its strengths, many of which seem like valid arguments to me. –  voithos Oct 14 '12 at 6:27

1 Answer 1

I've hesitated for quite a while, but decided there are a few points worth commenting on.

The first and most obvious is that (by design) this simply doesn't do much. It can log the types of statements encountered in the JSON you give to it, but that's all. To be of much real use, you'd normally want to create something at least vaguely AST-like, storing the JSON input data as a set of nodes with attributes describing the data in the nodes.

Alternatively, you could build some sort of call-back style framework where the parser called a specific function when each type of input was detected, and it would be up to the client code to decide how to store the data (and which data to store).

As it stands right now, however, the most it can produce is a description of the overall structure of the input file. If that's really all you want, I think it's worth considering whether you're trying to produce output to be read by people or by the machine for further processing.

If you want human-readable output, I think I'd prefer that somewhat more processing be done on the data before it was printed out. For example, as it stands now, an array like [1, 2, 3, 4] would produce a line of output for the JSON array, another for the value-list, and another for each item in the array.

If I'm going to read the file, I think I'd generally prefer those log entries coalesced into something like Array (4 Numbers). Given that the intent is for this to parse JSON, it's probably sufficient specify "JSON" in one place in the output, rather than prefixing every single line of output with "JSON". In short, as it stands right now, this produces output that's extremely verbose, leading to very low information density, so the reader needs to read and digest a great deal of output to understand even a relatively small amount of input--in fact, I'm pretty sure it would usually be easier to read the input file directly.

If your primary intent is to produce output for the computer to read for further processing, it's less necessary to go to extra work to coalesce the information, but still useful to keep the information compact. Given the small number of possibilities in a JSON file, I'd probably assign a single letter to each of the strings the parser can now produce, and just write those out. Alternatively, do some coalescing here as well, so repetitions of the same pattern are signaled by the pattern (in parentheses if it's more than one letter) followed by a number in brackets. For example, a map of 4 string/number pairs followed by an array of 6 numbers might come out something like: M(SN)[4]AN[6]. This is still somewhat human readable (if necessary), and a lot quicker for a parser on the receiving end to sort out (not to mention just being a lot less data to store, transmit, etc.)1

As far as the style of code itself goes, I really have only a few comments, and pretty minor ones at that.

  1. Given a list of alternatives like:

    production : 
               | Alternative1
               | Alternative2
    

    I prefer to add a semicolon to signal the end of the list:

    production : 
               | Alternative1
               | Alternative2
               ;
    
  2. This is at a large scale, and may even be out of your control, but I found this code:

    try
    {
        yyFlexLexer         lexer(&std::cin, &std::cout);
        yy::JsonParser      parser(lexer);
    
        std::cout << (parser.parse() == 0 ? "OK" : "FAIL") << "\n";
    }
    catch (std::exception const& e)
    {
        std::cout << "Exception: " << e.what() << "\n";
    }
    

    ...somewhat ugly. Combining return values and exceptions like this, and having to respond to both makes it seem rather...disjointed. I'd rather see one style adopted throughout, so you can depend on failure always being signaled by throwing an exception or else that it's always signaled by the return value. As it stands right now, we not only end up with this code being fairly ugly, but we also end up with rather uneven error reporting--errors reported via exception may have fairly detailed error messages, but those reported via return value all produce an identical (and probably unhelpful) "FAIL".


1. I haven't tried to work through all the details of ensuring that nested structures would remain unambiguous, but at least offhand it doesn't seem terribly difficult.

share|improve this answer

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Not the answer you're looking for? Browse other questions tagged or ask your own question.