2
\$\begingroup\$

I'm learning Rust and have decided to code up a toy YANG parser as an exercise. In case it's relevant, I mainly work in C and Python.

While working on the tokenizer/lexer, I tried to make it Rust-y, but have ended up with some rather clunky-looking match statements. They're of the form:

match fn_which_returns_result_my_type() {
    Err(err_str) => Err(err_str),
    Ok(MyType0(MyArg0)) => Ok(MyType0(MyArg0)), // This seems especially clunky :(
    Ok(MyType1) => Ok(MyType1),
    Ok(MyType2) => Ok(MyType2),
    Ok(_) => unreachable!()                     // Catch all other instances of the MyType Enum
}

I've highlighted the section I'm interested in feedback for. However, any general tips where I seem to be going down the wrong route (considering this is a work-in-progress) are welcome.

// @@@REMOVE
#![allow(dead_code)]
#![allow(unused_variables)]
#![allow(unused_imports)]

use std::convert::TryFrom;
use std::default::Default;
use std::fs::File;
use std::io::prelude::*;
use std::io::BufReader;
use std::path::Path;

/// This module takes a YANG file and tokenizes it (i.e. splits it into a series of tokens suitable 
/// for the yang_parser module)

static YANG_KEYWORDS: [&str; 86] = [
    ...truncated for clarity...
];

#[derive()]
struct Tokenizer {
    file: BufReader<File>,
}

impl Tokenizer {
    fn new(input: BufReader<File>) -> Self {
        Self { file: input }
    }
}

enum ExpectedToken {
    Keyword,
    ArgumentOrSemicolonOrOpenBrace,
    SemicolonOrOpenBrace,
    KeywordOrCloseBrace,
}

enum ActualToken {
    Keyword(String),
    Argument(String),
    Semicolon,
    OpenBrace,
    CloseBrace,
    EOF,
}

#[derive()]
pub struct YangTokens {
    tokens: Vec<ActualToken>,
}

impl YangTokens {
    fn get_keyword(&self, cursor: &mut Tokenizer) -> Result<ActualToken, &'static str> {
        todo!()
    }

    fn get_argument_or_semicolon_or_open_brace(
        &self,
        cursor: &mut Tokenizer,
    ) -> Result<ActualToken, &'static str> {
        todo!()
    }

    fn get_semicolon_or_close_brace(
        &self,
        cursor: &mut Tokenizer,
    ) -> Result<ActualToken, &'static str> {
        todo!()
    }

    fn get_keyword_or_close_brace(
        &self,
        cursor: &mut Tokenizer,
    ) -> Result<ActualToken, &'static str> {
        todo!()
    }

    // ==============================================================================
    // THIS IS WHERE I WANT SPECIFIC FEEDBACK
    fn get_next_token(
        &self,
        cursor: &mut Tokenizer,
        next_exp_token: &mut ExpectedToken,
    ) -> Result<ActualToken, &'static str> {
        match next_exp_token {
            ExpectedToken::Keyword => {
                *next_exp_token = ExpectedToken::ArgumentOrSemicolonOrOpenBrace;
                match self.get_keyword(cursor) {
                    Err(err_str) => Err(err_str),
                    Ok(ActualToken::Keyword(kw)) => Ok(ActualToken::Keyword(kw)),
                    Ok(ActualToken::EOF) => Ok(ActualToken::EOF),
                    Ok(_) => unreachable!(),
                }
            }
            ExpectedToken::ArgumentOrSemicolonOrOpenBrace => {
                match self.get_argument_or_semicolon_or_open_brace(cursor) {
                    Err(err_str) => Err(err_str),
                    Ok(ActualToken::Argument(arg)) => {
                        *next_exp_token = ExpectedToken::SemicolonOrOpenBrace;
                        Ok(ActualToken::Argument(arg))
                    },
                    Ok(ActualToken::Semicolon) => {
                        *next_exp_token = ExpectedToken::KeywordOrCloseBrace;
                        Ok(ActualToken::Semicolon)
                    },
                    Ok(ActualToken::OpenBrace) => {
                        *next_exp_token = ExpectedToken::KeywordOrCloseBrace;
                        Ok(ActualToken::OpenBrace)
                    },
                    Ok(_) => unreachable!(),
                }
            }
            ExpectedToken::SemicolonOrOpenBrace => {
                *next_exp_token = ExpectedToken::KeywordOrCloseBrace;
                match self.get_semicolon_or_close_brace(cursor) {
                    Err(err_str) => Err(err_str),
                    Ok(ActualToken::Semicolon) => Ok(ActualToken::Semicolon),
                    Ok(ActualToken::CloseBrace) => Ok(ActualToken::CloseBrace),
                    Ok(_) => unreachable!(),

                }
            }
            ExpectedToken::KeywordOrCloseBrace => {
                *next_exp_token = ExpectedToken::ArgumentOrSemicolonOrOpenBrace;
                match self.get_keyword_or_close_brace(cursor) {
                    Err(err_str) => Err(err_str),
                    Ok(ActualToken::Keyword(kw)) => Ok(ActualToken::Keyword(kw)),
                    Ok(ActualToken::EOF) => Ok(ActualToken::EOF),
                    Ok(ActualToken::CloseBrace) => Ok(ActualToken::CloseBrace),
                    Ok(_) => unreachable!(),
                }
            }
        }
    }
    // END OF WHERE I WANT SPECIFIC FEEDBACK
    // ==============================================================================
}

impl TryFrom<&mut Tokenizer> for YangTokens {
    type Error = &'static str;

    fn try_from(cursor: &mut Tokenizer) -> Result<Self, Self::Error> {
        let mut next_exp_token = ExpectedToken::Keyword;

        todo!()
    }
}

pub fn tokenize_yang_file(filename: &Path) -> Result<YangTokens, &str> {
    let file = File::open(&filename).unwrap(); //@@@ THIS SHOULDN'T PANIC
    let mut file = BufReader::new(file);
    let mut cursor = Tokenizer::new(file);

    YangTokens::try_from(&mut cursor)
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn new() {
        todo!()
    }
}
\$\endgroup\$
2
  • \$\begingroup\$ One thing I've noticed immediately upon posting it is that the responsibilities feel wrong - I don't think I need my Tokenizer, or it should be the one with all the impl functions. \$\endgroup\$ – Kyle_S-C Aug 23 '20 at 15:41
  • \$\begingroup\$ One of my colleagues has reminded me of the @ syntax in pattern matching (which I'd read about in the Rust book). Will refactor accordingly when I'm done at work and post a sample answer in case people are interested - other improvements definitely welcome though! \$\endgroup\$ – Kyle_S-C Aug 24 '20 at 14:22
0
\$\begingroup\$

For the function in question, the key thing was to remember @ bindings from the Rust book section 18.3.

They allow you to bind the matched value to a variable for use in the match arm.

    fn get_next_token(
        &self,
        cursor: &mut Tokenizer,
        next_exp_token: &mut ExpectedToken,
    ) -> Result<ActualToken, &'static str> {
        match next_exp_token {
            ExpectedToken::Keyword => {
                *next_exp_token = ExpectedToken::ArgumentOrSemicolonOrOpenBrace;
                match self.get_keyword(cursor) {
                    val @ Err(_) => val,
                    val @ Ok(ActualToken::Keyword(_)) => val,
                    val @ Ok(ActualToken::EOF) => val,
                    Ok(_) => unreachable!(),
                }
            }
            ExpectedToken::ArgumentOrSemicolonOrOpenBrace => {
                match self.get_argument_or_semicolon_or_open_brace(cursor) {
                    val @ Err(_) => val,
                    val @ Ok(ActualToken::Argument(_)) => {
                        *next_exp_token = ExpectedToken::SemicolonOrOpenBrace;
                        val
                    },
                    val @ Ok(ActualToken::Semicolon) => {
                        *next_exp_token = ExpectedToken::KeywordOrCloseBrace;
                        val
                    },
                    val @ Ok(ActualToken::OpenBrace) => {
                        *next_exp_token = ExpectedToken::KeywordOrCloseBrace;
                        val
                    },
                    Ok(_) => unreachable!(),
                }
            }
            ExpectedToken::SemicolonOrOpenBrace => {
                *next_exp_token = ExpectedToken::KeywordOrCloseBrace;
                match self.get_semicolon_or_close_brace(cursor) {
                    val @ Err(_) => val,
                    val @ Ok(ActualToken::Semicolon) => val,
                    val @ Ok(ActualToken::CloseBrace) => val,
                    Ok(_) => unreachable!(),

                }
            }
            ExpectedToken::KeywordOrCloseBrace => {
                *next_exp_token = ExpectedToken::ArgumentOrSemicolonOrOpenBrace;
                match self.get_keyword_or_close_brace(cursor) {
                    val @ Err(_) => val,
                    val @ Ok(ActualToken::Keyword(_)) => val,
                    val @ Ok(ActualToken::EOF) => val,
                    val @ Ok(ActualToken::CloseBrace) => val,
                    Ok(_) => unreachable!(),
                }
            }
        }
    }
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.