Please review the following Clojure tokenizer. The goal is to study Clojure, so sometimes I re-implement functions (it will be nice to see standard functions for this). Beyond of this, code seems quite cumbersome.
(defn space? [^Character c]
(Character/isWhitespace c))
(defn digit? [^Character c]
(= (Character/getType c)
(Character/DECIMAL_DIGIT_NUMBER)))
(defn to-int [ds]
(reduce #(+ (* 10 %1) %2) (map #(- (int %) (int \0)) ds)))
(defn take-int [s]
(when (digit? (first s))
(let [[digits rs] (split-with digit? s)]
[rs {:int (to-int digits)}])))
(defn take-sym [s]
(case (first s)
\+ [(rest s) {:sym :plus}]
\- [(rest s) {:sym :minus}]
\* [(rest s) {:sym :star}]
\/ [(rest s) {:sym :slash}]
nil))
(defn take-error [msg]
[nil {:error msg}])
(defn tokens [expr]
(loop [ex expr, ts []]
(let [nsp (drop-while space? ex)]
(if (empty? nsp)
(conj ts {:eof nil})
(if-let [[rs t] (or (take-int nsp)
(take-sym nsp)
(take-error (str "Invalid symbol " (first nsp))))]
(if (:error t)
t
(recur rs (conj ts t))))))))
(defn -main [& args]
; Work around dangerous default behavior in Clojure.
(alter-var-root #'*read-eval* (constantly false))
(println (tokens (seq " 20 + 32 * 4 ")))
)