1
\$\begingroup\$

Program generate sentences using Markov model and 3 text files

Text file example:

Antoine de Saint-Exupery, who was a French author, journalist and pilot wrote The Little Prince in 1943, one year before his death.

The Little Prince appears to be a simple children’s tale, some would say that it is actually a profound and deeply moving tale, written in riddles and laced with philosophy and poetic metaphor.

... ... ... ... ... ...

Input params: Model order: 2, sentence size: 4 steps

Example output:

we ought to get my single lamp at the latest the nest of a dying ...

I don't think it's good, help me improve code

Description:

Date.prototype.timeNow - just for debug

Dict - class for building words in Markov model Markov - class from building Markov model

_makeMarkovModel - build Markov model from array of words in order like in original text (This is cat -> ['This', 'is', 'cat'])

_makeMarkovModelFromJSON - build modle from json object

_prepareData - transfer plain text to array of words (This is cat -> ['This', 'is', 'cat'])

_getModelFromJSON - generate model from json file

_genJSONModelFile - generate json model file

_genJSONFromFileList - generate json models from file list

_concatModel - model + model = new model

buildModelFromFileList - generate model from files list and also create json version of them

const fs = require('fs')
const readFile = require('util').promisify(fs.readFile)

const nlp = require('compromise')
const _ = require('lodash')

// For DEBUG
Date.prototype.timeNow = function() {
  return (
    (this.getHours() < 10 ? '0' : '') +
    this.getHours() +
    ':' +
    (this.getMinutes() < 10 ? '0' : '') +
    this.getMinutes() +
    ':' +
    (this.getSeconds() < 10 ? '0' : '') +
    this.getSeconds()
  )
}

class Dict {
  constructor(data, dict = {}, tokens = 0, types = 0) {
    this.tokens = tokens
    this.types = types
    this.dict = dict

    this.update(data)
  }

  update(iterable) {
    iterable.map(item => {
      if (item in this.dict) {
        this.dict[item]++
        this.tokens++
      } else {
        this.dict[item] = 1
        this.tokens++
        this.types++
      }
    })
  }

  getRandom() {
    return _.sample(Object.keys(this.dict))
  }

  toObject() {
    return {
      tokens: this.tokens,
      types: this.types,
      dict: this.dict
    }
  }

  addDict(object) {
    const Dict = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
    const keys = Object.keys(object.dict)

    //console.log(this.dict)

    keys.map(key => {
      if (key in this.dict) {
        Dict.dict[key] += object.dict[key]
        Dict.tokens += object.dict[key]
      } else {
        Dict.dict[key] = object.dict[key]
        Dict.tokens += object.dict[key]
        Dict.types++
      }
    })

    //console.log(this.dict)

    return Dict
  }
}

class Markov {
  constructor(order = 1) {
    this.order = order
    this.model = {}
  }

  _prepareData(data) {
    let result = []

    nlp(data.toString())
      .normalize()
      .sentences()
      .out('array')
      .map(line => {
        result = [
          ...result,
          ...nlp(line)
            .terms()
            .out('array')
        ]
      })
    return result
  }

  _modelToJSON(model) {
    let copyModel = {}

    for (let key in model) {
      copyModel[key] = model[key].toObject()
    }
    console.log(`@_modelToJSON MODEL OBJECT CREATED ${new Date().timeNow()}`)
    return JSON.stringify(copyModel)
  }

  _makeMarkovModel(data) {
    const markovModel = {}

    for (let i = 0; i < data.length - this.order; i++) {
      let window = data.slice(i, i + this.order).join(' ')

      if (window in markovModel) {
        markovModel[window].update([data[i + this.order]])
      } else {
        markovModel[window] = new Dict([data[i + this.order]])
      }
    }

    return markovModel
  }

  _makeMarkovModelFromJSON(object) {
    const markovModel = Object.assign({}, object)

    for (let key in markovModel) {
      markovModel[key] = new Dict(
        [],
        markovModel[key].dict,
        markovModel[key].tokens,
        markovModel[key].types
      )
    }

    return markovModel
  }

  _getModelFromJSON(file) {
    return new Promise((resolve, reject) => {
      readFile(`${__dirname}${file}-${this.order}-.json`).then(data => {
        resolve(this._makeMarkovModelFromJSON(JSON.parse(data.toString())))
      })
    })
  }

  _genJSONModelFile(file) {
    return new Promise((resolve, reject) => {
      console.log(
        `@_genJSONModelFile IN GEN MODEL JSON ${new Date().timeNow()} ${file} ${
          this.order
        }`
      )

      readFile(`${__dirname}${file}`).then(data => {
        console.log(
          `@_genJSONModelFile FILE LOADED ${new Date().timeNow()} ${file} ${
            this.order
          }`
        )

        let model = this._makeMarkovModel(this._prepareData(data))

        console.log(
          `@_genJSONModelFile MODEL CREATED ${new Date().timeNow()} ${file} ${
            this.order
          }`
        )

        fs.writeFile(
          `${__dirname}${file}-${this.order}-.json`,
          this._modelToJSON(model),
          function(err) {
            if (err) reject(err)
            else resolve(data)
          }
        )
      })
    })
  }

  _genJSONFromFileList(list) {
    list = list.filter(
      file => !fs.existsSync(`${__dirname}${file}-${this.order}-.json`)
    )

    return new Promise((resolve, reject) => {
      Promise.all(list.map(file => this._genJSONModelFile(file))).then(data => {
        resolve(data)
      })
    })
  }

  _concatModel(firstModel, secondModel) {
    const firstModelKeys = Object.keys(firstModel)
    const secondModelKeys = Object.keys(secondModel)
    const resultModel = {}

    for (let i = 0; i < firstModelKeys.length; i++) {
      if (secondModelKeys.includes(firstModelKeys[i])) {
        resultModel[firstModelKeys[i]] = firstModel[firstModelKeys[i]].addDict(
          secondModel[firstModelKeys[i]]
        )
      } else {
        resultModel[firstModelKeys[i]] = firstModel[firstModelKeys[i]]
      }
    }

    for (let i = 0; i < secondModelKeys.length; i++) {
      if (!firstModelKeys.includes(secondModelKeys[i])) {
        resultModel[secondModelKeys[i]] = secondModel[secondModelKeys[i]]
      }
    }

    return resultModel
  }

  buildModelFromFileList(list) {
    return new Promise((resolve, reject) => {
      this._genJSONFromFileList(list).then(data => {
        Promise.all(list.map(element => this._getModelFromJSON(element))).then(
          values => {
            values.map(value => {
              this.model = this._concatModel(this.model, value)
            })

            resolve(this.model)
          }
        )
      })
    })
  }

  buildRandomSentence(model, steps) {
    let current = _.sample(Object.keys(model))
    const result = [current]

    for (let i = 0; i < steps - 1; i++) {
      let next = model[current].getRandom()

      if (next === undefined) {
        break
      }

      result.push(next)

      current = [...current.split(' '), next]

      current = current
        .slice(current.length - this.order, current.length)
        .join(' ')
    }

    return result.join(' ')
  }
}

const modelBuilder = new Markov(2)
const files = ['/data/1.txt', '/data/2.txt', '/data/3.txt']

modelBuilder.buildModelFromFileList(files).then(model => {
  console.log(modelBuilder.buildRandomSentence(model, 4))
})
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.