Browse Source

implemented text generation with markov chains, need to thing something to avoid sentence loops

pull/1/head
arnaucode 7 years ago
parent
commit
1475bb015c
4 changed files with 20648 additions and 11 deletions
  1. +1
    -0
      .gitignore
  2. +21
    -2
      main.go
  3. +16
    -5
      markov.go
  4. +20610
    -4
      text.txt

+ 1
- 0
.gitignore

@ -27,3 +27,4 @@ _testmain.go
flockConfig.json flockConfig.json
build/flockConfig.json build/flockConfig.json
temp temp
text.txt

+ 21
- 2
main.go

@ -4,6 +4,7 @@ import (
"bufio" "bufio"
"fmt" "fmt"
"os" "os"
"strconv"
"strings" "strings"
) )
@ -39,10 +40,28 @@ option to select: `
break break
case "2": case "2":
fmt.Println("selected 2 - Markov") fmt.Println("selected 2 - Markov")
fmt.Print("entry the first word: ")
newcommand := bufio.NewReader(os.Stdin)
firstWord, _ := newcommand.ReadString('\n')
firstWord = strings.TrimSpace(firstWord)
fmt.Print("first word: ")
c.Purple(firstWord)
c.Red("how many words you want on the text?")
newcommand = bufio.NewReader(os.Stdin)
answer, _ := newcommand.ReadString('\n')
answer = strings.TrimSpace(answer)
fmt.Print("Number of words on text to generate: ")
c.Purple(answer)
count, err := strconv.Atoi(answer)
if err != nil {
fmt.Println("incorrect entry, need a positive number")
}
states := markov.train("the", "text.txt") states := markov.train("the", "text.txt")
generatedText := markov.generateText(states, "Argus", 20)
generatedText := markov.generateText(states, firstWord, count)
c.Green(generatedText) c.Green(generatedText)
//fmt.Println(text)
break break
case "0": case "0":
fmt.Println("selected 0 - exit script") fmt.Println("selected 0 - exit script")

+ 16
- 5
markov.go

@ -63,7 +63,7 @@ func calcMarkovStates(words []string) []State {
states[i].NextStates[j].Prob = (float64(states[i].NextStates[j].Count) / float64(len(words)) * 100) states[i].NextStates[j].Prob = (float64(states[i].NextStates[j].Count) / float64(len(words)) * 100)
} }
} }
fmt.Println("total words: " + strconv.Itoa(len(words)))
fmt.Println("total words computed: " + strconv.Itoa(len(words)))
//fmt.Println(states) //fmt.Println(states)
return states return states
} }
@ -79,7 +79,8 @@ func readText(path string) (string, error) {
if err != nil { if err != nil {
//Do something //Do something
} }
content := string(data)
dataClean := strings.Replace(string(data), "\n", " ", -1)
content := string(dataClean)
return content, err return content, err
} }
@ -102,15 +103,25 @@ func getNextMarkovState(states []State, word string) string {
if iState < 0 { if iState < 0 {
return "word no exist on the memory" return "word no exist on the memory"
} }
fmt.Println(rand.Float64())
return word
var next State
next = states[iState].NextStates[0]
next.Prob = rand.Float64() * states[iState].Prob
for i := 0; i < len(states[iState].NextStates); i++ {
if (rand.Float64()*states[iState].NextStates[i].Prob) > next.Prob && states[iState-1].Word != states[iState].NextStates[i].Word {
next = states[iState].NextStates[i]
}
}
return next.Word
} }
func (markov Markov) generateText(states []State, initWord string, count int) string { func (markov Markov) generateText(states []State, initWord string, count int) string {
var generatedText []string var generatedText []string
word := initWord word := initWord
generatedText = append(generatedText, word)
for i := 0; i < count; i++ { for i := 0; i < count; i++ {
word = getNextMarkovState(states, word) word = getNextMarkovState(states, word)
generatedText = append(generatedText, word) generatedText = append(generatedText, word)
} }
return "a"
generatedText = append(generatedText, ".")
text := strings.Join(generatedText, " ")
return text
} }

+ 20610
- 4
text.txt
File diff suppressed because it is too large
View File


Loading…
Cancel
Save