sbrl
/
MarkovGrams
1
0
Fork 0
Code Issues Pull Requests Activity

Refactor CLI to make it much nicer, and add some features. Also, bugfix the weighted markov chain

Browse Source
master
Starbeamrainbowlabs 5 years ago
parent e7f67d7f5e
commit 0aab41f1b7
Signed by: sbrl
GPG Key ID: 1BE5172E637709C2
8 changed files with 181 additions and 108 deletions
Show Stats Download Patch File Download Diff File

1
MarkovGrams/MarkovGrams.csproj
Unescape View File

@ -39,6 +39,7 @@
<Compile Include="Utilities\WeightedRandom.cs" />
<Compile Include="WeightedMarkovChain.cs" />
<Compile Include="Utilities\LinqExtensions.cs" />
<Compile Include="Utilities\StreamReaderExtensions.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="Utilities\" />

7
MarkovGrams/NGrams.cs
Unescape View File

@ -15,14 +15,15 @@ namespace MarkovGrams
/// <param name="words">The words to turn into n-grams.</param>
/// <param name="order">The order of n-gram to generate..</param>
/// <returns>A unique list of n-grams found in the given list of words.</returns>
public static IEnumerable<string> GenerateFlat(IEnumerable<string> words, int order)
public static IEnumerable<string> GenerateFlat(IEnumerable<string> words, int order, bool distinct = true)
{
List<string> results = new List<string>();
foreach(string word in words)
foreach (string word in words)
{
results.AddRange(GenerateFlat(word, order));
}
return results.Distinct();
if (distinct) return results.Distinct();
return results;
}
/// <summary>

194
MarkovGrams/Program.cs
Unescape View File

@ -3,122 +3,138 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using MarkovGrams.Utilities;
namespace MarkovGrams
{
public enum Mode
{
Help,
NGrams,
Markov,
WeightedMarkov
}
class MainClass
{
public static int Main(string[] args)
{
if(args.Length < 1)
List<string> extras = new List<string>();
StreamReader wordlistSource = new StreamReader(Console.OpenStandardInput());
int order = 3, length = 8, count = 10;
bool splitOnWhitespace = true,
ngramsUnique = true,
convertLowercase = false,
startOnUppercase = false;
for (int i = 0; i < args.Length; i++)
{
Console.WriteLine("Usage:");
Console.WriteLine(" ./MarkovGrams.exe <command>");
Console.WriteLine();
Console.WriteLine("Available commands:");
Console.WriteLine(" markov:");
Console.WriteLine(" Generate new words using an unweighted markov chain.");
Console.WriteLine(" markov-w:");
Console.WriteLine(" Generate new words using a weighted markov chain.");
Console.WriteLine(" ngrams:");
Console.WriteLine(" Generate raw unique n-grams");
Console.WriteLine();
Console.WriteLine("Type just ./MarovGrams.exe <command> to see command-specific help.");
return 1;
if (!args[i].StartsWith("-"))
{
extras.Add(args[i]);
continue;
}
switch (args[i].TrimStart("-".ToCharArray()))
{
case "wordlist":
wordlistSource = new StreamReader(args[++i]);
break;
case "order":
order = int.Parse(args[++i]);
break;
case "length":
length = int.Parse(args[++i]);
break;
case "count":
count = int.Parse(args[++i]);
break;
case "no-split":
splitOnWhitespace = false;
break;
case "no-unique":
ngramsUnique = false;
break;
case "lowercase":
convertLowercase = true;
break;
case "start-uppercase":
startOnUppercase = true;
break;
default:
Console.Error.WriteLine($"Error: Unknown option '{args[i]}'.");
return 1;
}
}
string mode = args[0];
string wordlistFilename;
int order;
IEnumerable<string> words, ngrams;
Mode mode = extras.Count > 0 ? (Mode)Enum.Parse(typeof(Mode), extras.ShiftAt(0), true) : Mode.Help;
switch(mode)
{
case "markov":
if(args.Length != 5)
{
Console.WriteLine("markov command usage:");
Console.WriteLine(" ./MarkovGrams.exe markov <wordlist.txt> <order> <length> <count>");
Console.WriteLine();
Console.WriteLine("<wordlist.txt> The path to the wordlist to read from.");
Console.WriteLine("<order> The order of the n-grams to use.");
Console.WriteLine("<length> The length of word to generate.");
Console.WriteLine("<count> The number of words to generate.");
Console.WriteLine();
return 1;
}
wordlistFilename = args[1];
order = int.Parse(args[2]);
int desiredStringLength = int.Parse(args[3]);
int count = int.Parse(args[4]);
words = File.ReadLines(wordlistFilename).SelectMany(word => word.Trim().Split(' '));
ngrams = NGrams.GenerateFlat(words, order);
// ------------------------------------------------------------------------------------------
IEnumerable<string> words = wordlistSource.ReadAllLines().SelectMany((string word) => {
word = word.Trim();
if (convertLowercase)
word = word.ToLower();
if (splitOnWhitespace)
return word.Split(' ');
return new string[] { word.Trim() };
});
switch (mode)
{
case Mode.Markov:
Stopwatch utimer = Stopwatch.StartNew();
UnweightedMarkovChain chain = new UnweightedMarkovChain(ngrams);
UnweightedMarkovChain unweightedChain = new UnweightedMarkovChain(
NGrams.GenerateFlat(words, order)
);
unweightedChain.StartOnUppercase = startOnUppercase;
for(int i = 0; i < count; i++)
Console.WriteLine(chain.Generate(desiredStringLength));
for (int i = 0; i < count; i++)
Console.WriteLine(unweightedChain.Generate(length));
Console.Error.WriteLine($"{count} words in {utimer.ElapsedMilliseconds}ms");
break;
case "markov-w":
if (args.Length != 5)
{
Console.WriteLine("markov-w command usage:");
Console.WriteLine(" ./MarkovGrams.exe markov-w <wordlist.txt> <order> <length> <count>");
Console.WriteLine();
Console.WriteLine("<wordlist.txt> The path to the wordlist to read from.");
Console.WriteLine("<order> The order of the n-grams to use.");
Console.WriteLine("<length> The length of word to generate.");
Console.WriteLine("<count> The number of words to generate.");
Console.WriteLine();
return 1;
}
wordlistFilename = args[1];
order = int.Parse(args[2]);
int weightedDesiredStringLength = int.Parse(args[3]);
int weightedCount = int.Parse(args[4]);
words = File.ReadLines(wordlistFilename).SelectMany(word => word.Trim().Split(' '));
ngrams = NGrams.GenerateFlat(words, order);
case Mode.WeightedMarkov:
Stopwatch wtimer = Stopwatch.StartNew();
WeightedMarkovChain weightedChain = new WeightedMarkovChain(ngrams);
WeightedMarkovChain weightedChain = new WeightedMarkovChain(
NGrams.GenerateWeighted(words, order)
);
weightedChain.StartOnUppercase = startOnUppercase;
for (int i = 0; i < weightedCount; i++)
Console.WriteLine(weightedChain.Generate(weightedDesiredStringLength));
Console.Error.WriteLine($"{weightedCount} words in {wtimer.ElapsedMilliseconds}ms");
for (int i = 0; i < count; i++)
Console.WriteLine(weightedChain.Generate(length));
Console.Error.WriteLine($"{count} words in {wtimer.ElapsedMilliseconds}ms");
break;
case "ngrams":
if(args.Length != 3)
{
Console.WriteLine("ngrams command usage:");
Console.WriteLine(" ./MarkovGrams.exe <wordlist.txt> <order>");
Console.WriteLine();
Console.WriteLine("<wordlist.txt> The path to the wordlist to read from.");
Console.WriteLine("<order> The order of n-grams to generate.");
Console.WriteLine();
return 1;
}
wordlistFilename = args[1];
order = int.Parse(args[2]);
words = File.ReadLines(wordlistFilename).SelectMany(word => word.Trim().Split(' '));
ngrams = NGrams.GenerateFlat(words, order);
foreach(string ngram in ngrams)
case Mode.NGrams:
foreach (string ngram in NGrams.GenerateFlat(words, order, ngramsUnique))
Console.WriteLine(ngram);
break;
case Mode.Help:
default:
Console.WriteLine("Unknown command {0}.");
Console.WriteLine("Available commands:");
Console.WriteLine(" markov Generate words with a markov chain");
Console.WriteLine(" ngrams Generate unique ngrams from wordlists");
Console.WriteLine("Usage:");
Console.WriteLine(" ./MarkovGrams.exe <mode> [options]");
Console.WriteLine();
Console.WriteLine("Available modes:");
Console.WriteLine(" markov:");
Console.WriteLine(" Generate new words using an unweighted markov chain.");
Console.WriteLine(" markov-w:");
Console.WriteLine(" Generate new words using a weighted markov chain.");
Console.WriteLine(" ngrams:");
Console.WriteLine(" Generate raw unique n-grams");
Console.WriteLine();
Console.WriteLine("Available options:");
Console.WriteLine(" --wordlist {filename} Read the wordlist from the specified filename instead of stdin");
Console.WriteLine(" --order {number} Use the specified order when generating n-grams (default: 3)");
Console.WriteLine(" --length {number} The target length of word to generate (Not available in ngrams mode)");
Console.WriteLine(" --count {number} The number of words to generate (Not valid in ngrams mode)");
Console.WriteLine(" --no-split Don't split input words on whitespace - treat each line as a single word");
Console.WriteLine(" --lowercase Convert the input to lowercase before processing");
Console.WriteLine(" --start-uppercase Start the generating a word only with n-grams that start with a capital letter");
Console.WriteLine(" --no-unique Don't remove duplicates from the list of ngrams (Only valid in ngrams mode)");
Console.WriteLine("Type just ./MarkovGrams.exe <mode> to see mode-specific help.");
return 1;
}

15
MarkovGrams/UnweightedMarkovChain.cs
Unescape View File

@ -12,12 +12,18 @@ namespace MarkovGrams
/// <summary>
/// The random number generator
/// </summary>
Random rand = new Random();
private Random rand = new Random();
/// <summary>
/// The ngrams that this markov chain currently contains.
/// </summary>
List<string> ngrams;
private List<string> ngrams;
/// <summary>
/// Whether to always start generating a new word from an n-gram that starts with
/// an uppercase letter.
/// </summary>
public bool StartOnUppercase = false;
/// <summary>
/// Creates a new character-based markov chain.
@ -34,7 +40,10 @@ namespace MarkovGrams
/// <returns>A random ngram from this UnweightMarkovChain's cache of ngrams.</returns>
public string RandomNgram()
{
return ngrams[rand.Next(0, ngrams.Count)];
IEnumerable<string> validNGrams = StartOnUppercase ? ngrams.Where((ngram) => char.IsUpper(ngram[0])) : ngrams;
if (validNGrams.Count() == 0)
throw new Exception($"Error: No valid starting ngrams were found (StartOnUppercase: {StartOnUppercase}).");
return validNGrams.ElementAt(rand.Next(0, validNGrams.Count()));
}
/// <summary>

8
MarkovGrams/Utilities/LinqExtensions.cs
Unescape View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace MarkovGrams.Utilities
{
@ -12,5 +13,12 @@ namespace MarkovGrams.Utilities
action(item);
}
}
public static T ShiftAt<T>(this List<T> list, int index)
{
T item = list[index];
list.RemoveAt(index);
return item;
}
}
}

19
MarkovGrams/Utilities/StreamReaderExtensions.cs
Unescape View File

@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace MarkovGrams.Utilities
{
public static class StreamReaderExtensions
{
public static IEnumerable<string> ReadAllLines(this StreamReader streamReader)
{
string line;
while ((line = streamReader.ReadLine()) != null) {
yield return line;
}
}
}
}

5
MarkovGrams/Utilities/WeightedRandom.cs
Unescape View File

@ -16,6 +16,11 @@ namespace SBRL.Algorithms
/// <changelog>
/// v0.1 - 20th May 2017:
/// - Creation! :D
/// v0.2 - 17th Februrary 2018:
/// - Add Count property
/// - Add SetContents and ClearContents methods
/// - Add empty constructor
/// - Next() will now throw an InvalidOperationException if the generator's internal weights list is empty
/// </changelog>
public class WeightedRandom<ItemType>
{

40
MarkovGrams/WeightedMarkovChain.cs
Unescape View File

@ -16,32 +16,46 @@ namespace MarkovGrams
/// <summary>
/// The ngrams that this markov chain currently contains.
/// </summary>
Dictionary<string, double> ngrams;
private Dictionary<string, double> ngrams;
/// <summary>
/// Whether to always start generating a new word from an n-gram that starts with
/// an uppercase letter.
/// </summary>
public bool StartOnUppercase = false;
/// <summary>
/// Creates a new character-based markov chain.
/// </summary>
/// <param name="inNgrams">The ngrams to populate the new markov chain with.</param>
public WeightedMarkovChain(IEnumerable<string> inNgrams)
{
public WeightedMarkovChain(Dictionary<string, double> inNgrams) {
ngrams = inNgrams;
}
public WeightedMarkovChain(Dictionary<string, int> inNgrams) {
ngrams = new Dictionary<string, double>();
foreach (string ngram in inNgrams)
{
if (ngrams.ContainsKey(ngram))
ngrams[ngram]++;
else
ngrams.Add(ngram, 1);
}
foreach (KeyValuePair<string, int> ngram in inNgrams)
ngrams[ngram.Key] = ngram.Value;
}
/// <summary>
/// Returns a random ngram that's currently loaded into this WeightedMarkovChain.
/// </summary>
/// <returns>A random ngram from this UnweightMarkovChain's cache of ngrams.</returns>
/// <returns>A random ngram from this UnweightedMarkovChain's cache of ngrams.</returns>
public string RandomNgram()
{
if (wrandom.Count == 0)
wrandom.SetContents(ngrams);
if (wrandom.Count == 0) {
if (!StartOnUppercase)
wrandom.SetContents(ngrams);
else {
Dictionary<string, double> filteredNGrams = new Dictionary<string, double>();
foreach (KeyValuePair<string, double> pair in ngrams.Where((pair) => char.IsUpper(pair.Key[0])))
filteredNGrams.Add(pair.Key, pair.Value);
if (filteredNGrams.Count() == 0)
throw new Exception($"Error: No valid starting ngrams were found (StartOnUppercase: {StartOnUppercase}).");
wrandom.SetContents(filteredNGrams);
}
}
return wrandom.Next();
}

Write Preview
Loading…
Cancel
Save