From face6935549bd40f2938133b4e3089a14c61b69a Mon Sep 17 00:00:00 2001
From: Starbeamrainbowlabs <sbrl@starbeamrainbowlabs.com>
Date: Mon, 3 Sep 2018 15:48:27 +0100
Subject: [PATCH] Add initial word-level support

---
 MarkovGrams/NGrams.cs                | 39 ++++++++++++++++++++--------
 MarkovGrams/Program.cs               |  5 ++--
 MarkovGrams/UnweightedMarkovChain.cs |  2 +-
 MarkovGrams/WeightedMarkovChain.cs   |  2 +-
 4 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/MarkovGrams/NGrams.cs b/MarkovGrams/NGrams.cs
index 2c7bba5..6f2807c 100644
--- a/MarkovGrams/NGrams.cs
+++ b/MarkovGrams/NGrams.cs
@@ -15,6 +15,8 @@ namespace MarkovGrams
 	/// </summary>
 	public static class NGrams
 	{
+		public static bool Verbose { get; set; } = true;
+
 		/// <summary>
 		/// Generates a unique list of n-grams that the given list of words.
 		/// </summary>
@@ -38,7 +40,7 @@ namespace MarkovGrams
 		/// <param name="str">The string to n-gram-ise.</param>
 		/// <param name="order">The order of n-gram to generate.</param>
 		/// <returns>A unique list of n-grams found in the specified string.</returns>
-		public static IEnumerable<string> GenerateFlat(string str, int order, GenerationMode mode)
+		private static IEnumerable<string> GenerateFlat(string str, int order, GenerationMode mode)
 		{
 			List<string> results = new List<string>();
 			if (mode == GenerationMode.CharacterLevel) {
@@ -62,9 +64,15 @@ namespace MarkovGrams
 		/// <returns>The weighted dictionary of ngrams.</returns>
 		public static Dictionary<string, int> GenerateWeighted(IEnumerable<string> words, int order, GenerationMode mode)
 		{
+			List<string> wordList = new List<string>(words);
+			int wordCount = wordList.Count();
 			Dictionary<string, int> results = new Dictionary<string, int>();
-			foreach(string word in words)
-				GenerateWeighted(word, order, mode);
+			int i = 0;
+			foreach (string word in wordList) {
+				GenerateWeighted(word, order, mode, ref results);
+				i++;
+			}
+			Console.WriteLine(" - done");
 			return results;
 		}
 		/// <summary>
@@ -73,15 +81,24 @@ namespace MarkovGrams
 		/// <param name="str">The string to n-gram-ise.</param>
 		/// <param name="order">The order of n-grams to generate.</param>
 		/// <returns>The weighted dictionary of ngrams.</returns>
-		public static void GenerateWeighted(string str, int order, GenerationMode mode, Dictionary<string, int> results)
+		private static void GenerateWeighted(string str, int order, GenerationMode mode, ref Dictionary<string, int> results)
 		{
-			string[] parts = mode == GenerationMode.WordLevel ? str.Split(" ".ToCharArray()) : null;
-			for(int i = 0; i < str.Length - order; i++)
-			{
-				string ngram = mode == GenerationMode.CharacterLevel ? str.Substring(i, order) : string.Join(" ", parts.Skip(i).Take(order));
-				if(!results.ContainsKey(ngram))
-					results[ngram] = 0;
-				results[ngram]++;
+			if (mode == GenerationMode.CharacterLevel) {
+				for (int i = 0; i < str.Length - order; i++) {
+					string ngram = str.Substring(i, order);
+					if (!results.ContainsKey(ngram))
+						results[ngram] = 0;
+					results[ngram]++;
+				}
+			}
+			else {
+				string[] parts = str.Split(" ".ToCharArray());
+				for (int i = 0; i < parts.Length - order; i++) {
+					string ngram = string.Join(" ", parts.Skip(i).Take(order));
+					if (!results.ContainsKey(ngram))
+						results[ngram] = 0;
+					results[ngram]++;
+				}
 			}
 		}
 	}
diff --git a/MarkovGrams/Program.cs b/MarkovGrams/Program.cs
index 405b36a..b7d7871 100644
--- a/MarkovGrams/Program.cs
+++ b/MarkovGrams/Program.cs
@@ -65,6 +65,7 @@ namespace MarkovGrams
 						break;
 					case "words":
 						generationMode = GenerationMode.WordLevel;
+						splitOnWhitespace = false;
 						break;
 					case "help":
 						operationMode = Mode.Help;
@@ -141,9 +142,9 @@ namespace MarkovGrams
 					Console.WriteLine("    --help                  Show this message");
 					Console.WriteLine("    --wordlist {filename}   Read the wordlist from the specified filename instead of stdin");
 					Console.WriteLine("    --order {number}        Use the specified order when generating n-grams (default: 3)");
-					Console.WriteLine("    --length {number}       The target length of word to generate (Not available in ngrams mode)");
+					Console.WriteLine("    --length {number}       The target length of word to generate (Not available in ngrams mode; instead specifies the number of words to generate with --words)");
 					Console.WriteLine("    --count {number}        The number of words to generate (Not valid in ngrams mode)");
-					Console.WriteLine("    --words                 Generate ngrams on word-level instead of character-level (Applies to all modes)");
+					Console.WriteLine("    --words                 Generate ngrams on word-level instead of character-level (Applies to all modes; implies --no-split)");
 					Console.WriteLine("    --no-split              Don't split input words on whitespace - treat each line as a single word");
 					Console.WriteLine("    --lowercase             Convert the input to lowercase before processing");
 					Console.WriteLine("    --start-uppercase       Start the generating a word only with n-grams that start with a capital letter");
diff --git a/MarkovGrams/UnweightedMarkovChain.cs b/MarkovGrams/UnweightedMarkovChain.cs
index d5899e7..a232f57 100644
--- a/MarkovGrams/UnweightedMarkovChain.cs
+++ b/MarkovGrams/UnweightedMarkovChain.cs
@@ -69,7 +69,7 @@ namespace MarkovGrams
 		{
 			string result = RandomNgram();
 			string lastNgram = result;
-			while(result.Length < length)
+			while((Mode == GenerationMode.CharacterLevel ? result.Length : result.Split(' ').Length) < length)
 			{
 				// The substring that the next ngram in the chain needs to start with
 				string nextStartsWith = Mode == GenerationMode.CharacterLevel ? lastNgram.Substring(1) : lastNgram.Split(' ')[0];
diff --git a/MarkovGrams/WeightedMarkovChain.cs b/MarkovGrams/WeightedMarkovChain.cs
index 3e30d9e..295f44d 100644
--- a/MarkovGrams/WeightedMarkovChain.cs
+++ b/MarkovGrams/WeightedMarkovChain.cs
@@ -83,7 +83,7 @@ namespace MarkovGrams
 		{
 			string result = RandomNgram();
 			string lastNgram = result;
-			while(result.Length < length)
+			while((Mode == GenerationMode.CharacterLevel ? result.Length : result.Split(' ').Length) < length)
 			{
 				wrandom.ClearContents();
 				// The substring that the next ngram in the chain needs to start with
