Bugfix: Correct word-level generation to be a sliding-window

MarkovGrams/Program.cs

@@ -90,6 +90,7 @@ namespace MarkovGrams
 					return word.Split(' ');
 				return new string[] { word.Trim() };
 			});
+			List<float> choicePointRatios = new List<float>();
 
 			switch (operationMode)
 			{
@@ -101,9 +102,11 @@ namespace MarkovGrams
 					);
 					unweightedChain.StartOnUppercase = startOnUppercase;
 
-					for (int i = 0; i < count; i++)
+					for (int i = 0; i < count; i++) {
-						Console.WriteLine(unweightedChain.Generate(length));
+						Console.WriteLine(unweightedChain.Generate(length, out float nextChoicePathRatio));
-					Console.Error.WriteLine($"{count} words in {utimer.ElapsedMilliseconds}ms");
+						choicePointRatios.Add(nextChoicePathRatio);
+					}
+					Console.Error.WriteLine($"{count} words in {utimer.ElapsedMilliseconds}ms (average choice-point-ratio: {Math.Round(choicePointRatios.Sum()/count, 2)})");
 					break;
 
 				case Mode.WeightedMarkov:
@@ -114,9 +117,11 @@ namespace MarkovGrams
 					);
 					weightedChain.StartOnUppercase = startOnUppercase;
 
-					for (int i = 0; i < count; i++)
+					for (int i = 0; i < count; i++) {
-						Console.WriteLine(weightedChain.Generate(length));
+						Console.WriteLine(weightedChain.Generate(length, out float nextChoicePointRatio));
-					Console.Error.WriteLine($"{count} words in {wtimer.ElapsedMilliseconds}ms");
+						choicePointRatios.Add(nextChoicePointRatio);
+					}
+					Console.Error.WriteLine($"{count} words in {wtimer.ElapsedMilliseconds}ms (average choice-point-ratio: {Math.Round(choicePointRatios.Sum() / count, 2)})");
 					break;
 
 				case Mode.NGrams:

MarkovGrams/UnweightedMarkovChain.cs

@@ -66,15 +66,21 @@ namespace MarkovGrams
 		/// </param>
 		/// <returns>A new random string.</returns>
 		public string Generate(int length)
+		{
+			return Generate(length, out float noop);
+		}
+		public string Generate(int length, out float choicePointRatio)
 		{
 			string result = RandomNgram();
 			string lastNgram = result;
+			List<int> choiceCounts = new List<int>(); int i = 0;
 			while((Mode == GenerationMode.CharacterLevel ? result.Length : result.Split(' ').Length) < length)
 			{
 				// The substring that the next ngram in the chain needs to start with
-				string nextStartsWith = Mode == GenerationMode.CharacterLevel ? lastNgram.Substring(1) : lastNgram.Split(' ').Last();
+				string nextStartsWith = Mode == GenerationMode.CharacterLevel ? lastNgram.Substring(1) : string.Join(" ", lastNgram.Split(' ').Skip(1));
 				// Get a list of possible n-grams we could choose from next
 				List<string> nextNgrams = ngrams.FindAll(gram => gram.StartsWith(nextStartsWith));
+				choiceCounts.Add(nextNgrams.Count);
 				// If there aren't any choices left, we can't exactly keep adding to the new string any more :-(
 				if(nextNgrams.Count == 0)
 					break;
@@ -84,10 +90,15 @@ namespace MarkovGrams
 				if (Mode == GenerationMode.CharacterLevel)
 					result += nextNgram[nextNgram.Length - 1];
 				else
-					result += ' ' + string.Join(" ", nextNgram.Split(' ').Skip(1));
+					result += ' ' + nextNgram.Split(' ').Last();
-				lastNgram = nextNgram;
+				
+				lastNgram = nextNgram; i++;
 			}
 
+			if (choiceCounts.Sum() > 0)
+				choicePointRatio = (float)choiceCounts.Sum() / (float)(i + 1);
+			else
+				choicePointRatio = 0;
 			return result;
 		}
 	}

MarkovGrams/WeightedMarkovChain.cs

@@ -80,18 +80,25 @@ namespace MarkovGrams
 		/// </param>
 		/// <returns>A new random string.</returns>
 		public string Generate(int length)
+		{
+			return Generate(length, out float noop);
+		}
+		public string Generate(int length, out float choicePointRatio)
 		{
 			string result = RandomNgram();
 			string lastNgram = result;
+			List<int> choiceCounts = new List<int>(); int i = 0;
 			while((Mode == GenerationMode.CharacterLevel ? result.Length : result.Split(' ').Length) < length)
 			{
 				wrandom.ClearContents();
 				// The substring that the next ngram in the chain needs to start with
-				string nextStartsWith = Mode == GenerationMode.CharacterLevel ? lastNgram.Substring(1) : lastNgram.Split(' ').Last();
+				string nextStartsWith = Mode == GenerationMode.CharacterLevel ? lastNgram.Substring(1) : string.Join(" ", lastNgram.Split(' ').Skip(1));
 				// Get a list of possible n-grams we could choose from next
 				Dictionary<string, double> convNextNgrams = new Dictionary<string, double>();
 				ngrams.Where(gram_data => gram_data.Key.StartsWith(nextStartsWith))
 				      .ForEach((KeyValuePair<string, double> ngramData) => convNextNgrams.Add(ngramData.Key, ngramData.Value));
+
+				choiceCounts.Add(convNextNgrams.Count);
 				// If there aren't any choices left, we can't exactly keep adding to the new string any more :-(
 				if(convNextNgrams.Count() == 0)
 					break;
@@ -102,10 +109,15 @@ namespace MarkovGrams
 				if (Mode == GenerationMode.CharacterLevel)
 					result += nextNgram[nextNgram.Length - 1];
 				else
-					result += ' ' + string.Join(" ", nextNgram.Split(' ').Skip(1));
+					result += ' ' + nextNgram.Split(' ').Last();
-				lastNgram = nextNgram;
+				lastNgram = nextNgram; i++;
 			}
 			wrandom.ClearContents();
+
+			if (choiceCounts.Sum() > 0)
+				choicePointRatio = (float)choiceCounts.Sum() / (float)(i + 1);
+			else
+				choicePointRatio = 0;
 			return result;
 		}
 	}