There has to be a simpler way of implementing the HTML tag insertion. Let's try this again....

SearchBox/SearchBox.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
+using UnidecodeSharpFork;
 
 namespace LibSearchBox
 {
@@ -221,42 +222,23 @@ namespace LibSearchBox
 				currentLength += nextSnippet.Item2 - nextSnippet.Item1;
 			}
 
-			List<string> snippetsText = new List<string>(snippets.Select(((int, int) snippet) => {
+			List<string> snippetsText = new List<string>(
+				snippets.Select(((int, int) snippet) => {
 					string result = source.Substring(snippet.Item1, snippet.Item2 - snippet.Item1);
 					
-				if (string.IsNullOrWhiteSpace(result.Trim()))
-					return "";
-
 					if (settings.Html) {
-					List<string> parts = new List<string>() { result };
+						result = WebUtility.HtmlEncode(result);
-					int remainingStartIndex = snippet.Item1;
+						string resultSearchable = result.Unidecode().ToLower();
-					foreach ((int, int) tokenDef in tokenLocations) {
+						foreach ((int, string) nextToken in tokenizer.IterateTokens()) {
-						if (tokenDef.Item1 - remainingStartIndex < result.Length) {
+							// TODO: Insert html tags here
-							string remainingString = parts.Last();
+							throw new NotImplementedException("HTML tag insertion hasn't been implemented yet");
-							parts.RemoveAt(parts.Count - 1); // Remove the last element
-															 // The bit before the token
-							string nextPart = WebUtility.HtmlEncode(
-								remainingString.Substring(0, tokenDef.Item1 - remainingStartIndex)
-							);
-							// The token itself
-							nextPart += $"<span class='token'>{WebUtility.HtmlEncode(remainingString.Substring(tokenDef.Item1 - remainingStartIndex, tokenDef.Item2))}</span>";
-							parts.Add(nextPart);
-							// The bit after the token - this will be processed by the next loop, so ti doesn't need to be HTML-encoded (yet)
-							parts.Add(remainingString.Substring((tokenDef.Item1 - remainingStartIndex) + tokenDef.Item2));
-
-							// Update our marker as to where we've got up to
-							remainingStartIndex = (tokenDef.Item1 - remainingStartIndex) + tokenDef.Item2;
 						} 
 					}
 					
-					// HTML-encode the last part
-					parts[parts.Count - 1] = WebUtility.HtmlEncode(parts[parts.Count - 1]);
-
-					result = string.Join("", parts);
-				}
-
 					return result;
-			}).Where((string snippet) => !string.IsNullOrWhiteSpace(snippet)));
+				})
+				.Where((string snippet) => !string.IsNullOrWhiteSpace(snippet))
+			);
 
 			// Add the separator at the beginning and end if we aren't at the bounds of the source document
 			if (snippets.First().Item1 > 0)

SearchBox/Utilities/StringPlus.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 
 namespace LibSearchBox.Utilities
 {
@@ -14,5 +15,20 @@ namespace LibSearchBox.Utilities
 			}
 			return str;
 		}
+
+		// From https://stackoverflow.com/a/2641383/1460422
+		public static IEnumerable<int> AllIndexesOf(this string str, string value)
+		{
+			if (String.IsNullOrEmpty(value))
+				throw new ArgumentException("Error: The string to find may not be empty.", nameof(value));
+			
+			List<int> indexes = new List<int>();
+			for (int index = 0; ; index += value.Length) {
+				index = str.IndexOf(value, index);
+				if (index == -1)
+					break;
+				yield return index;
+			}
+		}
 	}
 }