From 2a5a50458ff70512df88a3481cf92b3bbb8f764f Mon Sep 17 00:00:00 2001
From: Starbeamrainbowlabs <sbrl@starbeamrainbowlabs.com>
Date: Tue, 18 Sep 2018 19:49:03 +0100
Subject: [PATCH] Add initial Query() method to SearchBox, but it's untested.

---
 SearchBox/DocumentMeta.cs  | 17 +++++++++++
 SearchBox/InvertedIndex.cs |  6 ++++
 SearchBox/QuerySettings.cs | 13 +++++++++
 SearchBox/SearchBox.cs     | 59 +++++++++++++++++++++++++++++++++++++-
 SearchBox/SearchBox.csproj |  4 ++-
 SearchBox/SearchResult.cs  | 16 +++++++++++
 6 files changed, 113 insertions(+), 2 deletions(-)
 create mode 100644 SearchBox/QuerySettings.cs
 create mode 100644 SearchBox/SearchResult.cs

diff --git a/SearchBox/DocumentMeta.cs b/SearchBox/DocumentMeta.cs
index f2d74f9..37376dd 100644
--- a/SearchBox/DocumentMeta.cs
+++ b/SearchBox/DocumentMeta.cs
@@ -1,5 +1,7 @@
 ﻿using System;
 using System.Collections.Generic;
+using Newtonsoft.Json;
+using UnidecodeSharpFork;
 
 namespace LibSearchBox
 {
@@ -8,6 +10,21 @@ namespace LibSearchBox
 		public string Title { get; set; }
 		public List<string> Tags { get; private set; } = new List<string>();
 
+		[JsonIgnore]
+		public string SearchableTitle {
+			get {
+				return Title.ToLower().Unidecode();
+			}
+		}
+
+		[JsonIgnore]
+		public IEnumerable<string> SearchableTags {
+			get {
+				foreach (string nextTag in Tags)
+					yield return nextTag.ToLower().Unidecode();
+			}
+		}
+
 		public DocumentMeta(string inTitle, IEnumerable<string> inTags)
 		{
 			Title = inTitle;
diff --git a/SearchBox/InvertedIndex.cs b/SearchBox/InvertedIndex.cs
index ca4e7b7..21a2454 100644
--- a/SearchBox/InvertedIndex.cs
+++ b/SearchBox/InvertedIndex.cs
@@ -56,5 +56,11 @@ namespace LibSearchBox
 			}
 			return true;
 		}
+
+		public ConcurrentDictionary<int, List<int>> Query(string normalisedToken)
+		{
+			invertedIndex.TryGetValue(normalisedToken, out ConcurrentDictionary<int, List<int>> result);
+			return result;
+		}
 	}
 }
diff --git a/SearchBox/QuerySettings.cs b/SearchBox/QuerySettings.cs
new file mode 100644
index 0000000..61db61e
--- /dev/null
+++ b/SearchBox/QuerySettings.cs
@@ -0,0 +1,13 @@
+﻿using System;
+namespace LibSearchBox
+{
+	public class QuerySettings
+	{
+		public int WeightTitleMatch = 100;
+		public int WeightTagMatch = 10;
+
+		public QuerySettings()
+		{
+		}
+	}
+}
diff --git a/SearchBox/SearchBox.cs b/SearchBox/SearchBox.cs
index 04eb84f..2788436 100644
--- a/SearchBox/SearchBox.cs
+++ b/SearchBox/SearchBox.cs
@@ -1,6 +1,8 @@
 ﻿using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 
 namespace LibSearchBox
@@ -30,11 +32,15 @@ namespace LibSearchBox
 		[JsonProperty]
 		public InvertedIndex index = new InvertedIndex();
 
+		public bool Verbose { get; set; } = false;
+
 		public SearchBox()
 		{
 
 		}
 
+		#region Index Management
+
 		public void AddDocument(string title, IEnumerable<string> tags, string content)
 		{
 			DocumentMeta info = new DocumentMeta(title, tags);
@@ -61,7 +67,58 @@ namespace LibSearchBox
 			metaTable.TryRemove(id, out DocumentMeta noop);
 			if (!index.RemoveById(id))
 				throw new SearchBoxException($"Failed to remove page with title '{title}' from inverted index.");
-			
+
 		}
+
+		#endregion
+
+		#region Query
+
+		public List<SearchResult> Query(string query, QuerySettings settings)
+		{
+			// pageId => token -> count
+			ConcurrentDictionary<int, ConcurrentDictionary<string, int>> matchingPages = new ConcurrentDictionary<int, ConcurrentDictionary<string, int>>();
+
+			Tokenizer tokenizer = new Tokenizer(query);
+			foreach(Tuple<int, string> token in tokenizer.IterateTokens())
+			{
+				ConcurrentDictionary<int, List<int>> tokenResults = index.Query(token.Item2);
+
+				Parallel.ForEach(tokenResults, (KeyValuePair<int, List<int>> pageTokenDef) => {
+					// pageTokenDef: pageId => List of token offsets
+					ConcurrentDictionary<string, int> pageData = matchingPages.GetOrAdd(pageTokenDef.Key, new ConcurrentDictionary<string, int>());
+					if (!pageData.TryAdd(token.Item2, pageTokenDef.Value.Count))
+						throw new SearchBoxException("Error: Failed to add token count to page data in search " +
+						    "results - the key already exists (are there duplicate tokens for this page id " +
+						    "in the inverted index?");
+				});
+			}
+
+			ConcurrentBag<SearchResult> resultsRaw = new ConcurrentBag<SearchResult>();
+			Parallel.ForEach(matchingPages, (KeyValuePair<int, ConcurrentDictionary<string, int>> pageDef) => {
+				int rank = pageDef.Value.Values.Sum(); // FUTURE: Linq is slow. Is theree a faster way of doing this?
+				if (!metaTable.TryGetValue(pageDef.Key, out DocumentMeta metaInfo)) {
+					if (Verbose) Console.Error.WriteLine($"Warning: Failed to fetch meta info for page id {pageDef.Key}");
+				}
+				else {
+					foreach (Tuple<int, string> token in tokenizer.IterateTokens()) {
+						if (metaInfo.SearchableTitle.Contains(token.Item2))
+							rank += settings.WeightTitleMatch; // FUTURE: Alter the amount added by the length of the word matched
+
+						foreach (string nextTag in metaInfo.SearchableTags)
+							if (nextTag.Contains(token.Item2))
+								rank += settings.WeightTagMatch;
+					}
+				}
+
+				resultsRaw.Add(new SearchResult(idMap.GetPageName(pageDef.Key), rank));
+			});
+
+			List<SearchResult> results = new List<SearchResult>(resultsRaw.AsEnumerable());
+			results.OrderBy((SearchResult result) => result.Rank);
+			return results;
+		}
+
+		#endregion
 	}
 }
diff --git a/SearchBox/SearchBox.csproj b/SearchBox/SearchBox.csproj
index fe61641..41e5bbb 100644
--- a/SearchBox/SearchBox.csproj
+++ b/SearchBox/SearchBox.csproj
@@ -5,7 +5,7 @@
     <Platform Condition=" '$(Platform)' == '' ">x86</Platform>
     <ProjectGuid>{5243F60A-F822-4C52-A333-E4089754EC6A}</ProjectGuid>
     <OutputType>Library</OutputType>
-    <RootNamespace>SearchBox</RootNamespace>
+    <RootNamespace>LibSearchBox</RootNamespace>
     <AssemblyName>SearchBox</AssemblyName>
     <TargetFrameworkVersion>v4.7</TargetFrameworkVersion>
   </PropertyGroup>
@@ -49,6 +49,8 @@
     <Compile Include="IdMap.cs" />
     <Compile Include="Utilities\BiDictionary.cs" />
     <Compile Include="DocumentMeta.cs" />
+    <Compile Include="SearchResult.cs" />
+    <Compile Include="QuerySettings.cs" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="EmbeddedFiles\" />
diff --git a/SearchBox/SearchResult.cs b/SearchBox/SearchResult.cs
new file mode 100644
index 0000000..e73ea9e
--- /dev/null
+++ b/SearchBox/SearchResult.cs
@@ -0,0 +1,16 @@
+﻿using System;
+
+namespace LibSearchBox
+{
+	public class SearchResult
+	{
+		public string PageName { get; private set; }
+		public float Rank { get; private set; }
+
+		public SearchResult(string inPageName, float inRank)
+		{
+			PageName = inPageName;
+			Rank = inRank;
+		}
+	}
+}
