sbrl/SearchBox
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add indexing.

Browse source
This commit is contained in:
Starbeamrainbowlabs 2018-09-02 16:36:37 +01:00
parent 0ef3b5958d
commit e8069c93ff
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
14 changed files with 807 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
SearchBox-CLI/Program.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using SearchBox;
namespace SearchBoxCLI
{
class MainClass
{
public static void Main(string[] args)
{
string input = Console.In.ReadToEnd();
Index index = new Index(input);
Console.WriteLine(index);
}
}
}

26
SearchBox-CLI/Properties/AssemblyInfo.cs Normal file
View file

@ -0,0 +1,26 @@
using System.Reflection;
using System.Runtime.CompilerServices;
// Information about this assembly is defined by the following attributes.
// Change them to the values specific to your project.
[assembly: AssemblyTitle("SearchBox-CLI")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("")]
[assembly: AssemblyCopyright("sbrl")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
// The form "{Major}.{Minor}.*" will automatically update the build and revision,
// and "{Major}.{Minor}.{Build}.*" will update just the revision.
[assembly: AssemblyVersion("1.0.*")]
// The following attributes are used to specify the signing key for the assembly,
// if desired. See the Mono documentation for more information about signing.
//[assembly: AssemblyDelaySign(false)]
//[assembly: AssemblyKeyFile("")]

45
SearchBox-CLI/SearchBox-CLI.csproj Normal file
View file

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProjectGuid>{1E2F5559-C918-4A5D-9C24-117F204708A4}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>SearchBoxCLI</RootNamespace>
<AssemblyName>SearchBox-CLI</AssemblyName>
<TargetFrameworkVersion>v4.7</TargetFrameworkVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug</OutputPath>
<DefineConstants>DEBUG;</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<ExternalConsole>true</ExternalConsole>
<PlatformTarget>x86</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
<Optimize>true</Optimize>
<OutputPath>bin\Release</OutputPath>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<ExternalConsole>true</ExternalConsole>
<PlatformTarget>x86</PlatformTarget>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\SearchBox\SearchBox.csproj">
<Project>{5243F60A-F822-4C52-A333-E4089754EC6A}</Project>
<Name>SearchBox</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
</Project>

16
SearchBox.sln
View file

@ -1,7 +1,9 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SearchBox", "SearchBox\SearchBox.csproj", "{6E470E38-B239-477F-BF65-D8522292D8C5}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SearchBox", "SearchBox\SearchBox.csproj", "{5243F60A-F822-4C52-A333-E4089754EC6A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SearchBox-CLI", "SearchBox-CLI\SearchBox-CLI.csproj", "{1E2F5559-C918-4A5D-9C24-117F204708A4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -9,9 +11,13 @@ Global
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{6E470E38-B239-477F-BF65-D8522292D8C5}.Debug|x86.ActiveCfg = Debug|x86
{6E470E38-B239-477F-BF65-D8522292D8C5}.Debug|x86.Build.0 = Debug|x86
{6E470E38-B239-477F-BF65-D8522292D8C5}.Release|x86.ActiveCfg = Release|x86
{6E470E38-B239-477F-BF65-D8522292D8C5}.Release|x86.Build.0 = Release|x86
{5243F60A-F822-4C52-A333-E4089754EC6A}.Debug|x86.ActiveCfg = Debug|x86
{5243F60A-F822-4C52-A333-E4089754EC6A}.Debug|x86.Build.0 = Debug|x86
{5243F60A-F822-4C52-A333-E4089754EC6A}.Release|x86.ActiveCfg = Release|x86
{5243F60A-F822-4C52-A333-E4089754EC6A}.Release|x86.Build.0 = Release|x86
{1E2F5559-C918-4A5D-9C24-117F204708A4}.Debug|x86.ActiveCfg = Debug|x86
{1E2F5559-C918-4A5D-9C24-117F204708A4}.Debug|x86.Build.0 = Debug|x86
{1E2F5559-C918-4A5D-9C24-117F204708A4}.Release|x86.ActiveCfg = Release|x86
{1E2F5559-C918-4A5D-9C24-117F204708A4}.Release|x86.Build.0 = Release|x86
EndGlobalSection
EndGlobal

311
SearchBox/EmbeddedFiles/Stopwords.txt Normal file
View file

@ -0,0 +1,311 @@
a
about
above
above
across
after
afterwards
again
against
all
almost
alone
along
already
also
although
always
am
among
amongst
amoungst
amount
an
and
another
any
anyhow
anyone
anything
anyway
anywhere
are
around
as
at
back
be
became
because
become
becomes
becoming
been
before
beforehand
behind
being
below
beside
besides
between
beyond
bill
both
bottom
but
by
call
can
cannot
cant
co
con
could
couldnt
cry
de
describe
detail
do
done
down
due
during
each
eg
eight
either
eleven
else
elsewhere
empty
enough
etc
even
ever
every
everyone
everything
everywhere
except
few
fill
find
fire
first
five
for
former
formerly
found
four
from
front
full
further
get
give
go
had
has
hasnt
have
he
hence
her
here
hereafter
hereby
herein
hereupon
hers
herself
him
himself
his
how
however
ie
if
in
inc
indeed
interest
into
is
it
its
itself
keep
last
latter
latterly
least
less
ltd
made
many
may
me
meanwhile
might
mine
more
moreover
most
mostly
move
much
must
my
myself
name
namely
neither
never
nevertheless
next
nine
no
none
nor
not
nothing
now
nowhere
of
off
often
on
once
one
only
onto
or
other
others
otherwise
our
ours
ourselves
out
over
own
part
per
perhaps
please
put
rather
re
same
see
seem
seemed
seeming
seems
serious
several
she
should
show
side
since
sincere
six
sixty
so
some
somehow
someone
something
sometime
sometimes
somewhere
still
such
system
take
ten
than
that
the
their
them
themselves
then
thence
there
thereafter
thereby
therefore
therein
thereupon
these
they
thickv
thin
third
this
those
though
three
through
throughout
thru
thus
to
together
too
top
toward
towards
twelve
twenty
two
un
under
until
up
upon
us
very
via
was
we
well
were
what
whatever
when
whence
whenever
where
whereafter
whereas
whereby
wherein
whereupon
wherever
whether
which
while
whither
who
whoever
whole
whom
whose
why
will
with
within
without
would
yet
you
your
yours
yourself
yourselves

71
SearchBox/Index.cs Normal file
View file

@ -0,0 +1,71 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using SBRL.Utilities;
namespace SearchBox
{
[Flags]
public enum IndexOptions
{
ExcludeStopwords = 1
}
public class Index
{
private Dictionary<string, List<int>> index = new Dictionary<string, List<int>>();
private StopwordTester stopwordTester;
public Index(string inSource, IEnumerable<string> stopwords, IndexOptions options)
{
if (options.HasFlag(IndexOptions.ExcludeStopwords))
stopwordTester = new StopwordTester(stopwords);
// Tokenize the input and file it in our index
Tokenizer tokenizer = new Tokenizer(inSource);
foreach (Tuple<int, string> token in tokenizer) {
if (stopwordTester.IsStopword(token.Item2)) continue;
insert(token.Item2, token.Item1);
}
}
public Index(string inSource, IndexOptions options)
: this(inSource, EmbeddedFiles.EnumerateLines("SearchBox.EmbeddedFiles.Stopwords.txt"), options)
{
}
public Index(string inSource) : this(inSource, IndexOptions.ExcludeStopwords)
{
}
public List<int> this[string key] {
get {
return index[key];
}
}
protected void insert(string token, int offset)
{
if (!index.ContainsKey(token))
index.Add(token, new List<int>());
index[token].Add(offset);
}
public override string ToString()
{
StringBuilder result = new StringBuilder("Index: \n");
foreach (KeyValuePair<string, List<int>> item in index)
result.AppendLine($"\t{item.Key}: {string.Join(", ", item.Value)}");
return result.ToString();
}
// --------------------------------------------------------------------------------------
public static Index FromFile(string filename)
{
return new Index(File.ReadAllText(filename));
}
}
}

12
SearchBox/Program.cs
View file

@ -1,12 +0,0 @@
using System;
namespace SearchBox
{
class MainClass
{
public static void Main(string[] args)
{
Console.WriteLine("Hello World!");
}
}
}

10
SearchBox/SearchBox.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
namespace SearchBox
{
public class SearchBox
{
public SearchBox()
{
}
}
}

24
SearchBox/SearchBox.csproj
View file

@ -3,8 +3,8 @@
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProjectGuid>{6E470E38-B239-477F-BF65-D8522292D8C5}</ProjectGuid>
<OutputType>Exe</OutputType>
<ProjectGuid>{5243F60A-F822-4C52-A333-E4089754EC6A}</ProjectGuid>
<OutputType>Library</OutputType>
<RootNamespace>SearchBox</RootNamespace>
<AssemblyName>SearchBox</AssemblyName>
<TargetFrameworkVersion>v4.7</TargetFrameworkVersion>
@ -30,10 +30,28 @@
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="UnidecodeSharpFork">
<HintPath>..\packages\UnidecodeSharpFork.1.0.0\lib\UnidecodeSharpFork.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Utilities\EmbeddedFiles.cs" />
<Compile Include="SearchBox.cs" />
<Compile Include="Tokenizer.cs" />
<Compile Include="Index.cs" />
<Compile Include="Utilities\StringPlus.cs" />
<Compile Include="StopwordTester.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="EmbeddedFiles\" />
<Folder Include="Utilities\" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="EmbeddedFiles\Stopwords.txt" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
</Project>

21
SearchBox/StopwordTester.cs Normal file
View file

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
namespace SearchBox
{
public class StopwordTester
{
private HashSet<string> words = new HashSet<string>();
public StopwordTester(IEnumerable<string> inWords)
{
foreach (string word in inWords)
words.Add(word);
}
public bool IsStopword(string word)
{
return words.Contains(word);
}
}
}

65
SearchBox/Tokenizer.cs Normal file
View file

@ -0,0 +1,65 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Net;
using System.Text.RegularExpressions;
using SearchBox.Utilities;
using UnidecodeSharpFork;
namespace SearchBox
{
[Flags]
public enum TokenizerOptions
{
Lowercase = 1,
Transliterate = 2,
HidePunctuation = 4,
DecodeHtmlEntities = 8
}
public class Tokenizer : IEnumerable<Tuple<int, string>>
{
private static Regex splitter = new Regex(
@"((^\p{P}+)|(\p{P}*\s+\p{P}*)|(\p{P}+$))|\|",
RegexOptions.Compiled | RegexOptions.Multiline
);
private string source;
public bool Verbose = false;
public Tokenizer(string inSource, TokenizerOptions options = TokenizerOptions.Transliterate | TokenizerOptions.Lowercase | TokenizerOptions.HidePunctuation)
{
if (options.HasFlag(TokenizerOptions.Transliterate)) inSource = inSource.Unidecode();
if (options.HasFlag(TokenizerOptions.Lowercase)) inSource = inSource.ToLower();
if (options.HasFlag(TokenizerOptions.HidePunctuation)) inSource = inSource.ReplaceMultiple(
@"[]{}|/\".ToCharArray(),
" ".ToCharArray()
);
if (options.HasFlag(TokenizerOptions.DecodeHtmlEntities)) inSource = WebUtility.HtmlDecode(inSource);
source = inSource;
}
public IEnumerable<Tuple<int, string>> IterateTokens()
{
int index = 0;
string[] parts = splitter.Split(source);
for (int i = 0; i < parts.Length; i++) {
index += parts[i].Length;
if (string.IsNullOrWhiteSpace(parts[i]) || splitter.IsMatch(parts[i])) continue;
if(Verbose) Console.WriteLine("#{0} @ {1}: '{2}'", i, index, parts[i]);
yield return new Tuple<int, string>(index, parts[i]);
}
}
public IEnumerator<Tuple<int, string>> GetEnumerator() {
return IterateTokens().GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

189
SearchBox/Utilities/EmbeddedFiles.cs Normal file
View file

@ -0,0 +1,189 @@
using System;
using System.Reflection;
using System.IO;
using System.Threading.Tasks;
using System.Collections.Generic;
namespace SBRL.Utilities
{
/// <summary>
/// A collection of static methods for manipulating embedded resources.
/// </summary>
/// <description>
/// v0.5, by Starbeamrainbowlabs <feedback@starbeamrainbowlabs.com>
/// Last updated 8th August 2016.
/// Licensed under MPL-2.0.
///
/// Changelog:
/// v0.1 (25th July 2016):
/// - Initial release.
/// v0.2 (8th August 2016):
/// - Changed namespace.
/// v0.3 (21st January 2017):
/// - Added GetRawReader().
/// v0.4 (8th April 2017):
/// - Removed unnecessary using statement.
/// v0.5 (3rd September 2017):
/// - Changed namespace
/// </description>
public static class EmbeddedFiles
{
/// <summary>
/// An array of the filenames of all the resources embedded in the calling assembly.
/// </summary>
/// <value>The resource list.</value>
public static string[] ResourceList {
get {
return Assembly.GetCallingAssembly().GetManifestResourceNames();
}
}
public static string GetResourceListText()
{
StringWriter result = new StringWriter();
result.WriteLine("Files embedded in {0}:", Assembly.GetCallingAssembly().GetName().Name);
foreach (string filename in ResourceList)
result.WriteLine(" - {0}", filename);
return result.ToString();
}
/// <summary>
/// Writes a list of embedded resources to the Console's standard output.
/// </summary>
public static void WriteResourceList()
{
Console.WriteLine(GetResourceListText());
}
/// <summary>
/// Gets a StreamReader attached to the specified embedded resource.
/// </summary>
/// <param name="filename">The filename of the embedded resource to get a StreamReader of.</param>
/// <returns>A StreamReader attached to the specified embedded resource.</returns>
public static StreamReader GetReader(string filename)
{
return new StreamReader(GetRawReader(filename));
}
/// <summary>
/// Gets a raw Stream that's attached to the specified embedded resource.
/// Useful when you want to copy an embedded resource to some other stream.
/// </summary>
/// <param name="filename">The path to the embedded resource.</param>
/// <returns>A raw Stream object attached to the specified file..</returns>
public static Stream GetRawReader(string filename)
{
return Assembly.GetCallingAssembly().GetManifestResourceStream(filename);
}
/// <summary>
/// Gets the specified embedded resource's content as a byte array.
/// </summary>
/// <param name="filename">The filename of the embedded resource to get conteent of.</param>
/// <returns>The specified embedded resource's content as a byte array.</returns>
public static byte[] ReadAllBytes(string filename)
{
// Referencing the Result property will block until the async method completes
return ReadAllBytesAsync(filename).Result;
}
/// <summary>
/// Gets the specified embedded resource's content as a byte array asynchronously.
/// </summary>
/// <param name="filename">The filename of the embedded resource to get conteent of.</param>
/// <returns>The specified embedded resource's content as a byte array.</returns>
public static async Task<byte[]> ReadAllBytesAsync(string filename)
{
using (Stream resourceStream = Assembly.GetCallingAssembly().GetManifestResourceStream(filename))
using (MemoryStream temp = new MemoryStream())
{
await resourceStream.CopyToAsync(temp);
return temp.ToArray();
}
}
/// <summary>
/// Gets all the text stored in the specified embedded resource.
/// </summary>
/// <param name="filename">The filename to fetch the content of.</param>
/// <returns>All the text stored in the specified embedded resource.</returns>
public static string ReadAllText(string filename)
{
using (StreamReader resourceReader = new StreamReader(Assembly.GetCallingAssembly().GetManifestResourceStream(filename)))
{
return resourceReader.ReadToEnd();
}
}
/// <summary>
/// Gets all the text stored in the specified embedded resource asynchronously.
/// </summary>
/// <param name="filename">The filename to fetch the content of.</param>
/// <returns>All the text stored in the specified embedded resource.</returns>
public static async Task<string> ReadAllTextAsync(string filename)
{
using (StreamReader resourceReader = new StreamReader(Assembly.GetCallingAssembly().GetManifestResourceStream(filename)))
{
return await resourceReader.ReadToEndAsync();
}
}
/// <summary>
/// Enumerates the lines of text in the specified embedded resource.
/// </summary>
/// <param name="filename">The filename of the embedded resource to enumerate.</param>
/// <returns>An IEnumerator that enumerates the specified embedded resource.</returns>
public static IEnumerable<string> EnumerateLines(string filename)
{
using (StreamReader resourceReader = new StreamReader(Assembly.GetCallingAssembly().GetManifestResourceStream(filename)))
{
string nextLine;
while ((nextLine = resourceReader.ReadLine()) != null)
{
yield return nextLine;
}
}
}
/// <summary>
/// Enumerates the lines of text in the specified embedded resource asynchronously.
/// Each successive call returns a task that, when complete, returns the next line of text stored
/// in the embedded resource.
/// </summary>
/// <param name="filename">The filename of the embedded resource to enumerate.</param>
/// <returns>An IEnumerator that enumerates the specified embedded resource.</returns>
public static IEnumerable<Task<string>> EnumerateLinesAsync(string filename)
{
using (StreamReader resourceReader = new StreamReader(Assembly.GetCallingAssembly().GetManifestResourceStream(filename)))
{
while (!resourceReader.EndOfStream)
{
yield return resourceReader.ReadLineAsync();
}
}
}
/// <summary>
/// Gets all the lines of text in the specified embedded resource.
/// You might find EnumerateLines(string filename) more useful depending on your situation.
/// </summary>
/// <param name="filename">The filename to obtain the lines of text from.</param>
/// <returns>A list of lines in the specified embedded resource.</returns>
public static List<string> GetAllLines(string filename)
{
// Referencing the Result property will block until the async method completes
return GetAllLinesAsync(filename).Result;
}
/// <summary>
/// Gets all the lines of text in the specified embedded resource asynchronously.
/// </summary>
/// <param name="filename">The filename to obtain the lines of text from.</param>
/// <returns>A list of lines in the specified embedded resource.</returns>
public static async Task<List<string>> GetAllLinesAsync(string filename)
{
List<string> lines = new List<string>();
IEnumerable<Task<string>> lineIterator = EnumerateLinesAsync(filename);
foreach (Task<string> nextLine in lineIterator)
{
lines.Add(await nextLine);
}
return lines;
}
}
}

18
SearchBox/Utilities/StringPlus.cs Normal file
View file

@ -0,0 +1,18 @@
using System;
namespace SearchBox.Utilities
{
public static class StringPlus
{
public static string ReplaceMultiple(this string str, char[] find, char[] replace)
{
for (int i = 0; i < find.Length; i++) {
str = str.Replace(
find[i],
i < replace.Length ? replace[i] : replace[replace.Length - 1]
);
}
return str;
}
}
}

4
SearchBox/packages.config Normal file
View file

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="UnidecodeSharpFork" version="1.0.0" targetFramework="net47" />
</packages>