sbrl-github/Pepperminty-Wiki
sbrl-github
/
Pepperminty-Wiki
mirror of https://github.com/sbrl/Pepperminty-Wiki.git
1
0
Fork 0
Code

Save the character offset, not the token offset in the inverted index

This commit is contained in:
Starbeamrainbowlabs 2018-06-30 11:19:38 +01:00
parent 1b3bc1cee7
commit 8955d6d131
Signed by: sbrl
GPG Key ID: 1BE5172E637709C2
1 changed files with 6 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
modules/feature-search.php
View File

@ -542,26 +542,17 @@ class search
$index = [];
$terms = self::tokenize($source);
$i = 0;
$terms = self::tokenize($source, true);
foreach($terms as $term)
{
$nterm = $term;
// Skip over stop words (see https://en.wikipedia.org/wiki/Stop_words)
if(in_array($nterm, self::$stop_words)) { $i++; continue; }
if(in_array($term[0], self::$stop_words)) continue;
if(!isset($index[$nterm]))
{
$index[$nterm] = [ "freq" => 0, "offsets" => [] ];
}
if(!isset($index[$term[0]]))
$index[$term[0]] = [ "freq" => 0, "offsets" => [] ];
// FIXME: Here we use the index of the token in the array, when we want the number of characters into the page!
$index[$nterm]["freq"]++;
$index[$nterm]["offsets"][] = $i;
$i++;
$index[$term[0]]["freq"]++;
$index[$term[0]]["offsets"][] = $term[1];
}
return $index;