Add debug idindex-show action.

Changelog.md

@@ -1,4 +1,4 @@
-# Changelog
+data# Changelog
 
 ## v0.12.1-dev
 

RestApiDocs/api_data.js

@@ -496,7 +496,7 @@ define({ "api": [
             "group": "Error 4xx",
             "optional": false,
             "field": "NonExistentPageError",
-            "description": "<p>The page doesn't exist and editing is disabled in the wiki's settings. If editing isn't disabled, you will be redirected to the edit apge instead.</p>"
+            "description": "<p>The page doesn't exist and editing is disabled in the wiki's settings. If editing isn't disabled, you will be redirected to the edit page instead.</p>"
           },
           {
             "group": "Error 4xx",
@@ -589,6 +589,24 @@ define({ "api": [
     "filename": "./modules/feature-search.php",
     "groupTitle": "Search"
   },
+  {
+    "type": "get",
+    "url": "?action=idindex-show",
+    "title": "Show the id index.",
+    "description": "<p>Outputs the id index. Useful if you need to verify that it's working as expected.</p>",
+    "name": "SearchShowIdIndex",
+    "group": "Search",
+    "permission": [
+      {
+        "name": "Anonymous",
+        "title": "Anybody may use this call.",
+        "description": ""
+      }
+    ],
+    "version": "0.0.0",
+    "filename": "./modules/feature-search.php",
+    "groupTitle": "Search"
+  },
   {
     "type": "get",
     "url": "?action=recentchanges",

RestApiDocs/api_data.json

@@ -496,7 +496,7 @@
             "group": "Error 4xx",
             "optional": false,
             "field": "NonExistentPageError",
-            "description": "<p>The page doesn't exist and editing is disabled in the wiki's settings. If editing isn't disabled, you will be redirected to the edit apge instead.</p>"
+            "description": "<p>The page doesn't exist and editing is disabled in the wiki's settings. If editing isn't disabled, you will be redirected to the edit page instead.</p>"
           },
           {
             "group": "Error 4xx",
@@ -589,6 +589,24 @@
     "filename": "./modules/feature-search.php",
     "groupTitle": "Search"
   },
+  {
+    "type": "get",
+    "url": "?action=idindex-show",
+    "title": "Show the id index.",
+    "description": "<p>Outputs the id index. Useful if you need to verify that it's working as expected.</p>",
+    "name": "SearchShowIdIndex",
+    "group": "Search",
+    "permission": [
+      {
+        "name": "Anonymous",
+        "title": "Anybody may use this call.",
+        "description": ""
+      }
+    ],
+    "version": "0.0.0",
+    "filename": "./modules/feature-search.php",
+    "groupTitle": "Search"
+  },
   {
     "type": "get",
     "url": "?action=recentchanges",

RestApiDocs/api_project.js

@@ -6,7 +6,7 @@ define({
   "apidoc": "0.2.0",
   "generator": {
     "name": "apidoc",
-    "time": "2016-06-15T18:29:14.696Z",
+    "time": "2016-08-26T16:47:22.344Z",
     "url": "http://apidocjs.com",
     "version": "0.16.1"
   }

RestApiDocs/api_project.json

@@ -6,7 +6,7 @@
   "apidoc": "0.2.0",
   "generator": {
     "name": "apidoc",
-    "time": "2016-06-15T18:29:14.696Z",
+    "time": "2016-08-26T16:47:22.344Z",
     "url": "http://apidocjs.com",
     "version": "0.16.1"
   }

build/index.php

@@ -2139,7 +2139,7 @@ register_module([
 		 * @apiDescription	Causes the inverted search index to be completely rebuilt from scratch. Can take a while for large wikis!
 		 * @apiName			SearchInvindexRebuild
 		 * @apiGroup		Search
-		 * @apiPermission Anonymous
+		 * @apiPermission	Anonymous
 		 */
 		
 		/*
@@ -2159,6 +2159,20 @@ register_module([
 			search::rebuild_invindex();
 		});
 		
+		
+		/**
+		 * @api {get} ?action=idindex-show Show the id index.
+		 * @apiDescription	Outputs the id index. Useful if you need to verify that it's working as expected.
+		 * @apiName			SearchShowIdIndex
+		 * @apiGroup		Search
+		 * @apiPermission	Anonymous
+		 */
+		add_action("idindex-show", function() {
+			global $idindex;
+			header("content-type: application/json; charset=UTF-8");
+			exit(json_encode($idindex, JSON_PRETTY_PRINT));
+		});
+		
 		/**
 		 * @api {get} ?action=search&query={text}	Search the wiki for a given query string
 		 * @apiName Search
@@ -2350,7 +2364,7 @@ class search
 	public static function tokenize($source)
 	{
 		$source = strtolower($source);
-		return preg_split("/((^\p{P}+)|(\p{P}*\s+\p{P}*)|(\p{P}+$))|\|/", $source, -1, PREG_SPLIT_NO_EMPTY);
+		return preg_split("/((^\p{P}+)|(\p{P}*\s+\p{P}*)|(\p{P}+$))|\|/u", $source, -1, PREG_SPLIT_NO_EMPTY);
 	}
 	
 	public static function strip_markup($source)
@@ -2362,15 +2376,21 @@ class search
 	{
 		global $pageindex, $env, $paths;
 		
+		header("content-type: text/event-stream");
+		
 		$invindex = [];
 		foreach($pageindex as $pagename => $pagedetails)
 		{
-			$pagesource = file_get_contents("$env->storage_prefix$pagename.md");
+			echo("Adding $pagename to the new search index.\n\n");
+			$pagesource = utf8_encode(file_get_contents("$env->storage_prefix$pagename.md"));
 			$index = self::index($pagesource);
 			
 			self::merge_into_invindex($invindex, ids::getid($pagename), $index);
 		}
 		
+		echo("Search index rebuilding complete.\n\n");
+		echo("Saving new search index to '$paths->searchindex'.\n\n");
+		
 		self::save_invindex($paths->searchindex, $invindex);
 	}
 	

module_index.json

@@ -68,7 +68,7 @@
         "author": "Starbeamrainbowlabs",
         "description": "Adds proper search functionality to Pepperminty Wiki using an inverted index to provide a full text search engine. If pages don't show up, then you might have hit a stop word. If not, try requesting the `invindex-rebuild` action to rebuild the inverted index from scratch.",
         "id": "feature-search",
-        "lastupdate": 1471791401,
+        "lastupdate": 1472229999,
         "optional": false
     },
     {

modules/feature-search.php

@@ -44,7 +44,7 @@ register_module([
 		 * @apiDescription	Causes the inverted search index to be completely rebuilt from scratch. Can take a while for large wikis!
 		 * @apiName			SearchInvindexRebuild
 		 * @apiGroup		Search
-		 * @apiPermission Anonymous
+		 * @apiPermission	Anonymous
 		 */
 		
 		/*
@@ -64,6 +64,20 @@ register_module([
 			search::rebuild_invindex();
 		});
 		
+		
+		/**
+		 * @api {get} ?action=idindex-show Show the id index.
+		 * @apiDescription	Outputs the id index. Useful if you need to verify that it's working as expected.
+		 * @apiName			SearchShowIdIndex
+		 * @apiGroup		Search
+		 * @apiPermission	Anonymous
+		 */
+		add_action("idindex-show", function() {
+			global $idindex;
+			header("content-type: application/json; charset=UTF-8");
+			exit(json_encode($idindex, JSON_PRETTY_PRINT));
+		});
+		
 		/**
 		 * @api {get} ?action=search&query={text}	Search the wiki for a given query string
 		 * @apiName Search
@@ -255,7 +269,7 @@ class search
 	public static function tokenize($source)
 	{
 		$source = strtolower($source);
-		return preg_split("/((^\p{P}+)|(\p{P}*\s+\p{P}*)|(\p{P}+$))|\|/", $source, -1, PREG_SPLIT_NO_EMPTY);
+		return preg_split("/((^\p{P}+)|(\p{P}*\s+\p{P}*)|(\p{P}+$))|\|/u", $source, -1, PREG_SPLIT_NO_EMPTY);
 	}
 	
 	public static function strip_markup($source)
@@ -267,15 +281,21 @@ class search
 	{
 		global $pageindex, $env, $paths;
 		
+		header("content-type: text/event-stream");
+		
 		$invindex = [];
 		foreach($pageindex as $pagename => $pagedetails)
 		{
-			$pagesource = file_get_contents("$env->storage_prefix$pagename.md");
+			echo("Adding $pagename to the new search index.\n\n");
+			$pagesource = utf8_encode(file_get_contents("$env->storage_prefix$pagename.md"));
 			$index = self::index($pagesource);
 			
 			self::merge_into_invindex($invindex, ids::getid($pagename), $index);
 		}
 		
+		echo("Search index rebuilding complete.\n\n");
+		echo("Saving new search index to '$paths->searchindex'.\n\n");
+		
 		self::save_invindex($paths->searchindex, $invindex);
 	}