Start removing search and lunr from core

This commit is contained in:
Samy Pessé
2015-10-05 15:55:39 +02:00
parent fa76029a17
commit 7f0e4803e2
11 changed files with 97 additions and 165 deletions
-37
View File
@@ -1,7 +1,6 @@
var Q = require('q');
var _ = require('lodash');
var path = require('path');
var lunr = require('lunr');
var parsers = require('gitbook-parsers');
var fs = require('./utils/fs');
@@ -78,16 +77,6 @@ var Book = function(root, context, parent) {
this.readmeFile = null;
this.langsFile = null;
// Search Index
this.searchIndexEnabled = true;
this.searchIndexSize = 0;
this.searchIndex = lunr(function () {
this.ref('url');
this.field('title', { boost: 10 });
this.field('body');
});
// Bind methods
_.bindAll(this);
};
@@ -595,7 +584,6 @@ Book.prototype.parsePage = function(filename, options) {
})
.then(function() {
that.indexPage(page);
return page;
});
};
@@ -755,31 +743,6 @@ Book.prototype.contentLink = function(link) {
return links.normalize(this.contentPath(link));
};
// Index a page into the search index
Book.prototype.indexPage = function(page) {
var nav = this.navigation[page.path];
if (!nav || !this.searchIndexEnabled) return;
this.log.debug.ln('index page', page.path);
// Extract text from the page
var text = pageUtil.extractText(page.sections);
// Limit size of index (to avoid #941)
this.searchIndexSize = this.searchIndexSize + text.length;
if (this.searchIndexSize > this.config.get('search.maxIndexSize')) {
this.log.warn.ln('search index is too big, indexing is now disabled');
this.searchIndexEnabled = false;
return;
}
this.searchIndex.add({
url: this.contentLink(page.path),
title: nav.title,
body: text
});
};
// Default structure paths to an extension
Book.prototype._defaultsStructure = function(filename) {
var that = this;
+1 -6
View File
@@ -7,7 +7,7 @@ var pkg = require('../package.json');
var i18n = require('./utils/i18n');
// Default plugins added to each books
var DEFAULT_PLUGINS = ['highlight', 'sharing', 'fontsettings'];
var DEFAULT_PLUGINS = ['highlight', 'search', 'sharing', 'fontsettings'];
// Check if a plugin is a default plugin
// Plugin should be in the list
@@ -201,11 +201,6 @@ Configuration.DEFAULT = {
// version of gitbook to use
'gitbook': '*',
// Search index
'search': {
'maxIndexSize': 1000000
},
// Structure
'structure': {
'langs': 'LANGS.md',
-9
View File
@@ -112,7 +112,6 @@ Generator.prototype.finish = function() {
return this.copyAssets()
.then(this.copyCover)
.then(this.writeGlossary)
.then(this.writeSearchIndex)
.then(this.writeLangsIndex);
};
@@ -167,14 +166,6 @@ Generator.prototype.writeGlossary = function() {
return this._writeTemplate(this.templates.glossary, {}, path.join(this.options.output, "GLOSSARY.html"));
};
// Write the search index
Generator.prototype.writeSearchIndex = function() {
return fs.writeFile(
path.join(this.options.output, "search_index.json"),
JSON.stringify(this.book.searchIndex)
);
};
// Convert a page into a normalized data set
Generator.prototype.normalizePage = function(page) {
var that = this;
+64 -78
View File
@@ -1,18 +1,18 @@
var Q = require("q");
var _ = require("lodash");
var url = require("url");
var path = require("path");
var cheerio = require("cheerio");
var domSerializer = require("dom-serializer");
var request = require("request");
var crc = require("crc");
var Q = require('q');
var _ = require('lodash');
var url = require('url');
var path = require('path');
var cheerio = require('cheerio');
var domSerializer = require('dom-serializer');
var request = require('request');
var crc = require('crc');
var links = require("./links");
var imgUtils = require("./images");
var fs = require("./fs");
var batch = require("./batch");
var links = require('./links');
var imgUtils = require('./images');
var fs = require('./fs');
var batch = require('./batch');
var parsableExtensions = require("gitbook-parsers").extensions;
var parsableExtensions = require('gitbook-parsers').extensions;
// Render a cheerio dom as html
var renderDom = function($, dom, options) {
@@ -59,7 +59,7 @@ function replaceText($, el, search, replace, text_only ) {
// robust way.
$(node).before( new_val );
// Don"t remove the node yet, or the loop will lose its place.
// Don't remove the node yet, or the loop will lose its place.
remove.push( node );
} else {
// The new value contains no HTML, so it can be set in this
@@ -79,7 +79,7 @@ function replaceText($, el, search, replace, text_only ) {
}
function pregQuote( str ) {
return (str+"").replace(/([\\\.\+\*\?\[\^\]\$\(\)\{\}\=\!\<\>\|\:])/g, "\\$1");
return (str+'').replace(/([\\\.\+\*\?\[\^\]\$\(\)\{\}\=\!\<\>\|\:])/g, '\\$1');
}
@@ -101,23 +101,23 @@ function normalizeHtml(src, options) {
// Find svg images to extract and process
if (options.convertImages) {
$("svg").each(function() {
$('svg').each(function() {
var content = renderDom($, $(this));
var svgId = _.uniqueId("svg");
var dest = svgId+".svg";
var svgId = _.uniqueId('svg');
var dest = svgId+'.svg';
// Generate filename
dest = "/"+fs.getUniqueFilename(outputRoot, dest);
dest = '/'+fs.getUniqueFilename(outputRoot, dest);
svgContent[dest] = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+content;
$(this).replaceWith($("<img>").attr("src", dest));
svgContent[dest] = '<?xml version="1.0" encoding="UTF-8"?>'+content;
$(this).replaceWith($('<img>').attr('src', dest));
});
}
// Find images to normalize
$("img").each(function() {
$('img').each(function() {
var origin;
var src = $(this).attr("src");
var src = $(this).attr('src');
if (!src) return;
var isExternal = links.isExternal(src);
@@ -132,13 +132,13 @@ function normalizeHtml(src, options) {
// If image is external and ebook, then downlaod the images
if (isExternal) {
origin = src;
src = "/"+crc.crc32(origin).toString(16)+path.extname(origin);
src = '/'+crc.crc32(origin).toString(16)+path.extname(origin);
src = links.toAbsolute(src, options.base, options.output);
isExternal = false;
}
var ext = path.extname(src);
var srcAbs = links.join("/", options.base, src);
var srcAbs = links.join('/', options.base, src);
// Test image extension
if (_.contains(imgUtils.INVALID, ext)) {
@@ -147,29 +147,29 @@ function normalizeHtml(src, options) {
src = imgConversionCache[outputRoot][srcAbs];
} else {
// Not converted yet
var dest = "";
var dest = '';
// Replace extension
dest = links.join(path.dirname(srcAbs), path.basename(srcAbs, ext)+".png");
dest = dest[0] == "/"? dest.slice(1) : dest;
dest = links.join(path.dirname(srcAbs), path.basename(srcAbs, ext)+'.png');
dest = dest[0] == '/'? dest.slice(1) : dest;
// Get a name that doesn"t exists
// Get a name that doesn't exists
dest = fs.getUniqueFilename(outputRoot, dest);
options.book.log.debug.ln("detect invalid image (will be converted to png):", srcAbs);
options.book.log.debug.ln('detect invalid image (will be converted to png):', srcAbs);
// Add to cache
imgConversionCache[outputRoot][srcAbs] = "/"+dest;
imgConversionCache[outputRoot][srcAbs] = '/'+dest;
// Push to convert
toConvert.push({
origin: origin,
content: svgContent[srcAbs],
source: isExternal? srcAbs : path.join("./", srcAbs),
dest: path.join("./", dest)
source: isExternal? srcAbs : path.join('./', srcAbs),
dest: path.join('./', dest)
});
src = links.join("/", dest);
src = links.join('/', dest);
}
// Reset as relative to output
@@ -180,17 +180,17 @@ function normalizeHtml(src, options) {
// Need to downlaod image
toConvert.push({
origin: origin,
source: path.join("./", srcAbs)
source: path.join('./', srcAbs)
});
}
}
$(this).attr("src", src);
$(this).attr('src', src);
});
// Normalize links
$("a").each(function() {
var href = $(this).attr("href");
$('a').each(function() {
var href = $(this).attr('href');
if (!href) return;
if (links.isAnchor(href)) {
@@ -198,7 +198,7 @@ function normalizeHtml(src, options) {
} else if (links.isRelative(href)) {
var parts = url.parse(href);
var absolutePath = links.join(options.base, parts.pathname);
var anchor = parts.hash || "";
var anchor = parts.hash || '';
// If is in navigation relative: transform as content
@@ -209,33 +209,33 @@ function normalizeHtml(src, options) {
// If md/adoc/rst files is not in summary
// or for ebook, signal all files that are outside the summary
else if (_.contains(parsableExtensions, path.extname(absolutePath)) ||
_.contains(["epub", "pdf", "mobi"], options.book.options.generator)) {
options.book.log.warn.ln("page", options.input, "contains an hyperlink to resource outside spine \""+href+"\"");
_.contains(['epub', 'pdf', 'mobi'], options.book.options.generator)) {
options.book.log.warn.ln('page', options.input, 'contains an hyperlink to resource outside spine \''+href+'\'');
}
// Transform as absolute
href = links.toAbsolute("/"+absolutePath, options.base, options.output)+anchor;
href = links.toAbsolute('/'+absolutePath, options.base, options.output)+anchor;
} else {
// External links
$(this).attr("target", "_blank");
$(this).attr('target', '_blank');
}
// Transform extension
$(this).attr("href", href);
$(this).attr('href', href);
});
// Highlight code blocks
$("code").each(function() {
$('code').each(function() {
// Normalize language
var lang = _.chain(
($(this).attr("class") || "").split(" ")
($(this).attr('class') || '').split(' ')
)
.map(function(cl) {
// Markdown
if (cl.search("lang-") === 0) return cl.slice("lang-".length);
if (cl.search('lang-') === 0) return cl.slice('lang-'.length);
// Asciidoc
if (cl.search("language-") === 0) return cl.slice("language-".length);
if (cl.search('language-') === 0) return cl.slice('language-'.length);
return null;
})
@@ -244,7 +244,7 @@ function normalizeHtml(src, options) {
.value();
var source = $(this).text();
var blk = options.book.template.applyBlock("code", {
var blk = options.book.template.applyBlock('code', {
body: source,
kwargs: {
language: lang
@@ -261,12 +261,12 @@ function normalizeHtml(src, options) {
});
_.each(glossary, function(term) {
var r = new RegExp( "\\b(" + pregQuote(term.name.toLowerCase()) + ")\\b" , "gi" );
var r = new RegExp( '\\b(' + pregQuote(term.name.toLowerCase()) + ')\\b' , 'gi' );
var includedInFiles = false;
$("*").each(function() {
$('*').each(function() {
// Ignore codeblocks
if (_.contains(["code", "pre", "a"], this.name.toLowerCase())) return;
if (_.contains(['code', 'pre', 'a'], this.name.toLowerCase())) return;
replaceText($, this, r, function(match) {
// Add to files index in glossary
@@ -275,7 +275,7 @@ function normalizeHtml(src, options) {
term.files = term.files || [];
term.files.push(options.navigation[options.input]);
}
return "<a href=\""+links.toAbsolute("/GLOSSARY.html", options.base, options.output) + "#" + term.id+"\" class=\"glossary-term\" title=\""+_.escape(term.description)+"\">"+match+"</a>";
return '<a href=\''+links.toAbsolute('/GLOSSARY.html', options.base, options.output) + '#' + term.id+'\' class=\'glossary-term\' title=\''+_.escape(term.description)+'\'>'+match+'</a>';
});
});
});
@@ -291,7 +291,7 @@ function convertImages(images, options) {
if (!options.convertImages) return Q();
var downloaded = [];
options.book.log.debug.ln("convert ", images.length, "images to png");
options.book.log.debug.ln('convert ', images.length, 'images to png');
return batch.execEach(images, {
max: 100,
@@ -303,13 +303,13 @@ function convertImages(images, options) {
// Write image if need to be download
.then(function() {
if (!image.origin && !_.contains(downloaded, image.origin)) return;
options.book.log.debug("download image", image.origin, "...");
options.book.log.debug('download image', image.origin, '...');
downloaded.push(image.origin);
return options.book.log.debug.promise(fs.writeStream(imgin, request(image.origin)))
.fail(function(err) {
if (!_.isError(err)) err = new Error(err);
err.message = "Fail downloading "+image.origin+": "+err.message;
err.message = 'Fail downloading '+image.origin+': '+err.message;
throw err;
});
})
@@ -324,13 +324,13 @@ function convertImages(images, options) {
.then(function() {
if (!image.dest) return;
var imgout = path.resolve(options.book.options.output, image.dest);
options.book.log.debug("convert image", image.source, "to", image.dest, "...");
options.book.log.debug('convert image', image.source, 'to', image.dest, '...');
return options.book.log.debug.promise(imgUtils.convertSVG(imgin, imgout));
});
}
})
.then(function() {
options.book.log.debug.ok(images.length+" images converted with success");
options.book.log.debug.ok(images.length+' images converted with success');
});
}
@@ -344,16 +344,16 @@ function normalizePage(sections, options) {
convertImages: false,
// Current file path
input: ".",
input: '.',
// Navigation to use to transform path
navigation: {},
// Directory parent of the file currently in rendering process
base: "./",
base: './',
// Directory parent from the html output
output: "./",
output: './',
// Glossary terms
glossary: []
@@ -363,7 +363,7 @@ function normalizePage(sections, options) {
var toConvert = [];
sections = _.map(sections, function(section) {
if (section.type != "normal") return section;
if (section.type != 'normal') return section;
var out = normalizeHtml(section.content, options);
@@ -374,27 +374,13 @@ function normalizePage(sections, options) {
return Q()
.then(function() {
toConvert = _.uniq(toConvert, "source");
toConvert = _.uniq(toConvert, 'source');
return convertImages(toConvert, options);
})
.thenResolve(sections);
}
// Extract text from sections
function extractText(sections) {
return _.reduce(sections, function(prev, section) {
if (section.type != "normal") return prev;
var $ = cheerio.load(section.content);
$("*").each(function() {
prev = prev+" "+$(this).text();
});
return prev;
}, "");
}
module.exports = {
normalize: normalizePage,
extractText: extractText
normalize: normalizePage
};
-1
View File
@@ -6,7 +6,6 @@
"main": "lib/index.js",
"dependencies": {
"q": "1.0.1",
"lunr": "0.5.12",
"lodash": "3.10.1",
"graceful-fs": "3.0.5",
"resolve": "0.6.3",
+1 -1
View File
File diff suppressed because one or more lines are too long
+20 -19
View File
@@ -1,38 +1,39 @@
define([
"jQuery",
"Mousetrap",
"core/navigation",
"core/sidebar",
"core/search"
], function($, Mousetrap, navigation, sidebar, search){
'jQuery',
'Mousetrap',
'core/navigation',
'core/sidebar'
], function($, Mousetrap, navigation, sidebar){
// Bind a keyboard shortcuts
function bindShortcut(keys, fn) {
Mousetrap.bind(keys, function(e) {
fn();
return false;
});
}
// Bind keyboard shortcuts
var init = function() {
// Next
Mousetrap.bind(['right'], function(e) {
bindShortcut(['right'], function(e) {
navigation.goNext();
return false;
});
// Prev
Mousetrap.bind(['left'], function(e) {
bindShortcut(['left'], function(e) {
navigation.goPrev();
return false;
});
// Toggle Summary
Mousetrap.bind(['s'], function(e) {
bindShortcut(['s'], function(e) {
sidebar.toggle();
return false;
});
// Toggle Search
Mousetrap.bind(['f'], function(e) {
search.toggle();
return false;
});
};
return {
init: init
init: init,
bind: bindShortcut
};
});
+2 -4
View File
@@ -4,9 +4,8 @@ define([
"core/events",
"core/state",
"core/progress",
"core/loading",
"core/search"
], function($, URL, events, state, progress, loading, search) {
"core/loading"
], function($, URL, events, state, progress, loading) {
var prev, next;
var usePushState = (typeof history.pushState !== "undefined");
@@ -67,7 +66,6 @@ define([
// Update state
state.update($("html"));
// recover search keyword
search.recover();
preparePage();
})
.fail(function (e) {
+9 -6
View File
@@ -9,18 +9,14 @@ define([
'core/navigation',
'core/progress',
'core/sidebar',
'core/search',
'apis/toolbar'
], function($, storage, dropdown, events, state,
keyboard, navigation, progress, sidebar, search, toolbar){
keyboard, navigation, progress, sidebar, toolbar){
var start = function(config) {
// Init sidebar
sidebar.init();
// Load search
search.init();
// Init keyboard
keyboard.init();
@@ -39,7 +35,14 @@ keyboard, navigation, progress, sidebar, search, toolbar){
events: events,
state: state,
// UI sections
toolbar: toolbar,
storage: storage
sidebar: sidebar,
// Read/Write the localstorage
storage: storage,
// Create keyboard shortcuts
keyboard: keyboard
};
});
@@ -1,7 +1,6 @@
<div class="book-header" role="navigation">
<!-- Actions Left -->
<a href="#" class="btn pull-left toggle-summary" aria-label="{{ __("SUMMARY_TOGGLE") }}"><i class="fa fa-align-justify"></i></a>
<a href="#" class="btn pull-left toggle-search" aria-label="{{ __("SEARCH_TOGGLE") }}"><i class="fa fa-search"></i></a>
{% if glossary.length > 0 %}
<a href="{{ basePath }}/GLOSSARY.html" class="btn pull-left" aria-label="{{ __("GLOSSARY_OPEN") }}"><i class="fa fa-sort-alpha-asc"></i></a>
{% endif %}
@@ -26,9 +26,6 @@
{% endmacro %}
<div class="book-summary">
<div class="book-search" role="search">
<input type="text" placeholder="{{ __("SEARCH_PLACEHOLDER") }}" class="form-control" />
</div>
<nav role="navigation">
<ul class="summary">
{% set _divider = false %}