smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

+ move unaccent.[h|cpp] to stringTools.[h|cpp] 

+ remove unaccent.[h|cpp]
+ put stringTools.[h|cpp] methods in the "kiwix" namespace
+ modify code & compilation script to keep everything compiling
This commit is contained in:
kelson42 2012-08-14 10:58:03 +00:00
parent 6cac5e6320
commit 68cf5e4977
10 changed files with 81 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/common/kiwix/indexer.cpp
View File

@ -154,11 +154,10 @@ namespace kiwix {
token.size = sizeStringStream.str();
/* Remove accent */
token.title = removeAccents(token.accentedTitle);
token.keywords = removeAccents(htmlParser.keywords);
token.content = removeAccents(htmlParser.dump);
token.title = kiwix::removeAccents(token.accentedTitle);
token.keywords = kiwix::removeAccents(htmlParser.keywords);
token.content = kiwix::removeAccents(htmlParser.dump);
self->pushToIndexQueue(token);
}
/* Test if the thread should be cancelled */

2
src/common/kiwix/indexer.h
View File

@ -35,7 +35,7 @@
#endif
#include <pthread.h>
#include <unaccent.h>
#include <stringTools.h>
#include <zim/file.h>
#include <zim/article.h>
#include <zim/fileiterator.h>

2
src/common/kiwix/library.cpp
View File

@ -62,7 +62,7 @@ namespace kiwix {
std::string Book::getHumanReadableIdFromPath() {
std::string id = path;
if (!id.empty()) {
removeAccents(id);
kiwix::removeAccents(id);
id = replaceRegex(id, "", "^.*/");
id = replaceRegex(id, "", "\\.zim[a-z]*$");
id = replaceRegex(id, "_", " ");

2
src/common/kiwix/library.h
View File

@ -27,7 +27,7 @@
#include <vector>
#include <stack>
#include <unaccent.h>
#include <stringTools.h>
#include <regexTools.h>
#define KIWIX_LIBRARY_VERSION "20110515"

6
src/common/kiwix/searcher.cpp
View File

@ -160,10 +160,10 @@ namespace kiwix {
result["snippet"] = this->resultOffset->snippet;
if (this->resultOffset->size >= 0)
result["size"] = ::beautifyInteger(this->resultOffset->size);
result["size"] = kiwix::beautifyInteger(this->resultOffset->size);
if (this->resultOffset->wordCount >= 0)
result["wordCount"] = ::beautifyInteger(this->resultOffset->wordCount);
result["wordCount"] = kiwix::beautifyInteger(this->resultOffset->wordCount);
resultsCDT.PushBack(result);
this->resultOffset++;
@ -195,7 +195,7 @@ namespace kiwix {
}
oData["pages"] = pagesCDT;
oData["count"] = ::beautifyInteger(this->estimatedResultCount);
oData["count"] = kiwix::beautifyInteger(this->estimatedResultCount);
oData["searchPattern"] = this->searchPattern;
oData["resultStart"] = this->resultStart + 1;
oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount ? this->estimatedResultCount : this->resultEnd);

1
src/common/kiwix/searcher.h
View File

@ -28,7 +28,6 @@
#include <locale>
#include <cctype>
#include <vector>
#include <unaccent.h>
#include <resourceTools.h>
#include <stringTools.h>

57
src/common/stringTools.cpp
View File

@ -20,8 +20,8 @@
#include "stringTools.h"
/* Prepare integer for display */
std::string beautifyInteger(const unsigned int number) {
stringstream numberStream;
std::string kiwix::beautifyInteger(const unsigned int number) {
std::stringstream numberStream;
numberStream << number;
std::string numberString = numberStream.str();
@ -35,14 +35,14 @@ std::string beautifyInteger(const unsigned int number) {
}
/* Split string in a token array */
std::vector<std::string> split(const std::string & str,
std::vector<std::string> kiwix::split(const std::string & str,
const std::string & delims=" *-")
{
string::size_type lastPos = str.find_first_not_of(delims, 0);
string::size_type pos = str.find_first_of(delims, lastPos);
vector<string> tokens;
std::string::size_type lastPos = str.find_first_not_of(delims, 0);
std::string::size_type pos = str.find_first_of(delims, lastPos);
std::vector<std::string> tokens;
while (string::npos != pos || string::npos != lastPos)
while (std::string::npos != pos || std::string::npos != lastPos)
{
tokens.push_back(str.substr(lastPos, pos - lastPos));
lastPos = str.find_first_not_of(delims, pos);
@ -52,15 +52,52 @@ std::vector<std::string> split(const std::string & str,
return tokens;
}
std::vector<std::string> split(const char* lhs, const char* rhs){
std::vector<std::string> kiwix::split(const char* lhs, const char* rhs){
const std::string m1 (lhs), m2 (rhs);
return split(m1, m2);
}
std::vector<std::string> split(const char* lhs, const std::string& rhs){
std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs){
return split(lhs, rhs.c_str());
}
std::vector<std::string> split(const std::string& lhs, const char* rhs){
std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs){
return split(lhs.c_str(), rhs);
}
std::string kiwix::removeAccents(const std::string &text) {
ucnv_setDefaultName("UTF-8");
UErrorCode status = U_ZERO_ERROR;
Transliterator *removeAccentsTrans = Transliterator::createInstance("Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
UnicodeString ustring = UnicodeString(text.c_str());
removeAccentsTrans->transliterate(ustring);
std::string unaccentedText;
ustring.toUTF8String(unaccentedText);
return unaccentedText;
}
void kiwix::printStringInHexadecimal(UnicodeString s) {
std::cout << std::showbase << std::hex;
for (int i=0; i<s.length(); i++) {
char c = (char)((s.getTerminatedBuffer())[i]);
if (c & 0x80)
std::cout << (c & 0xffff) << " ";
else
std::cout << c << " ";
}
std::cout << std::endl;
}
void kiwix::printStringInHexadecimal(const char *s) {
std::cout << std::showbase << std::hex;
for (char const* pc = s; *pc; ++pc) {
if (*pc & 0x80)
std::cout << (*pc & 0xffff);
else
std::cout << *pc;
std::cout << ' ';
}
std::cout << std::endl;
}

32
src/common/stringTools.h
View File

@ -1,5 +1,5 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright 2011-2012 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -17,16 +17,34 @@
* MA 02110-1301, USA.
*/
#ifndef KIWIX_STRINGTOOLS_H
#define KIWIX_STRINGTOOLS_H
#include <unicode/translit.h>
#include <unicode/normlzr.h>
#include <unicode/unistr.h>
#include <unicode/rep.h>
#include <unicode/translit.h>
#include <unicode/uniset.h>
#include <unicode/ustring.h>
#include <unicode/ucnv.h>
#include <iostream>
#include <vector>
#include <string>
#include <fstream>
#include <sstream>
using namespace std;
namespace kiwix {
std::string removeAccents(const std::string &text);
std::string beautifyInteger(const unsigned int number);
std::vector<std::string> split(const std::string&, const std::string&);
std::vector<std::string> split(const char*, const char*);
std::vector<std::string> split(const std::string&, const char*);
std::vector<std::string> split(const char*, const std::string&);
std::string beautifyInteger(const unsigned int number);
std::vector<std::string> split(const std::string&, const std::string&);
std::vector<std::string> split(const char*, const char*);
std::vector<std::string> split(const std::string&, const char*);
std::vector<std::string> split(const char*, std::string&);
void printStringInHexadecimal(const char *s);
void printStringInHexadecimal(UnicodeString s);
}
#endif

56
src/common/unaccent.cpp
View File

@ -1,56 +0,0 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "unaccent.h"
UErrorCode status = U_ZERO_ERROR;
Transliterator *trans = Transliterator::createInstance("Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
std::string removeAccents(const std::string &text) {
ucnv_setDefaultName("UTF-8");
UnicodeString ustring = UnicodeString(text.c_str());
trans->transliterate(ustring);
std::string unaccentedText;
ustring.toUTF8String(unaccentedText);
return unaccentedText;
}
void printStringInHexadecimal(UnicodeString s) {
std::cout << std::showbase << std::hex;
for (int i=0; i<s.length(); i++) {
char c = (char)((s.getTerminatedBuffer())[i]);
if (c & 0x80)
std::cout << (c & 0xffff) << " ";
else
std::cout << c << " ";
}
std::cout << std::endl;
}
void printStringInHexadecimal(const char *s) {
std::cout << std::showbase << std::hex;
for (char const* pc = s; *pc; ++pc) {
if (*pc & 0x80)
std::cout << (*pc & 0xffff);
else
std::cout << *pc;
std::cout << ' ';
}
std::cout << std::endl;
}

39
src/common/unaccent.h
View File

@ -1,39 +0,0 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_UNACCENT_H
#define KIWIX_UNACCENT_H
#include <unicode/translit.h>
#include <unicode/normlzr.h>
#include <unicode/unistr.h>
#include <unicode/rep.h>
#include <unicode/translit.h>
#include <unicode/uniset.h>
#include <unicode/ustring.h>
#include <unicode/ucnv.h>
#include <iostream>
#include <string>
std::string removeAccents(const std::string &text);
void printStringInHexadecimal(const char *s);
void printStringInHexadecimal(UnicodeString s);
#endif