From 8bb7c040b10074e0ad184383f68787264589ef62 Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Thu, 14 Nov 2013 18:50:26 +0000 Subject: [PATCH] Fixed CTPP2 loader to use resource instead of file --- src/common/kiwix/ctpp2/CTPP2TextLoader.hpp | 91 ------- .../kiwix/ctpp2/CTPP2VMStringLoader.cpp | 244 ++++++++++++++++++ ...TextLoader.cpp => CTPP2VMStringLoader.hpp} | 82 +++--- src/common/kiwix/searcher.cpp | 20 +- src/common/kiwix/searcher.h | 5 +- 5 files changed, 287 insertions(+), 155 deletions(-) delete mode 100644 src/common/kiwix/ctpp2/CTPP2TextLoader.hpp create mode 100644 src/common/kiwix/ctpp2/CTPP2VMStringLoader.cpp rename src/common/kiwix/ctpp2/{CTPP2TextLoader.cpp => CTPP2VMStringLoader.hpp} (64%) diff --git a/src/common/kiwix/ctpp2/CTPP2TextLoader.hpp b/src/common/kiwix/ctpp2/CTPP2TextLoader.hpp deleted file mode 100644 index d280cb0d3..000000000 --- a/src/common/kiwix/ctpp2/CTPP2TextLoader.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/*- - * Copyright (c) 2004 - 2010 CTPP Team - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the CTPP Team nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * CTPP2SourceLoader.hpp - * - * $CTPP$ - */ -#ifndef _CTPP2_TEXT_LOADER_H__ -#define _CTPP2_TEXT_LOADER_H__ 1 - -#include "CTPP2SourceLoader.hpp" - -#include "STLString.hpp" - -/** - @file CTPP2FileSourceLoader.hpp - @brief Load template from file -*/ - -namespace CTPP // C++ Template Engine -{ - -/** - @class CTPP2FileSourceLoader CTPP2FileSourceLoader.hpp> - @brief Load template from file -*/ -class CTPP2DECL CTPP2TextLoader: - public CTPP2SourceLoader -{ -public: - /** - @brief Constructor - */ - CTPP2TextLoader(); - - /** - @brief Load template with specified name - @param szTemplateName - template name - @return 0 if success, -1 if any error occured - */ - INT_32 LoadTemplate(CCHAR_P szTemplateName); - - /** - @brief Get template - @param iTemplateSize - template size [out] - @return pointer to start of template buffer if success, NULL - if any error occured - */ - CCHAR_P GetTemplate(UINT_32 & iTemplateSize); - - /** - @brief Clone loader object - @return clone to self - */ - CTPP2SourceLoader * Clone(); - - /** - @brief A destructor - */ - ~CTPP2TextLoader() throw(); - -private: - std::string templateText; -}; - -} // namespace CTPP -#endif // _CTPP2_FILE_SOURCE_LOADER_H__ -// End. diff --git a/src/common/kiwix/ctpp2/CTPP2VMStringLoader.cpp b/src/common/kiwix/ctpp2/CTPP2VMStringLoader.cpp new file mode 100644 index 000000000..cb910dce0 --- /dev/null +++ b/src/common/kiwix/ctpp2/CTPP2VMStringLoader.cpp @@ -0,0 +1,244 @@ +/*- + * Copyright (c) 2004 - 2011 CTPP Team + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the CTPP Team nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * CTPP2VMStringLoader.cpp + * + * $CTPP$ + */ + +#include "CTPP2VMStringLoader.hpp" + +#include "CTPP2Util.hpp" +#include "CTPP2Exception.hpp" +#include "CTPP2VMExecutable.hpp" +#include "CTPP2VMInstruction.hpp" +#include "CTPP2VMMemoryCore.hpp" + +#include +#include + +#include +#include +#include + +#include +#include +#include + +namespace CTPP // C++ Template Engine +{ + +// +// Convert byte order +// +static void ConvertExecutable(VMExecutable * oCore) +{ + // Code entry point + oCore -> entry_point = Swap32(oCore -> entry_point); + // Offset of code segment + oCore -> code_offset = Swap32(oCore -> code_offset); + + // Code segment size + oCore -> code_size = Swap32(oCore -> code_size); + + // Offset of static text segment + oCore -> syscalls_offset = Swap32(oCore -> syscalls_offset); + // Static text segment size + oCore -> syscalls_data_size = Swap32(oCore -> syscalls_data_size); + + // Offset of static text index segment + oCore -> syscalls_index_offset = Swap32(oCore -> syscalls_index_offset); + // Static text index segment size + oCore -> syscalls_index_size = Swap32(oCore -> syscalls_index_size); + + // Offset of static data segment + oCore -> static_data_offset = Swap32(oCore -> static_data_offset); + + // Static data segment size + oCore -> static_data_data_size = Swap32(oCore -> static_data_data_size); + + // Offset of static text segment + oCore -> static_text_offset = Swap32(oCore -> static_text_offset); + // Static text segment size + oCore -> static_text_data_size = Swap32(oCore -> static_text_data_size); + + // Offset of static text index segment + oCore -> static_text_index_offset = Swap32(oCore -> static_text_index_offset); + // Static text index segment size + oCore -> static_text_index_size = Swap32(oCore -> static_text_index_size); + + // Version 2.2+ + // Offset of static data bit index + oCore -> static_data_bit_index_offset = Swap32(oCore -> static_data_bit_index_offset); + /// Offset of static data bit index + oCore -> static_data_bit_index_size = Swap32(oCore -> static_data_bit_index_size); + + // Platform + oCore -> platform = Swap64(oCore -> platform); + + // Ugly-jolly hack! + // ... dereferencing type-punned pointer will break strict-aliasing rules ... + UINT_64 iTMP; + memcpy(&iTMP, &(oCore -> ieee754double), sizeof(UINT_64)); + iTMP = Swap64(iTMP); + memcpy(&(oCore -> ieee754double), &iTMP, sizeof(UINT_64)); + + // Cyclic Redundancy Check + oCore -> crc = 0; + + // Convert data structures + + // Convert code segment + VMInstruction * pInstructions = const_cast(VMExecutable::GetCodeSeg(oCore)); + UINT_32 iI = 0; + UINT_32 iSteps = oCore -> code_size / sizeof(VMInstruction); + for(iI = 0; iI < iSteps; ++iI) + { + pInstructions -> instruction = Swap32(pInstructions -> instruction); + pInstructions -> argument = Swap32(pInstructions -> argument); + pInstructions -> reserved = Swap64(pInstructions -> reserved); + ++pInstructions; + } + + // Convert syscalls index + TextDataIndex * pTextIndex = const_cast(VMExecutable::GetSyscallsIndexSeg(oCore)); + iSteps = oCore -> syscalls_index_size / sizeof(TextDataIndex); + for(iI = 0; iI < iSteps; ++iI) + { + pTextIndex -> offset = Swap32(pTextIndex -> offset); + pTextIndex -> length = Swap32(pTextIndex -> length); + ++pTextIndex; + } + + // Convert static text index + pTextIndex = const_cast(VMExecutable::GetStaticTextIndexSeg(oCore)); + iSteps = oCore -> static_text_index_size / sizeof(TextDataIndex); + for(iI = 0; iI < iSteps; ++iI) + { + pTextIndex -> offset = Swap32(pTextIndex -> offset); + pTextIndex -> length = Swap32(pTextIndex -> length); + ++pTextIndex; + } + + // Convert static data + StaticDataVar * pStaticDataVar = const_cast(VMExecutable::GetStaticDataSeg(oCore)); + iSteps = oCore -> static_data_data_size / sizeof(StaticDataVar); + for(iI = 0; iI < iSteps; ++iI) + { + (*pStaticDataVar).i_data = Swap64((*pStaticDataVar).i_data); + ++pStaticDataVar; + } +} + +// +// Constructor +// +VMStringLoader::VMStringLoader(CCHAR_P rawContent, size_t rawContentSize) +{ + oCore = (VMExecutable *)malloc(rawContentSize + 1); + memcpy(oCore, rawContent, rawContentSize); + + if (oCore -> magic[0] == 'C' && + oCore -> magic[1] == 'T' && + oCore -> magic[2] == 'P' && + oCore -> magic[3] == 'P') + { + // Check version + if (oCore -> version[0] >= 1) + { + // Platform-dependent data (byte order) + if (oCore -> platform == 0x4142434445464748ull) + { +#ifdef _DEBUG + fprintf(stderr, "Big/Little Endian conversion: Nothing to do\n"); +#endif + + // Nothing to do, only check crc + UINT_32 iCRC = oCore -> crc; + oCore -> crc = 0; + + // Calculate CRC of file + // KELSON: next line used to refer to oStat.st_size + // changed it to rawContentSize + if (iCRC != crc32((UCCHAR_P)oCore, rawContentSize)) + { + free(oCore); + throw CTPPLogicError("CRC checksum invalid"); + } + } + // Platform-dependent data (byte order) + else if (oCore -> platform == 0x4847464544434241ull) + { + // Need to reconvert data +#ifdef _DEBUG + fprintf(stderr, "Big/Little Endian conversion: Need to reconvert core\n"); +#endif + ConvertExecutable(oCore); + } + else + { + free(oCore); + throw CTPPLogicError("Conversion of middle-end architecture does not supported."); + } + + // Check IEEE 754 format + if (oCore -> ieee754double != 15839800103804824402926068484019465486336.0) + { + free(oCore); + throw CTPPLogicError("IEEE 754 format is broken, cannot convert file"); + } + } + + std::cout << "Last ?" << std::endl; + + pVMMemoryCore = new VMMemoryCore(oCore); + + std::cout << "last2" << std::endl; + } + else + { + free(oCore); + throw CTPPLogicError("Not an CTPP bytecode file."); + } +} + +// +// Get ready-to-run program +// +const VMMemoryCore * VMStringLoader::GetCore() const { return pVMMemoryCore; } + +// +// A destructor +// +VMStringLoader::~VMStringLoader() throw() +{ + delete pVMMemoryCore; + free(oCore); +} + +} // namespace CTPP +// End. diff --git a/src/common/kiwix/ctpp2/CTPP2TextLoader.cpp b/src/common/kiwix/ctpp2/CTPP2VMStringLoader.hpp similarity index 64% rename from src/common/kiwix/ctpp2/CTPP2TextLoader.cpp rename to src/common/kiwix/ctpp2/CTPP2VMStringLoader.hpp index 8f458b89f..9cfcec082 100644 --- a/src/common/kiwix/ctpp2/CTPP2TextLoader.cpp +++ b/src/common/kiwix/ctpp2/CTPP2VMStringLoader.hpp @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2004 - 2010 CTPP Team + * Copyright (c) 2004 - 2011 CTPP Team * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -25,66 +25,52 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * CTPP2FileSourceLoader.cpp + * CTPP2VMStringLoader.hpp * * $CTPP$ */ +#ifndef _CTPP2_VM_STRING_LOADER_HPP__ +#define _CTPP2_VM_STRING_LOADER_HPP__ 1 -#include "CTPP2TextLoader.hpp" - -#include "CTPP2Exception.hpp" - -#include -#include -#include -#include +#include "CTPP2VMLoader.hpp" /** - @file FileOutputCollector.hpp - @brief Virtual machine file output data collector + @file VMStringLoader.hpp + @brief Load program core from file */ namespace CTPP // C++ Template Engine { +// FWD +struct VMExecutable; -// -// Clone loader -// -CTPP2SourceLoader * CTPP2TextLoader::Clone() +/** + @class VMStringLoader CTPP2VMStringLoader.hpp + @brief Load program core from file +*/ +class CTPP2DECL VMStringLoader: + public VMLoader { - return new CTPP2TextLoader; -} +public: + /** + */ + VMStringLoader(CCHAR_P rawContent, size_t rawContentSize); + /** + @brief Get ready-to-run program + */ + const VMMemoryCore * GetCore() const; -// -// Constructor -// -CTPP2TextLoader::CTPP2TextLoader() { -} - -// -// Load template with specified name -// -INT_32 CTPP2TextLoader::LoadTemplate(CCHAR_P szTemplateName) -{ - this->templateText = std::string(szTemplateName); - return 0; -} - -// -// Get template -// -CCHAR_P CTPP2TextLoader::GetTemplate(UINT_32 & iITemplateSize) -{ - iITemplateSize = this->templateText.size(); - return this->templateText.data(); -} - -// -// A destructor -// -CTPP2TextLoader::~CTPP2TextLoader() throw() -{ -} + /** + @brief A destructor + */ + ~VMStringLoader() throw(); +private: + /** Program core */ + VMExecutable * oCore; + /** Ready-to-run program */ + VMMemoryCore * pVMMemoryCore; +}; } // namespace CTPP +#endif // _CTPP2_VM_STRING_LOADER_HPP__ // End. diff --git a/src/common/kiwix/searcher.cpp b/src/common/kiwix/searcher.cpp index cdffb336a..8d80403de 100644 --- a/src/common/kiwix/searcher.cpp +++ b/src/common/kiwix/searcher.cpp @@ -31,12 +31,9 @@ namespace kiwix { estimatedResultCount(0), resultStart(0), resultEnd(0), - resultRange(20), - template_ct2("_ctpp2results.ct2") + resultRange(20) { - if (!fileExists(template_ct2)) { - writeTextFile(template_ct2.c_str(), getResourceAsString("results.ct2")); - } + template_ct2 = getResourceAsString("results.ct2"); } /* Search strings in the database */ @@ -173,16 +170,13 @@ namespace kiwix { oData["searchProtocolPrefix"] = this->searchProtocolPrefix; oData["contentId"] = this->contentHumanReadableId; - // Load template file - VMFileLoader oLoader(this->template_ct2.c_str()); + VMStringLoader oLoader(template_ct2.c_str(), template_ct2.size()); - // Create logger object FileLogger oLogger(stderr); - // Execute template and write data to standard output - oSimpleVM.Run(oData, oLoader, stdout, oLogger); + // DEBUG only (write output to stdout) + // oSimpleVM.Run(oData, oLoader, stdout, oLogger); - // Execute template and write data to string std::string sResult; oSimpleVM.Run(oData, oLoader, sResult, oLogger); @@ -192,9 +186,7 @@ namespace kiwix { /* Destructor */ Searcher::~Searcher() { - if (fileExists(this->template_ct2)) { - remove(template_ct2.c_str()); - } + } } diff --git a/src/common/kiwix/searcher.h b/src/common/kiwix/searcher.h index e9d9094e3..efe4167d0 100644 --- a/src/common/kiwix/searcher.h +++ b/src/common/kiwix/searcher.h @@ -35,7 +35,8 @@ #include #include #include -#include + +#include "kiwix/ctpp2/CTPP2VMStringLoader.hpp" using namespace std; using namespace CTPP; @@ -79,7 +80,7 @@ namespace kiwix { std::string searchPattern; std::string protocolPrefix; std::string searchProtocolPrefix; - const std::string template_ct2; + std::string template_ct2; unsigned int resultCountPerPage; unsigned int estimatedResultCount; unsigned int resultStart;