diff --git a/include/entry.h b/include/entry.h index e2edf0a9a..c3a01521c 100644 --- a/include/entry.h +++ b/include/entry.h @@ -111,7 +111,7 @@ class Entry * The offset is the offset to read in the file. * Return <"",0> if is not possible to read directly. */ - std::pair getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); } + zim::Item::DirectAccessInfo getDirectAccessInfo() const { return entry.getItem().getDirectAccessInformation(); } /** * Get the size of the entry. diff --git a/include/reader.h b/include/reader.h index d36e44da9..94c108f9d 100644 --- a/include/reader.h +++ b/include/reader.h @@ -55,7 +55,11 @@ class Reader * unsplitted path as if the file were not splitted * (.zim extesion). */ - Reader(const string zimFilePath); + explicit Reader(const string zimFilePath); +#ifndef _WIN32 + explicit Reader(int fd); + Reader(int fd, zim::offset_type offset, zim::size_type size); +#endif ~Reader() = default; /** diff --git a/src/reader.cpp b/src/reader.cpp index 173dbbb93..523c1f766 100644 --- a/src/reader.cpp +++ b/src/reader.cpp @@ -86,6 +86,24 @@ Reader::Reader(const string zimFilePath) srand(time(nullptr)); } +#ifndef _WIN32 +Reader::Reader(int fd) + : zimArchive(new zim::Archive(fd)), + zimFilePath("") +{ + /* initialize random seed: */ + srand(time(nullptr)); +} + +Reader::Reader(int fd, zim::offset_type offset, zim::size_type size) + : zimArchive(new zim::Archive(fd, offset, size)), + zimFilePath("") +{ + /* initialize random seed: */ + srand(time(nullptr)); +} +#endif // #ifndef _WIN32 + zim::Archive* Reader::getZimArchive() const { return zimArchive.get(); diff --git a/src/wrapper/java/kiwixreader.cpp b/src/wrapper/java/kiwixreader.cpp index e55d54f8d..cc3c27706 100644 --- a/src/wrapper/java/kiwixreader.cpp +++ b/src/wrapper/java/kiwixreader.cpp @@ -45,6 +45,72 @@ JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReader( } } +namespace +{ + +int jni2fd(const jobject& fdObj, JNIEnv* env) +{ + jclass class_fdesc = env->FindClass("java/io/FileDescriptor"); + jfieldID field_fd = env->GetFieldID(class_fdesc, "fd", "I"); + if ( field_fd == NULL ) + { + env->ExceptionClear(); + // Under Android the (private) 'fd' field of java.io.FileDescriptor has been + // renamed to 'descriptor'. See, for example, + // https://android.googlesource.com/platform/libcore/+/refs/tags/android-8.1.0_r1/ojluni/src/main/java/java/io/FileDescriptor.java#55 + field_fd = env->GetFieldID(class_fdesc, "descriptor", "I"); + } + return env->GetIntField(fdObj, field_fd); +} + +} // unnamed namespace + +JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderByFD( + JNIEnv* env, jobject obj, jobject fdObj) +{ +#ifndef _WIN32 + int fd = jni2fd(fdObj, env); + + LOG("Attempting to create reader with fd: %d", fd); + Lock l; + try { + kiwix::Reader* reader = new kiwix::Reader(fd); + return reinterpret_cast(new Handle(reader)); + } catch (std::exception& e) { + LOG("Error opening ZIM file"); + LOG(e.what()); + return 0; + } +#else + jclass exception = env->FindClass("java/lang/UnsupportedOperationException"); + env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderByFD() is not supported under Windows"); + return 0; +#endif +} + +JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getNativeReaderEmbedded( + JNIEnv* env, jobject obj, jobject fdObj, jlong offset, jlong size) +{ +#ifndef _WIN32 + int fd = jni2fd(fdObj, env); + + LOG("Attempting to create reader with fd: %d", fd); + Lock l; + try { + kiwix::Reader* reader = new kiwix::Reader(fd, offset, size); + return reinterpret_cast(new Handle(reader)); + } catch (std::exception& e) { + LOG("Error opening ZIM file"); + LOG(e.what()); + return 0; + } +#else + jclass exception = env->FindClass("java/lang/UnsupportedOperationException"); + env->ThrowNew(exception, "org.kiwix.kiwixlib.JNIKiwixReader.getNativeReaderEmbedded() is not supported under Windows"); + return 0; +#endif +} + JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_dispose(JNIEnv* env, jobject obj) { @@ -325,22 +391,22 @@ JNIEXPORT jobject JNICALL Java_org_kiwix_kiwixlib_JNIKiwixReader_getDirectAccessInformation( JNIEnv* env, jobject obj, jstring url) { - jclass classPair = env->FindClass("org/kiwix/kiwixlib/Pair"); - jmethodID midPairinit = env->GetMethodID(classPair, "", "()V"); - jobject pair = env->NewObject(classPair, midPairinit); - setPairObjValue("", 0, pair, env); + jclass daiClass = env->FindClass("org/kiwix/kiwixlib/DirectAccessInfo"); + jmethodID daiInitMethod = env->GetMethodID(daiClass, "", "()V"); + jobject dai = env->NewObject(daiClass, daiInitMethod); + setDaiObjValue("", 0, dai, env); std::string cUrl = jni2c(url, env); try { auto entry = READER->getEntryFromEncodedPath(cUrl); entry = entry.getFinalEntry(); auto part_info = entry.getDirectAccessInfo(); - setPairObjValue(part_info.first, part_info.second, pair, env); + setDaiObjValue(part_info.first, part_info.second, dai, env); } catch (std::exception& e) { LOG("Unable to get direct access info for url: %s", cUrl.c_str()); LOG(e.what()); } - return pair; + return dai; } JNIEXPORT jboolean JNICALL diff --git a/src/wrapper/java/meson.build b/src/wrapper/java/meson.build index 034173045..de72e49b3 100644 --- a/src/wrapper/java/meson.build +++ b/src/wrapper/java/meson.build @@ -12,7 +12,7 @@ java_sources = files([ 'org/kiwix/kiwixlib/JNIKiwixString.java', 'org/kiwix/kiwixlib/JNIKiwixBool.java', 'org/kiwix/kiwixlib/JNIKiwixException.java', - 'org/kiwix/kiwixlib/Pair.java' + 'org/kiwix/kiwixlib/DirectAccessInfo.java' ]) kiwix_jni = custom_target('jni', diff --git a/src/wrapper/java/org/kiwix/kiwixlib/Pair.java b/src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java similarity index 96% rename from src/wrapper/java/org/kiwix/kiwixlib/Pair.java rename to src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java index 05aacb678..4ba137dc4 100644 --- a/src/wrapper/java/org/kiwix/kiwixlib/Pair.java +++ b/src/wrapper/java/org/kiwix/kiwixlib/DirectAccessInfo.java @@ -19,7 +19,7 @@ package org.kiwix.kiwixlib; -public class Pair +public class DirectAccessInfo { public String filename; public long offset; diff --git a/src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java b/src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java index f94c3dc6f..e11d72190 100644 --- a/src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java +++ b/src/wrapper/java/org/kiwix/kiwixlib/JNIKiwixReader.java @@ -24,7 +24,8 @@ import org.kiwix.kiwixlib.JNIKiwixException; import org.kiwix.kiwixlib.JNIKiwixString; import org.kiwix.kiwixlib.JNIKiwixInt; import org.kiwix.kiwixlib.JNIKiwixSearcher; -import org.kiwix.kiwixlib.Pair; +import org.kiwix.kiwixlib.DirectAccessInfo; +import java.io.FileDescriptor; public class JNIKiwixReader { @@ -102,13 +103,13 @@ public class JNIKiwixReader * the zim file (or zim part) and directly read the content from it (and so * bypassing the libzim). * - * Return a `Pair` (filename, offset) where the content is located. + * Return a `DirectAccessInfo` (filename, offset) where the content is located. * * If the content cannot be directly accessed (content is compressed or zim * file is cut in the middle of the content), the filename is an empty string * and offset is zero. */ - public native Pair getDirectAccessInformation(String url); + public native DirectAccessInfo getDirectAccessInformation(String url); public native boolean searchSuggestions(String prefix, int count); @@ -151,11 +152,31 @@ public class JNIKiwixReader throw new JNIKiwixException("Cannot open zimfile "+filename); } } + + public JNIKiwixReader(FileDescriptor fd) throws JNIKiwixException + { + nativeHandle = getNativeReaderByFD(fd); + if (nativeHandle == 0) { + throw new JNIKiwixException("Cannot open zimfile by fd "+fd.toString()); + } + } + + public JNIKiwixReader(FileDescriptor fd, long offset, long size) + throws JNIKiwixException + { + nativeHandle = getNativeReaderEmbedded(fd, offset, size); + if (nativeHandle == 0) { + throw new JNIKiwixException(String.format("Cannot open embedded zimfile (fd=%s, offset=%d, size=%d)", fd, offset, size)); + } + } + public JNIKiwixReader() { } public native void dispose(); private native long getNativeReader(String filename); + private native long getNativeReaderByFD(FileDescriptor fd); + private native long getNativeReaderEmbedded(FileDescriptor fd, long offset, long size); private long nativeHandle; } diff --git a/src/wrapper/java/org/kiwix/testing/catalog.xml b/src/wrapper/java/org/kiwix/testing/catalog.xml new file mode 100644 index 000000000..3e2f6ac20 --- /dev/null +++ b/src/wrapper/java/org/kiwix/testing/catalog.xml @@ -0,0 +1,19 @@ + + 00000000-0000-0000-0000-000000000000 + + Test ZIM file + urn:uuid:86c91e51-55bf-8882-464e-072aca37a3e8 + /meta?name=favicon&content=small + 2020-11-27:00::00:Z + en + This is a ZIM file used in libzim unit-tests + unit;test + + + Kiwix + + + + + + diff --git a/src/wrapper/java/org/kiwix/testing/compile_and_run_test.sh b/src/wrapper/java/org/kiwix/testing/compile_and_run_test.sh new file mode 100755 index 000000000..41899afaa --- /dev/null +++ b/src/wrapper/java/org/kiwix/testing/compile_and_run_test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/bash + +# This script compiles and runs the unit test to test the java wrapper. +# This is not integrated in meson because ... this is not so easy. + +die() +{ + echo >&2 "!!! ERROR: $*" + exit 1 +} + + +KIWIX_LIB_JAR=$1 +if [ -z $KIWIX_LIB_JAR ] +then + die "You must give the path to the kiwixlib.jar as first argument" +fi + +KIWIX_LIB_DIR=$2 +if [ -z $KIWIX_LIB_DIR ] +then + die "You must give the path to directory containing libkiwix.so as second argument" +fi + +KIWIX_LIB_JAR=$(readlink -f "$KIWIX_LIB_JAR") +KIWIX_LIB_DIR=$(readlink -f "$KIWIX_LIB_DIR") +TEST_SOURCE_DIR=$(dirname "$(readlink -f $0)") + +cd "$TEST_SOURCE_DIR" + +javac -g -d . -s . -cp "junit-4.13.jar:$KIWIX_LIB_JAR" test.java \ + || die "Compilation failed" + +java -Djava.library.path="$KIWIX_LIB_DIR" \ + -cp "junit-4.13.jar:hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:." \ + org.junit.runner.JUnitCore test \ + || die "Unit test failed" diff --git a/src/wrapper/java/org/kiwix/testing/compile_test.sh b/src/wrapper/java/org/kiwix/testing/compile_test.sh deleted file mode 100755 index a98569935..000000000 --- a/src/wrapper/java/org/kiwix/testing/compile_test.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/bash - -# This script compile the unit test to test the java wrapper. -# This is not integrated in meson because ... this is not so easy. - - -KIWIX_LIB_JAR=$1 -if [ -z $KIWIX_LIB_JAR ] -then - echo "You must give the path to the kiwixlib.jar as first argument" - exit 1 -fi - -KIWIX_LIB_DIR=$2 -if [ -z $KIWIX_LIB_DIR ] -then - echo "You must give the path to directory containing libkiwix.so as second argument" - exit 1 -fi -TEST_SOURCE_DIR=$(dirname $(readlink -f $0)) - - -javac -g -d . -s . -cp $TEST_SOURCE_DIR/junit-4.13.jar:$KIWIX_LIB_JAR $TEST_SOURCE_DIR/test.java - -java -Djava.library.path=$KIWIX_LIB_DIR -cp $TEST_SOURCE_DIR/junit-4.13.jar:$TEST_SOURCE_DIR/hamcrest-core-1.3.jar:$KIWIX_LIB_JAR:. org.junit.runner.JUnitCore test - diff --git a/src/wrapper/java/org/kiwix/testing/create_test_zimfiles b/src/wrapper/java/org/kiwix/testing/create_test_zimfiles new file mode 100755 index 000000000..4610b7da2 --- /dev/null +++ b/src/wrapper/java/org/kiwix/testing/create_test_zimfiles @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +die() +{ + echo >&2 "!!! ERROR: $*" + exit 1 +} + +cd "$(dirname "$0")" +rm -f small.zim +zimwriterfs --withoutFTIndex \ + -w main.html \ + -f favicon.png \ + -l en \ + -t "Test ZIM file" \ + -d "N/A" \ + -c "N/A" \ + -p "N/A" \ + small_zimfile_data \ + small.zim \ +&& echo 'small.zim was successfully created' \ +|| die 'Failed to create small.zim' + +printf "BEGINZIM" > small.zim.embedded \ +&& cat small.zim >> small.zim.embedded \ +&& printf "ENDZIM" >> small.zim.embedded \ +&& echo 'small.zim.embedded was successfully created' \ +|| die 'Failed to create small.zim.embedded' diff --git a/src/wrapper/java/org/kiwix/testing/small.zim b/src/wrapper/java/org/kiwix/testing/small.zim new file mode 100644 index 000000000..bce4fc57a Binary files /dev/null and b/src/wrapper/java/org/kiwix/testing/small.zim differ diff --git a/src/wrapper/java/org/kiwix/testing/small.zim.embedded b/src/wrapper/java/org/kiwix/testing/small.zim.embedded new file mode 100644 index 000000000..1bc1da2c2 Binary files /dev/null and b/src/wrapper/java/org/kiwix/testing/small.zim.embedded differ diff --git a/src/wrapper/java/org/kiwix/testing/small_zimfile_data/favicon.png b/src/wrapper/java/org/kiwix/testing/small_zimfile_data/favicon.png new file mode 100644 index 000000000..809454352 Binary files /dev/null and b/src/wrapper/java/org/kiwix/testing/small_zimfile_data/favicon.png differ diff --git a/src/wrapper/java/org/kiwix/testing/small_zimfile_data/main.html b/src/wrapper/java/org/kiwix/testing/small_zimfile_data/main.html new file mode 100644 index 000000000..c12e5739e --- /dev/null +++ b/src/wrapper/java/org/kiwix/testing/small_zimfile_data/main.html @@ -0,0 +1,11 @@ + + + + Test ZIM file + + + + + Test ZIM file + + diff --git a/src/wrapper/java/org/kiwix/testing/test.java b/src/wrapper/java/org/kiwix/testing/test.java index 671b0b46e..788729d12 100644 --- a/src/wrapper/java/org/kiwix/testing/test.java +++ b/src/wrapper/java/org/kiwix/testing/test.java @@ -10,36 +10,143 @@ static { System.loadLibrary("kiwix"); } -private static String getCatalogContent() +private static byte[] getFileContent(String path) throws IOException { - BufferedReader reader = new BufferedReader(new FileReader("catalog.xml")); - String line; - StringBuilder sb = new StringBuilder(); - while ((line = reader.readLine()) != null) - { - sb.append(line + "\n"); - } - reader.close(); - return sb.toString(); + File file = new File(path); + DataInputStream in = new DataInputStream( + new BufferedInputStream( + new FileInputStream(file))); + byte[] data = new byte[(int)file.length()]; + in.read(data); + return data; +} + +private static byte[] getFileContentPartial(String path, int offset, int size) +throws IOException +{ + File file = new File(path); + DataInputStream in = new DataInputStream( + new BufferedInputStream( + new FileInputStream(file))); + byte[] data = new byte[size]; + in.skipBytes(offset); + in.read(data, 0, size); + return data; +} + +private static String getTextFileContent(String path) +throws IOException +{ + return new String(getFileContent(path)); } @Test -public void testSome() +public void testReader() +throws JNIKiwixException, IOException +{ + JNIKiwixReader reader = new JNIKiwixReader("small.zim"); + assertEquals("Test ZIM file", reader.getTitle()); + assertEquals(45, reader.getFileSize()); // The file size is in KiB + assertEquals("A/main.html", reader.getMainPage()); + String s = getTextFileContent("small_zimfile_data/main.html"); + byte[] c = reader.getContent(new JNIKiwixString("A/main.html"), + new JNIKiwixString(), + new JNIKiwixString(), + new JNIKiwixInt()); + assertEquals(s, new String(c)); + + byte[] faviconData = getFileContent("small_zimfile_data/favicon.png"); + assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png")); + c = reader.getContent(new JNIKiwixString("I/favicon.png"), + new JNIKiwixString(), + new JNIKiwixString(), + new JNIKiwixInt()); + assertTrue(Arrays.equals(faviconData, c)); + + DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png"); + assertNotEquals("", dai.filename); + c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length); + assertTrue(Arrays.equals(faviconData, c)); +} + +@Test +public void testReaderByFd() +throws JNIKiwixException, IOException +{ + FileInputStream fis = new FileInputStream("small.zim"); + JNIKiwixReader reader = new JNIKiwixReader(fis.getFD()); + assertEquals("Test ZIM file", reader.getTitle()); + assertEquals(45, reader.getFileSize()); // The file size is in KiB + assertEquals("A/main.html", reader.getMainPage()); + String s = getTextFileContent("small_zimfile_data/main.html"); + byte[] c = reader.getContent(new JNIKiwixString("A/main.html"), + new JNIKiwixString(), + new JNIKiwixString(), + new JNIKiwixInt()); + assertEquals(s, new String(c)); + + byte[] faviconData = getFileContent("small_zimfile_data/favicon.png"); + assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png")); + c = reader.getContent(new JNIKiwixString("I/favicon.png"), + new JNIKiwixString(), + new JNIKiwixString(), + new JNIKiwixInt()); + assertTrue(Arrays.equals(faviconData, c)); + + DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png"); + assertNotEquals("", dai.filename); + c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length); + assertTrue(Arrays.equals(faviconData, c)); +} + +@Test +public void testReaderWithAnEmbeddedArchive() +throws JNIKiwixException, IOException +{ + File plainArchive = new File("small.zim"); + FileInputStream fis = new FileInputStream("small.zim.embedded"); + JNIKiwixReader reader = new JNIKiwixReader(fis.getFD(), 8, plainArchive.length()); + assertEquals("Test ZIM file", reader.getTitle()); + assertEquals(45, reader.getFileSize()); // The file size is in KiB + assertEquals("A/main.html", reader.getMainPage()); + String s = getTextFileContent("small_zimfile_data/main.html"); + byte[] c = reader.getContent(new JNIKiwixString("A/main.html"), + new JNIKiwixString(), + new JNIKiwixString(), + new JNIKiwixInt()); + assertEquals(s, new String(c)); + + byte[] faviconData = getFileContent("small_zimfile_data/favicon.png"); + assertEquals(faviconData.length, reader.getArticleSize("I/favicon.png")); + c = reader.getContent(new JNIKiwixString("I/favicon.png"), + new JNIKiwixString(), + new JNIKiwixString(), + new JNIKiwixInt()); + assertTrue(Arrays.equals(faviconData, c)); + + DirectAccessInfo dai = reader.getDirectAccessInformation("I/favicon.png"); + assertNotEquals("", dai.filename); + c = getFileContentPartial(dai.filename, (int)dai.offset, faviconData.length); + assertTrue(Arrays.equals(faviconData, c)); +} + +@Test +public void testLibrary() throws IOException { Library lib = new Library(); Manager manager = new Manager(lib); - String content = getCatalogContent(); - manager.readOpds(content, "https://library.kiwix.org"); - assertEquals(lib.getBookCount(true, true), 10); + String content = getTextFileContent("catalog.xml"); + manager.readOpds(content, "http://localhost"); + assertEquals(lib.getBookCount(true, true), 1); String[] bookIds = lib.getBooksIds(); - assertEquals(bookIds.length, 10); + assertEquals(bookIds.length, 1); Book book = lib.getBookById(bookIds[0]); - assertEquals(book.getTitle(), "Wikisource"); - assertEquals(book.getTags(), "wikisource;_category:wikisource;_pictures:no;_videos:no;_details:yes;_ftindex:yes"); - assertEquals(book.getFaviconUrl(), "https://library.kiwix.org/meta?name=favicon&content=wikisource_fr_all_nopic_2020-01"); - assertEquals(book.getUrl(), "http://download.kiwix.org/zim/wikisource/wikisource_fr_all_nopic_2020-01.zim.meta4"); + assertEquals(book.getTitle(), "Test ZIM file"); + assertEquals(book.getTags(), "unit;test"); + assertEquals(book.getFaviconUrl(), "http://localhost/meta?name=favicon&content=small"); + assertEquals(book.getUrl(), "http://localhost/small.zim"); } static diff --git a/src/wrapper/java/utils.h b/src/wrapper/java/utils.h index 00b5855a5..232071d54 100644 --- a/src/wrapper/java/utils.h +++ b/src/wrapper/java/utils.h @@ -246,7 +246,7 @@ inline void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env) env->SetIntField(obj, objFid, c2jni(value, env)); } -inline void setPairObjValue(const std::string& filename, const long offset, +inline void setDaiObjValue(const std::string& filename, const long offset, const jobject obj, JNIEnv* env) { jclass objClass = env->GetObjectClass(obj);