packages/lucene4
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Package init

Browse Source
This commit is contained in:
lei_ju 2020-08-19 14:23:46 +08:00
parent 2f9f9da699
commit 4b31ffefd9
10 changed files with 1985 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
0001-SOLR-11477-Disallow-resolving-of-external-entities-i.patch Normal file
View File

@ -0,0 +1,119 @@
From af208b7b3d55c6af5688132e206d73465e61104c Mon Sep 17 00:00:00 2001
From: Christine Poerschke <cpoerschke@apache.org>
Date: Fri, 13 Oct 2017 12:46:58 +0100
Subject: [PATCH] SOLR-11477: Disallow resolving of external entities in Lucene
---
.../apache/lucene/queryparser/xml/CoreParser.java | 65 +++++++++++++++++++---
1 file changed, 56 insertions(+), 9 deletions(-)
diff --git a/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java b/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
index 81c6b36..4e70aef 100644
--- a/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
+++ b/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java
@@ -6,10 +6,18 @@ import org.apache.lucene.queryparser.xml.builders.*;
import org.apache.lucene.search.Query;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
import java.io.InputStream;
+import java.util.Locale;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -117,6 +125,10 @@ public class CoreParser implements QueryBuilder {
queryFactory.addBuilder("SpanNot", snot);
}
+ /**
+ * Parses the given stream as XML file and returns a {@link Query}.
+ * By default this disallows external entities for security reasons.
+ */
public Query parse(InputStream xmlStream) throws ParserException {
return getQuery(parseXML(xmlStream).getDocumentElement());
}
@@ -129,23 +141,48 @@ public class CoreParser implements QueryBuilder {
filterFactory.addBuilder(nodeName, builder);
}
- private static Document parseXML(InputStream pXmlFile) throws ParserException {
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder db = null;
+ /**
+ * Returns a SAX {@link EntityResolver} to be used by {@link DocumentBuilder}.
+ * By default this returns {@link #DISALLOW_EXTERNAL_ENTITY_RESOLVER}, which disallows the
+ * expansion of external entities (for security reasons). To restore legacy behavior,
+ * override this method to return {@code null}.
+ */
+ protected EntityResolver getEntityResolver() {
+ return DISALLOW_EXTERNAL_ENTITY_RESOLVER;
+ }
+
+ /**
+ * Subclass and override to return a SAX {@link ErrorHandler} to be used by {@link DocumentBuilder}.
+ * By default this returns {@code null} so no error handler is used.
+ * This method can be used to redirect XML parse errors/warnings to a custom logger.
+ */
+ protected ErrorHandler getErrorHandler() {
+ return null;
+ }
+
+ private Document parseXML(InputStream pXmlFile) throws ParserException {
+ final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setValidating(false);
try {
- db = dbf.newDocumentBuilder();
+ dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ } catch (ParserConfigurationException e) {
+ // ignore since all implementations are required to support the
+ // {@link javax.xml.XMLConstants#FEATURE_SECURE_PROCESSING} feature
}
- catch (Exception se) {
- throw new ParserException("XML Parser configuration error", se);
+ final DocumentBuilder db;
+ try {
+ db = dbf.newDocumentBuilder();
+ } catch (Exception se) {
+ throw new ParserException("XML Parser configuration error.", se);
}
- org.w3c.dom.Document doc = null;
try {
- doc = db.parse(pXmlFile);
+ db.setEntityResolver(getEntityResolver());
+ db.setErrorHandler(getErrorHandler());
+ return db.parse(pXmlFile);
}
catch (Exception se) {
throw new ParserException("Error parsing XML stream:" + se, se);
}
- return doc;
}
@@ -153,4 +190,14 @@ public class CoreParser implements QueryBuilder {
public Query getQuery(Element e) throws ParserException {
return queryFactory.getQuery(e);
}
+
+ public static final EntityResolver DISALLOW_EXTERNAL_ENTITY_RESOLVER = new EntityResolver() {
+ @Override
+ public InputSource resolveEntity(String publicId, String systemId) throws SAXException {
+ throw new SAXException(String.format(Locale.ENGLISH,
+ "External Entity resolving unsupported: publicId=\"%s\" systemId=\"%s\"",
+ publicId, systemId));
+ }
+ };
+
}
--
2.13.6

74
0001-dependency-generation.patch Normal file
View File

@ -0,0 +1,74 @@
From 8d6fc5deeea8e870e3fdcb798a01940e48a0cbe3 Mon Sep 17 00:00:00 2001
From: rpm-build <rpm-build>
Date: Fri, 2 May 2014 11:07:23 +0200
Subject: [PATCH] dependency generation
Signed-off-by: rpm-build <rpm-build>
---
build.xml | 2 +-
common-build.xml | 7 +++----
.../org/apache/lucene/dependencies/GetMavenDependenciesTask.java | 2 ++
3 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/build.xml b/build.xml
index 3237141..28fffe3 100644
--- a/build.xml
+++ b/build.xml
@@ -497,7 +497,7 @@
<target name="generate-maven-artifacts" depends="-unpack-lucene-tgz">
<sequential>
<ant dir=".." target="resolve" inheritall="false"/>
- <antcall target="-filter-pom-templates" inheritall="false"/>
+ <antcall target="filter-pom-templates" inheritall="false"/>
<antcall target="-dist-maven" inheritall="false"/>
</sequential>
</target>
diff --git a/common-build.xml b/common-build.xml
index 7068fe6..b6d4b9f 100644
--- a/common-build.xml
+++ b/common-build.xml
@@ -1555,10 +1555,9 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
<property name="maven.dependencies.filters.file" location="${common.build.dir}/maven.dependencies.filters.properties"/>
- <target name="-get-maven-dependencies" depends="compile-tools,load-custom-tasks">
- <ant dir="${common.dir}/.." target="-append-all-modules-dependencies-properties" inheritall="false"/>
+ <target name="-get-maven-dependencies" depends="compile-tools,load-custom-tasks,-append-module-dependencies-properties">
<get-maven-dependencies-macro
- dir="${common.dir}/.."
+ dir="."
centralized.versions.file="${common.dir}/ivy-versions.properties"
module.dependencies.properties.file="${module.dependencies.properties.file}"
maven.dependencies.filters.file="${maven.dependencies.filters.file}"/>
@@ -1578,7 +1577,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
</copy>
</target>
- <target name="-filter-pom-templates" depends="-get-maven-dependencies">
+ <target name="filter-pom-templates" depends="-get-maven-dependencies">
<mkdir dir="${filtered.pom.templates.dir}"/>
<copy todir="${common.dir}/build/poms" overwrite="true" encoding="UTF-8" filtering="on">
<fileset dir="${common.dir}/../dev-tools/maven"/>
diff --git a/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java b/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
index 2069c7d..d891bc5 100644
--- a/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
+++ b/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
@@ -481,6 +481,7 @@ public class GetMavenDependenciesTask extends Task {
private Collection<String> getTransitiveDependenciesFromIvyCache
(String groupId, String artifactId, String version) {
SortedSet<String> transitiveDependencies = new TreeSet<>();
+ /*
// E.g. ~/.ivy2/cache/xerces/xercesImpl/ivy-2.9.1.xml
File ivyXmlFile = new File(new File(new File(ivyCacheDir, groupId), artifactId), "ivy-" + version + ".xml");
if ( ! ivyXmlFile.exists()) {
@@ -502,6 +503,7 @@ public class GetMavenDependenciesTask extends Task {
+ groupId + ':' + artifactId + ':' + version + " from "
+ ivyXmlFile.getAbsolutePath(), e);
}
+ */
return transitiveDependencies;
}
--
1.9.0

25
0001-disable-ivy-settings.patch Normal file
View File

@ -0,0 +1,25 @@
From 161ffbc7a22ed1941bf897a126b80ca6a7fbf3ca Mon Sep 17 00:00:00 2001
From: Michael Simacek <msimacek@redhat.com>
Date: Thu, 16 Jan 2014 16:27:10 +0100
Subject: [PATCH] disable ivy settings
Signed-off-by: Michael Simacek <msimacek@redhat.com>
---
common-build.xml | 1 -
1 file changed, 1 deletion(-)
diff --git a/common-build.xml b/common-build.xml
index f4720a8..fa7aee5 100644
--- a/common-build.xml
+++ b/common-build.xml
@@ -350,7 +350,6 @@
you have an idea, fix it.
unless="ivy.settings.uptodate" -->
<!-- override: just for safety, should be unnecessary -->
- <ivy:configure file="${common.dir}/ivy-settings.xml" override="true"/>
<!-- <property name="ivy.settings.uptodate" value="true"/> -->
</target>
--
1.8.4.2

382
RandomInts-oe.patch Normal file
View File

@ -0,0 +1,382 @@
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java 2014-02-19 13:42:17.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java 2020-06-09 01:50:25.176680050 +0000
@@ -26,7 +26,7 @@
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
public abstract class AbstractTestCompressionMode extends LuceneTestCase {
@@ -45,7 +45,7 @@
static byte[] randomArray(int length, int max) {
final byte[] arr = new byte[length];
for (int i = 0; i < arr.length; ++i) {
- arr[i] = (byte) RandomInts.randomIntBetween(random(), 0, max);
+ arr[i] = (byte) RandomNumbers.randomIntBetween(random(), 0, max);
}
return arr;
}
@@ -130,7 +130,7 @@
}
public void testIncompressible() throws IOException {
- final byte[] decompressed = new byte[RandomInts.randomIntBetween(random(), 20, 256)];
+ final byte[] decompressed = new byte[RandomNumbers.randomIntBetween(random(), 20, 256)];
for (int i = 0; i < decompressed.length; ++i) {
decompressed[i] = (byte) i;
}
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java 2014-03-31 12:29:44.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java 2020-06-09 01:50:25.172679991 +0000
@@ -20,7 +20,7 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
public abstract class AbstractTestLZ4CompressionMode extends AbstractTestCompressionMode {
@@ -88,7 +88,7 @@
public void testLongMatchs() throws IOException {
// match length >= 20
- final byte[] decompressed = new byte[RandomInts.randomIntBetween(random(), 300, 1024)];
+ final byte[] decompressed = new byte[RandomNumbers.randomIntBetween(random(), 300, 1024)];
for (int i = 0; i < decompressed.length; ++i) {
decompressed[i] = (byte) i;
}
@@ -97,10 +97,10 @@
public void testLongLiterals() throws IOException {
// long literals (length >= 16) which are not the last literals
- final byte[] decompressed = randomArray(RandomInts.randomIntBetween(random(), 400, 1024), 256);
+ final byte[] decompressed = randomArray(RandomNumbers.randomIntBetween(random(), 400, 1024), 256);
final int matchRef = random().nextInt(30);
- final int matchOff = RandomInts.randomIntBetween(random(), decompressed.length - 40, decompressed.length - 20);
- final int matchLength = RandomInts.randomIntBetween(random(), 4, 10);
+ final int matchOff = RandomNumbers.randomIntBetween(random(), decompressed.length - 40, decompressed.length - 20);
+ final int matchLength = RandomNumbers.randomIntBetween(random(), 4, 10);
System.arraycopy(decompressed, matchRef, decompressed, matchOff, matchLength);
test(decompressed);
}
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java 2014-09-19 13:24:09.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java 2020-06-09 01:50:25.176680050 +0000
@@ -34,7 +34,7 @@
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
@Repeat(iterations=5) // give it a chance to test various compression modes with different chunk sizes
public class TestCompressingStoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
@@ -52,7 +52,7 @@
((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
}
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
- iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
iwConf.setCodec(CompressingCodec.randomInstance(random()));
// disable CFS because this test checks file names
iwConf.setMergePolicy(newLogMergePolicy(false));
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/codecs/lucene41/TestForUtil.java lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/lucene41/TestForUtil.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/codecs/lucene41/TestForUtil.java 2012-10-12 02:00:19.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/codecs/lucene41/TestForUtil.java 2020-06-09 01:50:25.172679991 +0000
@@ -32,24 +32,24 @@
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.packed.PackedInts;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
public class TestForUtil extends LuceneTestCase {
public void testEncodeDecode() throws IOException {
- final int iterations = RandomInts.randomIntBetween(random(), 1, 1000);
+ final int iterations = RandomNumbers.randomIntBetween(random(), 1, 1000);
final float acceptableOverheadRatio = random().nextFloat();
final int[] values = new int[(iterations - 1) * BLOCK_SIZE + ForUtil.MAX_DATA_SIZE];
for (int i = 0; i < iterations; ++i) {
final int bpv = random().nextInt(32);
if (bpv == 0) {
- final int value = RandomInts.randomIntBetween(random(), 0, Integer.MAX_VALUE);
+ final int value = RandomNumbers.randomIntBetween(random(), 0, Integer.MAX_VALUE);
for (int j = 0; j < BLOCK_SIZE; ++j) {
values[i * BLOCK_SIZE + j] = value;
}
} else {
for (int j = 0; j < BLOCK_SIZE; ++j) {
- values[i * BLOCK_SIZE + j] = RandomInts.randomIntBetween(random(),
+ values[i * BLOCK_SIZE + j] = RandomNumbers.randomIntBetween(random(),
0, (int) PackedInts.maxValue(bpv));
}
}
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java lucene-4.10.4.new/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java 2014-04-05 10:09:42.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java 2020-06-09 01:50:25.176680050 +0000
@@ -31,7 +31,7 @@
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
/**
* This test creates an index with one segment that is a little larger than 4GB.
@@ -64,7 +64,7 @@
ft.setIndexed(false);
ft.setStored(true);
ft.freeze();
- final int valueLength = RandomInts.randomIntBetween(random(), 1 << 13, 1 << 20);
+ final int valueLength = RandomNumbers.randomIntBetween(random(), 1 << 13, 1 << 20);
final byte[] value = new byte[valueLength];
for (int i = 0; i < valueLength; ++i) {
// random so that even compressing codecs can't compress it
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/util/automaton/TestOperations.java lucene-4.10.4.new/core/src/test/org/apache/lucene/util/automaton/TestOperations.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/util/automaton/TestOperations.java 2014-11-04 22:29:27.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/util/automaton/TestOperations.java 2020-06-09 01:50:25.176680050 +0000
@@ -22,7 +22,7 @@
import org.apache.lucene.util.*;
import org.apache.lucene.util.fst.Util;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
@@ -30,7 +30,7 @@
/** Test string union. */
public void testStringUnion() {
List<BytesRef> strings = new ArrayList<>();
- for (int i = RandomInts.randomIntBetween(random(), 0, 1000); --i >= 0;) {
+ for (int i = RandomNumbers.randomIntBetween(random(), 0, 1000); --i >= 0;) {
strings.add(new BytesRef(TestUtil.randomUnicodeString(random())));
}
diff -urN lucene-4.10.4/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java lucene-4.10.4.new/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
--- lucene-4.10.4/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java 2014-07-01 14:59:52.000000000 +0000
+++ lucene-4.10.4.new/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java 2020-06-09 01:50:25.180680109 +0000
@@ -43,7 +43,7 @@
import org.apache.lucene.util.packed.PackedInts.Reader;
import org.junit.Ignore;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
@Slow
public class TestPackedInts extends LuceneTestCase {
@@ -51,7 +51,7 @@
public void testByteCount() {
final int iters = atLeast(3);
for (int i = 0; i < iters; ++i) {
- final int valueCount = RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE);
+ final int valueCount = RandomNumbers.randomIntBetween(random(), 1, Integer.MAX_VALUE);
for (PackedInts.Format format : PackedInts.Format.values()) {
for (int bpv = 1; bpv <= 64; ++bpv) {
final long byteCount = format.byteCount(PackedInts.VERSION_CURRENT, valueCount, bpv);
@@ -208,7 +208,7 @@
public void testEndPointer() throws IOException {
final Directory dir = newDirectory();
- final int valueCount = RandomInts.randomIntBetween(random(), 1, 1000);
+ final int valueCount = RandomNumbers.randomIntBetween(random(), 1, 1000);
final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random()));
for (int i = 0; i < valueCount; ++i) {
out.writeLong(0);
@@ -226,7 +226,7 @@
// test iterator
in.seek(0L);
- final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomInts.randomIntBetween(random(), 1, 1<<16));
+ final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomNumbers.randomIntBetween(random(), 1, 1<<16));
for (int i = 0; i < valueCount; ++i) {
it.next();
}
@@ -983,9 +983,9 @@
}
public void testPackedLongValues() {
- final long[] arr = new long[RandomInts.randomIntBetween(random(), 1, 1000000)];
+ final long[] arr = new long[RandomNumbers.randomIntBetween(random(), 1, 1000000)];
float[] ratioOptions = new float[]{PackedInts.DEFAULT, PackedInts.COMPACT, PackedInts.FAST};
- for (int bpv : new int[]{0, 1, 63, 64, RandomInts.randomIntBetween(random(), 2, 62)}) {
+ for (int bpv : new int[]{0, 1, 63, 64, RandomNumbers.randomIntBetween(random(), 2, 62)}) {
for (DataType dataType : Arrays.asList(DataType.DELTA_PACKED)) {
final int pageSize = 1 << TestUtil.nextInt(random(), 6, 20);
float acceptableOverheadRatio = ratioOptions[TestUtil.nextInt(random(), 0, ratioOptions.length - 1)];
@@ -1068,7 +1068,7 @@
final int[] bitsPerValues = new int[longs.length];
final boolean[] skip = new boolean[longs.length];
for (int i = 0; i < longs.length; ++i) {
- final int bpv = RandomInts.randomIntBetween(random(), 1, 64);
+ final int bpv = RandomNumbers.randomIntBetween(random(), 1, 64);
bitsPerValues[i] = random().nextBoolean() ? bpv : TestUtil.nextInt(random(), bpv, 64);
if (bpv == 64) {
longs[i] = random().nextLong();
diff -urN lucene-4.10.4/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java lucene-4.10.4.new/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
--- lucene-4.10.4/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java 2014-08-14 09:15:35.000000000 +0000
+++ lucene-4.10.4.new/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java 2020-06-09 01:50:25.180680109 +0000
@@ -39,7 +39,7 @@
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.packed.PackedInts;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
@SuppressCodecs("Lucene3x")
public class TestDocValuesFieldSources extends LuceneTestCase {
@@ -81,7 +81,7 @@
f.setBytesValue(new BytesRef((String) vals[i]));
break;
case NUMERIC:
- final int bitsPerValue = RandomInts.randomIntBetween(random(), 1, 31); // keep it an int
+ final int bitsPerValue = RandomNumbers.randomIntBetween(random(), 1, 31); // keep it an int
vals[i] = (long) random().nextInt((int) PackedInts.maxValue(bitsPerValue));
f.setLongValue((Long) vals[i]);
break;
diff -urN lucene-4.10.4/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java lucene-4.10.4.new/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
--- lucene-4.10.4/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java 2014-08-14 16:15:33.000000000 +0000
+++ lucene-4.10.4.new/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java 2020-06-09 01:50:25.180680109 +0000
@@ -25,7 +25,7 @@
import org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec;
import org.apache.lucene.codecs.lucene410.Lucene410Codec;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
/**
* A codec that uses {@link CompressingStoredFieldsFormat} for its stored
@@ -56,14 +56,14 @@
* suffix
*/
public static CompressingCodec randomInstance(Random random) {
- return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), false);
+ return randomInstance(random, RandomNumbers.randomIntBetween(random, 1, 500), false);
}
/**
* Creates a random {@link CompressingCodec} that is using a segment suffix
*/
public static CompressingCodec randomInstance(Random random, boolean withSegmentSuffix) {
- return randomInstance(random, RandomInts.randomIntBetween(random, 1, 500), withSegmentSuffix);
+ return randomInstance(random, RandomNumbers.randomIntBetween(random, 1, 500), withSegmentSuffix);
}
private final CompressingStoredFieldsFormat storedFieldsFormat;
diff -urN lucene-4.10.4/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java lucene-4.10.4.new/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
--- lucene-4.10.4/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java 2014-08-14 16:15:33.000000000 +0000
+++ lucene-4.10.4.new/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java 2020-06-09 01:50:25.184680168 +0000
@@ -59,7 +59,7 @@
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TestUtil;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
/**
@@ -330,7 +330,7 @@
public void testReadSkip() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
- iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
FieldType ft = new FieldType();
@@ -383,7 +383,7 @@
public void testEmptyDocs() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
- iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
// make sure that the fact that documents might be empty is not a problem
@@ -408,7 +408,7 @@
public void testConcurrentReads() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
- iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
// make sure the readers are properly cloned
@@ -496,15 +496,15 @@
}
Directory dir = newDirectory();
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
- iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
final int docCount = atLeast(200);
final byte[][][] data = new byte [docCount][][];
for (int i = 0; i < docCount; ++i) {
final int fieldCount = rarely()
- ? RandomInts.randomIntBetween(random(), 1, 500)
- : RandomInts.randomIntBetween(random(), 1, 5);
+ ? RandomNumbers.randomIntBetween(random(), 1, 500)
+ : RandomNumbers.randomIntBetween(random(), 1, 5);
data[i] = new byte[fieldCount][];
for (int j = 0; j < fieldCount; ++j) {
final int length = rarely()
@@ -589,7 +589,7 @@
// so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("testBigDocuments")));
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
- iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
if (dir instanceof MockDirectoryWrapper) {
@@ -609,12 +609,12 @@
onlyStored.setIndexed(false);
final Field smallField = new Field("fld", randomByteArray(random().nextInt(10), 256), onlyStored);
- final int numFields = RandomInts.randomIntBetween(random(), 500000, 1000000);
+ final int numFields = RandomNumbers.randomIntBetween(random(), 500000, 1000000);
for (int i = 0; i < numFields; ++i) {
bigDoc1.add(smallField);
}
- final Field bigField = new Field("fld", randomByteArray(RandomInts.randomIntBetween(random(), 1000000, 5000000), 2), onlyStored);
+ final Field bigField = new Field("fld", randomByteArray(RandomNumbers.randomIntBetween(random(), 1000000, 5000000), 2), onlyStored);
bigDoc2.add(bigField);
final int numDocs = atLeast(5);
diff -urN lucene-4.10.4/test-framework/src/java/org/apache/lucene/util/TestUtil.java lucene-4.10.4.new/test-framework/src/java/org/apache/lucene/util/TestUtil.java
--- lucene-4.10.4/test-framework/src/java/org/apache/lucene/util/TestUtil.java 2014-08-14 16:15:33.000000000 +0000
+++ lucene-4.10.4.new/test-framework/src/java/org/apache/lucene/util/TestUtil.java 2020-06-09 01:50:25.188680226 +0000
@@ -88,7 +88,7 @@
import org.apache.lucene.store.Directory;
import org.junit.Assert;
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
/**
@@ -243,7 +243,7 @@
/** start and end are BOTH inclusive */
public static int nextInt(Random r, int start, int end) {
- return RandomInts.randomIntBetween(r, start, end);
+ return RandomNumbers.randomIntBetween(r, start, end);
}
/** start and end are BOTH inclusive */
@@ -384,7 +384,7 @@
final StringBuilder regexp = new StringBuilder(maxLength);
for (int i = nextInt(r, 0, maxLength); i > 0; i--) {
if (r.nextBoolean()) {
- regexp.append((char) RandomInts.randomIntBetween(r, 'a', 'z'));
+ regexp.append((char) RandomNumbers.randomIntBetween(r, 'a', 'z'));
} else {
regexp.append(RandomPicks.randomFrom(r, ops));
}

BIN
dev-tools-4.10.4.tar.xz Normal file
View File

Binary file not shown.

179
lucene-4.10.4-morfologik-stemming.patch Normal file
View File

@ -0,0 +1,179 @@
diff -Nru lucene-4.10.4/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java lucene-4.10.4.morfologik-stemming/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java
--- lucene-4.10.4/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java 2014-08-21 05:12:52.000000000 +0200
+++ lucene-4.10.4.morfologik-stemming/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java 2016-04-30 01:39:55.894913112 +0200
@@ -20,6 +20,9 @@
import java.io.Reader;
+import morfologik.stemming.Dictionary;
+import morfologik.stemming.polish.PolishStemmer;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
@@ -31,7 +34,7 @@
* @see <a href="http://morfologik.blogspot.com/">Morfologik project page</a>
*/
public class MorfologikAnalyzer extends Analyzer {
- private final String dictionary;
+ private final Dictionary dictionary;
/**
* Builds an analyzer with an explicit dictionary resource.
@@ -43,32 +46,15 @@
*
* @see "http://morfologik.blogspot.com/"
*/
- public MorfologikAnalyzer(final String dictionaryResource) {
- this.dictionary = dictionaryResource;
- }
-
- /**
- * @deprecated Use {@link #MorfologikAnalyzer(String)}
- */
- @Deprecated
- public MorfologikAnalyzer(final Version version, final String dictionaryResource) {
- setVersion(version);
- this.dictionary = dictionaryResource;
+ public MorfologikAnalyzer(final Dictionary dictionary) {
+ this.dictionary = dictionary;
}
/**
* Builds an analyzer with the default Morfologik's Polish dictionary.
*/
public MorfologikAnalyzer() {
- this(MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE);
- }
-
- /**
- * @deprecated Use {@link #MorfologikAnalyzer()}
- */
- @Deprecated
- public MorfologikAnalyzer(final Version version) {
- this(version, MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE);
+ this(new PolishStemmer().getDictionary());
}
/**
@@ -88,6 +74,6 @@
return new TokenStreamComponents(
src,
- new MorfologikFilter(new StandardFilter(getVersion(), src), dictionary, getVersion()));
+ new MorfologikFilter(new StandardFilter(src), dictionary));
}
}
diff -Nru lucene-4.10.4/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java lucene-4.10.4.morfologik-stemming/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java
--- lucene-4.10.4/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java 2014-08-21 05:12:52.000000000 +0200
+++ lucene-4.10.4.morfologik-stemming/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java 2016-04-30 01:08:24.560899492 +0200
@@ -18,8 +18,13 @@
*/
import java.util.Map;
+import java.util.Objects;
import java.util.logging.Logger;
+import morfologik.stemming.Dictionary;
+import morfologik.stemming.DictionaryMetadata;
+import morfologik.stemming.polish.PolishStemmer;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -48,6 +53,9 @@
*/
private final String dictionaryResource;
+ /** Loaded {@link Dictionary}, initialized on {@link #inform(ResourceLoader)}. */
+ private Dictionary dictionary;
+
/** Schema attribute. */
@Deprecated
public static final String DICTIONARY_SCHEMA_ATTRIBUTE = "dictionary";
@@ -79,6 +87,6 @@
@Override
public TokenStream create(TokenStream ts) {
- return new MorfologikFilter(ts, dictionaryResource);
+ return new MorfologikFilter(ts, Objects.requireNonNull(dictionary, "MorfologikFilterFactory was not fully initialized."));
}
}
diff -Nru lucene-4.10.4/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java lucene-4.10.4.morfologik-stemming/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java
--- lucene-4.10.4/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java 2014-08-21 05:12:52.000000000 +0200
+++ lucene-4.10.4.morfologik-stemming/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java 2016-04-30 01:25:55.949415627 +0200
@@ -22,7 +22,11 @@
import java.util.*;
import java.util.regex.Pattern;
-import morfologik.stemming.*;
+import morfologik.stemming.Dictionary;
+import morfologik.stemming.DictionaryLookup;
+import morfologik.stemming.IStemmer;
+import morfologik.stemming.WordData;
+import morfologik.stemming.polish.PolishStemmer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -49,7 +53,7 @@
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
private final CharsRefBuilder scratch = new CharsRefBuilder();
- private final CharacterUtils charUtils;
+ private final CharacterUtils charUtils = CharacterUtils.getInstance();
private State current;
private final TokenStream input;
@@ -64,46 +68,20 @@
* Creates a filter with the default (Polish) dictionary.
*/
public MorfologikFilter(final TokenStream in) {
- this(in, MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE);
+ this(in, new PolishStemmer().getDictionary());
}
/**
- * @deprecated Use {@link #MorfologikFilter(TokenStream)}
- */
- @Deprecated
- public MorfologikFilter(final TokenStream in, final Version version) {
- this(in, MorfologikFilterFactory.DEFAULT_DICTIONARY_RESOURCE, version);
- }
-
- /**
- * Creates a filter with a given dictionary resource.
+ * Creates a filter with a given dictionary.
*
* @param in input token stream.
- * @param dict Dictionary resource from classpath.
+ * @param dict Dictionary to use for stemming.
*/
- public MorfologikFilter(final TokenStream in, final String dict) {
- this(in, dict, Version.LATEST);
- }
-
- /**
- * @deprecated Use {@link #MorfologikFilter(TokenStream,String)}
- */
- @Deprecated
- public MorfologikFilter(final TokenStream in, final String dict, final Version version) {
+ public MorfologikFilter(final TokenStream in, final Dictionary dict) {
super(in);
this.input = in;
-
- // SOLR-4007: temporarily substitute context class loader to allow finding dictionary resources.
- Thread me = Thread.currentThread();
- ClassLoader cl = me.getContextClassLoader();
- try {
- me.setContextClassLoader(morfologik.stemming.Dictionary.class.getClassLoader());
- this.stemmer = new DictionaryLookup(morfologik.stemming.Dictionary.getForLanguage(dict));
- this.charUtils = CharacterUtils.getInstance(version);
- this.lemmaList = Collections.emptyList();
- } finally {
- me.setContextClassLoader(cl);
- }
+ this.stemmer = new DictionaryLookup(dict);
+ this.lemmaList = Collections.emptyList();
}
/**

BIN
lucene-4.10.4-src.tgz Normal file
View File

Binary file not shown.

101
lucene4.spec Normal file
View File

@ -0,0 +1,101 @@
Summary: High-performance, full-featured text search engine
Name: lucene4
Version: 4.10.4
Release: 1
Epoch: 0
License: ASL 2.0
URL: http://lucene.apache.org/
Source0: http://archive.apache.org/dist/lucene/java/%{version}/lucene-%{version}-src.tgz
# svn export http://svn.apache.org/repos/asf/lucene/dev/tags/lucene_solr_4_10_4dev-tools/
# tar caf dev-tools-4.10.4.tar.xz dev-tools/
Source1: dev-tools-%{version}.tar.xz
Patch0: 0001-disable-ivy-settings.patch
Patch1: 0001-dependency-generation.patch
Patch2: lucene-4.10.4-morfologik-stemming.patch
Patch3: 0001-SOLR-11477-Disallow-resolving-of-external-entities-i.patch
Patch4: RandomInts-oe.patch
Patch5: spatial4j-oe.patch
BuildRequires: git subversion ant ivy-local apache-ivy icu4j httpcomponents-client
BuildRequires: jetty-continuation jetty-http jetty-io jetty-server jetty-servlet jetty-util
BuildRequires: morfologik-stemming uimaj uima-addons spatial4j nekohtml xerces-j2
BuildRequires: mvn(javax.servlet:javax.servlet-api) mvn(org.antlr:antlr-runtime) maven-local
BuildRequires: apache-parent buildnumber-maven-plugin maven-plugin-bundle regexp
BuildRequires: junit randomizedtesting-junit4-ant randomizedtesting-runner
Provides: %{name}-core = %{epoch}:%{version}-%{release}
BuildArch: noarch
%description
Apache Lucene is a high-performance, full-featured text search
engine library written entirely in Java. It is a technology suitable
for nearly any application that requires full-text search, especially
cross-platform.
%package javadoc
Summary: Javadoc for Lucene
%description javadoc
%{summary}.
%prep
%setup -q -n lucene-%{version}
mkdir lucene
find -maxdepth 1 \
! -name CHANGES.txt ! -name LICENSE.txt ! -name README.txt \
! -name NOTICE.txt ! -name MIGRATE.txt ! -name ivy-settings.xml \
! -path ./lucene -exec mv \{} lucene/ \;
tar xf %{SOURCE1}
pushd lucene
%patch0 -p1
%patch1 -p1
%patch3 -p1
%patch4 -p1
%patch5 -p1
find . -name "*.jar" -delete
rm sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestJakartaRegexpCapabilities.java
rm -r replicator/src/test/*
rm -r analysis/common/src/test/*
ln -s %{_sysconfdir}/ivy/ivysettings.xml
popd
%patch2 -p1
sed -i -e '/Export-Package/a<Import-Package>org.apache.lucene*;version="[${project.version},5.0.0)",org.tartarus*;version="[${project.version},5.0.0)",*</Import-Package>' \
dev-tools/maven/pom.xml.template
%mvn_alias :lucene-suggest :lucene-spellchecker
%mvn_alias :lucene-analyzers-common :lucene-analyzers
%mvn_compat_version : 4 %{version}
%build
pushd lucene
ant filter-pom-templates -Divy.mode=local -Dversion=%{version}
for pom in `find build/poms/lucene -name pom.xml`; do
sed 's/\${module-path}/${basedir}/g' "$pom" > "${pom##build/poms/lucene/}"
done
%pom_disable_module src/test core
%pom_disable_module src/test codecs
%pom_add_dep org.ow2.asm:asm::test demo
%pom_add_dep org.ow2.asm:asm-commons::test demo
%pom_add_dep org.antlr:antlr-runtime::test demo
popd
mv lucene/build/poms/pom.xml .
%pom_disable_module solr
%pom_remove_plugin :gmaven-plugin
%pom_remove_plugin -r :forbiddenapis
%pom_remove_dep org.eclipse.jetty.orbit:javax.servlet
%pom_change_dep org.eclipse.jetty.orbit:javax.servlet javax.servlet:javax.servlet-api:3.1.0 lucene/replicator
%pom_change_dep -r :servlet-api javax.servlet:javax.servlet-api:3.1.0
%pom_remove_plugin -r :maven-enforcer-plugin
%mvn_build -f
%install
%mvn_install
%files -f .mfiles
%doc CHANGES.txt README.txt MIGRATE.txt
%license LICENSE.txt NOTICE.txt
%files javadoc -f .mfiles-javadoc
%license LICENSE.txt NOTICE.txt
%changelog
* Thu Jul 30 2020 leiju <leiju4@huawei.com> - 4.10.4-1
- Package init

4
lucene4.yaml Normal file
View File

@ -0,0 +1,4 @@
version_control: NA
src_repo: NA
tag_prefix: NA
seperator: NA

1101
spatial4j-oe.patch Normal file
View File

File diff suppressed because it is too large Load Diff